1 /*
2 * Copyright (c) 2013-2021 The Linux Foundation. All rights reserved.
3 * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved.
4 *
5 * Permission to use, copy, modify, and/or distribute this software for
6 * any purpose with or without fee is hereby granted, provided that the
7 * above copyright notice and this permission notice appear in all
8 * copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
11 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
12 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
13 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
14 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
15 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
16 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
17 * PERFORMANCE OF THIS SOFTWARE.
18 */
19
20 #include "hif.h"
21 #include "hif_io32.h"
22 #include "ce_api.h"
23 #include "ce_main.h"
24 #include "ce_internal.h"
25 #include "ce_reg.h"
26 #include "qdf_lock.h"
27 #include "regtable.h"
28 #include "hif_main.h"
29 #include "hif_debug.h"
30 #include "hif_napi.h"
31 #include "qdf_module.h"
32 #include <qdf_tracepoint.h>
33
34 #ifdef IPA_OFFLOAD
35 #ifdef QCA_WIFI_3_0
36 #define CE_IPA_RING_INIT(ce_desc) \
37 do { \
38 ce_desc->gather = 0; \
39 ce_desc->enable_11h = 0; \
40 ce_desc->meta_data_low = 0; \
41 ce_desc->packet_result_offset = 64; \
42 ce_desc->toeplitz_hash_enable = 0; \
43 ce_desc->addr_y_search_disable = 0; \
44 ce_desc->addr_x_search_disable = 0; \
45 ce_desc->misc_int_disable = 0; \
46 ce_desc->target_int_disable = 0; \
47 ce_desc->host_int_disable = 0; \
48 ce_desc->dest_byte_swap = 0; \
49 ce_desc->byte_swap = 0; \
50 ce_desc->type = 2; \
51 ce_desc->tx_classify = 1; \
52 ce_desc->buffer_addr_hi = 0; \
53 ce_desc->meta_data = 0; \
54 ce_desc->nbytes = 128; \
55 } while (0)
56 #else
57 #define CE_IPA_RING_INIT(ce_desc) \
58 do { \
59 ce_desc->byte_swap = 0; \
60 ce_desc->nbytes = 60; \
61 ce_desc->gather = 0; \
62 } while (0)
63 #endif /* QCA_WIFI_3_0 */
64 #endif /* IPA_OFFLOAD */
65
66 static int war1_allow_sleep;
67 /* io32 write workaround */
68 static int hif_ce_war1;
69
70 /**
71 * hif_ce_war_disable() - disable ce war gobally
72 */
hif_ce_war_disable(void)73 void hif_ce_war_disable(void)
74 {
75 hif_ce_war1 = 0;
76 }
77
78 /**
79 * hif_ce_war_enable() - enable ce war gobally
80 */
hif_ce_war_enable(void)81 void hif_ce_war_enable(void)
82 {
83 hif_ce_war1 = 1;
84 }
85
86 /*
87 * Note: For MCL, #if defined (HIF_CONFIG_SLUB_DEBUG_ON) needs to be checked
88 * for defined here
89 */
90 #if defined(HIF_CONFIG_SLUB_DEBUG_ON) || defined(HIF_CE_DEBUG_DATA_BUF)
91
92 #define CE_DEBUG_PRINT_BUF_SIZE(x) (((x) * 3) - 1)
93 #define CE_DEBUG_DATA_PER_ROW 16
94
95 static const char *ce_event_type_to_str(enum hif_ce_event_type type);
96
get_next_record_index(qdf_atomic_t * table_index,int array_size)97 int get_next_record_index(qdf_atomic_t *table_index, int array_size)
98 {
99 int record_index = qdf_atomic_inc_return(table_index);
100
101 if (record_index == array_size)
102 qdf_atomic_sub(array_size, table_index);
103
104 while (record_index >= array_size)
105 record_index -= array_size;
106
107 return record_index;
108 }
109
110 qdf_export_symbol(get_next_record_index);
111
112 #ifdef HIF_CE_DEBUG_DATA_BUF
hif_ce_desc_data_record(struct hif_ce_desc_event * event,int len)113 void hif_ce_desc_data_record(struct hif_ce_desc_event *event, int len)
114 {
115 uint8_t *data = NULL;
116
117 if (!event->data) {
118 hif_err_rl("No ce debug memory allocated");
119 return;
120 }
121
122 if (event->memory && len > 0)
123 data = qdf_nbuf_data((qdf_nbuf_t)event->memory);
124
125 event->actual_data_len = 0;
126 qdf_mem_zero(event->data, CE_DEBUG_MAX_DATA_BUF_SIZE);
127
128 if (data && len > 0) {
129 qdf_mem_copy(event->data, data,
130 ((len < CE_DEBUG_MAX_DATA_BUF_SIZE) ?
131 len : CE_DEBUG_MAX_DATA_BUF_SIZE));
132 event->actual_data_len = len;
133 }
134 }
135
136 qdf_export_symbol(hif_ce_desc_data_record);
137
hif_clear_ce_desc_debug_data(struct hif_ce_desc_event * event)138 void hif_clear_ce_desc_debug_data(struct hif_ce_desc_event *event)
139 {
140 qdf_mem_zero(event,
141 offsetof(struct hif_ce_desc_event, data));
142 }
143
144 qdf_export_symbol(hif_clear_ce_desc_debug_data);
145 #else
hif_clear_ce_desc_debug_data(struct hif_ce_desc_event * event)146 void hif_clear_ce_desc_debug_data(struct hif_ce_desc_event *event)
147 {
148 qdf_mem_zero(event, sizeof(struct hif_ce_desc_event));
149 }
150
151 qdf_export_symbol(hif_clear_ce_desc_debug_data);
152 #endif /* HIF_CE_DEBUG_DATA_BUF */
153
154 #if defined(HIF_RECORD_PADDR)
hif_ce_desc_record_rx_paddr(struct hif_softc * scn,struct hif_ce_desc_event * event,qdf_nbuf_t memory)155 void hif_ce_desc_record_rx_paddr(struct hif_softc *scn,
156 struct hif_ce_desc_event *event,
157 qdf_nbuf_t memory)
158 {
159 if (memory) {
160 event->dma_addr = QDF_NBUF_CB_PADDR(memory);
161 event->dma_to_phy = qdf_mem_paddr_from_dmaaddr(
162 scn->qdf_dev,
163 event->dma_addr);
164
165 event->virt_to_phy =
166 virt_to_phys(qdf_nbuf_data(memory));
167 }
168 }
169 #endif /* HIF_RECORD_RX_PADDR */
170
hif_display_latest_desc_hist(struct hif_opaque_softc * hif_ctx)171 void hif_display_latest_desc_hist(struct hif_opaque_softc *hif_ctx)
172 {
173 struct hif_softc *scn = HIF_GET_SOFTC(hif_ctx);
174 struct ce_desc_hist *ce_hist;
175 struct latest_evt_history *evt;
176 int i, j;
177
178 if (!scn)
179 return;
180
181 ce_hist = &scn->hif_ce_desc_hist;
182
183 for (i = 0; i < HIF_CE_MAX_LATEST_HIST; i++) {
184 if (!ce_hist->enable[i + HIF_CE_MAX_LATEST_HIST])
185 continue;
186
187 for (j = 0; j < HIF_CE_MAX_LATEST_EVTS; j++) {
188 evt = &ce_hist->latest_evts[i][j];
189 hif_info_high("CE_id:%d event_idx:%d cpu_id:%d irq_entry:0x%llx tasklet_entry:0x%llx tasklet_resched:0x%llx tasklet_exit:0x%llx ce_work:0x%llx hp:%x tp:%x",
190 (i + HIF_CE_MAX_LATEST_HIST), j, evt->cpu_id,
191 evt->irq_entry_ts, evt->bh_entry_ts,
192 evt->bh_resched_ts, evt->bh_exit_ts,
193 evt->bh_work_ts, evt->ring_hp, evt->ring_tp);
194 }
195 }
196 }
197
hif_record_latest_evt(struct ce_desc_hist * ce_hist,uint8_t type,int ce_id,uint64_t time,uint32_t hp,uint32_t tp)198 void hif_record_latest_evt(struct ce_desc_hist *ce_hist,
199 uint8_t type,
200 int ce_id, uint64_t time,
201 uint32_t hp, uint32_t tp)
202 {
203 struct latest_evt_history *latest_evts;
204 int idx = 0;
205
206 if (ce_id != 2 && ce_id != 3)
207 return;
208
209 latest_evts = &ce_hist->latest_evts[ce_id - HIF_CE_MAX_LATEST_HIST][idx];
210
211 switch (type) {
212 case HIF_IRQ_EVENT:
213 if (latest_evts[idx].irq_entry_ts >
214 latest_evts[idx + 1].irq_entry_ts)
215 idx = 1;
216 latest_evts[idx].irq_entry_ts = time;
217 latest_evts[idx].cpu_id = qdf_get_cpu();
218 break;
219 case HIF_CE_TASKLET_ENTRY:
220 if (latest_evts[idx].bh_entry_ts >
221 latest_evts[idx + 1].bh_entry_ts)
222 idx = 1;
223 latest_evts[idx].bh_entry_ts = time;
224 break;
225 case HIF_CE_TASKLET_RESCHEDULE:
226 if (latest_evts[idx].bh_resched_ts >
227 latest_evts[idx + 1].bh_resched_ts)
228 idx = 1;
229 latest_evts[idx].bh_resched_ts = time;
230 break;
231 case HIF_CE_TASKLET_EXIT:
232 if (latest_evts[idx].bh_exit_ts >
233 latest_evts[idx + 1].bh_exit_ts)
234 idx = 1;
235 latest_evts[idx].bh_exit_ts = time;
236 break;
237 case HIF_TX_DESC_COMPLETION:
238 case HIF_CE_DEST_STATUS_RING_REAP:
239 if (latest_evts[idx].bh_work_ts >
240 latest_evts[idx + 1].bh_work_ts)
241 idx = 1;
242 latest_evts[idx].bh_work_ts = time;
243 latest_evts[idx].ring_hp = hp;
244 latest_evts[idx].ring_tp = tp;
245 break;
246 default:
247 break;
248 }
249 }
250
251 /**
252 * hif_record_ce_desc_event() - record ce descriptor events
253 * @scn: hif_softc
254 * @ce_id: which ce is the event occurring on
255 * @type: what happened
256 * @descriptor: pointer to the descriptor posted/completed
257 * @memory: virtual address of buffer related to the descriptor
258 * @index: index that the descriptor was/will be at.
259 * @len:
260 */
hif_record_ce_desc_event(struct hif_softc * scn,int ce_id,enum hif_ce_event_type type,union ce_desc * descriptor,void * memory,int index,int len)261 void hif_record_ce_desc_event(struct hif_softc *scn, int ce_id,
262 enum hif_ce_event_type type,
263 union ce_desc *descriptor,
264 void *memory, int index,
265 int len)
266 {
267 int record_index;
268 struct hif_ce_desc_event *event;
269
270 struct ce_desc_hist *ce_hist = &scn->hif_ce_desc_hist;
271 struct hif_ce_desc_event *hist_ev = NULL;
272
273 if (ce_id < CE_COUNT_MAX)
274 hist_ev = (struct hif_ce_desc_event *)ce_hist->hist_ev[ce_id];
275 else
276 return;
277
278 if (ce_id >= CE_COUNT_MAX)
279 return;
280
281 if (!ce_hist->enable[ce_id])
282 return;
283
284 if (!hist_ev)
285 return;
286
287 record_index = get_next_record_index(
288 &ce_hist->history_index[ce_id], HIF_CE_HISTORY_MAX);
289
290 event = &hist_ev[record_index];
291
292 hif_clear_ce_desc_debug_data(event);
293
294 event->type = type;
295 event->time = qdf_get_log_timestamp();
296 event->cpu_id = qdf_get_cpu();
297
298 if (descriptor)
299 qdf_mem_copy(&event->descriptor, descriptor,
300 sizeof(union ce_desc));
301
302 event->memory = memory;
303 event->index = index;
304
305 if (event->type == HIF_RX_DESC_POST ||
306 event->type == HIF_RX_DESC_COMPLETION)
307 hif_ce_desc_record_rx_paddr(scn, event, memory);
308
309 if (ce_hist->data_enable[ce_id])
310 hif_ce_desc_data_record(event, len);
311
312 hif_record_latest_evt(ce_hist, type, ce_id, event->time, 0, 0);
313 }
314 qdf_export_symbol(hif_record_ce_desc_event);
315
316 /**
317 * ce_init_ce_desc_event_log() - initialize the ce event log
318 * @scn: HIF context
319 * @ce_id: copy engine id for which we are initializing the log
320 * @size: size of array to dedicate
321 *
322 * Currently the passed size is ignored in favor of a precompiled value.
323 */
ce_init_ce_desc_event_log(struct hif_softc * scn,int ce_id,int size)324 void ce_init_ce_desc_event_log(struct hif_softc *scn, int ce_id, int size)
325 {
326 struct ce_desc_hist *ce_hist = &scn->hif_ce_desc_hist;
327 qdf_atomic_init(&ce_hist->history_index[ce_id]);
328 qdf_mutex_create(&ce_hist->ce_dbg_datamem_lock[ce_id]);
329 }
330
331 /**
332 * ce_deinit_ce_desc_event_log() - deinitialize the ce event log
333 * @scn: HIF context
334 * @ce_id: copy engine id for which we are deinitializing the log
335 *
336 */
ce_deinit_ce_desc_event_log(struct hif_softc * scn,int ce_id)337 inline void ce_deinit_ce_desc_event_log(struct hif_softc *scn, int ce_id)
338 {
339 struct ce_desc_hist *ce_hist = &scn->hif_ce_desc_hist;
340
341 qdf_mutex_destroy(&ce_hist->ce_dbg_datamem_lock[ce_id]);
342 }
343
344 #else /* (HIF_CONFIG_SLUB_DEBUG_ON) || defined(HIF_CE_DEBUG_DATA_BUF) */
hif_record_ce_desc_event(struct hif_softc * scn,int ce_id,enum hif_ce_event_type type,union ce_desc * descriptor,void * memory,int index,int len)345 void hif_record_ce_desc_event(struct hif_softc *scn,
346 int ce_id, enum hif_ce_event_type type,
347 union ce_desc *descriptor, void *memory,
348 int index, int len)
349 {
350 }
351 qdf_export_symbol(hif_record_ce_desc_event);
352
ce_init_ce_desc_event_log(struct hif_softc * scn,int ce_id,int size)353 inline void ce_init_ce_desc_event_log(struct hif_softc *scn, int ce_id,
354 int size)
355 {
356 }
357
ce_deinit_ce_desc_event_log(struct hif_softc * scn,int ce_id)358 void ce_deinit_ce_desc_event_log(struct hif_softc *scn, int ce_id)
359 {
360 }
361 #endif /*defined(HIF_CONFIG_SLUB_DEBUG_ON) || defined(HIF_CE_DEBUG_DATA_BUF) */
362
363 #ifdef NAPI_YIELD_BUDGET_BASED
hif_ce_service_should_yield(struct hif_softc * scn,struct CE_state * ce_state)364 bool hif_ce_service_should_yield(struct hif_softc *scn,
365 struct CE_state *ce_state)
366 {
367 bool yield = hif_max_num_receives_reached(scn, ce_state->receive_count);
368
369 /* Setting receive_count to MAX_NUM_OF_RECEIVES when this count goes
370 * beyond MAX_NUM_OF_RECEIVES for NAPI backet calculation issue. This
371 * can happen in fast path handling as processing is happening in
372 * batches.
373 */
374 if (yield)
375 ce_state->receive_count = MAX_NUM_OF_RECEIVES;
376
377 return yield;
378 }
379 #else
380 /**
381 * hif_ce_service_should_yield() - return true if the service is hogging the cpu
382 * @scn: hif context
383 * @ce_state: context of the copy engine being serviced
384 *
385 * Return: true if the service should yield
386 */
hif_ce_service_should_yield(struct hif_softc * scn,struct CE_state * ce_state)387 bool hif_ce_service_should_yield(struct hif_softc *scn,
388 struct CE_state *ce_state)
389 {
390 bool yield, time_limit_reached, rxpkt_thresh_reached = 0;
391
392 time_limit_reached = qdf_time_sched_clock() >
393 ce_state->ce_service_yield_time ? 1 : 0;
394
395 if (!time_limit_reached)
396 rxpkt_thresh_reached = hif_max_num_receives_reached
397 (scn, ce_state->receive_count);
398
399 /* Setting receive_count to MAX_NUM_OF_RECEIVES when this count goes
400 * beyond MAX_NUM_OF_RECEIVES for NAPI backet calculation issue. This
401 * can happen in fast path handling as processing is happening in
402 * batches.
403 */
404 if (rxpkt_thresh_reached)
405 ce_state->receive_count = MAX_NUM_OF_RECEIVES;
406
407 yield = time_limit_reached || rxpkt_thresh_reached;
408
409 if (yield &&
410 ce_state->htt_rx_data &&
411 hif_napi_enabled(GET_HIF_OPAQUE_HDL(scn), ce_state->id)) {
412 hif_napi_update_yield_stats(ce_state,
413 time_limit_reached,
414 rxpkt_thresh_reached);
415 }
416
417 return yield;
418 }
419 qdf_export_symbol(hif_ce_service_should_yield);
420 #endif
421
ce_flush_tx_ring_write_idx(struct CE_handle * ce_tx_hdl,bool force_flush)422 void ce_flush_tx_ring_write_idx(struct CE_handle *ce_tx_hdl, bool force_flush)
423 {
424 struct CE_state *ce_state = (struct CE_state *)ce_tx_hdl;
425 struct CE_ring_state *src_ring = ce_state->src_ring;
426 struct hif_softc *scn = ce_state->scn;
427
428 if (force_flush)
429 ce_ring_set_event(src_ring, CE_RING_FLUSH_EVENT);
430
431 if (ce_ring_get_clear_event(src_ring, CE_RING_FLUSH_EVENT)) {
432 qdf_spin_lock_bh(&ce_state->ce_index_lock);
433 CE_SRC_RING_WRITE_IDX_SET(scn, ce_state->ctrl_addr,
434 src_ring->write_index);
435 qdf_spin_unlock_bh(&ce_state->ce_index_lock);
436
437 src_ring->last_flush_ts = qdf_get_log_timestamp();
438 hif_debug("flushed");
439 }
440 }
441
442 /* Make sure this wrapper is called under ce_index_lock */
ce_tx_ring_write_idx_update_wrapper(struct CE_handle * ce_tx_hdl,int coalesce)443 void ce_tx_ring_write_idx_update_wrapper(struct CE_handle *ce_tx_hdl,
444 int coalesce)
445 {
446 struct CE_state *ce_state = (struct CE_state *)ce_tx_hdl;
447 struct CE_ring_state *src_ring = ce_state->src_ring;
448 struct hif_softc *scn = ce_state->scn;
449
450 if (!coalesce)
451 CE_SRC_RING_WRITE_IDX_SET(scn, ce_state->ctrl_addr,
452 src_ring->write_index);
453 }
454
455 /*
456 * Guts of ce_send, used by both ce_send and ce_sendlist_send.
457 * The caller takes responsibility for any needed locking.
458 */
459
war_ce_src_ring_write_idx_set(struct hif_softc * scn,u32 ctrl_addr,unsigned int write_index)460 void war_ce_src_ring_write_idx_set(struct hif_softc *scn,
461 u32 ctrl_addr, unsigned int write_index)
462 {
463 if (hif_ce_war1) {
464 void __iomem *indicator_addr;
465
466 indicator_addr = scn->mem + ctrl_addr + DST_WATERMARK_ADDRESS;
467
468 if (!war1_allow_sleep
469 && ctrl_addr == CE_BASE_ADDRESS(CDC_WAR_DATA_CE)) {
470 hif_write32_mb(scn, indicator_addr,
471 (CDC_WAR_MAGIC_STR | write_index));
472 } else {
473 unsigned long irq_flags;
474
475 local_irq_save(irq_flags);
476 hif_write32_mb(scn, indicator_addr, 1);
477
478 /*
479 * PCIE write waits for ACK in IPQ8K, there is no
480 * need to read back value.
481 */
482 (void)hif_read32_mb(scn, indicator_addr);
483 /* conservative */
484 (void)hif_read32_mb(scn, indicator_addr);
485
486 CE_SRC_RING_WRITE_IDX_SET(scn,
487 ctrl_addr, write_index);
488
489 hif_write32_mb(scn, indicator_addr, 0);
490 local_irq_restore(irq_flags);
491 }
492 } else {
493 CE_SRC_RING_WRITE_IDX_SET(scn, ctrl_addr, write_index);
494 }
495 }
496
497 qdf_export_symbol(war_ce_src_ring_write_idx_set);
498
499 QDF_STATUS
ce_send(struct CE_handle * copyeng,void * per_transfer_context,qdf_dma_addr_t buffer,uint32_t nbytes,uint32_t transfer_id,uint32_t flags,uint32_t user_flag)500 ce_send(struct CE_handle *copyeng,
501 void *per_transfer_context,
502 qdf_dma_addr_t buffer,
503 uint32_t nbytes,
504 uint32_t transfer_id,
505 uint32_t flags,
506 uint32_t user_flag)
507 {
508 struct CE_state *CE_state = (struct CE_state *)copyeng;
509 QDF_STATUS status;
510 struct HIF_CE_state *hif_state = HIF_GET_CE_STATE(CE_state->scn);
511
512 qdf_spin_lock_bh(&CE_state->ce_index_lock);
513 status = hif_state->ce_services->ce_send_nolock(copyeng,
514 per_transfer_context, buffer, nbytes,
515 transfer_id, flags, user_flag);
516 qdf_spin_unlock_bh(&CE_state->ce_index_lock);
517
518 return status;
519 }
520 qdf_export_symbol(ce_send);
521
ce_sendlist_sizeof(void)522 unsigned int ce_sendlist_sizeof(void)
523 {
524 return sizeof(struct ce_sendlist);
525 }
526
ce_sendlist_init(struct ce_sendlist * sendlist)527 void ce_sendlist_init(struct ce_sendlist *sendlist)
528 {
529 struct ce_sendlist_s *sl = (struct ce_sendlist_s *)sendlist;
530
531 sl->num_items = 0;
532 }
533
534 QDF_STATUS
ce_sendlist_buf_add(struct ce_sendlist * sendlist,qdf_dma_addr_t buffer,uint32_t nbytes,uint32_t flags,uint32_t user_flags)535 ce_sendlist_buf_add(struct ce_sendlist *sendlist,
536 qdf_dma_addr_t buffer,
537 uint32_t nbytes,
538 uint32_t flags,
539 uint32_t user_flags)
540 {
541 struct ce_sendlist_s *sl = (struct ce_sendlist_s *)sendlist;
542 unsigned int num_items = sl->num_items;
543 struct ce_sendlist_item *item;
544
545 if (num_items >= CE_SENDLIST_ITEMS_MAX) {
546 QDF_ASSERT(num_items < CE_SENDLIST_ITEMS_MAX);
547 return QDF_STATUS_E_RESOURCES;
548 }
549
550 item = &sl->item[num_items];
551 item->send_type = CE_SIMPLE_BUFFER_TYPE;
552 item->data = buffer;
553 item->u.nbytes = nbytes;
554 item->flags = flags;
555 item->user_flags = user_flags;
556 sl->num_items = num_items + 1;
557 return QDF_STATUS_SUCCESS;
558 }
559
560 QDF_STATUS
ce_sendlist_send(struct CE_handle * copyeng,void * per_transfer_context,struct ce_sendlist * sendlist,unsigned int transfer_id)561 ce_sendlist_send(struct CE_handle *copyeng,
562 void *per_transfer_context,
563 struct ce_sendlist *sendlist, unsigned int transfer_id)
564 {
565 struct CE_state *CE_state = (struct CE_state *)copyeng;
566 struct HIF_CE_state *hif_state = HIF_GET_CE_STATE(CE_state->scn);
567
568 return hif_state->ce_services->ce_sendlist_send(copyeng,
569 per_transfer_context, sendlist, transfer_id);
570 }
571
572 #ifndef AH_NEED_TX_DATA_SWAP
573 #define AH_NEED_TX_DATA_SWAP 0
574 #endif
575
576 /**
577 * ce_batch_send() - sends bunch of msdus at once
578 * @ce_tx_hdl : pointer to CE handle
579 * @msdu : list of msdus to be sent
580 * @transfer_id : transfer id
581 * @len : Downloaded length
582 * @sendhead : sendhead
583 *
584 * Assumption : Called with an array of MSDU's
585 * Function:
586 * For each msdu in the array
587 * 1. Send each msdu
588 * 2. Increment write index accordinlgy.
589 *
590 * Return: list of msds not sent
591 */
ce_batch_send(struct CE_handle * ce_tx_hdl,qdf_nbuf_t msdu,uint32_t transfer_id,u_int32_t len,uint32_t sendhead)592 qdf_nbuf_t ce_batch_send(struct CE_handle *ce_tx_hdl, qdf_nbuf_t msdu,
593 uint32_t transfer_id, u_int32_t len, uint32_t sendhead)
594 {
595 struct CE_state *ce_state = (struct CE_state *)ce_tx_hdl;
596 struct hif_softc *scn = ce_state->scn;
597 struct CE_ring_state *src_ring = ce_state->src_ring;
598 u_int32_t ctrl_addr = ce_state->ctrl_addr;
599 /* A_target_id_t targid = TARGID(scn);*/
600
601 uint32_t nentries_mask = src_ring->nentries_mask;
602 uint32_t sw_index, write_index;
603
604 struct CE_src_desc *src_desc_base =
605 (struct CE_src_desc *)src_ring->base_addr_owner_space;
606 uint32_t *src_desc;
607
608 struct CE_src_desc lsrc_desc = {0};
609 int deltacount = 0;
610 qdf_nbuf_t freelist = NULL, hfreelist = NULL, tempnext;
611
612 DATA_CE_UPDATE_SWINDEX(src_ring->sw_index, scn, ctrl_addr);
613 sw_index = src_ring->sw_index;
614 write_index = src_ring->write_index;
615
616 deltacount = CE_RING_DELTA(nentries_mask, write_index, sw_index-1);
617
618 while (msdu) {
619 tempnext = qdf_nbuf_next(msdu);
620
621 if (deltacount < 2) {
622 if (sendhead)
623 return msdu;
624 hif_err("Out of descriptors");
625 src_ring->write_index = write_index;
626 war_ce_src_ring_write_idx_set(scn, ctrl_addr,
627 write_index);
628
629 sw_index = src_ring->sw_index;
630 write_index = src_ring->write_index;
631
632 deltacount = CE_RING_DELTA(nentries_mask, write_index,
633 sw_index-1);
634 if (!freelist) {
635 freelist = msdu;
636 hfreelist = msdu;
637 } else {
638 qdf_nbuf_set_next(freelist, msdu);
639 freelist = msdu;
640 }
641 qdf_nbuf_set_next(msdu, NULL);
642 msdu = tempnext;
643 continue;
644 }
645
646 src_desc = (uint32_t *)CE_SRC_RING_TO_DESC(src_desc_base,
647 write_index);
648
649 src_desc[0] = qdf_nbuf_get_frag_paddr(msdu, 0);
650
651 lsrc_desc.meta_data = transfer_id;
652 if (len > msdu->len)
653 len = msdu->len;
654 lsrc_desc.nbytes = len;
655 /* Data packet is a byte stream, so disable byte swap */
656 lsrc_desc.byte_swap = AH_NEED_TX_DATA_SWAP;
657 lsrc_desc.gather = 0; /*For the last one, gather is not set*/
658
659 src_desc[1] = ((uint32_t *)&lsrc_desc)[1];
660
661
662 src_ring->per_transfer_context[write_index] = msdu;
663 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
664
665 if (sendhead)
666 break;
667 qdf_nbuf_set_next(msdu, NULL);
668 msdu = tempnext;
669
670 }
671
672
673 src_ring->write_index = write_index;
674 war_ce_src_ring_write_idx_set(scn, ctrl_addr, write_index);
675
676 return hfreelist;
677 }
678
679 /**
680 * ce_update_tx_ring() - Advance sw index.
681 * @ce_tx_hdl : pointer to CE handle
682 * @num_htt_cmpls : htt completions received.
683 *
684 * Function:
685 * Increment the value of sw index of src ring
686 * according to number of htt completions
687 * received.
688 *
689 * Return: void
690 */
691 #ifdef DATA_CE_SW_INDEX_NO_INLINE_UPDATE
ce_update_tx_ring(struct CE_handle * ce_tx_hdl,uint32_t num_htt_cmpls)692 void ce_update_tx_ring(struct CE_handle *ce_tx_hdl, uint32_t num_htt_cmpls)
693 {
694 struct CE_state *ce_state = (struct CE_state *)ce_tx_hdl;
695 struct CE_ring_state *src_ring = ce_state->src_ring;
696 uint32_t nentries_mask = src_ring->nentries_mask;
697 /*
698 * Advance the s/w index:
699 * This effectively simulates completing the CE ring descriptors
700 */
701 src_ring->sw_index =
702 CE_RING_IDX_ADD(nentries_mask, src_ring->sw_index,
703 num_htt_cmpls);
704 }
705 #else
ce_update_tx_ring(struct CE_handle * ce_tx_hdl,uint32_t num_htt_cmpls)706 void ce_update_tx_ring(struct CE_handle *ce_tx_hdl, uint32_t num_htt_cmpls)
707 {}
708 #endif
709
710 /**
711 * ce_send_single() - sends
712 * @ce_tx_hdl : pointer to CE handle
713 * @msdu : msdu to be sent
714 * @transfer_id : transfer id
715 * @len : Downloaded length
716 *
717 * Function:
718 * 1. Send one msdu
719 * 2. Increment write index of src ring accordinlgy.
720 *
721 * Return: QDF_STATUS: CE sent status
722 */
ce_send_single(struct CE_handle * ce_tx_hdl,qdf_nbuf_t msdu,uint32_t transfer_id,u_int32_t len)723 QDF_STATUS ce_send_single(struct CE_handle *ce_tx_hdl, qdf_nbuf_t msdu,
724 uint32_t transfer_id, u_int32_t len)
725 {
726 struct CE_state *ce_state = (struct CE_state *)ce_tx_hdl;
727 struct hif_softc *scn = ce_state->scn;
728 struct CE_ring_state *src_ring = ce_state->src_ring;
729 uint32_t ctrl_addr = ce_state->ctrl_addr;
730 /*A_target_id_t targid = TARGID(scn);*/
731
732 uint32_t nentries_mask = src_ring->nentries_mask;
733 uint32_t sw_index, write_index;
734
735 struct CE_src_desc *src_desc_base =
736 (struct CE_src_desc *)src_ring->base_addr_owner_space;
737 uint32_t *src_desc;
738
739 struct CE_src_desc lsrc_desc = {0};
740 enum hif_ce_event_type event_type;
741
742 DATA_CE_UPDATE_SWINDEX(src_ring->sw_index, scn, ctrl_addr);
743 sw_index = src_ring->sw_index;
744 write_index = src_ring->write_index;
745
746 if (qdf_unlikely(CE_RING_DELTA(nentries_mask, write_index,
747 sw_index-1) < 1)) {
748 hif_err("ce send fail %d %d %d", nentries_mask,
749 write_index, sw_index);
750 return QDF_STATUS_E_RESOURCES;
751 }
752
753 src_desc = (uint32_t *)CE_SRC_RING_TO_DESC(src_desc_base, write_index);
754
755 src_desc[0] = qdf_nbuf_get_frag_paddr(msdu, 0);
756
757 lsrc_desc.meta_data = transfer_id;
758 lsrc_desc.nbytes = len;
759 /* Data packet is a byte stream, so disable byte swap */
760 lsrc_desc.byte_swap = AH_NEED_TX_DATA_SWAP;
761 lsrc_desc.gather = 0; /* For the last one, gather is not set */
762
763 src_desc[1] = ((uint32_t *)&lsrc_desc)[1];
764
765
766 src_ring->per_transfer_context[write_index] = msdu;
767
768 if (((struct CE_src_desc *)src_desc)->gather)
769 event_type = HIF_TX_GATHER_DESC_POST;
770 else if (qdf_unlikely(ce_state->state != CE_RUNNING))
771 event_type = HIF_TX_DESC_SOFTWARE_POST;
772 else
773 event_type = HIF_TX_DESC_POST;
774
775 hif_record_ce_desc_event(scn, ce_state->id, event_type,
776 (union ce_desc *)src_desc, msdu,
777 write_index, len);
778
779 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
780
781 src_ring->write_index = write_index;
782
783 war_ce_src_ring_write_idx_set(scn, ctrl_addr, write_index);
784
785 return QDF_STATUS_SUCCESS;
786 }
787
788 /**
789 * ce_recv_buf_enqueue() - enqueue a recv buffer into a copy engine
790 * @copyeng: copy engine handle
791 * @per_recv_context: virtual address of the nbuf
792 * @buffer: physical address of the nbuf
793 *
794 * Return: QDF_STATUS_SUCCESS if the buffer is enqueued
795 */
796 QDF_STATUS
ce_recv_buf_enqueue(struct CE_handle * copyeng,void * per_recv_context,qdf_dma_addr_t buffer)797 ce_recv_buf_enqueue(struct CE_handle *copyeng,
798 void *per_recv_context, qdf_dma_addr_t buffer)
799 {
800 struct CE_state *CE_state = (struct CE_state *)copyeng;
801 struct HIF_CE_state *hif_state = HIF_GET_CE_STATE(CE_state->scn);
802
803 return hif_state->ce_services->ce_recv_buf_enqueue(copyeng,
804 per_recv_context, buffer);
805 }
806 qdf_export_symbol(ce_recv_buf_enqueue);
807
808 void
ce_send_watermarks_set(struct CE_handle * copyeng,unsigned int low_alert_nentries,unsigned int high_alert_nentries)809 ce_send_watermarks_set(struct CE_handle *copyeng,
810 unsigned int low_alert_nentries,
811 unsigned int high_alert_nentries)
812 {
813 struct CE_state *CE_state = (struct CE_state *)copyeng;
814 uint32_t ctrl_addr = CE_state->ctrl_addr;
815 struct hif_softc *scn = CE_state->scn;
816
817 CE_SRC_RING_LOWMARK_SET(scn, ctrl_addr, low_alert_nentries);
818 CE_SRC_RING_HIGHMARK_SET(scn, ctrl_addr, high_alert_nentries);
819 }
820
821 void
ce_recv_watermarks_set(struct CE_handle * copyeng,unsigned int low_alert_nentries,unsigned int high_alert_nentries)822 ce_recv_watermarks_set(struct CE_handle *copyeng,
823 unsigned int low_alert_nentries,
824 unsigned int high_alert_nentries)
825 {
826 struct CE_state *CE_state = (struct CE_state *)copyeng;
827 uint32_t ctrl_addr = CE_state->ctrl_addr;
828 struct hif_softc *scn = CE_state->scn;
829
830 CE_DEST_RING_LOWMARK_SET(scn, ctrl_addr,
831 low_alert_nentries);
832 CE_DEST_RING_HIGHMARK_SET(scn, ctrl_addr,
833 high_alert_nentries);
834 }
835
ce_send_entries_avail(struct CE_handle * copyeng)836 unsigned int ce_send_entries_avail(struct CE_handle *copyeng)
837 {
838 struct CE_state *CE_state = (struct CE_state *)copyeng;
839 struct CE_ring_state *src_ring = CE_state->src_ring;
840 unsigned int nentries_mask = src_ring->nentries_mask;
841 unsigned int sw_index;
842 unsigned int write_index;
843
844 qdf_spin_lock(&CE_state->ce_index_lock);
845 sw_index = src_ring->sw_index;
846 write_index = src_ring->write_index;
847 qdf_spin_unlock(&CE_state->ce_index_lock);
848
849 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
850 }
851
ce_recv_entries_avail(struct CE_handle * copyeng)852 unsigned int ce_recv_entries_avail(struct CE_handle *copyeng)
853 {
854 struct CE_state *CE_state = (struct CE_state *)copyeng;
855 struct CE_ring_state *dest_ring = CE_state->dest_ring;
856 unsigned int nentries_mask = dest_ring->nentries_mask;
857 unsigned int sw_index;
858 unsigned int write_index;
859
860 qdf_spin_lock(&CE_state->ce_index_lock);
861 sw_index = dest_ring->sw_index;
862 write_index = dest_ring->write_index;
863 qdf_spin_unlock(&CE_state->ce_index_lock);
864
865 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
866 }
867
868 /*
869 * Guts of ce_completed_recv_next.
870 * The caller takes responsibility for any necessary locking.
871 */
872 QDF_STATUS
ce_completed_recv_next(struct CE_handle * copyeng,void ** per_CE_contextp,void ** per_transfer_contextp,qdf_dma_addr_t * bufferp,unsigned int * nbytesp,unsigned int * transfer_idp,unsigned int * flagsp)873 ce_completed_recv_next(struct CE_handle *copyeng,
874 void **per_CE_contextp,
875 void **per_transfer_contextp,
876 qdf_dma_addr_t *bufferp,
877 unsigned int *nbytesp,
878 unsigned int *transfer_idp, unsigned int *flagsp)
879 {
880 struct CE_state *CE_state = (struct CE_state *)copyeng;
881 QDF_STATUS status;
882 struct hif_softc *scn = CE_state->scn;
883 struct HIF_CE_state *hif_state = HIF_GET_CE_STATE(scn);
884 struct ce_ops *ce_services;
885
886 ce_services = hif_state->ce_services;
887 qdf_spin_lock_bh(&CE_state->ce_index_lock);
888 status =
889 ce_services->ce_completed_recv_next_nolock(CE_state,
890 per_CE_contextp, per_transfer_contextp, bufferp,
891 nbytesp, transfer_idp, flagsp);
892 qdf_spin_unlock_bh(&CE_state->ce_index_lock);
893
894 return status;
895 }
896
897 QDF_STATUS
ce_revoke_recv_next(struct CE_handle * copyeng,void ** per_CE_contextp,void ** per_transfer_contextp,qdf_dma_addr_t * bufferp)898 ce_revoke_recv_next(struct CE_handle *copyeng,
899 void **per_CE_contextp,
900 void **per_transfer_contextp, qdf_dma_addr_t *bufferp)
901 {
902 struct CE_state *CE_state = (struct CE_state *)copyeng;
903 struct HIF_CE_state *hif_state = HIF_GET_CE_STATE(CE_state->scn);
904
905 return hif_state->ce_services->ce_revoke_recv_next(copyeng,
906 per_CE_contextp, per_transfer_contextp, bufferp);
907 }
908
909 QDF_STATUS
ce_cancel_send_next(struct CE_handle * copyeng,void ** per_CE_contextp,void ** per_transfer_contextp,qdf_dma_addr_t * bufferp,unsigned int * nbytesp,unsigned int * transfer_idp,uint32_t * toeplitz_hash_result)910 ce_cancel_send_next(struct CE_handle *copyeng,
911 void **per_CE_contextp,
912 void **per_transfer_contextp,
913 qdf_dma_addr_t *bufferp,
914 unsigned int *nbytesp,
915 unsigned int *transfer_idp,
916 uint32_t *toeplitz_hash_result)
917 {
918 struct CE_state *CE_state = (struct CE_state *)copyeng;
919 struct HIF_CE_state *hif_state = HIF_GET_CE_STATE(CE_state->scn);
920
921 return hif_state->ce_services->ce_cancel_send_next
922 (copyeng, per_CE_contextp, per_transfer_contextp,
923 bufferp, nbytesp, transfer_idp, toeplitz_hash_result);
924 }
925 qdf_export_symbol(ce_cancel_send_next);
926
927 QDF_STATUS
ce_completed_send_next(struct CE_handle * copyeng,void ** per_CE_contextp,void ** per_transfer_contextp,qdf_dma_addr_t * bufferp,unsigned int * nbytesp,unsigned int * transfer_idp,unsigned int * sw_idx,unsigned int * hw_idx,unsigned int * toeplitz_hash_result)928 ce_completed_send_next(struct CE_handle *copyeng,
929 void **per_CE_contextp,
930 void **per_transfer_contextp,
931 qdf_dma_addr_t *bufferp,
932 unsigned int *nbytesp,
933 unsigned int *transfer_idp,
934 unsigned int *sw_idx,
935 unsigned int *hw_idx,
936 unsigned int *toeplitz_hash_result)
937 {
938 struct CE_state *CE_state = (struct CE_state *)copyeng;
939 struct hif_softc *scn = CE_state->scn;
940 struct HIF_CE_state *hif_state = HIF_GET_CE_STATE(scn);
941 struct ce_ops *ce_services;
942 QDF_STATUS status;
943
944 ce_services = hif_state->ce_services;
945 qdf_spin_lock_bh(&CE_state->ce_index_lock);
946 status =
947 ce_services->ce_completed_send_next_nolock(CE_state,
948 per_CE_contextp, per_transfer_contextp,
949 bufferp, nbytesp, transfer_idp, sw_idx,
950 hw_idx, toeplitz_hash_result);
951 qdf_spin_unlock_bh(&CE_state->ce_index_lock);
952
953 return status;
954 }
955
956 #ifdef ATH_11AC_TXCOMPACT
957 /* CE engine descriptor reap
958 * Similar to ce_per_engine_service , Only difference is ce_per_engine_service
959 * does receive and reaping of completed descriptor ,
960 * This function only handles reaping of Tx complete descriptor.
961 * The Function is called from threshold reap poll routine
962 * hif_send_complete_check so should not contain receive functionality
963 * within it .
964 */
965
ce_per_engine_servicereap(struct hif_softc * scn,unsigned int ce_id)966 void ce_per_engine_servicereap(struct hif_softc *scn, unsigned int ce_id)
967 {
968 void *CE_context;
969 void *transfer_context;
970 qdf_dma_addr_t buf;
971 unsigned int nbytes;
972 unsigned int id;
973 unsigned int sw_idx, hw_idx;
974 uint32_t toeplitz_hash_result;
975 struct CE_state *CE_state = scn->ce_id_to_state[ce_id];
976 struct HIF_CE_state *hif_state = HIF_GET_CE_STATE(scn);
977
978 if (Q_TARGET_ACCESS_BEGIN(scn) < 0)
979 return;
980
981 hif_record_ce_desc_event(scn, ce_id, HIF_CE_REAP_ENTRY,
982 NULL, NULL, 0, 0);
983
984 /* Since this function is called from both user context and
985 * tasklet context the spinlock has to lock the bottom halves.
986 * This fix assumes that ATH_11AC_TXCOMPACT flag is always
987 * enabled in TX polling mode. If this is not the case, more
988 * bottom halve spin lock changes are needed. Due to data path
989 * performance concern, after internal discussion we've decided
990 * to make minimum change, i.e., only address the issue occurred
991 * in this function. The possible negative effect of this minimum
992 * change is that, in the future, if some other function will also
993 * be opened to let the user context to use, those cases need to be
994 * addressed by change spin_lock to spin_lock_bh also.
995 */
996
997 qdf_spin_lock_bh(&CE_state->ce_index_lock);
998
999 if (CE_state->send_cb) {
1000 {
1001 struct ce_ops *ce_services = hif_state->ce_services;
1002 /* Pop completed send buffers and call the
1003 * registered send callback for each
1004 */
1005 while (ce_services->ce_completed_send_next_nolock
1006 (CE_state, &CE_context,
1007 &transfer_context, &buf,
1008 &nbytes, &id, &sw_idx, &hw_idx,
1009 &toeplitz_hash_result) ==
1010 QDF_STATUS_SUCCESS) {
1011 if (ce_id != CE_HTT_H2T_MSG) {
1012 qdf_spin_unlock_bh(
1013 &CE_state->ce_index_lock);
1014 CE_state->send_cb(
1015 (struct CE_handle *)
1016 CE_state, CE_context,
1017 transfer_context, buf,
1018 nbytes, id, sw_idx, hw_idx,
1019 toeplitz_hash_result);
1020 qdf_spin_lock_bh(
1021 &CE_state->ce_index_lock);
1022 } else {
1023 struct HIF_CE_pipe_info *pipe_info =
1024 (struct HIF_CE_pipe_info *)
1025 CE_context;
1026
1027 qdf_spin_lock_bh(&pipe_info->
1028 completion_freeq_lock);
1029 pipe_info->num_sends_allowed++;
1030 qdf_spin_unlock_bh(&pipe_info->
1031 completion_freeq_lock);
1032 }
1033 }
1034 }
1035 }
1036
1037 qdf_spin_unlock_bh(&CE_state->ce_index_lock);
1038
1039 hif_record_ce_desc_event(scn, ce_id, HIF_CE_REAP_EXIT,
1040 NULL, NULL, 0, 0);
1041 Q_TARGET_ACCESS_END(scn);
1042 }
1043
1044 #endif /*ATH_11AC_TXCOMPACT */
1045
1046 #ifdef ENABLE_CE4_COMP_DISABLE_HTT_HTC_MISC_LIST
check_ce_id_and_epping_enabled(int CE_id,uint32_t mode)1047 static inline bool check_ce_id_and_epping_enabled(int CE_id, uint32_t mode)
1048 {
1049 // QDF_IS_EPPING_ENABLED is pre lithium feature
1050 // CE4 completion is enabled only lithium and later
1051 // so no need to check for EPPING
1052 return true;
1053 }
1054
1055 #else /* ENABLE_CE4_COMP_DISABLE_HTT_HTC_MISC_LIST */
1056
check_ce_id_and_epping_enabled(int CE_id,uint32_t mode)1057 static inline bool check_ce_id_and_epping_enabled(int CE_id, uint32_t mode)
1058 {
1059 if (CE_id != CE_HTT_H2T_MSG || QDF_IS_EPPING_ENABLED(mode))
1060 return true;
1061 else
1062 return false;
1063 }
1064
1065 #endif /* ENABLE_CE4_COMP_DISABLE_HTT_HTC_MISC_LIST */
1066
1067 /*
1068 * ce_engine_service_reg:
1069 *
1070 * Called from ce_per_engine_service and goes through the regular interrupt
1071 * handling that does not involve the WLAN fast path feature.
1072 *
1073 * Returns void
1074 */
ce_engine_service_reg(struct hif_softc * scn,int CE_id)1075 void ce_engine_service_reg(struct hif_softc *scn, int CE_id)
1076 {
1077 struct CE_state *CE_state = scn->ce_id_to_state[CE_id];
1078 uint32_t ctrl_addr = CE_state->ctrl_addr;
1079 struct HIF_CE_state *hif_state = HIF_GET_CE_STATE(scn);
1080 void *CE_context;
1081 void *transfer_context;
1082 qdf_dma_addr_t buf;
1083 unsigned int nbytes;
1084 unsigned int id;
1085 unsigned int flags;
1086 unsigned int more_comp_cnt = 0;
1087 unsigned int more_snd_comp_cnt = 0;
1088 unsigned int sw_idx, hw_idx;
1089 uint32_t toeplitz_hash_result;
1090 uint32_t mode = hif_get_conparam(scn);
1091
1092 more_completions:
1093 if (CE_state->recv_cb) {
1094
1095 /* Pop completed recv buffers and call
1096 * the registered recv callback for each
1097 */
1098 while (hif_state->ce_services->ce_completed_recv_next_nolock
1099 (CE_state, &CE_context, &transfer_context,
1100 &buf, &nbytes, &id, &flags) ==
1101 QDF_STATUS_SUCCESS) {
1102 qdf_spin_unlock(&CE_state->ce_index_lock);
1103 CE_state->recv_cb((struct CE_handle *)CE_state,
1104 CE_context, transfer_context, buf,
1105 nbytes, id, flags);
1106
1107 qdf_spin_lock(&CE_state->ce_index_lock);
1108 /*
1109 * EV #112693 -
1110 * [Peregrine][ES1][WB342][Win8x86][Performance]
1111 * BSoD_0x133 occurred in VHT80 UDP_DL
1112 * Break out DPC by force if number of loops in
1113 * hif_pci_ce_recv_data reaches MAX_NUM_OF_RECEIVES
1114 * to avoid spending too long time in
1115 * DPC for each interrupt handling. Schedule another
1116 * DPC to avoid data loss if we had taken
1117 * force-break action before apply to Windows OS
1118 * only currently, Linux/MAC os can expand to their
1119 * platform if necessary
1120 */
1121
1122 /* Break the receive processes by
1123 * force if force_break set up
1124 */
1125 if (qdf_unlikely(CE_state->force_break)) {
1126 qdf_atomic_set(&CE_state->rx_pending, 1);
1127 return;
1128 }
1129 }
1130 }
1131
1132 /*
1133 * Attention: We may experience potential infinite loop for below
1134 * While Loop during Sending Stress test.
1135 * Resolve the same way as Receive Case (Refer to EV #112693)
1136 */
1137
1138 if (CE_state->send_cb) {
1139 /* Pop completed send buffers and call
1140 * the registered send callback for each
1141 */
1142
1143 #ifdef ATH_11AC_TXCOMPACT
1144 while (hif_state->ce_services->ce_completed_send_next_nolock
1145 (CE_state, &CE_context,
1146 &transfer_context, &buf, &nbytes,
1147 &id, &sw_idx, &hw_idx,
1148 &toeplitz_hash_result) == QDF_STATUS_SUCCESS) {
1149
1150 if (check_ce_id_and_epping_enabled(CE_id, mode)) {
1151 qdf_spin_unlock(&CE_state->ce_index_lock);
1152 CE_state->send_cb((struct CE_handle *)CE_state,
1153 CE_context, transfer_context,
1154 buf, nbytes, id, sw_idx,
1155 hw_idx, toeplitz_hash_result);
1156 qdf_spin_lock(&CE_state->ce_index_lock);
1157 } else {
1158 struct HIF_CE_pipe_info *pipe_info =
1159 (struct HIF_CE_pipe_info *)CE_context;
1160
1161 qdf_spin_lock_bh(&pipe_info->
1162 completion_freeq_lock);
1163 pipe_info->num_sends_allowed++;
1164 qdf_spin_unlock_bh(&pipe_info->
1165 completion_freeq_lock);
1166 }
1167 }
1168 #else /*ATH_11AC_TXCOMPACT */
1169 while (hif_state->ce_services->ce_completed_send_next_nolock
1170 (CE_state, &CE_context,
1171 &transfer_context, &buf, &nbytes,
1172 &id, &sw_idx, &hw_idx,
1173 &toeplitz_hash_result) == QDF_STATUS_SUCCESS) {
1174 qdf_spin_unlock(&CE_state->ce_index_lock);
1175 CE_state->send_cb((struct CE_handle *)CE_state,
1176 CE_context, transfer_context, buf,
1177 nbytes, id, sw_idx, hw_idx,
1178 toeplitz_hash_result);
1179 qdf_spin_lock(&CE_state->ce_index_lock);
1180 }
1181 #endif /*ATH_11AC_TXCOMPACT */
1182 }
1183
1184 more_watermarks:
1185 if (CE_state->misc_cbs) {
1186 if (CE_state->watermark_cb &&
1187 hif_state->ce_services->watermark_int(CE_state,
1188 &flags)) {
1189 qdf_spin_unlock(&CE_state->ce_index_lock);
1190 /* Convert HW IS bits to software flags */
1191 CE_state->watermark_cb((struct CE_handle *)CE_state,
1192 CE_state->wm_context, flags);
1193 qdf_spin_lock(&CE_state->ce_index_lock);
1194 }
1195 }
1196
1197 /*
1198 * Clear the misc interrupts (watermark) that were handled above,
1199 * and that will be checked again below.
1200 * Clear and check for copy-complete interrupts again, just in case
1201 * more copy completions happened while the misc interrupts were being
1202 * handled.
1203 */
1204 if (!ce_srng_based(scn) && !CE_state->msi_supported) {
1205 if (TARGET_REGISTER_ACCESS_ALLOWED(scn)) {
1206 CE_ENGINE_INT_STATUS_CLEAR(scn, ctrl_addr,
1207 CE_WATERMARK_MASK |
1208 HOST_IS_COPY_COMPLETE_MASK);
1209 } else {
1210 qdf_atomic_set(&CE_state->rx_pending, 0);
1211 hif_err_rl("%s: target access is not allowed",
1212 __func__);
1213 return;
1214 }
1215 }
1216
1217 /*
1218 * Now that per-engine interrupts are cleared, verify that
1219 * no recv interrupts arrive while processing send interrupts,
1220 * and no recv or send interrupts happened while processing
1221 * misc interrupts.Go back and check again.Keep checking until
1222 * we find no more events to process.
1223 */
1224 if (CE_state->recv_cb &&
1225 hif_state->ce_services->ce_recv_entries_done_nolock(scn,
1226 CE_state)) {
1227 if (QDF_IS_EPPING_ENABLED(mode) ||
1228 more_comp_cnt++ < CE_TXRX_COMP_CHECK_THRESHOLD) {
1229 goto more_completions;
1230 } else {
1231 if (!ce_srng_based(scn) &&
1232 !CE_state->batch_intr_supported) {
1233 hif_err_rl(
1234 "Potential infinite loop detected during Rx processing id:%u nentries_mask:0x%x sw read_idx:0x%x hw read_idx:0x%x",
1235 CE_state->id,
1236 CE_state->dest_ring->nentries_mask,
1237 CE_state->dest_ring->sw_index,
1238 CE_DEST_RING_READ_IDX_GET(scn,
1239 CE_state->ctrl_addr));
1240 }
1241 }
1242 }
1243
1244 if (CE_state->send_cb &&
1245 hif_state->ce_services->ce_send_entries_done_nolock(scn,
1246 CE_state)) {
1247 if (QDF_IS_EPPING_ENABLED(mode) ||
1248 more_snd_comp_cnt++ < CE_TXRX_COMP_CHECK_THRESHOLD) {
1249 goto more_completions;
1250 } else {
1251 if (!ce_srng_based(scn) &&
1252 !CE_state->batch_intr_supported) {
1253 hif_err_rl(
1254 "Potential infinite loop detected during send completion id:%u mask:0x%x sw read_idx:0x%x hw_index:0x%x write_index: 0x%x hw read_idx:0x%x",
1255 CE_state->id,
1256 CE_state->src_ring->nentries_mask,
1257 CE_state->src_ring->sw_index,
1258 CE_state->src_ring->hw_index,
1259 CE_state->src_ring->write_index,
1260 CE_SRC_RING_READ_IDX_GET(scn,
1261 CE_state->ctrl_addr));
1262 }
1263 }
1264 }
1265
1266 if (CE_state->misc_cbs && CE_state->watermark_cb) {
1267 if (hif_state->ce_services->watermark_int(CE_state, &flags))
1268 goto more_watermarks;
1269 }
1270
1271 qdf_atomic_set(&CE_state->rx_pending, 0);
1272 }
1273
1274 #ifdef WLAN_TRACEPOINTS
1275 /**
1276 * ce_trace_tasklet_sched_latency() - Trace ce tasklet scheduling
1277 * latency
1278 * @ce_state: CE context
1279 *
1280 * Return: None
1281 */
1282 static inline
ce_trace_tasklet_sched_latency(struct CE_state * ce_state)1283 void ce_trace_tasklet_sched_latency(struct CE_state *ce_state)
1284 {
1285 qdf_trace_dp_ce_tasklet_sched_latency(ce_state->id,
1286 ce_state->ce_service_start_time -
1287 ce_state->ce_tasklet_sched_time);
1288 }
1289 #else
1290 static inline
ce_trace_tasklet_sched_latency(struct CE_state * ce_state)1291 void ce_trace_tasklet_sched_latency(struct CE_state *ce_state)
1292 {
1293 }
1294 #endif
1295
1296 /*
1297 * Guts of interrupt handler for per-engine interrupts on a particular CE.
1298 *
1299 * Invokes registered callbacks for recv_complete,
1300 * send_complete, and watermarks.
1301 *
1302 * Returns: number of messages processed
1303 */
ce_per_engine_service(struct hif_softc * scn,unsigned int CE_id)1304 int ce_per_engine_service(struct hif_softc *scn, unsigned int CE_id)
1305 {
1306 struct CE_state *CE_state = scn->ce_id_to_state[CE_id];
1307
1308 if (hif_is_nss_wifi_enabled(scn) && (CE_state->htt_rx_data))
1309 return CE_state->receive_count;
1310
1311 if (Q_TARGET_ACCESS_BEGIN(scn) < 0) {
1312 hif_err("[premature rc=0]");
1313 return 0; /* no work done */
1314 }
1315
1316 /* Clear force_break flag and re-initialize receive_count to 0 */
1317 CE_state->receive_count = 0;
1318 CE_state->force_break = 0;
1319 CE_state->ce_service_start_time = qdf_time_sched_clock();
1320 CE_state->ce_service_yield_time =
1321 CE_state->ce_service_start_time +
1322 hif_get_ce_service_max_yield_time(
1323 (struct hif_opaque_softc *)scn);
1324
1325 ce_trace_tasklet_sched_latency(CE_state);
1326
1327 qdf_spin_lock(&CE_state->ce_index_lock);
1328
1329 CE_state->service(scn, CE_id);
1330
1331 qdf_spin_unlock(&CE_state->ce_index_lock);
1332
1333 if (Q_TARGET_ACCESS_END(scn) < 0)
1334 hif_err("<--[premature rc=%d]", CE_state->receive_count);
1335 return CE_state->receive_count;
1336 }
1337 qdf_export_symbol(ce_per_engine_service);
1338
1339 /*
1340 * Handler for per-engine interrupts on ALL active CEs.
1341 * This is used in cases where the system is sharing a
1342 * single interrupt for all CEs
1343 */
1344
ce_per_engine_service_any(int irq,struct hif_softc * scn)1345 void ce_per_engine_service_any(int irq, struct hif_softc *scn)
1346 {
1347 int CE_id;
1348 uint32_t intr_summary;
1349
1350 if (Q_TARGET_ACCESS_BEGIN(scn) < 0)
1351 return;
1352
1353 if (!qdf_atomic_read(&scn->tasklet_from_intr)) {
1354 for (CE_id = 0; CE_id < scn->ce_count; CE_id++) {
1355 struct CE_state *CE_state = scn->ce_id_to_state[CE_id];
1356
1357 if (qdf_atomic_read(&CE_state->rx_pending)) {
1358 qdf_atomic_set(&CE_state->rx_pending, 0);
1359 ce_per_engine_service(scn, CE_id);
1360 }
1361 }
1362
1363 Q_TARGET_ACCESS_END(scn);
1364 return;
1365 }
1366
1367 intr_summary = CE_INTERRUPT_SUMMARY(scn);
1368
1369 for (CE_id = 0; intr_summary && (CE_id < scn->ce_count); CE_id++) {
1370 if (intr_summary & (1 << CE_id))
1371 intr_summary &= ~(1 << CE_id);
1372 else
1373 continue; /* no intr pending on this CE */
1374
1375 ce_per_engine_service(scn, CE_id);
1376 }
1377
1378 Q_TARGET_ACCESS_END(scn);
1379 }
1380
1381 /*Iterate the CE_state list and disable the compl interrupt
1382 * if it has been registered already.
1383 */
ce_disable_any_copy_compl_intr_nolock(struct hif_softc * scn)1384 void ce_disable_any_copy_compl_intr_nolock(struct hif_softc *scn)
1385 {
1386 int CE_id;
1387
1388 if (Q_TARGET_ACCESS_BEGIN(scn) < 0)
1389 return;
1390
1391 for (CE_id = 0; CE_id < scn->ce_count; CE_id++) {
1392 struct CE_state *CE_state = scn->ce_id_to_state[CE_id];
1393 uint32_t ctrl_addr = CE_state->ctrl_addr;
1394
1395 /* if the interrupt is currently enabled, disable it */
1396 if (!CE_state->disable_copy_compl_intr
1397 && (CE_state->send_cb || CE_state->recv_cb))
1398 CE_COPY_COMPLETE_INTR_DISABLE(scn, ctrl_addr);
1399
1400 if (CE_state->watermark_cb)
1401 CE_WATERMARK_INTR_DISABLE(scn, ctrl_addr);
1402 }
1403 Q_TARGET_ACCESS_END(scn);
1404 }
1405
ce_enable_any_copy_compl_intr_nolock(struct hif_softc * scn)1406 void ce_enable_any_copy_compl_intr_nolock(struct hif_softc *scn)
1407 {
1408 int CE_id;
1409
1410 if (Q_TARGET_ACCESS_BEGIN(scn) < 0)
1411 return;
1412
1413 for (CE_id = 0; CE_id < scn->ce_count; CE_id++) {
1414 struct CE_state *CE_state = scn->ce_id_to_state[CE_id];
1415 uint32_t ctrl_addr = CE_state->ctrl_addr;
1416
1417 /*
1418 * If the CE is supposed to have copy complete interrupts
1419 * enabled (i.e. there a callback registered, and the
1420 * "disable" flag is not set), then re-enable the interrupt.
1421 */
1422 if (!CE_state->disable_copy_compl_intr
1423 && (CE_state->send_cb || CE_state->recv_cb))
1424 CE_COPY_COMPLETE_INTR_ENABLE(scn, ctrl_addr);
1425
1426 if (CE_state->watermark_cb)
1427 CE_WATERMARK_INTR_ENABLE(scn, ctrl_addr);
1428 }
1429 Q_TARGET_ACCESS_END(scn);
1430 }
1431
1432 /**
1433 * ce_send_cb_register(): register completion handler
1434 * @copyeng: CE_state representing the ce we are adding the behavior to
1435 * @fn_ptr: callback that the ce should use when processing tx completions
1436 * @ce_send_context: context to pass back in the callback
1437 * @disable_interrupts: if the interrupts should be enabled or not.
1438 *
1439 * Caller should guarantee that no transactions are in progress before
1440 * switching the callback function.
1441 *
1442 * Registers the send context before the fn pointer so that if the cb is valid
1443 * the context should be valid.
1444 *
1445 * Beware that currently this function will enable completion interrupts.
1446 */
1447 void
ce_send_cb_register(struct CE_handle * copyeng,ce_send_cb fn_ptr,void * ce_send_context,int disable_interrupts)1448 ce_send_cb_register(struct CE_handle *copyeng,
1449 ce_send_cb fn_ptr,
1450 void *ce_send_context, int disable_interrupts)
1451 {
1452 struct CE_state *CE_state = (struct CE_state *)copyeng;
1453 struct hif_softc *scn;
1454 struct HIF_CE_state *hif_state;
1455
1456 if (!CE_state) {
1457 hif_err("Error CE state = NULL");
1458 return;
1459 }
1460 scn = CE_state->scn;
1461 hif_state = HIF_GET_CE_STATE(scn);
1462 if (!hif_state) {
1463 hif_err("Error HIF state = NULL");
1464 return;
1465 }
1466 CE_state->send_context = ce_send_context;
1467 CE_state->send_cb = fn_ptr;
1468 hif_state->ce_services->ce_per_engine_handler_adjust(CE_state,
1469 disable_interrupts);
1470 }
1471 qdf_export_symbol(ce_send_cb_register);
1472
1473 /**
1474 * ce_recv_cb_register(): register completion handler
1475 * @copyeng: CE_state representing the ce we are adding the behavior to
1476 * @fn_ptr: callback that the ce should use when processing rx completions
1477 * @CE_recv_context: context to pass back in the callback
1478 * @disable_interrupts: if the interrupts should be enabled or not.
1479 *
1480 * Registers the send context before the fn pointer so that if the cb is valid
1481 * the context should be valid.
1482 *
1483 * Caller should guarantee that no transactions are in progress before
1484 * switching the callback function.
1485 */
1486 void
ce_recv_cb_register(struct CE_handle * copyeng,CE_recv_cb fn_ptr,void * CE_recv_context,int disable_interrupts)1487 ce_recv_cb_register(struct CE_handle *copyeng,
1488 CE_recv_cb fn_ptr,
1489 void *CE_recv_context, int disable_interrupts)
1490 {
1491 struct CE_state *CE_state = (struct CE_state *)copyeng;
1492 struct hif_softc *scn;
1493 struct HIF_CE_state *hif_state;
1494
1495 if (!CE_state) {
1496 hif_err("ERROR CE state = NULL");
1497 return;
1498 }
1499 scn = CE_state->scn;
1500 hif_state = HIF_GET_CE_STATE(scn);
1501 if (!hif_state) {
1502 hif_err("Error HIF state = NULL");
1503 return;
1504 }
1505 CE_state->recv_context = CE_recv_context;
1506 CE_state->recv_cb = fn_ptr;
1507 hif_state->ce_services->ce_per_engine_handler_adjust(CE_state,
1508 disable_interrupts);
1509 }
1510 qdf_export_symbol(ce_recv_cb_register);
1511
1512 /**
1513 * ce_watermark_cb_register(): register completion handler
1514 * @copyeng: CE_state representing the ce we are adding the behavior to
1515 * @fn_ptr: callback that the ce should use when processing watermark events
1516 * @CE_wm_context: context to pass back in the callback
1517 *
1518 * Caller should guarantee that no watermark events are being processed before
1519 * switching the callback function.
1520 */
1521 void
ce_watermark_cb_register(struct CE_handle * copyeng,CE_watermark_cb fn_ptr,void * CE_wm_context)1522 ce_watermark_cb_register(struct CE_handle *copyeng,
1523 CE_watermark_cb fn_ptr, void *CE_wm_context)
1524 {
1525 struct CE_state *CE_state = (struct CE_state *)copyeng;
1526 struct hif_softc *scn = CE_state->scn;
1527 struct HIF_CE_state *hif_state = HIF_GET_CE_STATE(scn);
1528
1529 CE_state->watermark_cb = fn_ptr;
1530 CE_state->wm_context = CE_wm_context;
1531 hif_state->ce_services->ce_per_engine_handler_adjust(CE_state,
1532 0);
1533 if (fn_ptr)
1534 CE_state->misc_cbs = 1;
1535 }
1536
1537 #ifdef CUSTOM_CB_SCHEDULER_SUPPORT
1538 void
ce_register_custom_cb(struct CE_handle * copyeng,void (* custom_cb)(void *),void * custom_cb_context)1539 ce_register_custom_cb(struct CE_handle *copyeng, void (*custom_cb)(void *),
1540 void *custom_cb_context)
1541 {
1542 struct CE_state *CE_state = (struct CE_state *)copyeng;
1543
1544 CE_state->custom_cb = custom_cb;
1545 CE_state->custom_cb_context = custom_cb_context;
1546 qdf_atomic_init(&CE_state->custom_cb_pending);
1547 }
1548
1549 void
ce_unregister_custom_cb(struct CE_handle * copyeng)1550 ce_unregister_custom_cb(struct CE_handle *copyeng)
1551 {
1552 struct CE_state *CE_state = (struct CE_state *)copyeng;
1553
1554 qdf_assert_always(!qdf_atomic_read(&CE_state->custom_cb_pending));
1555 CE_state->custom_cb = NULL;
1556 CE_state->custom_cb_context = NULL;
1557 }
1558
1559 void
ce_enable_custom_cb(struct CE_handle * copyeng)1560 ce_enable_custom_cb(struct CE_handle *copyeng)
1561 {
1562 struct CE_state *CE_state = (struct CE_state *)copyeng;
1563 int32_t custom_cb_pending;
1564
1565 qdf_assert_always(CE_state->custom_cb);
1566 qdf_assert_always(CE_state->custom_cb_context);
1567
1568 custom_cb_pending = qdf_atomic_inc_return(&CE_state->custom_cb_pending);
1569 qdf_assert_always(custom_cb_pending >= 1);
1570 }
1571
1572 void
ce_disable_custom_cb(struct CE_handle * copyeng)1573 ce_disable_custom_cb(struct CE_handle *copyeng)
1574 {
1575 struct CE_state *CE_state = (struct CE_state *)copyeng;
1576
1577 qdf_assert_always(CE_state->custom_cb);
1578 qdf_assert_always(CE_state->custom_cb_context);
1579
1580 qdf_atomic_dec_if_positive(&CE_state->custom_cb_pending);
1581 }
1582 #endif /* CUSTOM_CB_SCHEDULER_SUPPORT */
1583
ce_get_rx_pending(struct hif_softc * scn)1584 bool ce_get_rx_pending(struct hif_softc *scn)
1585 {
1586 int CE_id;
1587
1588 for (CE_id = 0; CE_id < scn->ce_count; CE_id++) {
1589 struct CE_state *CE_state = scn->ce_id_to_state[CE_id];
1590
1591 if (qdf_atomic_read(&CE_state->rx_pending))
1592 return true;
1593 }
1594
1595 return false;
1596 }
1597
1598 /**
1599 * ce_check_rx_pending() - ce_check_rx_pending
1600 * @CE_state: context of the copy engine to check
1601 *
1602 * Return: true if there per_engine_service
1603 * didn't process all the rx descriptors.
1604 */
ce_check_rx_pending(struct CE_state * CE_state)1605 bool ce_check_rx_pending(struct CE_state *CE_state)
1606 {
1607 if (qdf_atomic_read(&CE_state->rx_pending))
1608 return true;
1609 else
1610 return false;
1611 }
1612 qdf_export_symbol(ce_check_rx_pending);
1613
1614 #ifdef IPA_OFFLOAD
1615 #ifdef QCN7605_SUPPORT
ce_ipa_get_wr_index_addr(struct CE_state * CE_state)1616 static qdf_dma_addr_t ce_ipa_get_wr_index_addr(struct CE_state *CE_state)
1617 {
1618 u_int32_t ctrl_addr = CE_state->ctrl_addr;
1619 struct hif_softc *scn = CE_state->scn;
1620 qdf_dma_addr_t wr_index_addr;
1621
1622 wr_index_addr = shadow_sr_wr_ind_addr(scn, ctrl_addr);
1623 return wr_index_addr;
1624 }
1625 #else
ce_ipa_get_wr_index_addr(struct CE_state * CE_state)1626 static qdf_dma_addr_t ce_ipa_get_wr_index_addr(struct CE_state *CE_state)
1627 {
1628 struct hif_softc *scn = CE_state->scn;
1629 qdf_dma_addr_t wr_index_addr;
1630
1631 wr_index_addr = CE_BASE_ADDRESS(CE_state->id) +
1632 SR_WR_INDEX_ADDRESS;
1633 return wr_index_addr;
1634 }
1635 #endif
1636
1637 /**
1638 * ce_ipa_get_resource() - get uc resource on copyengine
1639 * @ce: copyengine context
1640 * @ce_sr: copyengine source ring resource info
1641 * @ce_sr_ring_size: copyengine source ring size
1642 * @ce_reg_paddr: copyengine register physical address
1643 *
1644 * Copy engine should release resource to micro controller
1645 * Micro controller needs
1646 * - Copy engine source descriptor base address
1647 * - Copy engine source descriptor size
1648 * - PCI BAR address to access copy engine register
1649 *
1650 * Return: None
1651 */
ce_ipa_get_resource(struct CE_handle * ce,qdf_shared_mem_t ** ce_sr,uint32_t * ce_sr_ring_size,qdf_dma_addr_t * ce_reg_paddr)1652 void ce_ipa_get_resource(struct CE_handle *ce,
1653 qdf_shared_mem_t **ce_sr,
1654 uint32_t *ce_sr_ring_size,
1655 qdf_dma_addr_t *ce_reg_paddr)
1656 {
1657 struct CE_state *CE_state = (struct CE_state *)ce;
1658 uint32_t ring_loop;
1659 struct CE_src_desc *ce_desc;
1660 qdf_dma_addr_t phy_mem_base;
1661 struct hif_softc *scn = CE_state->scn;
1662
1663 if (CE_UNUSED == CE_state->state) {
1664 *qdf_mem_get_dma_addr_ptr(scn->qdf_dev,
1665 &CE_state->scn->ipa_ce_ring->mem_info) = 0;
1666 *ce_sr_ring_size = 0;
1667 return;
1668 }
1669
1670 /* Update default value for descriptor */
1671 for (ring_loop = 0; ring_loop < CE_state->src_ring->nentries;
1672 ring_loop++) {
1673 ce_desc = (struct CE_src_desc *)
1674 ((char *)CE_state->src_ring->base_addr_owner_space +
1675 ring_loop * (sizeof(struct CE_src_desc)));
1676 CE_IPA_RING_INIT(ce_desc);
1677 }
1678
1679 /* Get BAR address */
1680 hif_read_phy_mem_base(CE_state->scn, &phy_mem_base);
1681
1682 *ce_sr = CE_state->scn->ipa_ce_ring;
1683 *ce_sr_ring_size = (uint32_t)(CE_state->src_ring->nentries *
1684 sizeof(struct CE_src_desc));
1685 *ce_reg_paddr = phy_mem_base + ce_ipa_get_wr_index_addr(CE_state);
1686
1687 }
1688
1689 #endif /* IPA_OFFLOAD */
1690
1691 #ifdef HIF_CE_DEBUG_DATA_BUF
1692 /**
1693 * hif_dump_desc_data_buf() - record ce descriptor events
1694 * @buf: buffer to copy to
1695 * @pos: Current position till which the buf is filled
1696 * @data: Data to be copied
1697 * @data_len: Length of the data to be copied
1698 */
hif_dump_desc_data_buf(uint8_t * buf,ssize_t pos,uint8_t * data,uint32_t data_len)1699 static uint32_t hif_dump_desc_data_buf(uint8_t *buf, ssize_t pos,
1700 uint8_t *data, uint32_t data_len)
1701 {
1702 pos += snprintf(buf + pos, PAGE_SIZE - pos, "Data:(Max%dBytes)\n",
1703 CE_DEBUG_MAX_DATA_BUF_SIZE);
1704
1705 if ((data_len > 0) && data) {
1706 if (data_len < 16) {
1707 hex_dump_to_buffer(data,
1708 CE_DEBUG_DATA_PER_ROW,
1709 16, 1, buf + pos,
1710 (ssize_t)PAGE_SIZE - pos,
1711 false);
1712 pos += CE_DEBUG_PRINT_BUF_SIZE(data_len);
1713 pos += snprintf(buf + pos, PAGE_SIZE - pos, "\n");
1714 } else {
1715 uint32_t rows = (data_len / 16) + 1;
1716 uint32_t row = 0;
1717
1718 for (row = 0; row < rows; row++) {
1719 hex_dump_to_buffer(data + (row * 16),
1720 CE_DEBUG_DATA_PER_ROW,
1721 16, 1, buf + pos,
1722 (ssize_t)PAGE_SIZE
1723 - pos, false);
1724 pos +=
1725 CE_DEBUG_PRINT_BUF_SIZE(CE_DEBUG_DATA_PER_ROW);
1726 pos += snprintf(buf + pos, PAGE_SIZE - pos,
1727 "\n");
1728 }
1729 }
1730 }
1731
1732 return pos;
1733 }
1734 #endif
1735
1736 /*
1737 * Note: For MCL, #if defined (HIF_CONFIG_SLUB_DEBUG_ON) needs to be checked
1738 * for defined here
1739 */
1740 #if defined(HIF_CONFIG_SLUB_DEBUG_ON) || defined(HIF_CE_DEBUG_DATA_BUF)
ce_event_type_to_str(enum hif_ce_event_type type)1741 static const char *ce_event_type_to_str(enum hif_ce_event_type type)
1742 {
1743 switch (type) {
1744 case HIF_RX_DESC_POST:
1745 return "HIF_RX_DESC_POST";
1746 case HIF_RX_DESC_COMPLETION:
1747 return "HIF_RX_DESC_COMPLETION";
1748 case HIF_TX_GATHER_DESC_POST:
1749 return "HIF_TX_GATHER_DESC_POST";
1750 case HIF_TX_DESC_POST:
1751 return "HIF_TX_DESC_POST";
1752 case HIF_TX_DESC_SOFTWARE_POST:
1753 return "HIF_TX_DESC_SOFTWARE_POST";
1754 case HIF_TX_DESC_COMPLETION:
1755 return "HIF_TX_DESC_COMPLETION";
1756 case FAST_RX_WRITE_INDEX_UPDATE:
1757 return "FAST_RX_WRITE_INDEX_UPDATE";
1758 case FAST_RX_SOFTWARE_INDEX_UPDATE:
1759 return "FAST_RX_SOFTWARE_INDEX_UPDATE";
1760 case FAST_TX_WRITE_INDEX_UPDATE:
1761 return "FAST_TX_WRITE_INDEX_UPDATE";
1762 case FAST_TX_WRITE_INDEX_SOFTWARE_UPDATE:
1763 return "FAST_TX_WRITE_INDEX_SOFTWARE_UPDATE";
1764 case FAST_TX_SOFTWARE_INDEX_UPDATE:
1765 return "FAST_TX_SOFTWARE_INDEX_UPDATE";
1766 case RESUME_WRITE_INDEX_UPDATE:
1767 return "RESUME_WRITE_INDEX_UPDATE";
1768 case HIF_IRQ_EVENT:
1769 return "HIF_IRQ_EVENT";
1770 case HIF_CE_TASKLET_ENTRY:
1771 return "HIF_CE_TASKLET_ENTRY";
1772 case HIF_CE_TASKLET_RESCHEDULE:
1773 return "HIF_CE_TASKLET_RESCHEDULE";
1774 case HIF_CE_TASKLET_EXIT:
1775 return "HIF_CE_TASKLET_EXIT";
1776 case HIF_CE_REAP_ENTRY:
1777 return "HIF_CE_REAP_ENTRY";
1778 case HIF_CE_REAP_EXIT:
1779 return "HIF_CE_REAP_EXIT";
1780 case NAPI_SCHEDULE:
1781 return "NAPI_SCHEDULE";
1782 case NAPI_POLL_ENTER:
1783 return "NAPI_POLL_ENTER";
1784 case NAPI_COMPLETE:
1785 return "NAPI_COMPLETE";
1786 case NAPI_POLL_EXIT:
1787 return "NAPI_POLL_EXIT";
1788 case HIF_RX_NBUF_ALLOC_FAILURE:
1789 return "HIF_RX_NBUF_ALLOC_FAILURE";
1790 case HIF_RX_NBUF_MAP_FAILURE:
1791 return "HIF_RX_NBUF_MAP_FAILURE";
1792 case HIF_RX_NBUF_ENQUEUE_FAILURE:
1793 return "HIF_RX_NBUF_ENQUEUE_FAILURE";
1794 default:
1795 return "invalid";
1796 }
1797 }
1798
1799 /**
1800 * hif_dump_desc_event() - record ce descriptor events
1801 * @scn: HIF context
1802 * @buf: Buffer to which to be copied
1803 */
hif_dump_desc_event(struct hif_softc * scn,char * buf)1804 ssize_t hif_dump_desc_event(struct hif_softc *scn, char *buf)
1805 {
1806 struct hif_ce_desc_event *event;
1807 uint64_t secs, usecs;
1808 ssize_t len = 0;
1809 struct ce_desc_hist *ce_hist = NULL;
1810 struct hif_ce_desc_event *hist_ev = NULL;
1811
1812 if (!scn)
1813 return -EINVAL;
1814
1815 ce_hist = &scn->hif_ce_desc_hist;
1816
1817 if (ce_hist->hist_id >= CE_COUNT_MAX ||
1818 ce_hist->hist_index >= HIF_CE_HISTORY_MAX) {
1819 qdf_print("Invalid values");
1820 return -EINVAL;
1821 }
1822
1823 hist_ev =
1824 (struct hif_ce_desc_event *)ce_hist->hist_ev[ce_hist->hist_id];
1825
1826 if (!hist_ev) {
1827 qdf_print("Low Memory");
1828 return -EINVAL;
1829 }
1830
1831 event = &hist_ev[ce_hist->hist_index];
1832
1833 qdf_log_timestamp_to_secs(event->time, &secs, &usecs);
1834
1835 len += snprintf(buf, PAGE_SIZE - len,
1836 "\nTime:%lld.%06lld, CE:%d, EventType: %s, EventIndex: %d\nDataAddr=%pK",
1837 secs, usecs, ce_hist->hist_id,
1838 ce_event_type_to_str(event->type),
1839 event->index, event->memory);
1840 #ifdef HIF_CE_DEBUG_DATA_BUF
1841 len += snprintf(buf + len, PAGE_SIZE - len, ", Data len=%zu",
1842 event->actual_data_len);
1843 #endif
1844
1845 len += snprintf(buf + len, PAGE_SIZE - len, "\nCE descriptor: ");
1846
1847 hex_dump_to_buffer(&event->descriptor, sizeof(union ce_desc),
1848 16, 1, buf + len,
1849 (ssize_t)PAGE_SIZE - len, false);
1850 len += CE_DEBUG_PRINT_BUF_SIZE(sizeof(union ce_desc));
1851 len += snprintf(buf + len, PAGE_SIZE - len, "\n");
1852
1853 #ifdef HIF_CE_DEBUG_DATA_BUF
1854 if (ce_hist->data_enable[ce_hist->hist_id])
1855 len = hif_dump_desc_data_buf(buf, len, event->data,
1856 (event->actual_data_len <
1857 CE_DEBUG_MAX_DATA_BUF_SIZE) ?
1858 event->actual_data_len :
1859 CE_DEBUG_MAX_DATA_BUF_SIZE);
1860 #endif /*HIF_CE_DEBUG_DATA_BUF*/
1861
1862 len += snprintf(buf + len, PAGE_SIZE - len, "END\n");
1863
1864 return len;
1865 }
1866
1867 /*
1868 * hif_store_desc_trace_buf_index() -
1869 * API to get the CE id and CE debug storage buffer index
1870 *
1871 * @dev: network device
1872 * @attr: sysfs attribute
1873 * @buf: data got from the user
1874 *
1875 * Return total length
1876 */
hif_input_desc_trace_buf_index(struct hif_softc * scn,const char * buf,size_t size)1877 ssize_t hif_input_desc_trace_buf_index(struct hif_softc *scn,
1878 const char *buf, size_t size)
1879 {
1880 struct ce_desc_hist *ce_hist = NULL;
1881
1882 if (!scn)
1883 return -EINVAL;
1884
1885 ce_hist = &scn->hif_ce_desc_hist;
1886
1887 if (!size) {
1888 qdf_nofl_err("%s: Invalid input buffer.", __func__);
1889 return -EINVAL;
1890 }
1891
1892 if (sscanf(buf, "%u %u", (unsigned int *)&ce_hist->hist_id,
1893 (unsigned int *)&ce_hist->hist_index) != 2) {
1894 qdf_nofl_err("%s: Invalid input value.", __func__);
1895 return -EINVAL;
1896 }
1897 if ((ce_hist->hist_id >= CE_COUNT_MAX) ||
1898 (ce_hist->hist_index >= HIF_CE_HISTORY_MAX)) {
1899 qdf_print("Invalid values");
1900 return -EINVAL;
1901 }
1902
1903 return size;
1904 }
1905
1906 #endif /*defined(HIF_CONFIG_SLUB_DEBUG_ON) || defined(HIF_CE_DEBUG_DATA_BUF) */
1907
1908 #ifdef HIF_CE_DEBUG_DATA_BUF
1909 /*
1910 * hif_ce_en_desc_hist() -
1911 * API to enable recording the CE desc history
1912 *
1913 * @dev: network device
1914 * @attr: sysfs attribute
1915 * @buf: buffer to copy the data.
1916 *
1917 * Starts recording the ce desc history
1918 *
1919 * Return total length copied
1920 */
hif_ce_en_desc_hist(struct hif_softc * scn,const char * buf,size_t size)1921 ssize_t hif_ce_en_desc_hist(struct hif_softc *scn, const char *buf, size_t size)
1922 {
1923 struct ce_desc_hist *ce_hist = NULL;
1924 uint32_t cfg = 0;
1925 uint32_t ce_id = 0;
1926
1927 if (!scn)
1928 return -EINVAL;
1929
1930 ce_hist = &scn->hif_ce_desc_hist;
1931
1932 if (!size) {
1933 qdf_nofl_err("%s: Invalid input buffer.", __func__);
1934 return -EINVAL;
1935 }
1936
1937 if (sscanf(buf, "%u %u", (unsigned int *)&ce_id,
1938 (unsigned int *)&cfg) != 2) {
1939 qdf_nofl_err("%s: Invalid input: Enter CE Id<sp><1/0>.",
1940 __func__);
1941 return -EINVAL;
1942 }
1943 if (ce_id >= CE_COUNT_MAX) {
1944 qdf_print("Invalid value CE Id");
1945 return -EINVAL;
1946 }
1947
1948 if ((cfg > 1 || cfg < 0)) {
1949 qdf_print("Invalid values: enter 0 or 1");
1950 return -EINVAL;
1951 }
1952
1953 if (!ce_hist->hist_ev[ce_id])
1954 return -EINVAL;
1955
1956 qdf_mutex_acquire(&ce_hist->ce_dbg_datamem_lock[ce_id]);
1957 if (cfg == 1) {
1958 if (ce_hist->data_enable[ce_id] == 1) {
1959 qdf_debug("Already Enabled");
1960 } else {
1961 if (alloc_mem_ce_debug_hist_data(scn, ce_id)
1962 == QDF_STATUS_E_NOMEM){
1963 ce_hist->data_enable[ce_id] = 0;
1964 qdf_err("%s:Memory Alloc failed", __func__);
1965 } else
1966 ce_hist->data_enable[ce_id] = 1;
1967 }
1968 } else if (cfg == 0) {
1969 if (ce_hist->data_enable[ce_id] == 0) {
1970 qdf_debug("Already Disabled");
1971 } else {
1972 ce_hist->data_enable[ce_id] = 0;
1973 free_mem_ce_debug_hist_data(scn, ce_id);
1974 }
1975 }
1976 qdf_mutex_release(&ce_hist->ce_dbg_datamem_lock[ce_id]);
1977
1978 return size;
1979 }
1980
1981 /*
1982 * hif_disp_ce_enable_desc_data_hist() -
1983 * API to display value of data_enable
1984 *
1985 * @dev: network device
1986 * @attr: sysfs attribute
1987 * @buf: buffer to copy the data.
1988 *
1989 * Return total length copied
1990 */
hif_disp_ce_enable_desc_data_hist(struct hif_softc * scn,char * buf)1991 ssize_t hif_disp_ce_enable_desc_data_hist(struct hif_softc *scn, char *buf)
1992 {
1993 ssize_t len = 0;
1994 uint32_t ce_id = 0;
1995 struct ce_desc_hist *ce_hist = NULL;
1996
1997 if (!scn)
1998 return -EINVAL;
1999
2000 ce_hist = &scn->hif_ce_desc_hist;
2001
2002 for (ce_id = 0; ce_id < CE_COUNT_MAX; ce_id++) {
2003 len += snprintf(buf + len, PAGE_SIZE - len, " CE%d: %d\n",
2004 ce_id, ce_hist->data_enable[ce_id]);
2005 }
2006
2007 return len;
2008 }
2009 #endif /* HIF_CE_DEBUG_DATA_BUF */
2010
2011 #ifdef OL_ATH_SMART_LOGGING
2012 #define GUARD_SPACE 10
2013 #define LOG_ID_SZ 4
2014 /*
2015 * hif_log_src_ce_dump() - Copy all the CE SRC ring to buf
2016 * @src_ring: SRC ring state
2017 * @buf_cur: Current pointer in ring buffer
2018 * @buf_init:Start of the ring buffer
2019 * @buf_sz: Size of the ring buffer
2020 * @skb_sz: Max size of the SKB buffer to be copied
2021 *
2022 * Dumps all the CE SRC ring descriptors and buffers pointed by them in to
2023 * the given buf, skb_sz is the max buffer size to be copied
2024 *
2025 * Return: Current pointer in ring buffer
2026 */
hif_log_src_ce_dump(struct CE_ring_state * src_ring,uint8_t * buf_cur,uint8_t * buf_init,uint32_t buf_sz,uint32_t skb_sz)2027 static uint8_t *hif_log_src_ce_dump(struct CE_ring_state *src_ring,
2028 uint8_t *buf_cur, uint8_t *buf_init,
2029 uint32_t buf_sz, uint32_t skb_sz)
2030 {
2031 struct CE_src_desc *src_ring_base;
2032 uint32_t len, entry;
2033 struct CE_src_desc *src_desc;
2034 qdf_nbuf_t nbuf;
2035 uint32_t available_buf;
2036
2037 src_ring_base = (struct CE_src_desc *)src_ring->base_addr_owner_space;
2038 len = sizeof(struct CE_ring_state);
2039 available_buf = buf_sz - (buf_cur - buf_init);
2040 if (available_buf < (len + GUARD_SPACE)) {
2041 buf_cur = buf_init;
2042 }
2043
2044 qdf_mem_copy(buf_cur, src_ring, sizeof(struct CE_ring_state));
2045 buf_cur += sizeof(struct CE_ring_state);
2046
2047 for (entry = 0; entry < src_ring->nentries; entry++) {
2048 src_desc = CE_SRC_RING_TO_DESC(src_ring_base, entry);
2049 nbuf = src_ring->per_transfer_context[entry];
2050 if (nbuf) {
2051 uint32_t skb_len = qdf_nbuf_len(nbuf);
2052 uint32_t skb_cp_len = qdf_min(skb_len, skb_sz);
2053
2054 len = sizeof(struct CE_src_desc) + skb_cp_len
2055 + LOG_ID_SZ + sizeof(skb_cp_len);
2056 available_buf = buf_sz - (buf_cur - buf_init);
2057 if (available_buf < (len + GUARD_SPACE)) {
2058 buf_cur = buf_init;
2059 }
2060 qdf_mem_copy(buf_cur, src_desc,
2061 sizeof(struct CE_src_desc));
2062 buf_cur += sizeof(struct CE_src_desc);
2063
2064 available_buf = buf_sz - (buf_cur - buf_init);
2065 buf_cur += snprintf(buf_cur, available_buf, "SKB%d",
2066 skb_cp_len);
2067
2068 if (skb_cp_len) {
2069 qdf_mem_copy(buf_cur, qdf_nbuf_data(nbuf),
2070 skb_cp_len);
2071 buf_cur += skb_cp_len;
2072 }
2073 } else {
2074 len = sizeof(struct CE_src_desc) + LOG_ID_SZ;
2075 available_buf = buf_sz - (buf_cur - buf_init);
2076 if (available_buf < (len + GUARD_SPACE)) {
2077 buf_cur = buf_init;
2078 }
2079 qdf_mem_copy(buf_cur, src_desc,
2080 sizeof(struct CE_src_desc));
2081 buf_cur += sizeof(struct CE_src_desc);
2082 available_buf = buf_sz - (buf_cur - buf_init);
2083 buf_cur += snprintf(buf_cur, available_buf, "NUL");
2084 }
2085 }
2086
2087 return buf_cur;
2088 }
2089
2090 /*
2091 * hif_log_dest_ce_dump() - Copy all the CE DEST ring to buf
2092 * @dest_ring: SRC ring state
2093 * @buf_cur: Current pointer in ring buffer
2094 * @buf_init:Start of the ring buffer
2095 * @buf_sz: Size of the ring buffer
2096 * @skb_sz: Max size of the SKB buffer to be copied
2097 *
2098 * Dumps all the CE SRC ring descriptors and buffers pointed by them in to
2099 * the given buf, skb_sz is the max buffer size to be copied
2100 *
2101 * Return: Current pointer in ring buffer
2102 */
hif_log_dest_ce_dump(struct CE_ring_state * dest_ring,uint8_t * buf_cur,uint8_t * buf_init,uint32_t buf_sz,uint32_t skb_sz)2103 static uint8_t *hif_log_dest_ce_dump(struct CE_ring_state *dest_ring,
2104 uint8_t *buf_cur, uint8_t *buf_init,
2105 uint32_t buf_sz, uint32_t skb_sz)
2106 {
2107 struct CE_dest_desc *dest_ring_base;
2108 uint32_t len, entry;
2109 struct CE_dest_desc *dest_desc;
2110 qdf_nbuf_t nbuf;
2111 uint32_t available_buf;
2112
2113 dest_ring_base =
2114 (struct CE_dest_desc *)dest_ring->base_addr_owner_space;
2115
2116 len = sizeof(struct CE_ring_state);
2117 available_buf = buf_sz - (buf_cur - buf_init);
2118 if (available_buf < (len + GUARD_SPACE)) {
2119 buf_cur = buf_init;
2120 }
2121
2122 qdf_mem_copy(buf_cur, dest_ring, sizeof(struct CE_ring_state));
2123 buf_cur += sizeof(struct CE_ring_state);
2124
2125 for (entry = 0; entry < dest_ring->nentries; entry++) {
2126 dest_desc = CE_DEST_RING_TO_DESC(dest_ring_base, entry);
2127
2128 nbuf = dest_ring->per_transfer_context[entry];
2129 if (nbuf) {
2130 uint32_t skb_len = qdf_nbuf_len(nbuf);
2131 uint32_t skb_cp_len = qdf_min(skb_len, skb_sz);
2132
2133 len = sizeof(struct CE_dest_desc) + skb_cp_len
2134 + LOG_ID_SZ + sizeof(skb_cp_len);
2135
2136 available_buf = buf_sz - (buf_cur - buf_init);
2137 if (available_buf < (len + GUARD_SPACE)) {
2138 buf_cur = buf_init;
2139 }
2140
2141 qdf_mem_copy(buf_cur, dest_desc,
2142 sizeof(struct CE_dest_desc));
2143 buf_cur += sizeof(struct CE_dest_desc);
2144 available_buf = buf_sz - (buf_cur - buf_init);
2145 buf_cur += snprintf(buf_cur, available_buf, "SKB%d",
2146 skb_cp_len);
2147 if (skb_cp_len) {
2148 qdf_mem_copy(buf_cur, qdf_nbuf_data(nbuf),
2149 skb_cp_len);
2150 buf_cur += skb_cp_len;
2151 }
2152 } else {
2153 len = sizeof(struct CE_dest_desc) + LOG_ID_SZ;
2154 available_buf = buf_sz - (buf_cur - buf_init);
2155 if (available_buf < (len + GUARD_SPACE)) {
2156 buf_cur = buf_init;
2157 }
2158 qdf_mem_copy(buf_cur, dest_desc,
2159 sizeof(struct CE_dest_desc));
2160 buf_cur += sizeof(struct CE_dest_desc);
2161 available_buf = buf_sz - (buf_cur - buf_init);
2162 buf_cur += snprintf(buf_cur, available_buf, "NUL");
2163 }
2164 }
2165 return buf_cur;
2166 }
2167
2168 /**
2169 * hif_log_dump_ce() - Copy all the CE DEST ring to buf
2170 * @scn:
2171 * @buf_cur:
2172 * @buf_init:
2173 * @buf_sz:
2174 * @ce:
2175 * @skb_sz:
2176 *
2177 * Calls the respective function to dump all the CE SRC/DEST ring descriptors
2178 * and buffers pointed by them in to the given buf
2179 */
hif_log_dump_ce(struct hif_softc * scn,uint8_t * buf_cur,uint8_t * buf_init,uint32_t buf_sz,uint32_t ce,uint32_t skb_sz)2180 uint8_t *hif_log_dump_ce(struct hif_softc *scn, uint8_t *buf_cur,
2181 uint8_t *buf_init, uint32_t buf_sz,
2182 uint32_t ce, uint32_t skb_sz)
2183 {
2184 struct CE_state *ce_state;
2185 struct CE_ring_state *src_ring;
2186 struct CE_ring_state *dest_ring;
2187
2188 ce_state = scn->ce_id_to_state[ce];
2189 src_ring = ce_state->src_ring;
2190 dest_ring = ce_state->dest_ring;
2191
2192 if (src_ring) {
2193 buf_cur = hif_log_src_ce_dump(src_ring, buf_cur,
2194 buf_init, buf_sz, skb_sz);
2195 } else if (dest_ring) {
2196 buf_cur = hif_log_dest_ce_dump(dest_ring, buf_cur,
2197 buf_init, buf_sz, skb_sz);
2198 }
2199
2200 return buf_cur;
2201 }
2202
2203 qdf_export_symbol(hif_log_dump_ce);
2204 #endif /* OL_ATH_SMART_LOGGING */
2205
2206