1 /**
2 * IBM Accelerator Family 'GenWQE'
3 *
4 * (C) Copyright IBM Corp. 2013
5 *
6 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
7 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
8 * Author: Michael Jung <mijung@gmx.net>
9 * Author: Michael Ruettger <michael@ibmra.de>
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License (version 2 only)
13 * as published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 */
20
21 /*
22 * Miscelanous functionality used in the other GenWQE driver parts.
23 */
24
25 #include <linux/kernel.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/sched.h>
28 #include <linux/vmalloc.h>
29 #include <linux/page-flags.h>
30 #include <linux/scatterlist.h>
31 #include <linux/hugetlb.h>
32 #include <linux/iommu.h>
33 #include <linux/delay.h>
34 #include <linux/pci.h>
35 #include <linux/dma-mapping.h>
36 #include <linux/ctype.h>
37 #include <linux/module.h>
38 #include <linux/platform_device.h>
39 #include <linux/delay.h>
40 #include <asm/pgtable.h>
41
42 #include "genwqe_driver.h"
43 #include "card_base.h"
44 #include "card_ddcb.h"
45
46 /**
47 * __genwqe_writeq() - Write 64-bit register
48 * @cd: genwqe device descriptor
49 * @byte_offs: byte offset within BAR
50 * @val: 64-bit value
51 *
52 * Return: 0 if success; < 0 if error
53 */
__genwqe_writeq(struct genwqe_dev * cd,u64 byte_offs,u64 val)54 int __genwqe_writeq(struct genwqe_dev *cd, u64 byte_offs, u64 val)
55 {
56 struct pci_dev *pci_dev = cd->pci_dev;
57
58 if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
59 return -EIO;
60
61 if (cd->mmio == NULL)
62 return -EIO;
63
64 if (pci_channel_offline(pci_dev))
65 return -EIO;
66
67 __raw_writeq((__force u64)cpu_to_be64(val), cd->mmio + byte_offs);
68 return 0;
69 }
70
71 /**
72 * __genwqe_readq() - Read 64-bit register
73 * @cd: genwqe device descriptor
74 * @byte_offs: offset within BAR
75 *
76 * Return: value from register
77 */
__genwqe_readq(struct genwqe_dev * cd,u64 byte_offs)78 u64 __genwqe_readq(struct genwqe_dev *cd, u64 byte_offs)
79 {
80 if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
81 return 0xffffffffffffffffull;
82
83 if ((cd->err_inject & GENWQE_INJECT_GFIR_FATAL) &&
84 (byte_offs == IO_SLC_CFGREG_GFIR))
85 return 0x000000000000ffffull;
86
87 if ((cd->err_inject & GENWQE_INJECT_GFIR_INFO) &&
88 (byte_offs == IO_SLC_CFGREG_GFIR))
89 return 0x00000000ffff0000ull;
90
91 if (cd->mmio == NULL)
92 return 0xffffffffffffffffull;
93
94 return be64_to_cpu((__force __be64)__raw_readq(cd->mmio + byte_offs));
95 }
96
97 /**
98 * __genwqe_writel() - Write 32-bit register
99 * @cd: genwqe device descriptor
100 * @byte_offs: byte offset within BAR
101 * @val: 32-bit value
102 *
103 * Return: 0 if success; < 0 if error
104 */
__genwqe_writel(struct genwqe_dev * cd,u64 byte_offs,u32 val)105 int __genwqe_writel(struct genwqe_dev *cd, u64 byte_offs, u32 val)
106 {
107 struct pci_dev *pci_dev = cd->pci_dev;
108
109 if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
110 return -EIO;
111
112 if (cd->mmio == NULL)
113 return -EIO;
114
115 if (pci_channel_offline(pci_dev))
116 return -EIO;
117
118 __raw_writel((__force u32)cpu_to_be32(val), cd->mmio + byte_offs);
119 return 0;
120 }
121
122 /**
123 * __genwqe_readl() - Read 32-bit register
124 * @cd: genwqe device descriptor
125 * @byte_offs: offset within BAR
126 *
127 * Return: Value from register
128 */
__genwqe_readl(struct genwqe_dev * cd,u64 byte_offs)129 u32 __genwqe_readl(struct genwqe_dev *cd, u64 byte_offs)
130 {
131 if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
132 return 0xffffffff;
133
134 if (cd->mmio == NULL)
135 return 0xffffffff;
136
137 return be32_to_cpu((__force __be32)__raw_readl(cd->mmio + byte_offs));
138 }
139
140 /**
141 * genwqe_read_app_id() - Extract app_id
142 *
143 * app_unitcfg need to be filled with valid data first
144 */
genwqe_read_app_id(struct genwqe_dev * cd,char * app_name,int len)145 int genwqe_read_app_id(struct genwqe_dev *cd, char *app_name, int len)
146 {
147 int i, j;
148 u32 app_id = (u32)cd->app_unitcfg;
149
150 memset(app_name, 0, len);
151 for (i = 0, j = 0; j < min(len, 4); j++) {
152 char ch = (char)((app_id >> (24 - j*8)) & 0xff);
153
154 if (ch == ' ')
155 continue;
156 app_name[i++] = isprint(ch) ? ch : 'X';
157 }
158 return i;
159 }
160
161 /**
162 * genwqe_init_crc32() - Prepare a lookup table for fast crc32 calculations
163 *
164 * Existing kernel functions seem to use a different polynom,
165 * therefore we could not use them here.
166 *
167 * Genwqe's Polynomial = 0x20044009
168 */
169 #define CRC32_POLYNOMIAL 0x20044009
170 static u32 crc32_tab[256]; /* crc32 lookup table */
171
genwqe_init_crc32(void)172 void genwqe_init_crc32(void)
173 {
174 int i, j;
175 u32 crc;
176
177 for (i = 0; i < 256; i++) {
178 crc = i << 24;
179 for (j = 0; j < 8; j++) {
180 if (crc & 0x80000000)
181 crc = (crc << 1) ^ CRC32_POLYNOMIAL;
182 else
183 crc = (crc << 1);
184 }
185 crc32_tab[i] = crc;
186 }
187 }
188
189 /**
190 * genwqe_crc32() - Generate 32-bit crc as required for DDCBs
191 * @buff: pointer to data buffer
192 * @len: length of data for calculation
193 * @init: initial crc (0xffffffff at start)
194 *
195 * polynomial = x^32 * + x^29 + x^18 + x^14 + x^3 + 1 (0x20044009)
196
197 * Example: 4 bytes 0x01 0x02 0x03 0x04 with init=0xffffffff should
198 * result in a crc32 of 0xf33cb7d3.
199 *
200 * The existing kernel crc functions did not cover this polynom yet.
201 *
202 * Return: crc32 checksum.
203 */
genwqe_crc32(u8 * buff,size_t len,u32 init)204 u32 genwqe_crc32(u8 *buff, size_t len, u32 init)
205 {
206 int i;
207 u32 crc;
208
209 crc = init;
210 while (len--) {
211 i = ((crc >> 24) ^ *buff++) & 0xFF;
212 crc = (crc << 8) ^ crc32_tab[i];
213 }
214 return crc;
215 }
216
__genwqe_alloc_consistent(struct genwqe_dev * cd,size_t size,dma_addr_t * dma_handle)217 void *__genwqe_alloc_consistent(struct genwqe_dev *cd, size_t size,
218 dma_addr_t *dma_handle)
219 {
220 if (get_order(size) >= MAX_ORDER)
221 return NULL;
222
223 return dma_zalloc_coherent(&cd->pci_dev->dev, size, dma_handle,
224 GFP_KERNEL);
225 }
226
__genwqe_free_consistent(struct genwqe_dev * cd,size_t size,void * vaddr,dma_addr_t dma_handle)227 void __genwqe_free_consistent(struct genwqe_dev *cd, size_t size,
228 void *vaddr, dma_addr_t dma_handle)
229 {
230 if (vaddr == NULL)
231 return;
232
233 dma_free_coherent(&cd->pci_dev->dev, size, vaddr, dma_handle);
234 }
235
genwqe_unmap_pages(struct genwqe_dev * cd,dma_addr_t * dma_list,int num_pages)236 static void genwqe_unmap_pages(struct genwqe_dev *cd, dma_addr_t *dma_list,
237 int num_pages)
238 {
239 int i;
240 struct pci_dev *pci_dev = cd->pci_dev;
241
242 for (i = 0; (i < num_pages) && (dma_list[i] != 0x0); i++) {
243 pci_unmap_page(pci_dev, dma_list[i],
244 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
245 dma_list[i] = 0x0;
246 }
247 }
248
genwqe_map_pages(struct genwqe_dev * cd,struct page ** page_list,int num_pages,dma_addr_t * dma_list)249 static int genwqe_map_pages(struct genwqe_dev *cd,
250 struct page **page_list, int num_pages,
251 dma_addr_t *dma_list)
252 {
253 int i;
254 struct pci_dev *pci_dev = cd->pci_dev;
255
256 /* establish DMA mapping for requested pages */
257 for (i = 0; i < num_pages; i++) {
258 dma_addr_t daddr;
259
260 dma_list[i] = 0x0;
261 daddr = pci_map_page(pci_dev, page_list[i],
262 0, /* map_offs */
263 PAGE_SIZE,
264 PCI_DMA_BIDIRECTIONAL); /* FIXME rd/rw */
265
266 if (pci_dma_mapping_error(pci_dev, daddr)) {
267 dev_err(&pci_dev->dev,
268 "[%s] err: no dma addr daddr=%016llx!\n",
269 __func__, (long long)daddr);
270 goto err;
271 }
272
273 dma_list[i] = daddr;
274 }
275 return 0;
276
277 err:
278 genwqe_unmap_pages(cd, dma_list, num_pages);
279 return -EIO;
280 }
281
genwqe_sgl_size(int num_pages)282 static int genwqe_sgl_size(int num_pages)
283 {
284 int len, num_tlb = num_pages / 7;
285
286 len = sizeof(struct sg_entry) * (num_pages+num_tlb + 1);
287 return roundup(len, PAGE_SIZE);
288 }
289
290 /**
291 * genwqe_alloc_sync_sgl() - Allocate memory for sgl and overlapping pages
292 *
293 * Allocates memory for sgl and overlapping pages. Pages which might
294 * overlap other user-space memory blocks are being cached for DMAs,
295 * such that we do not run into syncronization issues. Data is copied
296 * from user-space into the cached pages.
297 */
genwqe_alloc_sync_sgl(struct genwqe_dev * cd,struct genwqe_sgl * sgl,void __user * user_addr,size_t user_size,int write)298 int genwqe_alloc_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl,
299 void __user *user_addr, size_t user_size, int write)
300 {
301 int ret = -ENOMEM;
302 struct pci_dev *pci_dev = cd->pci_dev;
303
304 sgl->fpage_offs = offset_in_page((unsigned long)user_addr);
305 sgl->fpage_size = min_t(size_t, PAGE_SIZE-sgl->fpage_offs, user_size);
306 sgl->nr_pages = DIV_ROUND_UP(sgl->fpage_offs + user_size, PAGE_SIZE);
307 sgl->lpage_size = (user_size - sgl->fpage_size) % PAGE_SIZE;
308
309 dev_dbg(&pci_dev->dev, "[%s] uaddr=%p usize=%8ld nr_pages=%ld fpage_offs=%lx fpage_size=%ld lpage_size=%ld\n",
310 __func__, user_addr, user_size, sgl->nr_pages,
311 sgl->fpage_offs, sgl->fpage_size, sgl->lpage_size);
312
313 sgl->user_addr = user_addr;
314 sgl->user_size = user_size;
315 sgl->write = write;
316 sgl->sgl_size = genwqe_sgl_size(sgl->nr_pages);
317
318 if (get_order(sgl->sgl_size) > MAX_ORDER) {
319 dev_err(&pci_dev->dev,
320 "[%s] err: too much memory requested!\n", __func__);
321 return ret;
322 }
323
324 sgl->sgl = __genwqe_alloc_consistent(cd, sgl->sgl_size,
325 &sgl->sgl_dma_addr);
326 if (sgl->sgl == NULL) {
327 dev_err(&pci_dev->dev,
328 "[%s] err: no memory available!\n", __func__);
329 return ret;
330 }
331
332 /* Only use buffering on incomplete pages */
333 if ((sgl->fpage_size != 0) && (sgl->fpage_size != PAGE_SIZE)) {
334 sgl->fpage = __genwqe_alloc_consistent(cd, PAGE_SIZE,
335 &sgl->fpage_dma_addr);
336 if (sgl->fpage == NULL)
337 goto err_out;
338
339 /* Sync with user memory */
340 if (copy_from_user(sgl->fpage + sgl->fpage_offs,
341 user_addr, sgl->fpage_size)) {
342 ret = -EFAULT;
343 goto err_out;
344 }
345 }
346 if (sgl->lpage_size != 0) {
347 sgl->lpage = __genwqe_alloc_consistent(cd, PAGE_SIZE,
348 &sgl->lpage_dma_addr);
349 if (sgl->lpage == NULL)
350 goto err_out1;
351
352 /* Sync with user memory */
353 if (copy_from_user(sgl->lpage, user_addr + user_size -
354 sgl->lpage_size, sgl->lpage_size)) {
355 ret = -EFAULT;
356 goto err_out2;
357 }
358 }
359 return 0;
360
361 err_out2:
362 __genwqe_free_consistent(cd, PAGE_SIZE, sgl->lpage,
363 sgl->lpage_dma_addr);
364 sgl->lpage = NULL;
365 sgl->lpage_dma_addr = 0;
366 err_out1:
367 __genwqe_free_consistent(cd, PAGE_SIZE, sgl->fpage,
368 sgl->fpage_dma_addr);
369 sgl->fpage = NULL;
370 sgl->fpage_dma_addr = 0;
371 err_out:
372 __genwqe_free_consistent(cd, sgl->sgl_size, sgl->sgl,
373 sgl->sgl_dma_addr);
374 sgl->sgl = NULL;
375 sgl->sgl_dma_addr = 0;
376 sgl->sgl_size = 0;
377
378 return ret;
379 }
380
genwqe_setup_sgl(struct genwqe_dev * cd,struct genwqe_sgl * sgl,dma_addr_t * dma_list)381 int genwqe_setup_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl,
382 dma_addr_t *dma_list)
383 {
384 int i = 0, j = 0, p;
385 unsigned long dma_offs, map_offs;
386 dma_addr_t prev_daddr = 0;
387 struct sg_entry *s, *last_s = NULL;
388 size_t size = sgl->user_size;
389
390 dma_offs = 128; /* next block if needed/dma_offset */
391 map_offs = sgl->fpage_offs; /* offset in first page */
392
393 s = &sgl->sgl[0]; /* first set of 8 entries */
394 p = 0; /* page */
395 while (p < sgl->nr_pages) {
396 dma_addr_t daddr;
397 unsigned int size_to_map;
398
399 /* always write the chaining entry, cleanup is done later */
400 j = 0;
401 s[j].target_addr = cpu_to_be64(sgl->sgl_dma_addr + dma_offs);
402 s[j].len = cpu_to_be32(128);
403 s[j].flags = cpu_to_be32(SG_CHAINED);
404 j++;
405
406 while (j < 8) {
407 /* DMA mapping for requested page, offs, size */
408 size_to_map = min(size, PAGE_SIZE - map_offs);
409
410 if ((p == 0) && (sgl->fpage != NULL)) {
411 daddr = sgl->fpage_dma_addr + map_offs;
412
413 } else if ((p == sgl->nr_pages - 1) &&
414 (sgl->lpage != NULL)) {
415 daddr = sgl->lpage_dma_addr;
416 } else {
417 daddr = dma_list[p] + map_offs;
418 }
419
420 size -= size_to_map;
421 map_offs = 0;
422
423 if (prev_daddr == daddr) {
424 u32 prev_len = be32_to_cpu(last_s->len);
425
426 /* pr_info("daddr combining: "
427 "%016llx/%08x -> %016llx\n",
428 prev_daddr, prev_len, daddr); */
429
430 last_s->len = cpu_to_be32(prev_len +
431 size_to_map);
432
433 p++; /* process next page */
434 if (p == sgl->nr_pages)
435 goto fixup; /* nothing to do */
436
437 prev_daddr = daddr + size_to_map;
438 continue;
439 }
440
441 /* start new entry */
442 s[j].target_addr = cpu_to_be64(daddr);
443 s[j].len = cpu_to_be32(size_to_map);
444 s[j].flags = cpu_to_be32(SG_DATA);
445 prev_daddr = daddr + size_to_map;
446 last_s = &s[j];
447 j++;
448
449 p++; /* process next page */
450 if (p == sgl->nr_pages)
451 goto fixup; /* nothing to do */
452 }
453 dma_offs += 128;
454 s += 8; /* continue 8 elements further */
455 }
456 fixup:
457 if (j == 1) { /* combining happened on last entry! */
458 s -= 8; /* full shift needed on previous sgl block */
459 j = 7; /* shift all elements */
460 }
461
462 for (i = 0; i < j; i++) /* move elements 1 up */
463 s[i] = s[i + 1];
464
465 s[i].target_addr = cpu_to_be64(0);
466 s[i].len = cpu_to_be32(0);
467 s[i].flags = cpu_to_be32(SG_END_LIST);
468 return 0;
469 }
470
471 /**
472 * genwqe_free_sync_sgl() - Free memory for sgl and overlapping pages
473 *
474 * After the DMA transfer has been completed we free the memory for
475 * the sgl and the cached pages. Data is being transferred from cached
476 * pages into user-space buffers.
477 */
genwqe_free_sync_sgl(struct genwqe_dev * cd,struct genwqe_sgl * sgl)478 int genwqe_free_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl)
479 {
480 int rc = 0;
481 size_t offset;
482 unsigned long res;
483 struct pci_dev *pci_dev = cd->pci_dev;
484
485 if (sgl->fpage) {
486 if (sgl->write) {
487 res = copy_to_user(sgl->user_addr,
488 sgl->fpage + sgl->fpage_offs, sgl->fpage_size);
489 if (res) {
490 dev_err(&pci_dev->dev,
491 "[%s] err: copying fpage! (res=%lu)\n",
492 __func__, res);
493 rc = -EFAULT;
494 }
495 }
496 __genwqe_free_consistent(cd, PAGE_SIZE, sgl->fpage,
497 sgl->fpage_dma_addr);
498 sgl->fpage = NULL;
499 sgl->fpage_dma_addr = 0;
500 }
501 if (sgl->lpage) {
502 if (sgl->write) {
503 offset = sgl->user_size - sgl->lpage_size;
504 res = copy_to_user(sgl->user_addr + offset, sgl->lpage,
505 sgl->lpage_size);
506 if (res) {
507 dev_err(&pci_dev->dev,
508 "[%s] err: copying lpage! (res=%lu)\n",
509 __func__, res);
510 rc = -EFAULT;
511 }
512 }
513 __genwqe_free_consistent(cd, PAGE_SIZE, sgl->lpage,
514 sgl->lpage_dma_addr);
515 sgl->lpage = NULL;
516 sgl->lpage_dma_addr = 0;
517 }
518 __genwqe_free_consistent(cd, sgl->sgl_size, sgl->sgl,
519 sgl->sgl_dma_addr);
520
521 sgl->sgl = NULL;
522 sgl->sgl_dma_addr = 0x0;
523 sgl->sgl_size = 0;
524 return rc;
525 }
526
527 /**
528 * genwqe_free_user_pages() - Give pinned pages back
529 *
530 * Documentation of get_user_pages is in mm/gup.c:
531 *
532 * If the page is written to, set_page_dirty (or set_page_dirty_lock,
533 * as appropriate) must be called after the page is finished with, and
534 * before put_page is called.
535 */
genwqe_free_user_pages(struct page ** page_list,unsigned int nr_pages,int dirty)536 static int genwqe_free_user_pages(struct page **page_list,
537 unsigned int nr_pages, int dirty)
538 {
539 unsigned int i;
540
541 for (i = 0; i < nr_pages; i++) {
542 if (page_list[i] != NULL) {
543 if (dirty)
544 set_page_dirty_lock(page_list[i]);
545 put_page(page_list[i]);
546 }
547 }
548 return 0;
549 }
550
551 /**
552 * genwqe_user_vmap() - Map user-space memory to virtual kernel memory
553 * @cd: pointer to genwqe device
554 * @m: mapping params
555 * @uaddr: user virtual address
556 * @size: size of memory to be mapped
557 *
558 * We need to think about how we could speed this up. Of course it is
559 * not a good idea to do this over and over again, like we are
560 * currently doing it. Nevertheless, I am curious where on the path
561 * the performance is spend. Most probably within the memory
562 * allocation functions, but maybe also in the DMA mapping code.
563 *
564 * Restrictions: The maximum size of the possible mapping currently depends
565 * on the amount of memory we can get using kzalloc() for the
566 * page_list and pci_alloc_consistent for the sg_list.
567 * The sg_list is currently itself not scattered, which could
568 * be fixed with some effort. The page_list must be split into
569 * PAGE_SIZE chunks too. All that will make the complicated
570 * code more complicated.
571 *
572 * Return: 0 if success
573 */
genwqe_user_vmap(struct genwqe_dev * cd,struct dma_mapping * m,void * uaddr,unsigned long size)574 int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,
575 unsigned long size)
576 {
577 int rc = -EINVAL;
578 unsigned long data, offs;
579 struct pci_dev *pci_dev = cd->pci_dev;
580
581 if ((uaddr == NULL) || (size == 0)) {
582 m->size = 0; /* mark unused and not added */
583 return -EINVAL;
584 }
585 m->u_vaddr = uaddr;
586 m->size = size;
587
588 /* determine space needed for page_list. */
589 data = (unsigned long)uaddr;
590 offs = offset_in_page(data);
591 if (size > ULONG_MAX - PAGE_SIZE - offs) {
592 m->size = 0; /* mark unused and not added */
593 return -EINVAL;
594 }
595 m->nr_pages = DIV_ROUND_UP(offs + size, PAGE_SIZE);
596
597 m->page_list = kcalloc(m->nr_pages,
598 sizeof(struct page *) + sizeof(dma_addr_t),
599 GFP_KERNEL);
600 if (!m->page_list) {
601 dev_err(&pci_dev->dev, "err: alloc page_list failed\n");
602 m->nr_pages = 0;
603 m->u_vaddr = NULL;
604 m->size = 0; /* mark unused and not added */
605 return -ENOMEM;
606 }
607 m->dma_list = (dma_addr_t *)(m->page_list + m->nr_pages);
608
609 /* pin user pages in memory */
610 rc = get_user_pages_fast(data & PAGE_MASK, /* page aligned addr */
611 m->nr_pages,
612 m->write, /* readable/writable */
613 m->page_list); /* ptrs to pages */
614 if (rc < 0)
615 goto fail_get_user_pages;
616
617 /* assumption: get_user_pages can be killed by signals. */
618 if (rc < m->nr_pages) {
619 genwqe_free_user_pages(m->page_list, rc, m->write);
620 rc = -EFAULT;
621 goto fail_get_user_pages;
622 }
623
624 rc = genwqe_map_pages(cd, m->page_list, m->nr_pages, m->dma_list);
625 if (rc != 0)
626 goto fail_free_user_pages;
627
628 return 0;
629
630 fail_free_user_pages:
631 genwqe_free_user_pages(m->page_list, m->nr_pages, m->write);
632
633 fail_get_user_pages:
634 kfree(m->page_list);
635 m->page_list = NULL;
636 m->dma_list = NULL;
637 m->nr_pages = 0;
638 m->u_vaddr = NULL;
639 m->size = 0; /* mark unused and not added */
640 return rc;
641 }
642
643 /**
644 * genwqe_user_vunmap() - Undo mapping of user-space mem to virtual kernel
645 * memory
646 * @cd: pointer to genwqe device
647 * @m: mapping params
648 */
genwqe_user_vunmap(struct genwqe_dev * cd,struct dma_mapping * m)649 int genwqe_user_vunmap(struct genwqe_dev *cd, struct dma_mapping *m)
650 {
651 struct pci_dev *pci_dev = cd->pci_dev;
652
653 if (!dma_mapping_used(m)) {
654 dev_err(&pci_dev->dev, "[%s] err: mapping %p not used!\n",
655 __func__, m);
656 return -EINVAL;
657 }
658
659 if (m->dma_list)
660 genwqe_unmap_pages(cd, m->dma_list, m->nr_pages);
661
662 if (m->page_list) {
663 genwqe_free_user_pages(m->page_list, m->nr_pages, m->write);
664
665 kfree(m->page_list);
666 m->page_list = NULL;
667 m->dma_list = NULL;
668 m->nr_pages = 0;
669 }
670
671 m->u_vaddr = NULL;
672 m->size = 0; /* mark as unused and not added */
673 return 0;
674 }
675
676 /**
677 * genwqe_card_type() - Get chip type SLU Configuration Register
678 * @cd: pointer to the genwqe device descriptor
679 * Return: 0: Altera Stratix-IV 230
680 * 1: Altera Stratix-IV 530
681 * 2: Altera Stratix-V A4
682 * 3: Altera Stratix-V A7
683 */
genwqe_card_type(struct genwqe_dev * cd)684 u8 genwqe_card_type(struct genwqe_dev *cd)
685 {
686 u64 card_type = cd->slu_unitcfg;
687
688 return (u8)((card_type & IO_SLU_UNITCFG_TYPE_MASK) >> 20);
689 }
690
691 /**
692 * genwqe_card_reset() - Reset the card
693 * @cd: pointer to the genwqe device descriptor
694 */
genwqe_card_reset(struct genwqe_dev * cd)695 int genwqe_card_reset(struct genwqe_dev *cd)
696 {
697 u64 softrst;
698 struct pci_dev *pci_dev = cd->pci_dev;
699
700 if (!genwqe_is_privileged(cd))
701 return -ENODEV;
702
703 /* new SL */
704 __genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, 0x1ull);
705 msleep(1000);
706 __genwqe_readq(cd, IO_HSU_FIR_CLR);
707 __genwqe_readq(cd, IO_APP_FIR_CLR);
708 __genwqe_readq(cd, IO_SLU_FIR_CLR);
709
710 /*
711 * Read-modify-write to preserve the stealth bits
712 *
713 * For SL >= 039, Stealth WE bit allows removing
714 * the read-modify-wrote.
715 * r-m-w may require a mask 0x3C to avoid hitting hard
716 * reset again for error reset (should be 0, chicken).
717 */
718 softrst = __genwqe_readq(cd, IO_SLC_CFGREG_SOFTRESET) & 0x3cull;
719 __genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, softrst | 0x2ull);
720
721 /* give ERRORRESET some time to finish */
722 msleep(50);
723
724 if (genwqe_need_err_masking(cd)) {
725 dev_info(&pci_dev->dev,
726 "[%s] masking errors for old bitstreams\n", __func__);
727 __genwqe_writeq(cd, IO_SLC_MISC_DEBUG, 0x0aull);
728 }
729 return 0;
730 }
731
genwqe_read_softreset(struct genwqe_dev * cd)732 int genwqe_read_softreset(struct genwqe_dev *cd)
733 {
734 u64 bitstream;
735
736 if (!genwqe_is_privileged(cd))
737 return -ENODEV;
738
739 bitstream = __genwqe_readq(cd, IO_SLU_BITSTREAM) & 0x1;
740 cd->softreset = (bitstream == 0) ? 0x8ull : 0xcull;
741 return 0;
742 }
743
744 /**
745 * genwqe_set_interrupt_capability() - Configure MSI capability structure
746 * @cd: pointer to the device
747 * Return: 0 if no error
748 */
genwqe_set_interrupt_capability(struct genwqe_dev * cd,int count)749 int genwqe_set_interrupt_capability(struct genwqe_dev *cd, int count)
750 {
751 int rc;
752
753 rc = pci_alloc_irq_vectors(cd->pci_dev, 1, count, PCI_IRQ_MSI);
754 if (rc < 0)
755 return rc;
756 return 0;
757 }
758
759 /**
760 * genwqe_reset_interrupt_capability() - Undo genwqe_set_interrupt_capability()
761 * @cd: pointer to the device
762 */
genwqe_reset_interrupt_capability(struct genwqe_dev * cd)763 void genwqe_reset_interrupt_capability(struct genwqe_dev *cd)
764 {
765 pci_free_irq_vectors(cd->pci_dev);
766 }
767
768 /**
769 * set_reg_idx() - Fill array with data. Ignore illegal offsets.
770 * @cd: card device
771 * @r: debug register array
772 * @i: index to desired entry
773 * @m: maximum possible entries
774 * @addr: addr which is read
775 * @index: index in debug array
776 * @val: read value
777 */
set_reg_idx(struct genwqe_dev * cd,struct genwqe_reg * r,unsigned int * i,unsigned int m,u32 addr,u32 idx,u64 val)778 static int set_reg_idx(struct genwqe_dev *cd, struct genwqe_reg *r,
779 unsigned int *i, unsigned int m, u32 addr, u32 idx,
780 u64 val)
781 {
782 if (WARN_ON_ONCE(*i >= m))
783 return -EFAULT;
784
785 r[*i].addr = addr;
786 r[*i].idx = idx;
787 r[*i].val = val;
788 ++*i;
789 return 0;
790 }
791
set_reg(struct genwqe_dev * cd,struct genwqe_reg * r,unsigned int * i,unsigned int m,u32 addr,u64 val)792 static int set_reg(struct genwqe_dev *cd, struct genwqe_reg *r,
793 unsigned int *i, unsigned int m, u32 addr, u64 val)
794 {
795 return set_reg_idx(cd, r, i, m, addr, 0, val);
796 }
797
genwqe_read_ffdc_regs(struct genwqe_dev * cd,struct genwqe_reg * regs,unsigned int max_regs,int all)798 int genwqe_read_ffdc_regs(struct genwqe_dev *cd, struct genwqe_reg *regs,
799 unsigned int max_regs, int all)
800 {
801 unsigned int i, j, idx = 0;
802 u32 ufir_addr, ufec_addr, sfir_addr, sfec_addr;
803 u64 gfir, sluid, appid, ufir, ufec, sfir, sfec;
804
805 /* Global FIR */
806 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
807 set_reg(cd, regs, &idx, max_regs, IO_SLC_CFGREG_GFIR, gfir);
808
809 /* UnitCfg for SLU */
810 sluid = __genwqe_readq(cd, IO_SLU_UNITCFG); /* 0x00000000 */
811 set_reg(cd, regs, &idx, max_regs, IO_SLU_UNITCFG, sluid);
812
813 /* UnitCfg for APP */
814 appid = __genwqe_readq(cd, IO_APP_UNITCFG); /* 0x02000000 */
815 set_reg(cd, regs, &idx, max_regs, IO_APP_UNITCFG, appid);
816
817 /* Check all chip Units */
818 for (i = 0; i < GENWQE_MAX_UNITS; i++) {
819
820 /* Unit FIR */
821 ufir_addr = (i << 24) | 0x008;
822 ufir = __genwqe_readq(cd, ufir_addr);
823 set_reg(cd, regs, &idx, max_regs, ufir_addr, ufir);
824
825 /* Unit FEC */
826 ufec_addr = (i << 24) | 0x018;
827 ufec = __genwqe_readq(cd, ufec_addr);
828 set_reg(cd, regs, &idx, max_regs, ufec_addr, ufec);
829
830 for (j = 0; j < 64; j++) {
831 /* wherever there is a primary 1, read the 2ndary */
832 if (!all && (!(ufir & (1ull << j))))
833 continue;
834
835 sfir_addr = (i << 24) | (0x100 + 8 * j);
836 sfir = __genwqe_readq(cd, sfir_addr);
837 set_reg(cd, regs, &idx, max_regs, sfir_addr, sfir);
838
839 sfec_addr = (i << 24) | (0x300 + 8 * j);
840 sfec = __genwqe_readq(cd, sfec_addr);
841 set_reg(cd, regs, &idx, max_regs, sfec_addr, sfec);
842 }
843 }
844
845 /* fill with invalid data until end */
846 for (i = idx; i < max_regs; i++) {
847 regs[i].addr = 0xffffffff;
848 regs[i].val = 0xffffffffffffffffull;
849 }
850 return idx;
851 }
852
853 /**
854 * genwqe_ffdc_buff_size() - Calculates the number of dump registers
855 */
genwqe_ffdc_buff_size(struct genwqe_dev * cd,int uid)856 int genwqe_ffdc_buff_size(struct genwqe_dev *cd, int uid)
857 {
858 int entries = 0, ring, traps, traces, trace_entries;
859 u32 eevptr_addr, l_addr, d_len, d_type;
860 u64 eevptr, val, addr;
861
862 eevptr_addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_ERROR_POINTER;
863 eevptr = __genwqe_readq(cd, eevptr_addr);
864
865 if ((eevptr != 0x0) && (eevptr != -1ull)) {
866 l_addr = GENWQE_UID_OFFS(uid) | eevptr;
867
868 while (1) {
869 val = __genwqe_readq(cd, l_addr);
870
871 if ((val == 0x0) || (val == -1ull))
872 break;
873
874 /* 38:24 */
875 d_len = (val & 0x0000007fff000000ull) >> 24;
876
877 /* 39 */
878 d_type = (val & 0x0000008000000000ull) >> 36;
879
880 if (d_type) { /* repeat */
881 entries += d_len;
882 } else { /* size in bytes! */
883 entries += d_len >> 3;
884 }
885
886 l_addr += 8;
887 }
888 }
889
890 for (ring = 0; ring < 8; ring++) {
891 addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_DIAG_MAP(ring);
892 val = __genwqe_readq(cd, addr);
893
894 if ((val == 0x0ull) || (val == -1ull))
895 continue;
896
897 traps = (val >> 24) & 0xff;
898 traces = (val >> 16) & 0xff;
899 trace_entries = val & 0xffff;
900
901 entries += traps + (traces * trace_entries);
902 }
903 return entries;
904 }
905
906 /**
907 * genwqe_ffdc_buff_read() - Implements LogoutExtendedErrorRegisters procedure
908 */
genwqe_ffdc_buff_read(struct genwqe_dev * cd,int uid,struct genwqe_reg * regs,unsigned int max_regs)909 int genwqe_ffdc_buff_read(struct genwqe_dev *cd, int uid,
910 struct genwqe_reg *regs, unsigned int max_regs)
911 {
912 int i, traps, traces, trace, trace_entries, trace_entry, ring;
913 unsigned int idx = 0;
914 u32 eevptr_addr, l_addr, d_addr, d_len, d_type;
915 u64 eevptr, e, val, addr;
916
917 eevptr_addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_ERROR_POINTER;
918 eevptr = __genwqe_readq(cd, eevptr_addr);
919
920 if ((eevptr != 0x0) && (eevptr != 0xffffffffffffffffull)) {
921 l_addr = GENWQE_UID_OFFS(uid) | eevptr;
922 while (1) {
923 e = __genwqe_readq(cd, l_addr);
924 if ((e == 0x0) || (e == 0xffffffffffffffffull))
925 break;
926
927 d_addr = (e & 0x0000000000ffffffull); /* 23:0 */
928 d_len = (e & 0x0000007fff000000ull) >> 24; /* 38:24 */
929 d_type = (e & 0x0000008000000000ull) >> 36; /* 39 */
930 d_addr |= GENWQE_UID_OFFS(uid);
931
932 if (d_type) {
933 for (i = 0; i < (int)d_len; i++) {
934 val = __genwqe_readq(cd, d_addr);
935 set_reg_idx(cd, regs, &idx, max_regs,
936 d_addr, i, val);
937 }
938 } else {
939 d_len >>= 3; /* Size in bytes! */
940 for (i = 0; i < (int)d_len; i++, d_addr += 8) {
941 val = __genwqe_readq(cd, d_addr);
942 set_reg_idx(cd, regs, &idx, max_regs,
943 d_addr, 0, val);
944 }
945 }
946 l_addr += 8;
947 }
948 }
949
950 /*
951 * To save time, there are only 6 traces poplulated on Uid=2,
952 * Ring=1. each with iters=512.
953 */
954 for (ring = 0; ring < 8; ring++) { /* 0 is fls, 1 is fds,
955 2...7 are ASI rings */
956 addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_DIAG_MAP(ring);
957 val = __genwqe_readq(cd, addr);
958
959 if ((val == 0x0ull) || (val == -1ull))
960 continue;
961
962 traps = (val >> 24) & 0xff; /* Number of Traps */
963 traces = (val >> 16) & 0xff; /* Number of Traces */
964 trace_entries = val & 0xffff; /* Entries per trace */
965
966 /* Note: This is a combined loop that dumps both the traps */
967 /* (for the trace == 0 case) as well as the traces 1 to */
968 /* 'traces'. */
969 for (trace = 0; trace <= traces; trace++) {
970 u32 diag_sel =
971 GENWQE_EXTENDED_DIAG_SELECTOR(ring, trace);
972
973 addr = (GENWQE_UID_OFFS(uid) |
974 IO_EXTENDED_DIAG_SELECTOR);
975 __genwqe_writeq(cd, addr, diag_sel);
976
977 for (trace_entry = 0;
978 trace_entry < (trace ? trace_entries : traps);
979 trace_entry++) {
980 addr = (GENWQE_UID_OFFS(uid) |
981 IO_EXTENDED_DIAG_READ_MBX);
982 val = __genwqe_readq(cd, addr);
983 set_reg_idx(cd, regs, &idx, max_regs, addr,
984 (diag_sel<<16) | trace_entry, val);
985 }
986 }
987 }
988 return 0;
989 }
990
991 /**
992 * genwqe_write_vreg() - Write register in virtual window
993 *
994 * Note, these registers are only accessible to the PF through the
995 * VF-window. It is not intended for the VF to access.
996 */
genwqe_write_vreg(struct genwqe_dev * cd,u32 reg,u64 val,int func)997 int genwqe_write_vreg(struct genwqe_dev *cd, u32 reg, u64 val, int func)
998 {
999 __genwqe_writeq(cd, IO_PF_SLC_VIRTUAL_WINDOW, func & 0xf);
1000 __genwqe_writeq(cd, reg, val);
1001 return 0;
1002 }
1003
1004 /**
1005 * genwqe_read_vreg() - Read register in virtual window
1006 *
1007 * Note, these registers are only accessible to the PF through the
1008 * VF-window. It is not intended for the VF to access.
1009 */
genwqe_read_vreg(struct genwqe_dev * cd,u32 reg,int func)1010 u64 genwqe_read_vreg(struct genwqe_dev *cd, u32 reg, int func)
1011 {
1012 __genwqe_writeq(cd, IO_PF_SLC_VIRTUAL_WINDOW, func & 0xf);
1013 return __genwqe_readq(cd, reg);
1014 }
1015
1016 /**
1017 * genwqe_base_clock_frequency() - Deteremine base clock frequency of the card
1018 *
1019 * Note: From a design perspective it turned out to be a bad idea to
1020 * use codes here to specifiy the frequency/speed values. An old
1021 * driver cannot understand new codes and is therefore always a
1022 * problem. Better is to measure out the value or put the
1023 * speed/frequency directly into a register which is always a valid
1024 * value for old as well as for new software.
1025 *
1026 * Return: Card clock in MHz
1027 */
genwqe_base_clock_frequency(struct genwqe_dev * cd)1028 int genwqe_base_clock_frequency(struct genwqe_dev *cd)
1029 {
1030 u16 speed; /* MHz MHz MHz MHz */
1031 static const int speed_grade[] = { 250, 200, 166, 175 };
1032
1033 speed = (u16)((cd->slu_unitcfg >> 28) & 0x0full);
1034 if (speed >= ARRAY_SIZE(speed_grade))
1035 return 0; /* illegal value */
1036
1037 return speed_grade[speed];
1038 }
1039
1040 /**
1041 * genwqe_stop_traps() - Stop traps
1042 *
1043 * Before reading out the analysis data, we need to stop the traps.
1044 */
genwqe_stop_traps(struct genwqe_dev * cd)1045 void genwqe_stop_traps(struct genwqe_dev *cd)
1046 {
1047 __genwqe_writeq(cd, IO_SLC_MISC_DEBUG_SET, 0xcull);
1048 }
1049
1050 /**
1051 * genwqe_start_traps() - Start traps
1052 *
1053 * After having read the data, we can/must enable the traps again.
1054 */
genwqe_start_traps(struct genwqe_dev * cd)1055 void genwqe_start_traps(struct genwqe_dev *cd)
1056 {
1057 __genwqe_writeq(cd, IO_SLC_MISC_DEBUG_CLR, 0xcull);
1058
1059 if (genwqe_need_err_masking(cd))
1060 __genwqe_writeq(cd, IO_SLC_MISC_DEBUG, 0x0aull);
1061 }
1062