1 /**
2  * IBM Accelerator Family 'GenWQE'
3  *
4  * (C) Copyright IBM Corp. 2013
5  *
6  * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
7  * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
8  * Author: Michael Jung <mijung@gmx.net>
9  * Author: Michael Ruettger <michael@ibmra.de>
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License (version 2 only)
13  * as published by the Free Software Foundation.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18  * GNU General Public License for more details.
19  */
20 
21 /*
22  * Miscelanous functionality used in the other GenWQE driver parts.
23  */
24 
25 #include <linux/kernel.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/sched.h>
28 #include <linux/vmalloc.h>
29 #include <linux/page-flags.h>
30 #include <linux/scatterlist.h>
31 #include <linux/hugetlb.h>
32 #include <linux/iommu.h>
33 #include <linux/delay.h>
34 #include <linux/pci.h>
35 #include <linux/dma-mapping.h>
36 #include <linux/ctype.h>
37 #include <linux/module.h>
38 #include <linux/platform_device.h>
39 #include <linux/delay.h>
40 #include <asm/pgtable.h>
41 
42 #include "genwqe_driver.h"
43 #include "card_base.h"
44 #include "card_ddcb.h"
45 
46 /**
47  * __genwqe_writeq() - Write 64-bit register
48  * @cd:	        genwqe device descriptor
49  * @byte_offs:  byte offset within BAR
50  * @val:        64-bit value
51  *
52  * Return: 0 if success; < 0 if error
53  */
__genwqe_writeq(struct genwqe_dev * cd,u64 byte_offs,u64 val)54 int __genwqe_writeq(struct genwqe_dev *cd, u64 byte_offs, u64 val)
55 {
56 	struct pci_dev *pci_dev = cd->pci_dev;
57 
58 	if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
59 		return -EIO;
60 
61 	if (cd->mmio == NULL)
62 		return -EIO;
63 
64 	if (pci_channel_offline(pci_dev))
65 		return -EIO;
66 
67 	__raw_writeq((__force u64)cpu_to_be64(val), cd->mmio + byte_offs);
68 	return 0;
69 }
70 
71 /**
72  * __genwqe_readq() - Read 64-bit register
73  * @cd:         genwqe device descriptor
74  * @byte_offs:  offset within BAR
75  *
76  * Return: value from register
77  */
__genwqe_readq(struct genwqe_dev * cd,u64 byte_offs)78 u64 __genwqe_readq(struct genwqe_dev *cd, u64 byte_offs)
79 {
80 	if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
81 		return 0xffffffffffffffffull;
82 
83 	if ((cd->err_inject & GENWQE_INJECT_GFIR_FATAL) &&
84 	    (byte_offs == IO_SLC_CFGREG_GFIR))
85 		return 0x000000000000ffffull;
86 
87 	if ((cd->err_inject & GENWQE_INJECT_GFIR_INFO) &&
88 	    (byte_offs == IO_SLC_CFGREG_GFIR))
89 		return 0x00000000ffff0000ull;
90 
91 	if (cd->mmio == NULL)
92 		return 0xffffffffffffffffull;
93 
94 	return be64_to_cpu((__force __be64)__raw_readq(cd->mmio + byte_offs));
95 }
96 
97 /**
98  * __genwqe_writel() - Write 32-bit register
99  * @cd:	        genwqe device descriptor
100  * @byte_offs:  byte offset within BAR
101  * @val:        32-bit value
102  *
103  * Return: 0 if success; < 0 if error
104  */
__genwqe_writel(struct genwqe_dev * cd,u64 byte_offs,u32 val)105 int __genwqe_writel(struct genwqe_dev *cd, u64 byte_offs, u32 val)
106 {
107 	struct pci_dev *pci_dev = cd->pci_dev;
108 
109 	if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
110 		return -EIO;
111 
112 	if (cd->mmio == NULL)
113 		return -EIO;
114 
115 	if (pci_channel_offline(pci_dev))
116 		return -EIO;
117 
118 	__raw_writel((__force u32)cpu_to_be32(val), cd->mmio + byte_offs);
119 	return 0;
120 }
121 
122 /**
123  * __genwqe_readl() - Read 32-bit register
124  * @cd:         genwqe device descriptor
125  * @byte_offs:  offset within BAR
126  *
127  * Return: Value from register
128  */
__genwqe_readl(struct genwqe_dev * cd,u64 byte_offs)129 u32 __genwqe_readl(struct genwqe_dev *cd, u64 byte_offs)
130 {
131 	if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE)
132 		return 0xffffffff;
133 
134 	if (cd->mmio == NULL)
135 		return 0xffffffff;
136 
137 	return be32_to_cpu((__force __be32)__raw_readl(cd->mmio + byte_offs));
138 }
139 
140 /**
141  * genwqe_read_app_id() - Extract app_id
142  *
143  * app_unitcfg need to be filled with valid data first
144  */
genwqe_read_app_id(struct genwqe_dev * cd,char * app_name,int len)145 int genwqe_read_app_id(struct genwqe_dev *cd, char *app_name, int len)
146 {
147 	int i, j;
148 	u32 app_id = (u32)cd->app_unitcfg;
149 
150 	memset(app_name, 0, len);
151 	for (i = 0, j = 0; j < min(len, 4); j++) {
152 		char ch = (char)((app_id >> (24 - j*8)) & 0xff);
153 
154 		if (ch == ' ')
155 			continue;
156 		app_name[i++] = isprint(ch) ? ch : 'X';
157 	}
158 	return i;
159 }
160 
161 /**
162  * genwqe_init_crc32() - Prepare a lookup table for fast crc32 calculations
163  *
164  * Existing kernel functions seem to use a different polynom,
165  * therefore we could not use them here.
166  *
167  * Genwqe's Polynomial = 0x20044009
168  */
169 #define CRC32_POLYNOMIAL	0x20044009
170 static u32 crc32_tab[256];	/* crc32 lookup table */
171 
genwqe_init_crc32(void)172 void genwqe_init_crc32(void)
173 {
174 	int i, j;
175 	u32 crc;
176 
177 	for (i = 0;  i < 256;  i++) {
178 		crc = i << 24;
179 		for (j = 0;  j < 8;  j++) {
180 			if (crc & 0x80000000)
181 				crc = (crc << 1) ^ CRC32_POLYNOMIAL;
182 			else
183 				crc = (crc << 1);
184 		}
185 		crc32_tab[i] = crc;
186 	}
187 }
188 
189 /**
190  * genwqe_crc32() - Generate 32-bit crc as required for DDCBs
191  * @buff:       pointer to data buffer
192  * @len:        length of data for calculation
193  * @init:       initial crc (0xffffffff at start)
194  *
195  * polynomial = x^32 * + x^29 + x^18 + x^14 + x^3 + 1 (0x20044009)
196 
197  * Example: 4 bytes 0x01 0x02 0x03 0x04 with init=0xffffffff should
198  * result in a crc32 of 0xf33cb7d3.
199  *
200  * The existing kernel crc functions did not cover this polynom yet.
201  *
202  * Return: crc32 checksum.
203  */
genwqe_crc32(u8 * buff,size_t len,u32 init)204 u32 genwqe_crc32(u8 *buff, size_t len, u32 init)
205 {
206 	int i;
207 	u32 crc;
208 
209 	crc = init;
210 	while (len--) {
211 		i = ((crc >> 24) ^ *buff++) & 0xFF;
212 		crc = (crc << 8) ^ crc32_tab[i];
213 	}
214 	return crc;
215 }
216 
__genwqe_alloc_consistent(struct genwqe_dev * cd,size_t size,dma_addr_t * dma_handle)217 void *__genwqe_alloc_consistent(struct genwqe_dev *cd, size_t size,
218 			       dma_addr_t *dma_handle)
219 {
220 	if (get_order(size) >= MAX_ORDER)
221 		return NULL;
222 
223 	return dma_zalloc_coherent(&cd->pci_dev->dev, size, dma_handle,
224 				   GFP_KERNEL);
225 }
226 
__genwqe_free_consistent(struct genwqe_dev * cd,size_t size,void * vaddr,dma_addr_t dma_handle)227 void __genwqe_free_consistent(struct genwqe_dev *cd, size_t size,
228 			     void *vaddr, dma_addr_t dma_handle)
229 {
230 	if (vaddr == NULL)
231 		return;
232 
233 	dma_free_coherent(&cd->pci_dev->dev, size, vaddr, dma_handle);
234 }
235 
genwqe_unmap_pages(struct genwqe_dev * cd,dma_addr_t * dma_list,int num_pages)236 static void genwqe_unmap_pages(struct genwqe_dev *cd, dma_addr_t *dma_list,
237 			      int num_pages)
238 {
239 	int i;
240 	struct pci_dev *pci_dev = cd->pci_dev;
241 
242 	for (i = 0; (i < num_pages) && (dma_list[i] != 0x0); i++) {
243 		pci_unmap_page(pci_dev, dma_list[i],
244 			       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
245 		dma_list[i] = 0x0;
246 	}
247 }
248 
genwqe_map_pages(struct genwqe_dev * cd,struct page ** page_list,int num_pages,dma_addr_t * dma_list)249 static int genwqe_map_pages(struct genwqe_dev *cd,
250 			   struct page **page_list, int num_pages,
251 			   dma_addr_t *dma_list)
252 {
253 	int i;
254 	struct pci_dev *pci_dev = cd->pci_dev;
255 
256 	/* establish DMA mapping for requested pages */
257 	for (i = 0; i < num_pages; i++) {
258 		dma_addr_t daddr;
259 
260 		dma_list[i] = 0x0;
261 		daddr = pci_map_page(pci_dev, page_list[i],
262 				     0,	 /* map_offs */
263 				     PAGE_SIZE,
264 				     PCI_DMA_BIDIRECTIONAL);  /* FIXME rd/rw */
265 
266 		if (pci_dma_mapping_error(pci_dev, daddr)) {
267 			dev_err(&pci_dev->dev,
268 				"[%s] err: no dma addr daddr=%016llx!\n",
269 				__func__, (long long)daddr);
270 			goto err;
271 		}
272 
273 		dma_list[i] = daddr;
274 	}
275 	return 0;
276 
277  err:
278 	genwqe_unmap_pages(cd, dma_list, num_pages);
279 	return -EIO;
280 }
281 
genwqe_sgl_size(int num_pages)282 static int genwqe_sgl_size(int num_pages)
283 {
284 	int len, num_tlb = num_pages / 7;
285 
286 	len = sizeof(struct sg_entry) * (num_pages+num_tlb + 1);
287 	return roundup(len, PAGE_SIZE);
288 }
289 
290 /**
291  * genwqe_alloc_sync_sgl() - Allocate memory for sgl and overlapping pages
292  *
293  * Allocates memory for sgl and overlapping pages. Pages which might
294  * overlap other user-space memory blocks are being cached for DMAs,
295  * such that we do not run into syncronization issues. Data is copied
296  * from user-space into the cached pages.
297  */
genwqe_alloc_sync_sgl(struct genwqe_dev * cd,struct genwqe_sgl * sgl,void __user * user_addr,size_t user_size,int write)298 int genwqe_alloc_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl,
299 			  void __user *user_addr, size_t user_size, int write)
300 {
301 	int ret = -ENOMEM;
302 	struct pci_dev *pci_dev = cd->pci_dev;
303 
304 	sgl->fpage_offs = offset_in_page((unsigned long)user_addr);
305 	sgl->fpage_size = min_t(size_t, PAGE_SIZE-sgl->fpage_offs, user_size);
306 	sgl->nr_pages = DIV_ROUND_UP(sgl->fpage_offs + user_size, PAGE_SIZE);
307 	sgl->lpage_size = (user_size - sgl->fpage_size) % PAGE_SIZE;
308 
309 	dev_dbg(&pci_dev->dev, "[%s] uaddr=%p usize=%8ld nr_pages=%ld fpage_offs=%lx fpage_size=%ld lpage_size=%ld\n",
310 		__func__, user_addr, user_size, sgl->nr_pages,
311 		sgl->fpage_offs, sgl->fpage_size, sgl->lpage_size);
312 
313 	sgl->user_addr = user_addr;
314 	sgl->user_size = user_size;
315 	sgl->write = write;
316 	sgl->sgl_size = genwqe_sgl_size(sgl->nr_pages);
317 
318 	if (get_order(sgl->sgl_size) > MAX_ORDER) {
319 		dev_err(&pci_dev->dev,
320 			"[%s] err: too much memory requested!\n", __func__);
321 		return ret;
322 	}
323 
324 	sgl->sgl = __genwqe_alloc_consistent(cd, sgl->sgl_size,
325 					     &sgl->sgl_dma_addr);
326 	if (sgl->sgl == NULL) {
327 		dev_err(&pci_dev->dev,
328 			"[%s] err: no memory available!\n", __func__);
329 		return ret;
330 	}
331 
332 	/* Only use buffering on incomplete pages */
333 	if ((sgl->fpage_size != 0) && (sgl->fpage_size != PAGE_SIZE)) {
334 		sgl->fpage = __genwqe_alloc_consistent(cd, PAGE_SIZE,
335 						       &sgl->fpage_dma_addr);
336 		if (sgl->fpage == NULL)
337 			goto err_out;
338 
339 		/* Sync with user memory */
340 		if (copy_from_user(sgl->fpage + sgl->fpage_offs,
341 				   user_addr, sgl->fpage_size)) {
342 			ret = -EFAULT;
343 			goto err_out;
344 		}
345 	}
346 	if (sgl->lpage_size != 0) {
347 		sgl->lpage = __genwqe_alloc_consistent(cd, PAGE_SIZE,
348 						       &sgl->lpage_dma_addr);
349 		if (sgl->lpage == NULL)
350 			goto err_out1;
351 
352 		/* Sync with user memory */
353 		if (copy_from_user(sgl->lpage, user_addr + user_size -
354 				   sgl->lpage_size, sgl->lpage_size)) {
355 			ret = -EFAULT;
356 			goto err_out2;
357 		}
358 	}
359 	return 0;
360 
361  err_out2:
362 	__genwqe_free_consistent(cd, PAGE_SIZE, sgl->lpage,
363 				 sgl->lpage_dma_addr);
364 	sgl->lpage = NULL;
365 	sgl->lpage_dma_addr = 0;
366  err_out1:
367 	__genwqe_free_consistent(cd, PAGE_SIZE, sgl->fpage,
368 				 sgl->fpage_dma_addr);
369 	sgl->fpage = NULL;
370 	sgl->fpage_dma_addr = 0;
371  err_out:
372 	__genwqe_free_consistent(cd, sgl->sgl_size, sgl->sgl,
373 				 sgl->sgl_dma_addr);
374 	sgl->sgl = NULL;
375 	sgl->sgl_dma_addr = 0;
376 	sgl->sgl_size = 0;
377 
378 	return ret;
379 }
380 
genwqe_setup_sgl(struct genwqe_dev * cd,struct genwqe_sgl * sgl,dma_addr_t * dma_list)381 int genwqe_setup_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl,
382 		     dma_addr_t *dma_list)
383 {
384 	int i = 0, j = 0, p;
385 	unsigned long dma_offs, map_offs;
386 	dma_addr_t prev_daddr = 0;
387 	struct sg_entry *s, *last_s = NULL;
388 	size_t size = sgl->user_size;
389 
390 	dma_offs = 128;		/* next block if needed/dma_offset */
391 	map_offs = sgl->fpage_offs; /* offset in first page */
392 
393 	s = &sgl->sgl[0];	/* first set of 8 entries */
394 	p = 0;			/* page */
395 	while (p < sgl->nr_pages) {
396 		dma_addr_t daddr;
397 		unsigned int size_to_map;
398 
399 		/* always write the chaining entry, cleanup is done later */
400 		j = 0;
401 		s[j].target_addr = cpu_to_be64(sgl->sgl_dma_addr + dma_offs);
402 		s[j].len	 = cpu_to_be32(128);
403 		s[j].flags	 = cpu_to_be32(SG_CHAINED);
404 		j++;
405 
406 		while (j < 8) {
407 			/* DMA mapping for requested page, offs, size */
408 			size_to_map = min(size, PAGE_SIZE - map_offs);
409 
410 			if ((p == 0) && (sgl->fpage != NULL)) {
411 				daddr = sgl->fpage_dma_addr + map_offs;
412 
413 			} else if ((p == sgl->nr_pages - 1) &&
414 				   (sgl->lpage != NULL)) {
415 				daddr = sgl->lpage_dma_addr;
416 			} else {
417 				daddr = dma_list[p] + map_offs;
418 			}
419 
420 			size -= size_to_map;
421 			map_offs = 0;
422 
423 			if (prev_daddr == daddr) {
424 				u32 prev_len = be32_to_cpu(last_s->len);
425 
426 				/* pr_info("daddr combining: "
427 					"%016llx/%08x -> %016llx\n",
428 					prev_daddr, prev_len, daddr); */
429 
430 				last_s->len = cpu_to_be32(prev_len +
431 							  size_to_map);
432 
433 				p++; /* process next page */
434 				if (p == sgl->nr_pages)
435 					goto fixup;  /* nothing to do */
436 
437 				prev_daddr = daddr + size_to_map;
438 				continue;
439 			}
440 
441 			/* start new entry */
442 			s[j].target_addr = cpu_to_be64(daddr);
443 			s[j].len	 = cpu_to_be32(size_to_map);
444 			s[j].flags	 = cpu_to_be32(SG_DATA);
445 			prev_daddr = daddr + size_to_map;
446 			last_s = &s[j];
447 			j++;
448 
449 			p++;	/* process next page */
450 			if (p == sgl->nr_pages)
451 				goto fixup;  /* nothing to do */
452 		}
453 		dma_offs += 128;
454 		s += 8;		/* continue 8 elements further */
455 	}
456  fixup:
457 	if (j == 1) {		/* combining happened on last entry! */
458 		s -= 8;		/* full shift needed on previous sgl block */
459 		j =  7;		/* shift all elements */
460 	}
461 
462 	for (i = 0; i < j; i++)	/* move elements 1 up */
463 		s[i] = s[i + 1];
464 
465 	s[i].target_addr = cpu_to_be64(0);
466 	s[i].len	 = cpu_to_be32(0);
467 	s[i].flags	 = cpu_to_be32(SG_END_LIST);
468 	return 0;
469 }
470 
471 /**
472  * genwqe_free_sync_sgl() - Free memory for sgl and overlapping pages
473  *
474  * After the DMA transfer has been completed we free the memory for
475  * the sgl and the cached pages. Data is being transferred from cached
476  * pages into user-space buffers.
477  */
genwqe_free_sync_sgl(struct genwqe_dev * cd,struct genwqe_sgl * sgl)478 int genwqe_free_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl)
479 {
480 	int rc = 0;
481 	size_t offset;
482 	unsigned long res;
483 	struct pci_dev *pci_dev = cd->pci_dev;
484 
485 	if (sgl->fpage) {
486 		if (sgl->write) {
487 			res = copy_to_user(sgl->user_addr,
488 				sgl->fpage + sgl->fpage_offs, sgl->fpage_size);
489 			if (res) {
490 				dev_err(&pci_dev->dev,
491 					"[%s] err: copying fpage! (res=%lu)\n",
492 					__func__, res);
493 				rc = -EFAULT;
494 			}
495 		}
496 		__genwqe_free_consistent(cd, PAGE_SIZE, sgl->fpage,
497 					 sgl->fpage_dma_addr);
498 		sgl->fpage = NULL;
499 		sgl->fpage_dma_addr = 0;
500 	}
501 	if (sgl->lpage) {
502 		if (sgl->write) {
503 			offset = sgl->user_size - sgl->lpage_size;
504 			res = copy_to_user(sgl->user_addr + offset, sgl->lpage,
505 					   sgl->lpage_size);
506 			if (res) {
507 				dev_err(&pci_dev->dev,
508 					"[%s] err: copying lpage! (res=%lu)\n",
509 					__func__, res);
510 				rc = -EFAULT;
511 			}
512 		}
513 		__genwqe_free_consistent(cd, PAGE_SIZE, sgl->lpage,
514 					 sgl->lpage_dma_addr);
515 		sgl->lpage = NULL;
516 		sgl->lpage_dma_addr = 0;
517 	}
518 	__genwqe_free_consistent(cd, sgl->sgl_size, sgl->sgl,
519 				 sgl->sgl_dma_addr);
520 
521 	sgl->sgl = NULL;
522 	sgl->sgl_dma_addr = 0x0;
523 	sgl->sgl_size = 0;
524 	return rc;
525 }
526 
527 /**
528  * genwqe_free_user_pages() - Give pinned pages back
529  *
530  * Documentation of get_user_pages is in mm/gup.c:
531  *
532  * If the page is written to, set_page_dirty (or set_page_dirty_lock,
533  * as appropriate) must be called after the page is finished with, and
534  * before put_page is called.
535  */
genwqe_free_user_pages(struct page ** page_list,unsigned int nr_pages,int dirty)536 static int genwqe_free_user_pages(struct page **page_list,
537 			unsigned int nr_pages, int dirty)
538 {
539 	unsigned int i;
540 
541 	for (i = 0; i < nr_pages; i++) {
542 		if (page_list[i] != NULL) {
543 			if (dirty)
544 				set_page_dirty_lock(page_list[i]);
545 			put_page(page_list[i]);
546 		}
547 	}
548 	return 0;
549 }
550 
551 /**
552  * genwqe_user_vmap() - Map user-space memory to virtual kernel memory
553  * @cd:         pointer to genwqe device
554  * @m:          mapping params
555  * @uaddr:      user virtual address
556  * @size:       size of memory to be mapped
557  *
558  * We need to think about how we could speed this up. Of course it is
559  * not a good idea to do this over and over again, like we are
560  * currently doing it. Nevertheless, I am curious where on the path
561  * the performance is spend. Most probably within the memory
562  * allocation functions, but maybe also in the DMA mapping code.
563  *
564  * Restrictions: The maximum size of the possible mapping currently depends
565  *               on the amount of memory we can get using kzalloc() for the
566  *               page_list and pci_alloc_consistent for the sg_list.
567  *               The sg_list is currently itself not scattered, which could
568  *               be fixed with some effort. The page_list must be split into
569  *               PAGE_SIZE chunks too. All that will make the complicated
570  *               code more complicated.
571  *
572  * Return: 0 if success
573  */
genwqe_user_vmap(struct genwqe_dev * cd,struct dma_mapping * m,void * uaddr,unsigned long size)574 int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,
575 		     unsigned long size)
576 {
577 	int rc = -EINVAL;
578 	unsigned long data, offs;
579 	struct pci_dev *pci_dev = cd->pci_dev;
580 
581 	if ((uaddr == NULL) || (size == 0)) {
582 		m->size = 0;	/* mark unused and not added */
583 		return -EINVAL;
584 	}
585 	m->u_vaddr = uaddr;
586 	m->size    = size;
587 
588 	/* determine space needed for page_list. */
589 	data = (unsigned long)uaddr;
590 	offs = offset_in_page(data);
591 	if (size > ULONG_MAX - PAGE_SIZE - offs) {
592 		m->size = 0;	/* mark unused and not added */
593 		return -EINVAL;
594 	}
595 	m->nr_pages = DIV_ROUND_UP(offs + size, PAGE_SIZE);
596 
597 	m->page_list = kcalloc(m->nr_pages,
598 			       sizeof(struct page *) + sizeof(dma_addr_t),
599 			       GFP_KERNEL);
600 	if (!m->page_list) {
601 		dev_err(&pci_dev->dev, "err: alloc page_list failed\n");
602 		m->nr_pages = 0;
603 		m->u_vaddr = NULL;
604 		m->size = 0;	/* mark unused and not added */
605 		return -ENOMEM;
606 	}
607 	m->dma_list = (dma_addr_t *)(m->page_list + m->nr_pages);
608 
609 	/* pin user pages in memory */
610 	rc = get_user_pages_fast(data & PAGE_MASK, /* page aligned addr */
611 				 m->nr_pages,
612 				 m->write,		/* readable/writable */
613 				 m->page_list);	/* ptrs to pages */
614 	if (rc < 0)
615 		goto fail_get_user_pages;
616 
617 	/* assumption: get_user_pages can be killed by signals. */
618 	if (rc < m->nr_pages) {
619 		genwqe_free_user_pages(m->page_list, rc, m->write);
620 		rc = -EFAULT;
621 		goto fail_get_user_pages;
622 	}
623 
624 	rc = genwqe_map_pages(cd, m->page_list, m->nr_pages, m->dma_list);
625 	if (rc != 0)
626 		goto fail_free_user_pages;
627 
628 	return 0;
629 
630  fail_free_user_pages:
631 	genwqe_free_user_pages(m->page_list, m->nr_pages, m->write);
632 
633  fail_get_user_pages:
634 	kfree(m->page_list);
635 	m->page_list = NULL;
636 	m->dma_list = NULL;
637 	m->nr_pages = 0;
638 	m->u_vaddr = NULL;
639 	m->size = 0;		/* mark unused and not added */
640 	return rc;
641 }
642 
643 /**
644  * genwqe_user_vunmap() - Undo mapping of user-space mem to virtual kernel
645  *                        memory
646  * @cd:         pointer to genwqe device
647  * @m:          mapping params
648  */
genwqe_user_vunmap(struct genwqe_dev * cd,struct dma_mapping * m)649 int genwqe_user_vunmap(struct genwqe_dev *cd, struct dma_mapping *m)
650 {
651 	struct pci_dev *pci_dev = cd->pci_dev;
652 
653 	if (!dma_mapping_used(m)) {
654 		dev_err(&pci_dev->dev, "[%s] err: mapping %p not used!\n",
655 			__func__, m);
656 		return -EINVAL;
657 	}
658 
659 	if (m->dma_list)
660 		genwqe_unmap_pages(cd, m->dma_list, m->nr_pages);
661 
662 	if (m->page_list) {
663 		genwqe_free_user_pages(m->page_list, m->nr_pages, m->write);
664 
665 		kfree(m->page_list);
666 		m->page_list = NULL;
667 		m->dma_list = NULL;
668 		m->nr_pages = 0;
669 	}
670 
671 	m->u_vaddr = NULL;
672 	m->size = 0;		/* mark as unused and not added */
673 	return 0;
674 }
675 
676 /**
677  * genwqe_card_type() - Get chip type SLU Configuration Register
678  * @cd:         pointer to the genwqe device descriptor
679  * Return: 0: Altera Stratix-IV 230
680  *         1: Altera Stratix-IV 530
681  *         2: Altera Stratix-V A4
682  *         3: Altera Stratix-V A7
683  */
genwqe_card_type(struct genwqe_dev * cd)684 u8 genwqe_card_type(struct genwqe_dev *cd)
685 {
686 	u64 card_type = cd->slu_unitcfg;
687 
688 	return (u8)((card_type & IO_SLU_UNITCFG_TYPE_MASK) >> 20);
689 }
690 
691 /**
692  * genwqe_card_reset() - Reset the card
693  * @cd:         pointer to the genwqe device descriptor
694  */
genwqe_card_reset(struct genwqe_dev * cd)695 int genwqe_card_reset(struct genwqe_dev *cd)
696 {
697 	u64 softrst;
698 	struct pci_dev *pci_dev = cd->pci_dev;
699 
700 	if (!genwqe_is_privileged(cd))
701 		return -ENODEV;
702 
703 	/* new SL */
704 	__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, 0x1ull);
705 	msleep(1000);
706 	__genwqe_readq(cd, IO_HSU_FIR_CLR);
707 	__genwqe_readq(cd, IO_APP_FIR_CLR);
708 	__genwqe_readq(cd, IO_SLU_FIR_CLR);
709 
710 	/*
711 	 * Read-modify-write to preserve the stealth bits
712 	 *
713 	 * For SL >= 039, Stealth WE bit allows removing
714 	 * the read-modify-wrote.
715 	 * r-m-w may require a mask 0x3C to avoid hitting hard
716 	 * reset again for error reset (should be 0, chicken).
717 	 */
718 	softrst = __genwqe_readq(cd, IO_SLC_CFGREG_SOFTRESET) & 0x3cull;
719 	__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, softrst | 0x2ull);
720 
721 	/* give ERRORRESET some time to finish */
722 	msleep(50);
723 
724 	if (genwqe_need_err_masking(cd)) {
725 		dev_info(&pci_dev->dev,
726 			 "[%s] masking errors for old bitstreams\n", __func__);
727 		__genwqe_writeq(cd, IO_SLC_MISC_DEBUG, 0x0aull);
728 	}
729 	return 0;
730 }
731 
genwqe_read_softreset(struct genwqe_dev * cd)732 int genwqe_read_softreset(struct genwqe_dev *cd)
733 {
734 	u64 bitstream;
735 
736 	if (!genwqe_is_privileged(cd))
737 		return -ENODEV;
738 
739 	bitstream = __genwqe_readq(cd, IO_SLU_BITSTREAM) & 0x1;
740 	cd->softreset = (bitstream == 0) ? 0x8ull : 0xcull;
741 	return 0;
742 }
743 
744 /**
745  * genwqe_set_interrupt_capability() - Configure MSI capability structure
746  * @cd:         pointer to the device
747  * Return: 0 if no error
748  */
genwqe_set_interrupt_capability(struct genwqe_dev * cd,int count)749 int genwqe_set_interrupt_capability(struct genwqe_dev *cd, int count)
750 {
751 	int rc;
752 
753 	rc = pci_alloc_irq_vectors(cd->pci_dev, 1, count, PCI_IRQ_MSI);
754 	if (rc < 0)
755 		return rc;
756 	return 0;
757 }
758 
759 /**
760  * genwqe_reset_interrupt_capability() - Undo genwqe_set_interrupt_capability()
761  * @cd:         pointer to the device
762  */
genwqe_reset_interrupt_capability(struct genwqe_dev * cd)763 void genwqe_reset_interrupt_capability(struct genwqe_dev *cd)
764 {
765 	pci_free_irq_vectors(cd->pci_dev);
766 }
767 
768 /**
769  * set_reg_idx() - Fill array with data. Ignore illegal offsets.
770  * @cd:         card device
771  * @r:          debug register array
772  * @i:          index to desired entry
773  * @m:          maximum possible entries
774  * @addr:       addr which is read
775  * @index:      index in debug array
776  * @val:        read value
777  */
set_reg_idx(struct genwqe_dev * cd,struct genwqe_reg * r,unsigned int * i,unsigned int m,u32 addr,u32 idx,u64 val)778 static int set_reg_idx(struct genwqe_dev *cd, struct genwqe_reg *r,
779 		       unsigned int *i, unsigned int m, u32 addr, u32 idx,
780 		       u64 val)
781 {
782 	if (WARN_ON_ONCE(*i >= m))
783 		return -EFAULT;
784 
785 	r[*i].addr = addr;
786 	r[*i].idx = idx;
787 	r[*i].val = val;
788 	++*i;
789 	return 0;
790 }
791 
set_reg(struct genwqe_dev * cd,struct genwqe_reg * r,unsigned int * i,unsigned int m,u32 addr,u64 val)792 static int set_reg(struct genwqe_dev *cd, struct genwqe_reg *r,
793 		   unsigned int *i, unsigned int m, u32 addr, u64 val)
794 {
795 	return set_reg_idx(cd, r, i, m, addr, 0, val);
796 }
797 
genwqe_read_ffdc_regs(struct genwqe_dev * cd,struct genwqe_reg * regs,unsigned int max_regs,int all)798 int genwqe_read_ffdc_regs(struct genwqe_dev *cd, struct genwqe_reg *regs,
799 			 unsigned int max_regs, int all)
800 {
801 	unsigned int i, j, idx = 0;
802 	u32 ufir_addr, ufec_addr, sfir_addr, sfec_addr;
803 	u64 gfir, sluid, appid, ufir, ufec, sfir, sfec;
804 
805 	/* Global FIR */
806 	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
807 	set_reg(cd, regs, &idx, max_regs, IO_SLC_CFGREG_GFIR, gfir);
808 
809 	/* UnitCfg for SLU */
810 	sluid = __genwqe_readq(cd, IO_SLU_UNITCFG); /* 0x00000000 */
811 	set_reg(cd, regs, &idx, max_regs, IO_SLU_UNITCFG, sluid);
812 
813 	/* UnitCfg for APP */
814 	appid = __genwqe_readq(cd, IO_APP_UNITCFG); /* 0x02000000 */
815 	set_reg(cd, regs, &idx, max_regs, IO_APP_UNITCFG, appid);
816 
817 	/* Check all chip Units */
818 	for (i = 0; i < GENWQE_MAX_UNITS; i++) {
819 
820 		/* Unit FIR */
821 		ufir_addr = (i << 24) | 0x008;
822 		ufir = __genwqe_readq(cd, ufir_addr);
823 		set_reg(cd, regs, &idx, max_regs, ufir_addr, ufir);
824 
825 		/* Unit FEC */
826 		ufec_addr = (i << 24) | 0x018;
827 		ufec = __genwqe_readq(cd, ufec_addr);
828 		set_reg(cd, regs, &idx, max_regs, ufec_addr, ufec);
829 
830 		for (j = 0; j < 64; j++) {
831 			/* wherever there is a primary 1, read the 2ndary */
832 			if (!all && (!(ufir & (1ull << j))))
833 				continue;
834 
835 			sfir_addr = (i << 24) | (0x100 + 8 * j);
836 			sfir = __genwqe_readq(cd, sfir_addr);
837 			set_reg(cd, regs, &idx, max_regs, sfir_addr, sfir);
838 
839 			sfec_addr = (i << 24) | (0x300 + 8 * j);
840 			sfec = __genwqe_readq(cd, sfec_addr);
841 			set_reg(cd, regs, &idx, max_regs, sfec_addr, sfec);
842 		}
843 	}
844 
845 	/* fill with invalid data until end */
846 	for (i = idx; i < max_regs; i++) {
847 		regs[i].addr = 0xffffffff;
848 		regs[i].val = 0xffffffffffffffffull;
849 	}
850 	return idx;
851 }
852 
853 /**
854  * genwqe_ffdc_buff_size() - Calculates the number of dump registers
855  */
genwqe_ffdc_buff_size(struct genwqe_dev * cd,int uid)856 int genwqe_ffdc_buff_size(struct genwqe_dev *cd, int uid)
857 {
858 	int entries = 0, ring, traps, traces, trace_entries;
859 	u32 eevptr_addr, l_addr, d_len, d_type;
860 	u64 eevptr, val, addr;
861 
862 	eevptr_addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_ERROR_POINTER;
863 	eevptr = __genwqe_readq(cd, eevptr_addr);
864 
865 	if ((eevptr != 0x0) && (eevptr != -1ull)) {
866 		l_addr = GENWQE_UID_OFFS(uid) | eevptr;
867 
868 		while (1) {
869 			val = __genwqe_readq(cd, l_addr);
870 
871 			if ((val == 0x0) || (val == -1ull))
872 				break;
873 
874 			/* 38:24 */
875 			d_len  = (val & 0x0000007fff000000ull) >> 24;
876 
877 			/* 39 */
878 			d_type = (val & 0x0000008000000000ull) >> 36;
879 
880 			if (d_type) {	/* repeat */
881 				entries += d_len;
882 			} else {	/* size in bytes! */
883 				entries += d_len >> 3;
884 			}
885 
886 			l_addr += 8;
887 		}
888 	}
889 
890 	for (ring = 0; ring < 8; ring++) {
891 		addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_DIAG_MAP(ring);
892 		val = __genwqe_readq(cd, addr);
893 
894 		if ((val == 0x0ull) || (val == -1ull))
895 			continue;
896 
897 		traps = (val >> 24) & 0xff;
898 		traces = (val >> 16) & 0xff;
899 		trace_entries = val & 0xffff;
900 
901 		entries += traps + (traces * trace_entries);
902 	}
903 	return entries;
904 }
905 
906 /**
907  * genwqe_ffdc_buff_read() - Implements LogoutExtendedErrorRegisters procedure
908  */
genwqe_ffdc_buff_read(struct genwqe_dev * cd,int uid,struct genwqe_reg * regs,unsigned int max_regs)909 int genwqe_ffdc_buff_read(struct genwqe_dev *cd, int uid,
910 			  struct genwqe_reg *regs, unsigned int max_regs)
911 {
912 	int i, traps, traces, trace, trace_entries, trace_entry, ring;
913 	unsigned int idx = 0;
914 	u32 eevptr_addr, l_addr, d_addr, d_len, d_type;
915 	u64 eevptr, e, val, addr;
916 
917 	eevptr_addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_ERROR_POINTER;
918 	eevptr = __genwqe_readq(cd, eevptr_addr);
919 
920 	if ((eevptr != 0x0) && (eevptr != 0xffffffffffffffffull)) {
921 		l_addr = GENWQE_UID_OFFS(uid) | eevptr;
922 		while (1) {
923 			e = __genwqe_readq(cd, l_addr);
924 			if ((e == 0x0) || (e == 0xffffffffffffffffull))
925 				break;
926 
927 			d_addr = (e & 0x0000000000ffffffull);	    /* 23:0 */
928 			d_len  = (e & 0x0000007fff000000ull) >> 24; /* 38:24 */
929 			d_type = (e & 0x0000008000000000ull) >> 36; /* 39 */
930 			d_addr |= GENWQE_UID_OFFS(uid);
931 
932 			if (d_type) {
933 				for (i = 0; i < (int)d_len; i++) {
934 					val = __genwqe_readq(cd, d_addr);
935 					set_reg_idx(cd, regs, &idx, max_regs,
936 						    d_addr, i, val);
937 				}
938 			} else {
939 				d_len >>= 3; /* Size in bytes! */
940 				for (i = 0; i < (int)d_len; i++, d_addr += 8) {
941 					val = __genwqe_readq(cd, d_addr);
942 					set_reg_idx(cd, regs, &idx, max_regs,
943 						    d_addr, 0, val);
944 				}
945 			}
946 			l_addr += 8;
947 		}
948 	}
949 
950 	/*
951 	 * To save time, there are only 6 traces poplulated on Uid=2,
952 	 * Ring=1. each with iters=512.
953 	 */
954 	for (ring = 0; ring < 8; ring++) { /* 0 is fls, 1 is fds,
955 					      2...7 are ASI rings */
956 		addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_DIAG_MAP(ring);
957 		val = __genwqe_readq(cd, addr);
958 
959 		if ((val == 0x0ull) || (val == -1ull))
960 			continue;
961 
962 		traps = (val >> 24) & 0xff;	/* Number of Traps	*/
963 		traces = (val >> 16) & 0xff;	/* Number of Traces	*/
964 		trace_entries = val & 0xffff;	/* Entries per trace	*/
965 
966 		/* Note: This is a combined loop that dumps both the traps */
967 		/* (for the trace == 0 case) as well as the traces 1 to    */
968 		/* 'traces'.						   */
969 		for (trace = 0; trace <= traces; trace++) {
970 			u32 diag_sel =
971 				GENWQE_EXTENDED_DIAG_SELECTOR(ring, trace);
972 
973 			addr = (GENWQE_UID_OFFS(uid) |
974 				IO_EXTENDED_DIAG_SELECTOR);
975 			__genwqe_writeq(cd, addr, diag_sel);
976 
977 			for (trace_entry = 0;
978 			     trace_entry < (trace ? trace_entries : traps);
979 			     trace_entry++) {
980 				addr = (GENWQE_UID_OFFS(uid) |
981 					IO_EXTENDED_DIAG_READ_MBX);
982 				val = __genwqe_readq(cd, addr);
983 				set_reg_idx(cd, regs, &idx, max_regs, addr,
984 					    (diag_sel<<16) | trace_entry, val);
985 			}
986 		}
987 	}
988 	return 0;
989 }
990 
991 /**
992  * genwqe_write_vreg() - Write register in virtual window
993  *
994  * Note, these registers are only accessible to the PF through the
995  * VF-window. It is not intended for the VF to access.
996  */
genwqe_write_vreg(struct genwqe_dev * cd,u32 reg,u64 val,int func)997 int genwqe_write_vreg(struct genwqe_dev *cd, u32 reg, u64 val, int func)
998 {
999 	__genwqe_writeq(cd, IO_PF_SLC_VIRTUAL_WINDOW, func & 0xf);
1000 	__genwqe_writeq(cd, reg, val);
1001 	return 0;
1002 }
1003 
1004 /**
1005  * genwqe_read_vreg() - Read register in virtual window
1006  *
1007  * Note, these registers are only accessible to the PF through the
1008  * VF-window. It is not intended for the VF to access.
1009  */
genwqe_read_vreg(struct genwqe_dev * cd,u32 reg,int func)1010 u64 genwqe_read_vreg(struct genwqe_dev *cd, u32 reg, int func)
1011 {
1012 	__genwqe_writeq(cd, IO_PF_SLC_VIRTUAL_WINDOW, func & 0xf);
1013 	return __genwqe_readq(cd, reg);
1014 }
1015 
1016 /**
1017  * genwqe_base_clock_frequency() - Deteremine base clock frequency of the card
1018  *
1019  * Note: From a design perspective it turned out to be a bad idea to
1020  * use codes here to specifiy the frequency/speed values. An old
1021  * driver cannot understand new codes and is therefore always a
1022  * problem. Better is to measure out the value or put the
1023  * speed/frequency directly into a register which is always a valid
1024  * value for old as well as for new software.
1025  *
1026  * Return: Card clock in MHz
1027  */
genwqe_base_clock_frequency(struct genwqe_dev * cd)1028 int genwqe_base_clock_frequency(struct genwqe_dev *cd)
1029 {
1030 	u16 speed;		/*         MHz  MHz  MHz  MHz */
1031 	static const int speed_grade[] = { 250, 200, 166, 175 };
1032 
1033 	speed = (u16)((cd->slu_unitcfg >> 28) & 0x0full);
1034 	if (speed >= ARRAY_SIZE(speed_grade))
1035 		return 0;	/* illegal value */
1036 
1037 	return speed_grade[speed];
1038 }
1039 
1040 /**
1041  * genwqe_stop_traps() - Stop traps
1042  *
1043  * Before reading out the analysis data, we need to stop the traps.
1044  */
genwqe_stop_traps(struct genwqe_dev * cd)1045 void genwqe_stop_traps(struct genwqe_dev *cd)
1046 {
1047 	__genwqe_writeq(cd, IO_SLC_MISC_DEBUG_SET, 0xcull);
1048 }
1049 
1050 /**
1051  * genwqe_start_traps() - Start traps
1052  *
1053  * After having read the data, we can/must enable the traps again.
1054  */
genwqe_start_traps(struct genwqe_dev * cd)1055 void genwqe_start_traps(struct genwqe_dev *cd)
1056 {
1057 	__genwqe_writeq(cd, IO_SLC_MISC_DEBUG_CLR, 0xcull);
1058 
1059 	if (genwqe_need_err_masking(cd))
1060 		__genwqe_writeq(cd, IO_SLC_MISC_DEBUG, 0x0aull);
1061 }
1062