1 /**
2  * IBM Accelerator Family 'GenWQE'
3  *
4  * (C) Copyright IBM Corp. 2013
5  *
6  * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
7  * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
8  * Author: Michael Jung <mijung@gmx.net>
9  * Author: Michael Ruettger <michael@ibmra.de>
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License (version 2 only)
13  * as published by the Free Software Foundation.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18  * GNU General Public License for more details.
19  */
20 
21 /*
22  * Device Driver Control Block (DDCB) queue support. Definition of
23  * interrupt handlers for queue support as well as triggering the
24  * health monitor code in case of problems. The current hardware uses
25  * an MSI interrupt which is shared between error handling and
26  * functional code.
27  */
28 
29 #include <linux/types.h>
30 #include <linux/module.h>
31 #include <linux/sched.h>
32 #include <linux/wait.h>
33 #include <linux/pci.h>
34 #include <linux/string.h>
35 #include <linux/dma-mapping.h>
36 #include <linux/delay.h>
37 #include <linux/module.h>
38 #include <linux/interrupt.h>
39 #include <linux/crc-itu-t.h>
40 
41 #include "card_base.h"
42 #include "card_ddcb.h"
43 
44 /*
45  * N: next DDCB, this is where the next DDCB will be put.
46  * A: active DDCB, this is where the code will look for the next completion.
47  * x: DDCB is enqueued, we are waiting for its completion.
48 
49  * Situation (1): Empty queue
50  *  +---+---+---+---+---+---+---+---+
51  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
52  *  |   |   |   |   |   |   |   |   |
53  *  +---+---+---+---+---+---+---+---+
54  *           A/N
55  *  enqueued_ddcbs = A - N = 2 - 2 = 0
56  *
57  * Situation (2): Wrapped, N > A
58  *  +---+---+---+---+---+---+---+---+
59  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
60  *  |   |   | x | x |   |   |   |   |
61  *  +---+---+---+---+---+---+---+---+
62  *            A       N
63  *  enqueued_ddcbs = N - A = 4 - 2 = 2
64  *
65  * Situation (3): Queue wrapped, A > N
66  *  +---+---+---+---+---+---+---+---+
67  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
68  *  | x | x |   |   | x | x | x | x |
69  *  +---+---+---+---+---+---+---+---+
70  *            N       A
71  *  enqueued_ddcbs = queue_max  - (A - N) = 8 - (4 - 2) = 6
72  *
73  * Situation (4a): Queue full N > A
74  *  +---+---+---+---+---+---+---+---+
75  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
76  *  | x | x | x | x | x | x | x |   |
77  *  +---+---+---+---+---+---+---+---+
78  *    A                           N
79  *
80  *  enqueued_ddcbs = N - A = 7 - 0 = 7
81  *
82  * Situation (4a): Queue full A > N
83  *  +---+---+---+---+---+---+---+---+
84  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
85  *  | x | x | x |   | x | x | x | x |
86  *  +---+---+---+---+---+---+---+---+
87  *                N   A
88  *  enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7
89  */
90 
queue_empty(struct ddcb_queue * queue)91 static int queue_empty(struct ddcb_queue *queue)
92 {
93 	return queue->ddcb_next == queue->ddcb_act;
94 }
95 
queue_enqueued_ddcbs(struct ddcb_queue * queue)96 static int queue_enqueued_ddcbs(struct ddcb_queue *queue)
97 {
98 	if (queue->ddcb_next >= queue->ddcb_act)
99 		return queue->ddcb_next - queue->ddcb_act;
100 
101 	return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next);
102 }
103 
queue_free_ddcbs(struct ddcb_queue * queue)104 static int queue_free_ddcbs(struct ddcb_queue *queue)
105 {
106 	int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1;
107 
108 	if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */
109 		return 0;
110 	}
111 	return free_ddcbs;
112 }
113 
114 /*
115  * Use of the PRIV field in the DDCB for queue debugging:
116  *
117  * (1) Trying to get rid of a DDCB which saw a timeout:
118  *     pddcb->priv[6] = 0xcc;   # cleared
119  *
120  * (2) Append a DDCB via NEXT bit:
121  *     pddcb->priv[7] = 0xaa;	# appended
122  *
123  * (3) DDCB needed tapping:
124  *     pddcb->priv[7] = 0xbb;   # tapped
125  *
126  * (4) DDCB marked as correctly finished:
127  *     pddcb->priv[6] = 0xff;	# finished
128  */
129 
ddcb_mark_tapped(struct ddcb * pddcb)130 static inline void ddcb_mark_tapped(struct ddcb *pddcb)
131 {
132 	pddcb->priv[7] = 0xbb;  /* tapped */
133 }
134 
ddcb_mark_appended(struct ddcb * pddcb)135 static inline void ddcb_mark_appended(struct ddcb *pddcb)
136 {
137 	pddcb->priv[7] = 0xaa;	/* appended */
138 }
139 
ddcb_mark_cleared(struct ddcb * pddcb)140 static inline void ddcb_mark_cleared(struct ddcb *pddcb)
141 {
142 	pddcb->priv[6] = 0xcc; /* cleared */
143 }
144 
ddcb_mark_finished(struct ddcb * pddcb)145 static inline void ddcb_mark_finished(struct ddcb *pddcb)
146 {
147 	pddcb->priv[6] = 0xff;	/* finished */
148 }
149 
ddcb_mark_unused(struct ddcb * pddcb)150 static inline void ddcb_mark_unused(struct ddcb *pddcb)
151 {
152 	pddcb->priv_64 = cpu_to_be64(0); /* not tapped */
153 }
154 
155 /**
156  * genwqe_crc16() - Generate 16-bit crc as required for DDCBs
157  * @buff:       pointer to data buffer
158  * @len:        length of data for calculation
159  * @init:       initial crc (0xffff at start)
160  *
161  * Polynomial = x^16 + x^12 + x^5 + 1   (0x1021)
162  * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff
163  *          should result in a crc16 of 0x89c3
164  *
165  * Return: crc16 checksum in big endian format !
166  */
genwqe_crc16(const u8 * buff,size_t len,u16 init)167 static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init)
168 {
169 	return crc_itu_t(init, buff, len);
170 }
171 
print_ddcb_info(struct genwqe_dev * cd,struct ddcb_queue * queue)172 static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)
173 {
174 	int i;
175 	struct ddcb *pddcb;
176 	unsigned long flags;
177 	struct pci_dev *pci_dev = cd->pci_dev;
178 
179 	spin_lock_irqsave(&cd->print_lock, flags);
180 
181 	dev_info(&pci_dev->dev,
182 		 "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n",
183 		 cd->card_idx, queue->ddcb_act, queue->ddcb_next);
184 
185 	pddcb = queue->ddcb_vaddr;
186 	for (i = 0; i < queue->ddcb_max; i++) {
187 		dev_err(&pci_dev->dev,
188 			"  %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n",
189 			i == queue->ddcb_act ? '>' : ' ',
190 			i,
191 			be16_to_cpu(pddcb->retc_16),
192 			be16_to_cpu(pddcb->seqnum_16),
193 			pddcb->hsi,
194 			pddcb->shi,
195 			be64_to_cpu(pddcb->priv_64),
196 			pddcb->cmd);
197 		pddcb++;
198 	}
199 	spin_unlock_irqrestore(&cd->print_lock, flags);
200 }
201 
ddcb_requ_alloc(void)202 struct genwqe_ddcb_cmd *ddcb_requ_alloc(void)
203 {
204 	struct ddcb_requ *req;
205 
206 	req = kzalloc(sizeof(*req), GFP_KERNEL);
207 	if (!req)
208 		return NULL;
209 
210 	return &req->cmd;
211 }
212 
ddcb_requ_free(struct genwqe_ddcb_cmd * cmd)213 void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd)
214 {
215 	struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
216 
217 	kfree(req);
218 }
219 
ddcb_requ_get_state(struct ddcb_requ * req)220 static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req)
221 {
222 	return req->req_state;
223 }
224 
ddcb_requ_set_state(struct ddcb_requ * req,enum genwqe_requ_state new_state)225 static inline void ddcb_requ_set_state(struct ddcb_requ *req,
226 				       enum genwqe_requ_state new_state)
227 {
228 	req->req_state = new_state;
229 }
230 
ddcb_requ_collect_debug_data(struct ddcb_requ * req)231 static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req)
232 {
233 	return req->cmd.ddata_addr != 0x0;
234 }
235 
236 /**
237  * ddcb_requ_finished() - Returns the hardware state of the associated DDCB
238  * @cd:          pointer to genwqe device descriptor
239  * @req:         DDCB work request
240  *
241  * Status of ddcb_requ mirrors this hardware state, but is copied in
242  * the ddcb_requ on interrupt/polling function. The lowlevel code
243  * should check the hardware state directly, the higher level code
244  * should check the copy.
245  *
246  * This function will also return true if the state of the queue is
247  * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the
248  * shutdown case.
249  */
ddcb_requ_finished(struct genwqe_dev * cd,struct ddcb_requ * req)250 static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)
251 {
252 	return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) ||
253 		(cd->card_state != GENWQE_CARD_USED);
254 }
255 
256 /**
257  * enqueue_ddcb() - Enqueue a DDCB
258  * @cd:         pointer to genwqe device descriptor
259  * @queue:	queue this operation should be done on
260  * @ddcb_no:    pointer to ddcb number being tapped
261  *
262  * Start execution of DDCB by tapping or append to queue via NEXT
263  * bit. This is done by an atomic 'compare and swap' instruction and
264  * checking SHI and HSI of the previous DDCB.
265  *
266  * This function must only be called with ddcb_lock held.
267  *
268  * Return: 1 if new DDCB is appended to previous
269  *         2 if DDCB queue is tapped via register/simulation
270  */
271 #define RET_DDCB_APPENDED 1
272 #define RET_DDCB_TAPPED   2
273 
enqueue_ddcb(struct genwqe_dev * cd,struct ddcb_queue * queue,struct ddcb * pddcb,int ddcb_no)274 static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,
275 			struct ddcb *pddcb, int ddcb_no)
276 {
277 	unsigned int try;
278 	int prev_no;
279 	struct ddcb *prev_ddcb;
280 	__be32 old, new, icrc_hsi_shi;
281 	u64 num;
282 
283 	/*
284 	 * For performance checks a Dispatch Timestamp can be put into
285 	 * DDCB It is supposed to use the SLU's free running counter,
286 	 * but this requires PCIe cycles.
287 	 */
288 	ddcb_mark_unused(pddcb);
289 
290 	/* check previous DDCB if already fetched */
291 	prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1;
292 	prev_ddcb = &queue->ddcb_vaddr[prev_no];
293 
294 	/*
295 	 * It might have happened that the HSI.FETCHED bit is
296 	 * set. Retry in this case. Therefore I expect maximum 2 times
297 	 * trying.
298 	 */
299 	ddcb_mark_appended(pddcb);
300 	for (try = 0; try < 2; try++) {
301 		old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
302 
303 		/* try to append via NEXT bit if prev DDCB is not completed */
304 		if ((old & DDCB_COMPLETED_BE32) != 0x00000000)
305 			break;
306 
307 		new = (old | DDCB_NEXT_BE32);
308 
309 		wmb();		/* need to ensure write ordering */
310 		icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new);
311 
312 		if (icrc_hsi_shi == old)
313 			return RET_DDCB_APPENDED; /* appended to queue */
314 	}
315 
316 	/* Queue must be re-started by updating QUEUE_OFFSET */
317 	ddcb_mark_tapped(pddcb);
318 	num = (u64)ddcb_no << 8;
319 
320 	wmb();			/* need to ensure write ordering */
321 	__genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */
322 
323 	return RET_DDCB_TAPPED;
324 }
325 
326 /**
327  * copy_ddcb_results() - Copy output state from real DDCB to request
328  *
329  * Copy DDCB ASV to request struct. There is no endian
330  * conversion made, since data structure in ASV is still
331  * unknown here.
332  *
333  * This is needed by:
334  *   - genwqe_purge_ddcb()
335  *   - genwqe_check_ddcb_queue()
336  */
copy_ddcb_results(struct ddcb_requ * req,int ddcb_no)337 static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no)
338 {
339 	struct ddcb_queue *queue = req->queue;
340 	struct ddcb *pddcb = &queue->ddcb_vaddr[req->num];
341 
342 	memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH);
343 
344 	/* copy status flags of the variant part */
345 	req->cmd.vcrc     = be16_to_cpu(pddcb->vcrc_16);
346 	req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64);
347 	req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64);
348 
349 	req->cmd.attn     = be16_to_cpu(pddcb->attn_16);
350 	req->cmd.progress = be32_to_cpu(pddcb->progress_32);
351 	req->cmd.retc     = be16_to_cpu(pddcb->retc_16);
352 
353 	if (ddcb_requ_collect_debug_data(req)) {
354 		int prev_no = (ddcb_no == 0) ?
355 			queue->ddcb_max - 1 : ddcb_no - 1;
356 		struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no];
357 
358 		memcpy(&req->debug_data.ddcb_finished, pddcb,
359 		       sizeof(req->debug_data.ddcb_finished));
360 		memcpy(&req->debug_data.ddcb_prev, prev_pddcb,
361 		       sizeof(req->debug_data.ddcb_prev));
362 	}
363 }
364 
365 /**
366  * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work equests.
367  * @cd:         pointer to genwqe device descriptor
368  *
369  * Return: Number of DDCBs which were finished
370  */
genwqe_check_ddcb_queue(struct genwqe_dev * cd,struct ddcb_queue * queue)371 static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,
372 				   struct ddcb_queue *queue)
373 {
374 	unsigned long flags;
375 	int ddcbs_finished = 0;
376 	struct pci_dev *pci_dev = cd->pci_dev;
377 
378 	spin_lock_irqsave(&queue->ddcb_lock, flags);
379 
380 	/* FIXME avoid soft locking CPU */
381 	while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) {
382 
383 		struct ddcb *pddcb;
384 		struct ddcb_requ *req;
385 		u16 vcrc, vcrc_16, retc_16;
386 
387 		pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
388 
389 		if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) ==
390 		    0x00000000)
391 			goto go_home; /* not completed, continue waiting */
392 
393 		wmb();  /*  Add sync to decouple prev. read operations */
394 
395 		/* Note: DDCB could be purged */
396 		req = queue->ddcb_req[queue->ddcb_act];
397 		if (req == NULL) {
398 			/* this occurs if DDCB is purged, not an error */
399 			/* Move active DDCB further; Nothing to do anymore. */
400 			goto pick_next_one;
401 		}
402 
403 		/*
404 		 * HSI=0x44 (fetched and completed), but RETC is
405 		 * 0x101, or even worse 0x000.
406 		 *
407 		 * In case of seeing the queue in inconsistent state
408 		 * we read the errcnts and the queue status to provide
409 		 * a trigger for our PCIe analyzer stop capturing.
410 		 */
411 		retc_16 = be16_to_cpu(pddcb->retc_16);
412 		if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) {
413 			u64 errcnts, status;
414 			u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr;
415 
416 			errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);
417 			status  = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
418 
419 			dev_err(&pci_dev->dev,
420 				"[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n",
421 				__func__, be16_to_cpu(pddcb->seqnum_16),
422 				pddcb->hsi, retc_16, errcnts, status,
423 				queue->ddcb_daddr + ddcb_offs);
424 		}
425 
426 		copy_ddcb_results(req, queue->ddcb_act);
427 		queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */
428 
429 		dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num);
430 		genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
431 
432 		ddcb_mark_finished(pddcb);
433 
434 		/* calculate CRC_16 to see if VCRC is correct */
435 		vcrc = genwqe_crc16(pddcb->asv,
436 				   VCRC_LENGTH(req->cmd.asv_length),
437 				   0xffff);
438 		vcrc_16 = be16_to_cpu(pddcb->vcrc_16);
439 		if (vcrc != vcrc_16) {
440 			printk_ratelimited(KERN_ERR
441 				"%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n",
442 				GENWQE_DEVNAME, dev_name(&pci_dev->dev),
443 				pddcb->pre, VCRC_LENGTH(req->cmd.asv_length),
444 				vcrc, vcrc_16);
445 		}
446 
447 		ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
448 		queue->ddcbs_completed++;
449 		queue->ddcbs_in_flight--;
450 
451 		/* wake up process waiting for this DDCB, and
452                    processes on the busy queue */
453 		wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
454 		wake_up_interruptible(&queue->busy_waitq);
455 
456 pick_next_one:
457 		queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max;
458 		ddcbs_finished++;
459 	}
460 
461  go_home:
462 	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
463 	return ddcbs_finished;
464 }
465 
466 /**
467  * __genwqe_wait_ddcb(): Waits until DDCB is completed
468  * @cd:         pointer to genwqe device descriptor
469  * @req:        pointer to requsted DDCB parameters
470  *
471  * The Service Layer will update the RETC in DDCB when processing is
472  * pending or done.
473  *
474  * Return: > 0 remaining jiffies, DDCB completed
475  *           -ETIMEDOUT	when timeout
476  *           -ERESTARTSYS when ^C
477  *           -EINVAL when unknown error condition
478  *
479  * When an error is returned the called needs to ensure that
480  * purge_ddcb() is being called to get the &req removed from the
481  * queue.
482  */
__genwqe_wait_ddcb(struct genwqe_dev * cd,struct ddcb_requ * req)483 int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
484 {
485 	int rc;
486 	unsigned int ddcb_no;
487 	struct ddcb_queue *queue;
488 	struct pci_dev *pci_dev = cd->pci_dev;
489 
490 	if (req == NULL)
491 		return -EINVAL;
492 
493 	queue = req->queue;
494 	if (queue == NULL)
495 		return -EINVAL;
496 
497 	ddcb_no = req->num;
498 	if (ddcb_no >= queue->ddcb_max)
499 		return -EINVAL;
500 
501 	rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],
502 				ddcb_requ_finished(cd, req),
503 				GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ);
504 
505 	/*
506 	 * We need to distinguish 3 cases here:
507 	 *   1. rc == 0              timeout occured
508 	 *   2. rc == -ERESTARTSYS   signal received
509 	 *   3. rc > 0               remaining jiffies condition is true
510 	 */
511 	if (rc == 0) {
512 		struct ddcb_queue *queue = req->queue;
513 		struct ddcb *pddcb;
514 
515 		/*
516 		 * Timeout may be caused by long task switching time.
517 		 * When timeout happens, check if the request has
518 		 * meanwhile completed.
519 		 */
520 		genwqe_check_ddcb_queue(cd, req->queue);
521 		if (ddcb_requ_finished(cd, req))
522 			return rc;
523 
524 		dev_err(&pci_dev->dev,
525 			"[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n",
526 			__func__, req->num, rc,	ddcb_requ_get_state(req),
527 			req);
528 		dev_err(&pci_dev->dev,
529 			"[%s]      IO_QUEUE_STATUS=0x%016llx\n", __func__,
530 			__genwqe_readq(cd, queue->IO_QUEUE_STATUS));
531 
532 		pddcb = &queue->ddcb_vaddr[req->num];
533 		genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
534 
535 		print_ddcb_info(cd, req->queue);
536 		return -ETIMEDOUT;
537 
538 	} else if (rc == -ERESTARTSYS) {
539 		return rc;
540 		/*
541 		 * EINTR:       Stops the application
542 		 * ERESTARTSYS: Restartable systemcall; called again
543 		 */
544 
545 	} else if (rc < 0) {
546 		dev_err(&pci_dev->dev,
547 			"[%s] err: DDCB#%d unknown result (rc=%d) %d!\n",
548 			__func__, req->num, rc, ddcb_requ_get_state(req));
549 		return -EINVAL;
550 	}
551 
552 	/* Severe error occured. Driver is forced to stop operation */
553 	if (cd->card_state != GENWQE_CARD_USED) {
554 		dev_err(&pci_dev->dev,
555 			"[%s] err: DDCB#%d forced to stop (rc=%d)\n",
556 			__func__, req->num, rc);
557 		return -EIO;
558 	}
559 	return rc;
560 }
561 
562 /**
563  * get_next_ddcb() - Get next available DDCB
564  * @cd:         pointer to genwqe device descriptor
565  *
566  * DDCB's content is completely cleared but presets for PRE and
567  * SEQNUM. This function must only be called when ddcb_lock is held.
568  *
569  * Return: NULL if no empty DDCB available otherwise ptr to next DDCB.
570  */
get_next_ddcb(struct genwqe_dev * cd,struct ddcb_queue * queue,int * num)571 static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,
572 				  struct ddcb_queue *queue,
573 				  int *num)
574 {
575 	u64 *pu64;
576 	struct ddcb *pddcb;
577 
578 	if (queue_free_ddcbs(queue) == 0) /* queue is  full */
579 		return NULL;
580 
581 	/* find new ddcb */
582 	pddcb = &queue->ddcb_vaddr[queue->ddcb_next];
583 
584 	/* if it is not completed, we are not allowed to use it */
585 	/* barrier(); */
586 	if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000)
587 		return NULL;
588 
589 	*num = queue->ddcb_next;	/* internal DDCB number */
590 	queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max;
591 
592 	/* clear important DDCB fields */
593 	pu64 = (u64 *)pddcb;
594 	pu64[0] = 0ULL;		/* offs 0x00 (ICRC,HSI,SHI,...) */
595 	pu64[1] = 0ULL;		/* offs 0x01 (ACFUNC,CMD...) */
596 
597 	/* destroy previous results in ASV */
598 	pu64[0x80/8] = 0ULL;	/* offs 0x80 (ASV + 0) */
599 	pu64[0x88/8] = 0ULL;	/* offs 0x88 (ASV + 0x08) */
600 	pu64[0x90/8] = 0ULL;	/* offs 0x90 (ASV + 0x10) */
601 	pu64[0x98/8] = 0ULL;	/* offs 0x98 (ASV + 0x18) */
602 	pu64[0xd0/8] = 0ULL;	/* offs 0xd0 (RETC,ATTN...) */
603 
604 	pddcb->pre = DDCB_PRESET_PRE; /* 128 */
605 	pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++);
606 	return pddcb;
607 }
608 
609 /**
610  * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue
611  * @cd:         genwqe device descriptor
612  * @req:        DDCB request
613  *
614  * This will fail when the request was already FETCHED. In this case
615  * we need to wait until it is finished. Else the DDCB can be
616  * reused. This function also ensures that the request data structure
617  * is removed from ddcb_req[].
618  *
619  * Do not forget to call this function when genwqe_wait_ddcb() fails,
620  * such that the request gets really removed from ddcb_req[].
621  *
622  * Return: 0 success
623  */
__genwqe_purge_ddcb(struct genwqe_dev * cd,struct ddcb_requ * req)624 int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
625 {
626 	struct ddcb *pddcb = NULL;
627 	unsigned int t;
628 	unsigned long flags;
629 	struct ddcb_queue *queue = req->queue;
630 	struct pci_dev *pci_dev = cd->pci_dev;
631 	u64 queue_status;
632 	__be32 icrc_hsi_shi = 0x0000;
633 	__be32 old, new;
634 
635 	/* unsigned long flags; */
636 	if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) {
637 		dev_err(&pci_dev->dev,
638 			"[%s] err: software timeout is not set!\n", __func__);
639 		return -EFAULT;
640 	}
641 
642 	pddcb = &queue->ddcb_vaddr[req->num];
643 
644 	for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) {
645 
646 		spin_lock_irqsave(&queue->ddcb_lock, flags);
647 
648 		/* Check if req was meanwhile finished */
649 		if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED)
650 			goto go_home;
651 
652 		/* try to set PURGE bit if FETCHED/COMPLETED are not set */
653 		old = pddcb->icrc_hsi_shi_32;	/* read SHI/HSI in BE32 */
654 		if ((old & DDCB_FETCHED_BE32) == 0x00000000) {
655 
656 			new = (old | DDCB_PURGE_BE32);
657 			icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32,
658 					       old, new);
659 			if (icrc_hsi_shi == old)
660 				goto finish_ddcb;
661 		}
662 
663 		/* normal finish with HSI bit */
664 		barrier();
665 		icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
666 		if (icrc_hsi_shi & DDCB_COMPLETED_BE32)
667 			goto finish_ddcb;
668 
669 		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
670 
671 		/*
672 		 * Here the check_ddcb() function will most likely
673 		 * discover this DDCB to be finished some point in
674 		 * time. It will mark the req finished and free it up
675 		 * in the list.
676 		 */
677 
678 		copy_ddcb_results(req, req->num); /* for the failing case */
679 		msleep(100); /* sleep for 1/10 second and try again */
680 		continue;
681 
682 finish_ddcb:
683 		copy_ddcb_results(req, req->num);
684 		ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
685 		queue->ddcbs_in_flight--;
686 		queue->ddcb_req[req->num] = NULL; /* delete from array */
687 		ddcb_mark_cleared(pddcb);
688 
689 		/* Move active DDCB further; Nothing to do here anymore. */
690 
691 		/*
692 		 * We need to ensure that there is at least one free
693 		 * DDCB in the queue. To do that, we must update
694 		 * ddcb_act only if the COMPLETED bit is set for the
695 		 * DDCB we are working on else we treat that DDCB even
696 		 * if we PURGED it as occupied (hardware is supposed
697 		 * to set the COMPLETED bit yet!).
698 		 */
699 		icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
700 		if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) &&
701 		    (queue->ddcb_act == req->num)) {
702 			queue->ddcb_act = ((queue->ddcb_act + 1) %
703 					   queue->ddcb_max);
704 		}
705 go_home:
706 		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
707 		return 0;
708 	}
709 
710 	/*
711 	 * If the card is dead and the queue is forced to stop, we
712 	 * might see this in the queue status register.
713 	 */
714 	queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
715 
716 	dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num);
717 	genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
718 
719 	dev_err(&pci_dev->dev,
720 		"[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",
721 		__func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT,
722 		queue_status);
723 
724 	print_ddcb_info(cd, req->queue);
725 
726 	return -EFAULT;
727 }
728 
genwqe_init_debug_data(struct genwqe_dev * cd,struct genwqe_debug_data * d)729 int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)
730 {
731 	int len;
732 	struct pci_dev *pci_dev = cd->pci_dev;
733 
734 	if (d == NULL) {
735 		dev_err(&pci_dev->dev,
736 			"[%s] err: invalid memory for debug data!\n",
737 			__func__);
738 		return -EFAULT;
739 	}
740 
741 	len  = sizeof(d->driver_version);
742 	snprintf(d->driver_version, len, "%s", DRV_VERSION);
743 	d->slu_unitcfg = cd->slu_unitcfg;
744 	d->app_unitcfg = cd->app_unitcfg;
745 	return 0;
746 }
747 
748 /**
749  * __genwqe_enqueue_ddcb() - Enqueue a DDCB
750  * @cd:         pointer to genwqe device descriptor
751  * @req:        pointer to DDCB execution request
752  * @f_flags:    file mode: blocking, non-blocking
753  *
754  * Return: 0 if enqueuing succeeded
755  *         -EIO if card is unusable/PCIe problems
756  *         -EBUSY if enqueuing failed
757  */
__genwqe_enqueue_ddcb(struct genwqe_dev * cd,struct ddcb_requ * req,unsigned int f_flags)758 int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
759 			  unsigned int f_flags)
760 {
761 	struct ddcb *pddcb;
762 	unsigned long flags;
763 	struct ddcb_queue *queue;
764 	struct pci_dev *pci_dev = cd->pci_dev;
765 	u16 icrc;
766 
767  retry:
768 	if (cd->card_state != GENWQE_CARD_USED) {
769 		printk_ratelimited(KERN_ERR
770 			"%s %s: [%s] Card is unusable/PCIe problem Req#%d\n",
771 			GENWQE_DEVNAME, dev_name(&pci_dev->dev),
772 			__func__, req->num);
773 		return -EIO;
774 	}
775 
776 	queue = req->queue = &cd->queue;
777 
778 	/* FIXME circumvention to improve performance when no irq is
779 	 * there.
780 	 */
781 	if (GENWQE_POLLING_ENABLED)
782 		genwqe_check_ddcb_queue(cd, queue);
783 
784 	/*
785 	 * It must be ensured to process all DDCBs in successive
786 	 * order. Use a lock here in order to prevent nested DDCB
787 	 * enqueuing.
788 	 */
789 	spin_lock_irqsave(&queue->ddcb_lock, flags);
790 
791 	pddcb = get_next_ddcb(cd, queue, &req->num);	/* get ptr and num */
792 	if (pddcb == NULL) {
793 		int rc;
794 
795 		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
796 
797 		if (f_flags & O_NONBLOCK) {
798 			queue->return_on_busy++;
799 			return -EBUSY;
800 		}
801 
802 		queue->wait_on_busy++;
803 		rc = wait_event_interruptible(queue->busy_waitq,
804 					      queue_free_ddcbs(queue) != 0);
805 		dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n",
806 			__func__, rc);
807 		if (rc == -ERESTARTSYS)
808 			return rc;  /* interrupted by a signal */
809 
810 		goto retry;
811 	}
812 
813 	if (queue->ddcb_req[req->num] != NULL) {
814 		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
815 
816 		dev_err(&pci_dev->dev,
817 			"[%s] picked DDCB %d with req=%p still in use!!\n",
818 			__func__, req->num, req);
819 		return -EFAULT;
820 	}
821 	ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED);
822 	queue->ddcb_req[req->num] = req;
823 
824 	pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts);
825 	pddcb->cmd = req->cmd.cmd;
826 	pddcb->acfunc = req->cmd.acfunc;	/* functional unit */
827 
828 	/*
829 	 * We know that we can get retc 0x104 with CRC error, do not
830 	 * stop the queue in those cases for this command. XDIR = 1
831 	 * does not work for old SLU versions.
832 	 *
833 	 * Last bitstream with the old XDIR behavior had SLU_ID
834 	 * 0x34199.
835 	 */
836 	if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)
837 		pddcb->xdir = 0x1;
838 	else
839 		pddcb->xdir = 0x0;
840 
841 
842 	pddcb->psp = (((req->cmd.asiv_length / 8) << 4) |
843 		      ((req->cmd.asv_length  / 8)));
844 	pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts);
845 
846 	/*
847 	 * If copying the whole DDCB_ASIV_LENGTH is impacting
848 	 * performance we need to change it to
849 	 * req->cmd.asiv_length. But simulation benefits from some
850 	 * non-architectured bits behind the architectured content.
851 	 *
852 	 * How much data is copied depends on the availability of the
853 	 * ATS field, which was introduced late. If the ATS field is
854 	 * supported ASIV is 8 bytes shorter than it used to be. Since
855 	 * the ATS field is copied too, the code should do exactly
856 	 * what it did before, but I wanted to make copying of the ATS
857 	 * field very explicit.
858 	 */
859 	if (genwqe_get_slu_id(cd) <= 0x2) {
860 		memcpy(&pddcb->__asiv[0],	/* destination */
861 		       &req->cmd.__asiv[0],	/* source */
862 		       DDCB_ASIV_LENGTH);	/* req->cmd.asiv_length */
863 	} else {
864 		pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats);
865 		memcpy(&pddcb->n.asiv[0],	/* destination */
866 			&req->cmd.asiv[0],	/* source */
867 			DDCB_ASIV_LENGTH_ATS);	/* req->cmd.asiv_length */
868 	}
869 
870 	pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */
871 
872 	/*
873 	 * Calculate CRC_16 for corresponding range PSP(7:4). Include
874 	 * empty 4 bytes prior to the data.
875 	 */
876 	icrc = genwqe_crc16((const u8 *)pddcb,
877 			   ICRC_LENGTH(req->cmd.asiv_length), 0xffff);
878 	pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);
879 
880 	/* enable DDCB completion irq */
881 	if (!GENWQE_POLLING_ENABLED)
882 		pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;
883 
884 	dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);
885 	genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
886 
887 	if (ddcb_requ_collect_debug_data(req)) {
888 		/* use the kernel copy of debug data. copying back to
889 		   user buffer happens later */
890 
891 		genwqe_init_debug_data(cd, &req->debug_data);
892 		memcpy(&req->debug_data.ddcb_before, pddcb,
893 		       sizeof(req->debug_data.ddcb_before));
894 	}
895 
896 	enqueue_ddcb(cd, queue, pddcb, req->num);
897 	queue->ddcbs_in_flight++;
898 
899 	if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight)
900 		queue->ddcbs_max_in_flight = queue->ddcbs_in_flight;
901 
902 	ddcb_requ_set_state(req, GENWQE_REQU_TAPPED);
903 	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
904 	wake_up_interruptible(&cd->queue_waitq);
905 
906 	return 0;
907 }
908 
909 /**
910  * __genwqe_execute_raw_ddcb() - Setup and execute DDCB
911  * @cd:         pointer to genwqe device descriptor
912  * @req:        user provided DDCB request
913  * @f_flags:    file mode: blocking, non-blocking
914  */
__genwqe_execute_raw_ddcb(struct genwqe_dev * cd,struct genwqe_ddcb_cmd * cmd,unsigned int f_flags)915 int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,
916 			      struct genwqe_ddcb_cmd *cmd,
917 			      unsigned int f_flags)
918 {
919 	int rc = 0;
920 	struct pci_dev *pci_dev = cd->pci_dev;
921 	struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
922 
923 	if (cmd->asiv_length > DDCB_ASIV_LENGTH) {
924 		dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n",
925 			__func__, cmd->asiv_length);
926 		return -EINVAL;
927 	}
928 	if (cmd->asv_length > DDCB_ASV_LENGTH) {
929 		dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n",
930 			__func__, cmd->asiv_length);
931 		return -EINVAL;
932 	}
933 	rc = __genwqe_enqueue_ddcb(cd, req, f_flags);
934 	if (rc != 0)
935 		return rc;
936 
937 	rc = __genwqe_wait_ddcb(cd, req);
938 	if (rc < 0)		/* error or signal interrupt */
939 		goto err_exit;
940 
941 	if (ddcb_requ_collect_debug_data(req)) {
942 		if (copy_to_user((struct genwqe_debug_data __user *)
943 				 (unsigned long)cmd->ddata_addr,
944 				 &req->debug_data,
945 				 sizeof(struct genwqe_debug_data)))
946 			return -EFAULT;
947 	}
948 
949 	/*
950 	 * Higher values than 0x102 indicate completion with faults,
951 	 * lower values than 0x102 indicate processing faults. Note
952 	 * that DDCB might have been purged. E.g. Cntl+C.
953 	 */
954 	if (cmd->retc != DDCB_RETC_COMPLETE) {
955 		/* This might happen e.g. flash read, and needs to be
956 		   handled by the upper layer code. */
957 		rc = -EBADMSG;	/* not processed/error retc */
958 	}
959 
960 	return rc;
961 
962  err_exit:
963 	__genwqe_purge_ddcb(cd, req);
964 
965 	if (ddcb_requ_collect_debug_data(req)) {
966 		if (copy_to_user((struct genwqe_debug_data __user *)
967 				 (unsigned long)cmd->ddata_addr,
968 				 &req->debug_data,
969 				 sizeof(struct genwqe_debug_data)))
970 			return -EFAULT;
971 	}
972 	return rc;
973 }
974 
975 /**
976  * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished
977  *
978  * We use this as condition for our wait-queue code.
979  */
genwqe_next_ddcb_ready(struct genwqe_dev * cd)980 static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)
981 {
982 	unsigned long flags;
983 	struct ddcb *pddcb;
984 	struct ddcb_queue *queue = &cd->queue;
985 
986 	spin_lock_irqsave(&queue->ddcb_lock, flags);
987 
988 	if (queue_empty(queue)) { /* emtpy queue */
989 		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
990 		return 0;
991 	}
992 
993 	pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
994 	if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */
995 		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
996 		return 1;
997 	}
998 
999 	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1000 	return 0;
1001 }
1002 
1003 /**
1004  * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight
1005  *
1006  * Keep track on the number of DDCBs which ware currently in the
1007  * queue. This is needed for statistics as well as conditon if we want
1008  * to wait or better do polling in case of no interrupts available.
1009  */
genwqe_ddcbs_in_flight(struct genwqe_dev * cd)1010 int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)
1011 {
1012 	unsigned long flags;
1013 	int ddcbs_in_flight = 0;
1014 	struct ddcb_queue *queue = &cd->queue;
1015 
1016 	spin_lock_irqsave(&queue->ddcb_lock, flags);
1017 	ddcbs_in_flight += queue->ddcbs_in_flight;
1018 	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1019 
1020 	return ddcbs_in_flight;
1021 }
1022 
setup_ddcb_queue(struct genwqe_dev * cd,struct ddcb_queue * queue)1023 static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1024 {
1025 	int rc, i;
1026 	struct ddcb *pddcb;
1027 	u64 val64;
1028 	unsigned int queue_size;
1029 	struct pci_dev *pci_dev = cd->pci_dev;
1030 
1031 	if (GENWQE_DDCB_MAX < 2)
1032 		return -EINVAL;
1033 
1034 	queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE);
1035 
1036 	queue->ddcbs_in_flight = 0;  /* statistics */
1037 	queue->ddcbs_max_in_flight = 0;
1038 	queue->ddcbs_completed = 0;
1039 	queue->return_on_busy = 0;
1040 	queue->wait_on_busy = 0;
1041 
1042 	queue->ddcb_seq	  = 0x100; /* start sequence number */
1043 	queue->ddcb_max	  = GENWQE_DDCB_MAX;
1044 	queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,
1045 						&queue->ddcb_daddr);
1046 	if (queue->ddcb_vaddr == NULL) {
1047 		dev_err(&pci_dev->dev,
1048 			"[%s] **err: could not allocate DDCB **\n", __func__);
1049 		return -ENOMEM;
1050 	}
1051 	queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *),
1052 				  GFP_KERNEL);
1053 	if (!queue->ddcb_req) {
1054 		rc = -ENOMEM;
1055 		goto free_ddcbs;
1056 	}
1057 
1058 	queue->ddcb_waitqs = kcalloc(queue->ddcb_max,
1059 				     sizeof(wait_queue_head_t),
1060 				     GFP_KERNEL);
1061 	if (!queue->ddcb_waitqs) {
1062 		rc = -ENOMEM;
1063 		goto free_requs;
1064 	}
1065 
1066 	for (i = 0; i < queue->ddcb_max; i++) {
1067 		pddcb = &queue->ddcb_vaddr[i];		     /* DDCBs */
1068 		pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32;
1069 		pddcb->retc_16 = cpu_to_be16(0xfff);
1070 
1071 		queue->ddcb_req[i] = NULL;		     /* requests */
1072 		init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */
1073 	}
1074 
1075 	queue->ddcb_act  = 0;
1076 	queue->ddcb_next = 0;	/* queue is empty */
1077 
1078 	spin_lock_init(&queue->ddcb_lock);
1079 	init_waitqueue_head(&queue->busy_waitq);
1080 
1081 	val64 = ((u64)(queue->ddcb_max - 1) <<  8); /* lastptr */
1082 	__genwqe_writeq(cd, queue->IO_QUEUE_CONFIG,  0x07);  /* iCRC/vCRC */
1083 	__genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);
1084 	__genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);
1085 	__genwqe_writeq(cd, queue->IO_QUEUE_WRAP,    val64);
1086 	return 0;
1087 
1088  free_requs:
1089 	kfree(queue->ddcb_req);
1090 	queue->ddcb_req = NULL;
1091  free_ddcbs:
1092 	__genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1093 				queue->ddcb_daddr);
1094 	queue->ddcb_vaddr = NULL;
1095 	queue->ddcb_daddr = 0ull;
1096 	return -ENODEV;
1097 
1098 }
1099 
ddcb_queue_initialized(struct ddcb_queue * queue)1100 static int ddcb_queue_initialized(struct ddcb_queue *queue)
1101 {
1102 	return queue->ddcb_vaddr != NULL;
1103 }
1104 
free_ddcb_queue(struct genwqe_dev * cd,struct ddcb_queue * queue)1105 static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1106 {
1107 	unsigned int queue_size;
1108 
1109 	queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
1110 
1111 	kfree(queue->ddcb_req);
1112 	queue->ddcb_req = NULL;
1113 
1114 	if (queue->ddcb_vaddr) {
1115 		__genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1116 					queue->ddcb_daddr);
1117 		queue->ddcb_vaddr = NULL;
1118 		queue->ddcb_daddr = 0ull;
1119 	}
1120 }
1121 
genwqe_pf_isr(int irq,void * dev_id)1122 static irqreturn_t genwqe_pf_isr(int irq, void *dev_id)
1123 {
1124 	u64 gfir;
1125 	struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1126 	struct pci_dev *pci_dev = cd->pci_dev;
1127 
1128 	/*
1129 	 * In case of fatal FIR error the queue is stopped, such that
1130 	 * we can safely check it without risking anything.
1131 	 */
1132 	cd->irqs_processed++;
1133 	wake_up_interruptible(&cd->queue_waitq);
1134 
1135 	/*
1136 	 * Checking for errors before kicking the queue might be
1137 	 * safer, but slower for the good-case ... See above.
1138 	 */
1139 	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
1140 	if (((gfir & GFIR_ERR_TRIGGER) != 0x0) &&
1141 	    !pci_channel_offline(pci_dev)) {
1142 
1143 		if (cd->use_platform_recovery) {
1144 			/*
1145 			 * Since we use raw accessors, EEH errors won't be
1146 			 * detected by the platform until we do a non-raw
1147 			 * MMIO or config space read
1148 			 */
1149 			readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1150 
1151 			/* Don't do anything if the PCI channel is frozen */
1152 			if (pci_channel_offline(pci_dev))
1153 				goto exit;
1154 		}
1155 
1156 		wake_up_interruptible(&cd->health_waitq);
1157 
1158 		/*
1159 		 * By default GFIRs causes recovery actions. This
1160 		 * count is just for debug when recovery is masked.
1161 		 */
1162 		dev_err_ratelimited(&pci_dev->dev,
1163 				    "[%s] GFIR=%016llx\n",
1164 				    __func__, gfir);
1165 	}
1166 
1167  exit:
1168 	return IRQ_HANDLED;
1169 }
1170 
genwqe_vf_isr(int irq,void * dev_id)1171 static irqreturn_t genwqe_vf_isr(int irq, void *dev_id)
1172 {
1173 	struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1174 
1175 	cd->irqs_processed++;
1176 	wake_up_interruptible(&cd->queue_waitq);
1177 
1178 	return IRQ_HANDLED;
1179 }
1180 
1181 /**
1182  * genwqe_card_thread() - Work thread for the DDCB queue
1183  *
1184  * The idea is to check if there are DDCBs in processing. If there are
1185  * some finished DDCBs, we process them and wakeup the
1186  * requestors. Otherwise we give other processes time using
1187  * cond_resched().
1188  */
genwqe_card_thread(void * data)1189 static int genwqe_card_thread(void *data)
1190 {
1191 	int should_stop = 0, rc = 0;
1192 	struct genwqe_dev *cd = (struct genwqe_dev *)data;
1193 
1194 	while (!kthread_should_stop()) {
1195 
1196 		genwqe_check_ddcb_queue(cd, &cd->queue);
1197 
1198 		if (GENWQE_POLLING_ENABLED) {
1199 			rc = wait_event_interruptible_timeout(
1200 				cd->queue_waitq,
1201 				genwqe_ddcbs_in_flight(cd) ||
1202 				(should_stop = kthread_should_stop()), 1);
1203 		} else {
1204 			rc = wait_event_interruptible_timeout(
1205 				cd->queue_waitq,
1206 				genwqe_next_ddcb_ready(cd) ||
1207 				(should_stop = kthread_should_stop()), HZ);
1208 		}
1209 		if (should_stop)
1210 			break;
1211 
1212 		/*
1213 		 * Avoid soft lockups on heavy loads; we do not want
1214 		 * to disable our interrupts.
1215 		 */
1216 		cond_resched();
1217 	}
1218 	return 0;
1219 }
1220 
1221 /**
1222  * genwqe_setup_service_layer() - Setup DDCB queue
1223  * @cd:         pointer to genwqe device descriptor
1224  *
1225  * Allocate DDCBs. Configure Service Layer Controller (SLC).
1226  *
1227  * Return: 0 success
1228  */
genwqe_setup_service_layer(struct genwqe_dev * cd)1229 int genwqe_setup_service_layer(struct genwqe_dev *cd)
1230 {
1231 	int rc;
1232 	struct ddcb_queue *queue;
1233 	struct pci_dev *pci_dev = cd->pci_dev;
1234 
1235 	if (genwqe_is_privileged(cd)) {
1236 		rc = genwqe_card_reset(cd);
1237 		if (rc < 0) {
1238 			dev_err(&pci_dev->dev,
1239 				"[%s] err: reset failed.\n", __func__);
1240 			return rc;
1241 		}
1242 		genwqe_read_softreset(cd);
1243 	}
1244 
1245 	queue = &cd->queue;
1246 	queue->IO_QUEUE_CONFIG  = IO_SLC_QUEUE_CONFIG;
1247 	queue->IO_QUEUE_STATUS  = IO_SLC_QUEUE_STATUS;
1248 	queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT;
1249 	queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN;
1250 	queue->IO_QUEUE_OFFSET  = IO_SLC_QUEUE_OFFSET;
1251 	queue->IO_QUEUE_WRAP    = IO_SLC_QUEUE_WRAP;
1252 	queue->IO_QUEUE_WTIME   = IO_SLC_QUEUE_WTIME;
1253 	queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS;
1254 	queue->IO_QUEUE_LRW     = IO_SLC_QUEUE_LRW;
1255 
1256 	rc = setup_ddcb_queue(cd, queue);
1257 	if (rc != 0) {
1258 		rc = -ENODEV;
1259 		goto err_out;
1260 	}
1261 
1262 	init_waitqueue_head(&cd->queue_waitq);
1263 	cd->card_thread = kthread_run(genwqe_card_thread, cd,
1264 				      GENWQE_DEVNAME "%d_thread",
1265 				      cd->card_idx);
1266 	if (IS_ERR(cd->card_thread)) {
1267 		rc = PTR_ERR(cd->card_thread);
1268 		cd->card_thread = NULL;
1269 		goto stop_free_queue;
1270 	}
1271 
1272 	rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);
1273 	if (rc)
1274 		goto stop_kthread;
1275 
1276 	/*
1277 	 * We must have all wait-queues initialized when we enable the
1278 	 * interrupts. Otherwise we might crash if we get an early
1279 	 * irq.
1280 	 */
1281 	init_waitqueue_head(&cd->health_waitq);
1282 
1283 	if (genwqe_is_privileged(cd)) {
1284 		rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED,
1285 				 GENWQE_DEVNAME, cd);
1286 	} else {
1287 		rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED,
1288 				 GENWQE_DEVNAME, cd);
1289 	}
1290 	if (rc < 0) {
1291 		dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq);
1292 		goto stop_irq_cap;
1293 	}
1294 
1295 	cd->card_state = GENWQE_CARD_USED;
1296 	return 0;
1297 
1298  stop_irq_cap:
1299 	genwqe_reset_interrupt_capability(cd);
1300  stop_kthread:
1301 	kthread_stop(cd->card_thread);
1302 	cd->card_thread = NULL;
1303  stop_free_queue:
1304 	free_ddcb_queue(cd, queue);
1305  err_out:
1306 	return rc;
1307 }
1308 
1309 /**
1310  * queue_wake_up_all() - Handles fatal error case
1311  *
1312  * The PCI device got unusable and we have to stop all pending
1313  * requests as fast as we can. The code after this must purge the
1314  * DDCBs in question and ensure that all mappings are freed.
1315  */
queue_wake_up_all(struct genwqe_dev * cd)1316 static int queue_wake_up_all(struct genwqe_dev *cd)
1317 {
1318 	unsigned int i;
1319 	unsigned long flags;
1320 	struct ddcb_queue *queue = &cd->queue;
1321 
1322 	spin_lock_irqsave(&queue->ddcb_lock, flags);
1323 
1324 	for (i = 0; i < queue->ddcb_max; i++)
1325 		wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
1326 
1327 	wake_up_interruptible(&queue->busy_waitq);
1328 	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1329 
1330 	return 0;
1331 }
1332 
1333 /**
1334  * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces
1335  *
1336  * Relies on the pre-condition that there are no users of the card
1337  * device anymore e.g. with open file-descriptors.
1338  *
1339  * This function must be robust enough to be called twice.
1340  */
genwqe_finish_queue(struct genwqe_dev * cd)1341 int genwqe_finish_queue(struct genwqe_dev *cd)
1342 {
1343 	int i, rc = 0, in_flight;
1344 	int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT;
1345 	struct pci_dev *pci_dev = cd->pci_dev;
1346 	struct ddcb_queue *queue = &cd->queue;
1347 
1348 	if (!ddcb_queue_initialized(queue))
1349 		return 0;
1350 
1351 	/* Do not wipe out the error state. */
1352 	if (cd->card_state == GENWQE_CARD_USED)
1353 		cd->card_state = GENWQE_CARD_UNUSED;
1354 
1355 	/* Wake up all requests in the DDCB queue such that they
1356 	   should be removed nicely. */
1357 	queue_wake_up_all(cd);
1358 
1359 	/* We must wait to get rid of the DDCBs in flight */
1360 	for (i = 0; i < waitmax; i++) {
1361 		in_flight = genwqe_ddcbs_in_flight(cd);
1362 
1363 		if (in_flight == 0)
1364 			break;
1365 
1366 		dev_dbg(&pci_dev->dev,
1367 			"  DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n",
1368 			i, waitmax, in_flight);
1369 
1370 		/*
1371 		 * Severe severe error situation: The card itself has
1372 		 * 16 DDCB queues, each queue has e.g. 32 entries,
1373 		 * each DDBC has a hardware timeout of currently 250
1374 		 * msec but the PFs have a hardware timeout of 8 sec
1375 		 * ... so I take something large.
1376 		 */
1377 		msleep(1000);
1378 	}
1379 	if (i == waitmax) {
1380 		dev_err(&pci_dev->dev, "  [%s] err: queue is not empty!!\n",
1381 			__func__);
1382 		rc = -EIO;
1383 	}
1384 	return rc;
1385 }
1386 
1387 /**
1388  * genwqe_release_service_layer() - Shutdown DDCB queue
1389  * @cd:       genwqe device descriptor
1390  *
1391  * This function must be robust enough to be called twice.
1392  */
genwqe_release_service_layer(struct genwqe_dev * cd)1393 int genwqe_release_service_layer(struct genwqe_dev *cd)
1394 {
1395 	struct pci_dev *pci_dev = cd->pci_dev;
1396 
1397 	if (!ddcb_queue_initialized(&cd->queue))
1398 		return 1;
1399 
1400 	free_irq(pci_dev->irq, cd);
1401 	genwqe_reset_interrupt_capability(cd);
1402 
1403 	if (cd->card_thread != NULL) {
1404 		kthread_stop(cd->card_thread);
1405 		cd->card_thread = NULL;
1406 	}
1407 
1408 	free_ddcb_queue(cd, &cd->queue);
1409 	return 0;
1410 }
1411