1 /*
2  * Asynchronous RAID-6 recovery calculations ASYNC_TX API.
3  * Copyright(c) 2009 Intel Corporation
4  *
5  * based on raid6recov.c:
6  *   Copyright 2002 H. Peter Anvin
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms of the GNU General Public License as published by the Free
10  * Software Foundation; either version 2 of the License, or (at your option)
11  * any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along with
19  * this program; if not, write to the Free Software Foundation, Inc., 51
20  * Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
21  *
22  */
23 #include <linux/kernel.h>
24 #include <linux/interrupt.h>
25 #include <linux/module.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/raid/pq.h>
28 #include <linux/async_tx.h>
29 #include <linux/dmaengine.h>
30 
31 static struct dma_async_tx_descriptor *
async_sum_product(struct page * dest,struct page ** srcs,unsigned char * coef,size_t len,struct async_submit_ctl * submit)32 async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef,
33 		  size_t len, struct async_submit_ctl *submit)
34 {
35 	struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
36 						      &dest, 1, srcs, 2, len);
37 	struct dma_device *dma = chan ? chan->device : NULL;
38 	struct dmaengine_unmap_data *unmap = NULL;
39 	const u8 *amul, *bmul;
40 	u8 ax, bx;
41 	u8 *a, *b, *c;
42 
43 	if (dma)
44 		unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);
45 
46 	if (unmap) {
47 		struct device *dev = dma->dev;
48 		dma_addr_t pq[2];
49 		struct dma_async_tx_descriptor *tx;
50 		enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
51 
52 		if (submit->flags & ASYNC_TX_FENCE)
53 			dma_flags |= DMA_PREP_FENCE;
54 		unmap->addr[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE);
55 		unmap->addr[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE);
56 		unmap->to_cnt = 2;
57 
58 		unmap->addr[2] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
59 		unmap->bidi_cnt = 1;
60 		/* engine only looks at Q, but expects it to follow P */
61 		pq[1] = unmap->addr[2];
62 
63 		unmap->len = len;
64 		tx = dma->device_prep_dma_pq(chan, pq, unmap->addr, 2, coef,
65 					     len, dma_flags);
66 		if (tx) {
67 			dma_set_unmap(tx, unmap);
68 			async_tx_submit(chan, tx, submit);
69 			dmaengine_unmap_put(unmap);
70 			return tx;
71 		}
72 
73 		/* could not get a descriptor, unmap and fall through to
74 		 * the synchronous path
75 		 */
76 		dmaengine_unmap_put(unmap);
77 	}
78 
79 	/* run the operation synchronously */
80 	async_tx_quiesce(&submit->depend_tx);
81 	amul = raid6_gfmul[coef[0]];
82 	bmul = raid6_gfmul[coef[1]];
83 	a = page_address(srcs[0]);
84 	b = page_address(srcs[1]);
85 	c = page_address(dest);
86 
87 	while (len--) {
88 		ax    = amul[*a++];
89 		bx    = bmul[*b++];
90 		*c++ = ax ^ bx;
91 	}
92 
93 	return NULL;
94 }
95 
96 static struct dma_async_tx_descriptor *
async_mult(struct page * dest,struct page * src,u8 coef,size_t len,struct async_submit_ctl * submit)97 async_mult(struct page *dest, struct page *src, u8 coef, size_t len,
98 	   struct async_submit_ctl *submit)
99 {
100 	struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
101 						      &dest, 1, &src, 1, len);
102 	struct dma_device *dma = chan ? chan->device : NULL;
103 	struct dmaengine_unmap_data *unmap = NULL;
104 	const u8 *qmul; /* Q multiplier table */
105 	u8 *d, *s;
106 
107 	if (dma)
108 		unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOWAIT);
109 
110 	if (unmap) {
111 		dma_addr_t dma_dest[2];
112 		struct device *dev = dma->dev;
113 		struct dma_async_tx_descriptor *tx;
114 		enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
115 
116 		if (submit->flags & ASYNC_TX_FENCE)
117 			dma_flags |= DMA_PREP_FENCE;
118 		unmap->addr[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE);
119 		unmap->to_cnt++;
120 		unmap->addr[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
121 		dma_dest[1] = unmap->addr[1];
122 		unmap->bidi_cnt++;
123 		unmap->len = len;
124 
125 		/* this looks funny, but the engine looks for Q at
126 		 * dma_dest[1] and ignores dma_dest[0] as a dest
127 		 * due to DMA_PREP_PQ_DISABLE_P
128 		 */
129 		tx = dma->device_prep_dma_pq(chan, dma_dest, unmap->addr,
130 					     1, &coef, len, dma_flags);
131 
132 		if (tx) {
133 			dma_set_unmap(tx, unmap);
134 			dmaengine_unmap_put(unmap);
135 			async_tx_submit(chan, tx, submit);
136 			return tx;
137 		}
138 
139 		/* could not get a descriptor, unmap and fall through to
140 		 * the synchronous path
141 		 */
142 		dmaengine_unmap_put(unmap);
143 	}
144 
145 	/* no channel available, or failed to allocate a descriptor, so
146 	 * perform the operation synchronously
147 	 */
148 	async_tx_quiesce(&submit->depend_tx);
149 	qmul  = raid6_gfmul[coef];
150 	d = page_address(dest);
151 	s = page_address(src);
152 
153 	while (len--)
154 		*d++ = qmul[*s++];
155 
156 	return NULL;
157 }
158 
159 static struct dma_async_tx_descriptor *
__2data_recov_4(int disks,size_t bytes,int faila,int failb,struct page ** blocks,struct async_submit_ctl * submit)160 __2data_recov_4(int disks, size_t bytes, int faila, int failb,
161 		struct page **blocks, struct async_submit_ctl *submit)
162 {
163 	struct dma_async_tx_descriptor *tx = NULL;
164 	struct page *p, *q, *a, *b;
165 	struct page *srcs[2];
166 	unsigned char coef[2];
167 	enum async_tx_flags flags = submit->flags;
168 	dma_async_tx_callback cb_fn = submit->cb_fn;
169 	void *cb_param = submit->cb_param;
170 	void *scribble = submit->scribble;
171 
172 	p = blocks[disks-2];
173 	q = blocks[disks-1];
174 
175 	a = blocks[faila];
176 	b = blocks[failb];
177 
178 	/* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
179 	/* Dx = A*(P+Pxy) + B*(Q+Qxy) */
180 	srcs[0] = p;
181 	srcs[1] = q;
182 	coef[0] = raid6_gfexi[failb-faila];
183 	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
184 	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
185 	tx = async_sum_product(b, srcs, coef, bytes, submit);
186 
187 	/* Dy = P+Pxy+Dx */
188 	srcs[0] = p;
189 	srcs[1] = b;
190 	init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
191 			  cb_param, scribble);
192 	tx = async_xor(a, srcs, 0, 2, bytes, submit);
193 
194 	return tx;
195 
196 }
197 
198 static struct dma_async_tx_descriptor *
__2data_recov_5(int disks,size_t bytes,int faila,int failb,struct page ** blocks,struct async_submit_ctl * submit)199 __2data_recov_5(int disks, size_t bytes, int faila, int failb,
200 		struct page **blocks, struct async_submit_ctl *submit)
201 {
202 	struct dma_async_tx_descriptor *tx = NULL;
203 	struct page *p, *q, *g, *dp, *dq;
204 	struct page *srcs[2];
205 	unsigned char coef[2];
206 	enum async_tx_flags flags = submit->flags;
207 	dma_async_tx_callback cb_fn = submit->cb_fn;
208 	void *cb_param = submit->cb_param;
209 	void *scribble = submit->scribble;
210 	int good_srcs, good, i;
211 
212 	good_srcs = 0;
213 	good = -1;
214 	for (i = 0; i < disks-2; i++) {
215 		if (blocks[i] == NULL)
216 			continue;
217 		if (i == faila || i == failb)
218 			continue;
219 		good = i;
220 		good_srcs++;
221 	}
222 	BUG_ON(good_srcs > 1);
223 
224 	p = blocks[disks-2];
225 	q = blocks[disks-1];
226 	g = blocks[good];
227 
228 	/* Compute syndrome with zero for the missing data pages
229 	 * Use the dead data pages as temporary storage for delta p and
230 	 * delta q
231 	 */
232 	dp = blocks[faila];
233 	dq = blocks[failb];
234 
235 	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
236 	tx = async_memcpy(dp, g, 0, 0, bytes, submit);
237 	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
238 	tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
239 
240 	/* compute P + Pxy */
241 	srcs[0] = dp;
242 	srcs[1] = p;
243 	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
244 			  NULL, NULL, scribble);
245 	tx = async_xor(dp, srcs, 0, 2, bytes, submit);
246 
247 	/* compute Q + Qxy */
248 	srcs[0] = dq;
249 	srcs[1] = q;
250 	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
251 			  NULL, NULL, scribble);
252 	tx = async_xor(dq, srcs, 0, 2, bytes, submit);
253 
254 	/* Dx = A*(P+Pxy) + B*(Q+Qxy) */
255 	srcs[0] = dp;
256 	srcs[1] = dq;
257 	coef[0] = raid6_gfexi[failb-faila];
258 	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
259 	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
260 	tx = async_sum_product(dq, srcs, coef, bytes, submit);
261 
262 	/* Dy = P+Pxy+Dx */
263 	srcs[0] = dp;
264 	srcs[1] = dq;
265 	init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
266 			  cb_param, scribble);
267 	tx = async_xor(dp, srcs, 0, 2, bytes, submit);
268 
269 	return tx;
270 }
271 
272 static struct dma_async_tx_descriptor *
__2data_recov_n(int disks,size_t bytes,int faila,int failb,struct page ** blocks,struct async_submit_ctl * submit)273 __2data_recov_n(int disks, size_t bytes, int faila, int failb,
274 	      struct page **blocks, struct async_submit_ctl *submit)
275 {
276 	struct dma_async_tx_descriptor *tx = NULL;
277 	struct page *p, *q, *dp, *dq;
278 	struct page *srcs[2];
279 	unsigned char coef[2];
280 	enum async_tx_flags flags = submit->flags;
281 	dma_async_tx_callback cb_fn = submit->cb_fn;
282 	void *cb_param = submit->cb_param;
283 	void *scribble = submit->scribble;
284 
285 	p = blocks[disks-2];
286 	q = blocks[disks-1];
287 
288 	/* Compute syndrome with zero for the missing data pages
289 	 * Use the dead data pages as temporary storage for
290 	 * delta p and delta q
291 	 */
292 	dp = blocks[faila];
293 	blocks[faila] = NULL;
294 	blocks[disks-2] = dp;
295 	dq = blocks[failb];
296 	blocks[failb] = NULL;
297 	blocks[disks-1] = dq;
298 
299 	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
300 	tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
301 
302 	/* Restore pointer table */
303 	blocks[faila]   = dp;
304 	blocks[failb]   = dq;
305 	blocks[disks-2] = p;
306 	blocks[disks-1] = q;
307 
308 	/* compute P + Pxy */
309 	srcs[0] = dp;
310 	srcs[1] = p;
311 	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
312 			  NULL, NULL, scribble);
313 	tx = async_xor(dp, srcs, 0, 2, bytes, submit);
314 
315 	/* compute Q + Qxy */
316 	srcs[0] = dq;
317 	srcs[1] = q;
318 	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
319 			  NULL, NULL, scribble);
320 	tx = async_xor(dq, srcs, 0, 2, bytes, submit);
321 
322 	/* Dx = A*(P+Pxy) + B*(Q+Qxy) */
323 	srcs[0] = dp;
324 	srcs[1] = dq;
325 	coef[0] = raid6_gfexi[failb-faila];
326 	coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
327 	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
328 	tx = async_sum_product(dq, srcs, coef, bytes, submit);
329 
330 	/* Dy = P+Pxy+Dx */
331 	srcs[0] = dp;
332 	srcs[1] = dq;
333 	init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
334 			  cb_param, scribble);
335 	tx = async_xor(dp, srcs, 0, 2, bytes, submit);
336 
337 	return tx;
338 }
339 
340 /**
341  * async_raid6_2data_recov - asynchronously calculate two missing data blocks
342  * @disks: number of disks in the RAID-6 array
343  * @bytes: block size
344  * @faila: first failed drive index
345  * @failb: second failed drive index
346  * @blocks: array of source pointers where the last two entries are p and q
347  * @submit: submission/completion modifiers
348  */
349 struct dma_async_tx_descriptor *
async_raid6_2data_recov(int disks,size_t bytes,int faila,int failb,struct page ** blocks,struct async_submit_ctl * submit)350 async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
351 			struct page **blocks, struct async_submit_ctl *submit)
352 {
353 	void *scribble = submit->scribble;
354 	int non_zero_srcs, i;
355 
356 	BUG_ON(faila == failb);
357 	if (failb < faila)
358 		swap(faila, failb);
359 
360 	pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
361 
362 	/* if a dma resource is not available or a scribble buffer is not
363 	 * available punt to the synchronous path.  In the 'dma not
364 	 * available' case be sure to use the scribble buffer to
365 	 * preserve the content of 'blocks' as the caller intended.
366 	 */
367 	if (!async_dma_find_channel(DMA_PQ) || !scribble) {
368 		void **ptrs = scribble ? scribble : (void **) blocks;
369 
370 		async_tx_quiesce(&submit->depend_tx);
371 		for (i = 0; i < disks; i++)
372 			if (blocks[i] == NULL)
373 				ptrs[i] = (void *) raid6_empty_zero_page;
374 			else
375 				ptrs[i] = page_address(blocks[i]);
376 
377 		raid6_2data_recov(disks, bytes, faila, failb, ptrs);
378 
379 		async_tx_sync_epilog(submit);
380 
381 		return NULL;
382 	}
383 
384 	non_zero_srcs = 0;
385 	for (i = 0; i < disks-2 && non_zero_srcs < 4; i++)
386 		if (blocks[i])
387 			non_zero_srcs++;
388 	switch (non_zero_srcs) {
389 	case 0:
390 	case 1:
391 		/* There must be at least 2 sources - the failed devices. */
392 		BUG();
393 
394 	case 2:
395 		/* dma devices do not uniformly understand a zero source pq
396 		 * operation (in contrast to the synchronous case), so
397 		 * explicitly handle the special case of a 4 disk array with
398 		 * both data disks missing.
399 		 */
400 		return __2data_recov_4(disks, bytes, faila, failb, blocks, submit);
401 	case 3:
402 		/* dma devices do not uniformly understand a single
403 		 * source pq operation (in contrast to the synchronous
404 		 * case), so explicitly handle the special case of a 5 disk
405 		 * array with 2 of 3 data disks missing.
406 		 */
407 		return __2data_recov_5(disks, bytes, faila, failb, blocks, submit);
408 	default:
409 		return __2data_recov_n(disks, bytes, faila, failb, blocks, submit);
410 	}
411 }
412 EXPORT_SYMBOL_GPL(async_raid6_2data_recov);
413 
414 /**
415  * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
416  * @disks: number of disks in the RAID-6 array
417  * @bytes: block size
418  * @faila: failed drive index
419  * @blocks: array of source pointers where the last two entries are p and q
420  * @submit: submission/completion modifiers
421  */
422 struct dma_async_tx_descriptor *
async_raid6_datap_recov(int disks,size_t bytes,int faila,struct page ** blocks,struct async_submit_ctl * submit)423 async_raid6_datap_recov(int disks, size_t bytes, int faila,
424 			struct page **blocks, struct async_submit_ctl *submit)
425 {
426 	struct dma_async_tx_descriptor *tx = NULL;
427 	struct page *p, *q, *dq;
428 	u8 coef;
429 	enum async_tx_flags flags = submit->flags;
430 	dma_async_tx_callback cb_fn = submit->cb_fn;
431 	void *cb_param = submit->cb_param;
432 	void *scribble = submit->scribble;
433 	int good_srcs, good, i;
434 	struct page *srcs[2];
435 
436 	pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
437 
438 	/* if a dma resource is not available or a scribble buffer is not
439 	 * available punt to the synchronous path.  In the 'dma not
440 	 * available' case be sure to use the scribble buffer to
441 	 * preserve the content of 'blocks' as the caller intended.
442 	 */
443 	if (!async_dma_find_channel(DMA_PQ) || !scribble) {
444 		void **ptrs = scribble ? scribble : (void **) blocks;
445 
446 		async_tx_quiesce(&submit->depend_tx);
447 		for (i = 0; i < disks; i++)
448 			if (blocks[i] == NULL)
449 				ptrs[i] = (void*)raid6_empty_zero_page;
450 			else
451 				ptrs[i] = page_address(blocks[i]);
452 
453 		raid6_datap_recov(disks, bytes, faila, ptrs);
454 
455 		async_tx_sync_epilog(submit);
456 
457 		return NULL;
458 	}
459 
460 	good_srcs = 0;
461 	good = -1;
462 	for (i = 0; i < disks-2; i++) {
463 		if (i == faila)
464 			continue;
465 		if (blocks[i]) {
466 			good = i;
467 			good_srcs++;
468 			if (good_srcs > 1)
469 				break;
470 		}
471 	}
472 	BUG_ON(good_srcs == 0);
473 
474 	p = blocks[disks-2];
475 	q = blocks[disks-1];
476 
477 	/* Compute syndrome with zero for the missing data page
478 	 * Use the dead data page as temporary storage for delta q
479 	 */
480 	dq = blocks[faila];
481 	blocks[faila] = NULL;
482 	blocks[disks-1] = dq;
483 
484 	/* in the 4-disk case we only need to perform a single source
485 	 * multiplication with the one good data block.
486 	 */
487 	if (good_srcs == 1) {
488 		struct page *g = blocks[good];
489 
490 		init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
491 				  scribble);
492 		tx = async_memcpy(p, g, 0, 0, bytes, submit);
493 
494 		init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
495 				  scribble);
496 		tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
497 	} else {
498 		init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
499 				  scribble);
500 		tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
501 	}
502 
503 	/* Restore pointer table */
504 	blocks[faila]   = dq;
505 	blocks[disks-1] = q;
506 
507 	/* calculate g^{-faila} */
508 	coef = raid6_gfinv[raid6_gfexp[faila]];
509 
510 	srcs[0] = dq;
511 	srcs[1] = q;
512 	init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
513 			  NULL, NULL, scribble);
514 	tx = async_xor(dq, srcs, 0, 2, bytes, submit);
515 
516 	init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
517 	tx = async_mult(dq, dq, coef, bytes, submit);
518 
519 	srcs[0] = p;
520 	srcs[1] = dq;
521 	init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
522 			  cb_param, scribble);
523 	tx = async_xor(p, srcs, 0, 2, bytes, submit);
524 
525 	return tx;
526 }
527 EXPORT_SYMBOL_GPL(async_raid6_datap_recov);
528 
529 MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
530 MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
531 MODULE_LICENSE("GPL");
532