1 /*
2  * Copyright (C) 2012 Intel Corporation
3  * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License
7  * as published by the Free Software Foundation; version 2
8  * of the License.
9  */
10 
11 #include <linux/raid/pq.h>
12 
13 #ifdef __KERNEL__
14 #include <asm/neon.h>
15 #else
16 #define kernel_neon_begin()
17 #define kernel_neon_end()
18 #define cpu_has_neon()		(1)
19 #endif
20 
raid6_has_neon(void)21 static int raid6_has_neon(void)
22 {
23 	return cpu_has_neon();
24 }
25 
26 void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
27 			      uint8_t *dq, const uint8_t *pbmul,
28 			      const uint8_t *qmul);
29 
30 void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
31 			      const uint8_t *qmul);
32 
raid6_2data_recov_neon(int disks,size_t bytes,int faila,int failb,void ** ptrs)33 static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
34 		int failb, void **ptrs)
35 {
36 	u8 *p, *q, *dp, *dq;
37 	const u8 *pbmul;	/* P multiplier table for B data */
38 	const u8 *qmul;		/* Q multiplier table (for both) */
39 
40 	p = (u8 *)ptrs[disks - 2];
41 	q = (u8 *)ptrs[disks - 1];
42 
43 	/*
44 	 * Compute syndrome with zero for the missing data pages
45 	 * Use the dead data pages as temporary storage for
46 	 * delta p and delta q
47 	 */
48 	dp = (u8 *)ptrs[faila];
49 	ptrs[faila] = (void *)raid6_empty_zero_page;
50 	ptrs[disks - 2] = dp;
51 	dq = (u8 *)ptrs[failb];
52 	ptrs[failb] = (void *)raid6_empty_zero_page;
53 	ptrs[disks - 1] = dq;
54 
55 	raid6_call.gen_syndrome(disks, bytes, ptrs);
56 
57 	/* Restore pointer table */
58 	ptrs[faila]     = dp;
59 	ptrs[failb]     = dq;
60 	ptrs[disks - 2] = p;
61 	ptrs[disks - 1] = q;
62 
63 	/* Now, pick the proper data tables */
64 	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
65 	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
66 					 raid6_gfexp[failb]]];
67 
68 	kernel_neon_begin();
69 	__raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
70 	kernel_neon_end();
71 }
72 
raid6_datap_recov_neon(int disks,size_t bytes,int faila,void ** ptrs)73 static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
74 		void **ptrs)
75 {
76 	u8 *p, *q, *dq;
77 	const u8 *qmul;		/* Q multiplier table */
78 
79 	p = (u8 *)ptrs[disks - 2];
80 	q = (u8 *)ptrs[disks - 1];
81 
82 	/*
83 	 * Compute syndrome with zero for the missing data page
84 	 * Use the dead data page as temporary storage for delta q
85 	 */
86 	dq = (u8 *)ptrs[faila];
87 	ptrs[faila] = (void *)raid6_empty_zero_page;
88 	ptrs[disks - 1] = dq;
89 
90 	raid6_call.gen_syndrome(disks, bytes, ptrs);
91 
92 	/* Restore pointer table */
93 	ptrs[faila]     = dq;
94 	ptrs[disks - 1] = q;
95 
96 	/* Now, pick the proper data tables */
97 	qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
98 
99 	kernel_neon_begin();
100 	__raid6_datap_recov_neon(bytes, p, q, dq, qmul);
101 	kernel_neon_end();
102 }
103 
104 const struct raid6_recov_calls raid6_recov_neon = {
105 	.data2		= raid6_2data_recov_neon,
106 	.datap		= raid6_datap_recov_neon,
107 	.valid		= raid6_has_neon,
108 	.name		= "neon",
109 	.priority	= 10,
110 };
111