1 /* -*- linux-c -*- ------------------------------------------------------- *
2 *
3 * Copyright 2002 H. Peter Anvin - All Rights Reserved
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, Inc., 53 Temple Place Ste 330,
8 * Boston MA 02111-1307, USA; either version 2 of the License, or
9 * (at your option) any later version; incorporated herein by reference.
10 *
11 * ----------------------------------------------------------------------- */
12
13 /*
14 * raid6/sse2.c
15 *
16 * SSE-2 implementation of RAID-6 syndrome functions
17 *
18 */
19
20 #include <linux/raid/pq.h>
21 #include "x86.h"
22
23 static const struct raid6_sse_constants {
24 u64 x1d[2];
25 } raid6_sse_constants __attribute__((aligned(16))) = {
26 { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL },
27 };
28
raid6_have_sse2(void)29 static int raid6_have_sse2(void)
30 {
31 /* Not really boot_cpu but "all_cpus" */
32 return boot_cpu_has(X86_FEATURE_MMX) &&
33 boot_cpu_has(X86_FEATURE_FXSR) &&
34 boot_cpu_has(X86_FEATURE_XMM) &&
35 boot_cpu_has(X86_FEATURE_XMM2);
36 }
37
38 /*
39 * Plain SSE2 implementation
40 */
raid6_sse21_gen_syndrome(int disks,size_t bytes,void ** ptrs)41 static void raid6_sse21_gen_syndrome(int disks, size_t bytes, void **ptrs)
42 {
43 u8 **dptr = (u8 **)ptrs;
44 u8 *p, *q;
45 int d, z, z0;
46
47 z0 = disks - 3; /* Highest data disk */
48 p = dptr[z0+1]; /* XOR parity */
49 q = dptr[z0+2]; /* RS syndrome */
50
51 kernel_fpu_begin();
52
53 asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
54 asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
55
56 for ( d = 0 ; d < bytes ; d += 16 ) {
57 asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
58 asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d])); /* P[0] */
59 asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
60 asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
61 asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z0-1][d]));
62 for ( z = z0-2 ; z >= 0 ; z-- ) {
63 asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
64 asm volatile("pcmpgtb %xmm4,%xmm5");
65 asm volatile("paddb %xmm4,%xmm4");
66 asm volatile("pand %xmm0,%xmm5");
67 asm volatile("pxor %xmm5,%xmm4");
68 asm volatile("pxor %xmm5,%xmm5");
69 asm volatile("pxor %xmm6,%xmm2");
70 asm volatile("pxor %xmm6,%xmm4");
71 asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z][d]));
72 }
73 asm volatile("pcmpgtb %xmm4,%xmm5");
74 asm volatile("paddb %xmm4,%xmm4");
75 asm volatile("pand %xmm0,%xmm5");
76 asm volatile("pxor %xmm5,%xmm4");
77 asm volatile("pxor %xmm5,%xmm5");
78 asm volatile("pxor %xmm6,%xmm2");
79 asm volatile("pxor %xmm6,%xmm4");
80
81 asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
82 asm volatile("pxor %xmm2,%xmm2");
83 asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
84 asm volatile("pxor %xmm4,%xmm4");
85 }
86
87 asm volatile("sfence" : : : "memory");
88 kernel_fpu_end();
89 }
90
91
raid6_sse21_xor_syndrome(int disks,int start,int stop,size_t bytes,void ** ptrs)92 static void raid6_sse21_xor_syndrome(int disks, int start, int stop,
93 size_t bytes, void **ptrs)
94 {
95 u8 **dptr = (u8 **)ptrs;
96 u8 *p, *q;
97 int d, z, z0;
98
99 z0 = stop; /* P/Q right side optimization */
100 p = dptr[disks-2]; /* XOR parity */
101 q = dptr[disks-1]; /* RS syndrome */
102
103 kernel_fpu_begin();
104
105 asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
106
107 for ( d = 0 ; d < bytes ; d += 16 ) {
108 asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
109 asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
110 asm volatile("pxor %xmm4,%xmm2");
111 /* P/Q data pages */
112 for ( z = z0-1 ; z >= start ; z-- ) {
113 asm volatile("pxor %xmm5,%xmm5");
114 asm volatile("pcmpgtb %xmm4,%xmm5");
115 asm volatile("paddb %xmm4,%xmm4");
116 asm volatile("pand %xmm0,%xmm5");
117 asm volatile("pxor %xmm5,%xmm4");
118 asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
119 asm volatile("pxor %xmm5,%xmm2");
120 asm volatile("pxor %xmm5,%xmm4");
121 }
122 /* P/Q left side optimization */
123 for ( z = start-1 ; z >= 0 ; z-- ) {
124 asm volatile("pxor %xmm5,%xmm5");
125 asm volatile("pcmpgtb %xmm4,%xmm5");
126 asm volatile("paddb %xmm4,%xmm4");
127 asm volatile("pand %xmm0,%xmm5");
128 asm volatile("pxor %xmm5,%xmm4");
129 }
130 asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
131 /* Don't use movntdq for r/w memory area < cache line */
132 asm volatile("movdqa %%xmm4,%0" : "=m" (q[d]));
133 asm volatile("movdqa %%xmm2,%0" : "=m" (p[d]));
134 }
135
136 asm volatile("sfence" : : : "memory");
137 kernel_fpu_end();
138 }
139
140 const struct raid6_calls raid6_sse2x1 = {
141 raid6_sse21_gen_syndrome,
142 raid6_sse21_xor_syndrome,
143 raid6_have_sse2,
144 "sse2x1",
145 1 /* Has cache hints */
146 };
147
148 /*
149 * Unrolled-by-2 SSE2 implementation
150 */
raid6_sse22_gen_syndrome(int disks,size_t bytes,void ** ptrs)151 static void raid6_sse22_gen_syndrome(int disks, size_t bytes, void **ptrs)
152 {
153 u8 **dptr = (u8 **)ptrs;
154 u8 *p, *q;
155 int d, z, z0;
156
157 z0 = disks - 3; /* Highest data disk */
158 p = dptr[z0+1]; /* XOR parity */
159 q = dptr[z0+2]; /* RS syndrome */
160
161 kernel_fpu_begin();
162
163 asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
164 asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
165 asm volatile("pxor %xmm7,%xmm7"); /* Zero temp */
166
167 /* We uniformly assume a single prefetch covers at least 32 bytes */
168 for ( d = 0 ; d < bytes ; d += 32 ) {
169 asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
170 asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d])); /* P[0] */
171 asm volatile("movdqa %0,%%xmm3" : : "m" (dptr[z0][d+16])); /* P[1] */
172 asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
173 asm volatile("movdqa %xmm3,%xmm6"); /* Q[1] */
174 for ( z = z0-1 ; z >= 0 ; z-- ) {
175 asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
176 asm volatile("pcmpgtb %xmm4,%xmm5");
177 asm volatile("pcmpgtb %xmm6,%xmm7");
178 asm volatile("paddb %xmm4,%xmm4");
179 asm volatile("paddb %xmm6,%xmm6");
180 asm volatile("pand %xmm0,%xmm5");
181 asm volatile("pand %xmm0,%xmm7");
182 asm volatile("pxor %xmm5,%xmm4");
183 asm volatile("pxor %xmm7,%xmm6");
184 asm volatile("movdqa %0,%%xmm5" : : "m" (dptr[z][d]));
185 asm volatile("movdqa %0,%%xmm7" : : "m" (dptr[z][d+16]));
186 asm volatile("pxor %xmm5,%xmm2");
187 asm volatile("pxor %xmm7,%xmm3");
188 asm volatile("pxor %xmm5,%xmm4");
189 asm volatile("pxor %xmm7,%xmm6");
190 asm volatile("pxor %xmm5,%xmm5");
191 asm volatile("pxor %xmm7,%xmm7");
192 }
193 asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
194 asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
195 asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
196 asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
197 }
198
199 asm volatile("sfence" : : : "memory");
200 kernel_fpu_end();
201 }
202
raid6_sse22_xor_syndrome(int disks,int start,int stop,size_t bytes,void ** ptrs)203 static void raid6_sse22_xor_syndrome(int disks, int start, int stop,
204 size_t bytes, void **ptrs)
205 {
206 u8 **dptr = (u8 **)ptrs;
207 u8 *p, *q;
208 int d, z, z0;
209
210 z0 = stop; /* P/Q right side optimization */
211 p = dptr[disks-2]; /* XOR parity */
212 q = dptr[disks-1]; /* RS syndrome */
213
214 kernel_fpu_begin();
215
216 asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
217
218 for ( d = 0 ; d < bytes ; d += 32 ) {
219 asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
220 asm volatile("movdqa %0,%%xmm6" :: "m" (dptr[z0][d+16]));
221 asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
222 asm volatile("movdqa %0,%%xmm3" : : "m" (p[d+16]));
223 asm volatile("pxor %xmm4,%xmm2");
224 asm volatile("pxor %xmm6,%xmm3");
225 /* P/Q data pages */
226 for ( z = z0-1 ; z >= start ; z-- ) {
227 asm volatile("pxor %xmm5,%xmm5");
228 asm volatile("pxor %xmm7,%xmm7");
229 asm volatile("pcmpgtb %xmm4,%xmm5");
230 asm volatile("pcmpgtb %xmm6,%xmm7");
231 asm volatile("paddb %xmm4,%xmm4");
232 asm volatile("paddb %xmm6,%xmm6");
233 asm volatile("pand %xmm0,%xmm5");
234 asm volatile("pand %xmm0,%xmm7");
235 asm volatile("pxor %xmm5,%xmm4");
236 asm volatile("pxor %xmm7,%xmm6");
237 asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
238 asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
239 asm volatile("pxor %xmm5,%xmm2");
240 asm volatile("pxor %xmm7,%xmm3");
241 asm volatile("pxor %xmm5,%xmm4");
242 asm volatile("pxor %xmm7,%xmm6");
243 }
244 /* P/Q left side optimization */
245 for ( z = start-1 ; z >= 0 ; z-- ) {
246 asm volatile("pxor %xmm5,%xmm5");
247 asm volatile("pxor %xmm7,%xmm7");
248 asm volatile("pcmpgtb %xmm4,%xmm5");
249 asm volatile("pcmpgtb %xmm6,%xmm7");
250 asm volatile("paddb %xmm4,%xmm4");
251 asm volatile("paddb %xmm6,%xmm6");
252 asm volatile("pand %xmm0,%xmm5");
253 asm volatile("pand %xmm0,%xmm7");
254 asm volatile("pxor %xmm5,%xmm4");
255 asm volatile("pxor %xmm7,%xmm6");
256 }
257 asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
258 asm volatile("pxor %0,%%xmm6" : : "m" (q[d+16]));
259 /* Don't use movntdq for r/w memory area < cache line */
260 asm volatile("movdqa %%xmm4,%0" : "=m" (q[d]));
261 asm volatile("movdqa %%xmm6,%0" : "=m" (q[d+16]));
262 asm volatile("movdqa %%xmm2,%0" : "=m" (p[d]));
263 asm volatile("movdqa %%xmm3,%0" : "=m" (p[d+16]));
264 }
265
266 asm volatile("sfence" : : : "memory");
267 kernel_fpu_end();
268 }
269
270 const struct raid6_calls raid6_sse2x2 = {
271 raid6_sse22_gen_syndrome,
272 raid6_sse22_xor_syndrome,
273 raid6_have_sse2,
274 "sse2x2",
275 1 /* Has cache hints */
276 };
277
278 #ifdef CONFIG_X86_64
279
280 /*
281 * Unrolled-by-4 SSE2 implementation
282 */
raid6_sse24_gen_syndrome(int disks,size_t bytes,void ** ptrs)283 static void raid6_sse24_gen_syndrome(int disks, size_t bytes, void **ptrs)
284 {
285 u8 **dptr = (u8 **)ptrs;
286 u8 *p, *q;
287 int d, z, z0;
288
289 z0 = disks - 3; /* Highest data disk */
290 p = dptr[z0+1]; /* XOR parity */
291 q = dptr[z0+2]; /* RS syndrome */
292
293 kernel_fpu_begin();
294
295 asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
296 asm volatile("pxor %xmm2,%xmm2"); /* P[0] */
297 asm volatile("pxor %xmm3,%xmm3"); /* P[1] */
298 asm volatile("pxor %xmm4,%xmm4"); /* Q[0] */
299 asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
300 asm volatile("pxor %xmm6,%xmm6"); /* Q[1] */
301 asm volatile("pxor %xmm7,%xmm7"); /* Zero temp */
302 asm volatile("pxor %xmm10,%xmm10"); /* P[2] */
303 asm volatile("pxor %xmm11,%xmm11"); /* P[3] */
304 asm volatile("pxor %xmm12,%xmm12"); /* Q[2] */
305 asm volatile("pxor %xmm13,%xmm13"); /* Zero temp */
306 asm volatile("pxor %xmm14,%xmm14"); /* Q[3] */
307 asm volatile("pxor %xmm15,%xmm15"); /* Zero temp */
308
309 for ( d = 0 ; d < bytes ; d += 64 ) {
310 for ( z = z0 ; z >= 0 ; z-- ) {
311 /* The second prefetch seems to improve performance... */
312 asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
313 asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32]));
314 asm volatile("pcmpgtb %xmm4,%xmm5");
315 asm volatile("pcmpgtb %xmm6,%xmm7");
316 asm volatile("pcmpgtb %xmm12,%xmm13");
317 asm volatile("pcmpgtb %xmm14,%xmm15");
318 asm volatile("paddb %xmm4,%xmm4");
319 asm volatile("paddb %xmm6,%xmm6");
320 asm volatile("paddb %xmm12,%xmm12");
321 asm volatile("paddb %xmm14,%xmm14");
322 asm volatile("pand %xmm0,%xmm5");
323 asm volatile("pand %xmm0,%xmm7");
324 asm volatile("pand %xmm0,%xmm13");
325 asm volatile("pand %xmm0,%xmm15");
326 asm volatile("pxor %xmm5,%xmm4");
327 asm volatile("pxor %xmm7,%xmm6");
328 asm volatile("pxor %xmm13,%xmm12");
329 asm volatile("pxor %xmm15,%xmm14");
330 asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
331 asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
332 asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32]));
333 asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48]));
334 asm volatile("pxor %xmm5,%xmm2");
335 asm volatile("pxor %xmm7,%xmm3");
336 asm volatile("pxor %xmm13,%xmm10");
337 asm volatile("pxor %xmm15,%xmm11");
338 asm volatile("pxor %xmm5,%xmm4");
339 asm volatile("pxor %xmm7,%xmm6");
340 asm volatile("pxor %xmm13,%xmm12");
341 asm volatile("pxor %xmm15,%xmm14");
342 asm volatile("pxor %xmm5,%xmm5");
343 asm volatile("pxor %xmm7,%xmm7");
344 asm volatile("pxor %xmm13,%xmm13");
345 asm volatile("pxor %xmm15,%xmm15");
346 }
347 asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
348 asm volatile("pxor %xmm2,%xmm2");
349 asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
350 asm volatile("pxor %xmm3,%xmm3");
351 asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32]));
352 asm volatile("pxor %xmm10,%xmm10");
353 asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48]));
354 asm volatile("pxor %xmm11,%xmm11");
355 asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
356 asm volatile("pxor %xmm4,%xmm4");
357 asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
358 asm volatile("pxor %xmm6,%xmm6");
359 asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32]));
360 asm volatile("pxor %xmm12,%xmm12");
361 asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
362 asm volatile("pxor %xmm14,%xmm14");
363 }
364
365 asm volatile("sfence" : : : "memory");
366 kernel_fpu_end();
367 }
368
raid6_sse24_xor_syndrome(int disks,int start,int stop,size_t bytes,void ** ptrs)369 static void raid6_sse24_xor_syndrome(int disks, int start, int stop,
370 size_t bytes, void **ptrs)
371 {
372 u8 **dptr = (u8 **)ptrs;
373 u8 *p, *q;
374 int d, z, z0;
375
376 z0 = stop; /* P/Q right side optimization */
377 p = dptr[disks-2]; /* XOR parity */
378 q = dptr[disks-1]; /* RS syndrome */
379
380 kernel_fpu_begin();
381
382 asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
383
384 for ( d = 0 ; d < bytes ; d += 64 ) {
385 asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
386 asm volatile("movdqa %0,%%xmm6" :: "m" (dptr[z0][d+16]));
387 asm volatile("movdqa %0,%%xmm12" :: "m" (dptr[z0][d+32]));
388 asm volatile("movdqa %0,%%xmm14" :: "m" (dptr[z0][d+48]));
389 asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
390 asm volatile("movdqa %0,%%xmm3" : : "m" (p[d+16]));
391 asm volatile("movdqa %0,%%xmm10" : : "m" (p[d+32]));
392 asm volatile("movdqa %0,%%xmm11" : : "m" (p[d+48]));
393 asm volatile("pxor %xmm4,%xmm2");
394 asm volatile("pxor %xmm6,%xmm3");
395 asm volatile("pxor %xmm12,%xmm10");
396 asm volatile("pxor %xmm14,%xmm11");
397 /* P/Q data pages */
398 for ( z = z0-1 ; z >= start ; z-- ) {
399 asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
400 asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32]));
401 asm volatile("pxor %xmm5,%xmm5");
402 asm volatile("pxor %xmm7,%xmm7");
403 asm volatile("pxor %xmm13,%xmm13");
404 asm volatile("pxor %xmm15,%xmm15");
405 asm volatile("pcmpgtb %xmm4,%xmm5");
406 asm volatile("pcmpgtb %xmm6,%xmm7");
407 asm volatile("pcmpgtb %xmm12,%xmm13");
408 asm volatile("pcmpgtb %xmm14,%xmm15");
409 asm volatile("paddb %xmm4,%xmm4");
410 asm volatile("paddb %xmm6,%xmm6");
411 asm volatile("paddb %xmm12,%xmm12");
412 asm volatile("paddb %xmm14,%xmm14");
413 asm volatile("pand %xmm0,%xmm5");
414 asm volatile("pand %xmm0,%xmm7");
415 asm volatile("pand %xmm0,%xmm13");
416 asm volatile("pand %xmm0,%xmm15");
417 asm volatile("pxor %xmm5,%xmm4");
418 asm volatile("pxor %xmm7,%xmm6");
419 asm volatile("pxor %xmm13,%xmm12");
420 asm volatile("pxor %xmm15,%xmm14");
421 asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
422 asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
423 asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32]));
424 asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48]));
425 asm volatile("pxor %xmm5,%xmm2");
426 asm volatile("pxor %xmm7,%xmm3");
427 asm volatile("pxor %xmm13,%xmm10");
428 asm volatile("pxor %xmm15,%xmm11");
429 asm volatile("pxor %xmm5,%xmm4");
430 asm volatile("pxor %xmm7,%xmm6");
431 asm volatile("pxor %xmm13,%xmm12");
432 asm volatile("pxor %xmm15,%xmm14");
433 }
434 asm volatile("prefetchnta %0" :: "m" (q[d]));
435 asm volatile("prefetchnta %0" :: "m" (q[d+32]));
436 /* P/Q left side optimization */
437 for ( z = start-1 ; z >= 0 ; z-- ) {
438 asm volatile("pxor %xmm5,%xmm5");
439 asm volatile("pxor %xmm7,%xmm7");
440 asm volatile("pxor %xmm13,%xmm13");
441 asm volatile("pxor %xmm15,%xmm15");
442 asm volatile("pcmpgtb %xmm4,%xmm5");
443 asm volatile("pcmpgtb %xmm6,%xmm7");
444 asm volatile("pcmpgtb %xmm12,%xmm13");
445 asm volatile("pcmpgtb %xmm14,%xmm15");
446 asm volatile("paddb %xmm4,%xmm4");
447 asm volatile("paddb %xmm6,%xmm6");
448 asm volatile("paddb %xmm12,%xmm12");
449 asm volatile("paddb %xmm14,%xmm14");
450 asm volatile("pand %xmm0,%xmm5");
451 asm volatile("pand %xmm0,%xmm7");
452 asm volatile("pand %xmm0,%xmm13");
453 asm volatile("pand %xmm0,%xmm15");
454 asm volatile("pxor %xmm5,%xmm4");
455 asm volatile("pxor %xmm7,%xmm6");
456 asm volatile("pxor %xmm13,%xmm12");
457 asm volatile("pxor %xmm15,%xmm14");
458 }
459 asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
460 asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
461 asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32]));
462 asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48]));
463 asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
464 asm volatile("pxor %0,%%xmm6" : : "m" (q[d+16]));
465 asm volatile("pxor %0,%%xmm12" : : "m" (q[d+32]));
466 asm volatile("pxor %0,%%xmm14" : : "m" (q[d+48]));
467 asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
468 asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
469 asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32]));
470 asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
471 }
472 asm volatile("sfence" : : : "memory");
473 kernel_fpu_end();
474 }
475
476
477 const struct raid6_calls raid6_sse2x4 = {
478 raid6_sse24_gen_syndrome,
479 raid6_sse24_xor_syndrome,
480 raid6_have_sse2,
481 "sse2x4",
482 1 /* Has cache hints */
483 };
484
485 #endif /* CONFIG_X86_64 */
486