471 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/* -*- linux-c -*- ------------------------------------------------------- *3 *4 * Copyright (C) 2012 Intel Corporation5 * Author: Yuanhan Liu <yuanhan.liu@linux.intel.com>6 *7 * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved8 *9 * ----------------------------------------------------------------------- */10 11/*12 * AVX2 implementation of RAID-6 syndrome functions13 *14 */15 16#include <linux/raid/pq.h>17#include "x86.h"18 19static const struct raid6_avx2_constants {20 u64 x1d[4];21} raid6_avx2_constants __aligned(32) = {22 { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,23 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},24};25 26static int raid6_have_avx2(void)27{28 return boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX);29}30 31/*32 * Plain AVX2 implementation33 */34static void raid6_avx21_gen_syndrome(int disks, size_t bytes, void **ptrs)35{36 u8 **dptr = (u8 **)ptrs;37 u8 *p, *q;38 int d, z, z0;39 40 z0 = disks - 3; /* Highest data disk */41 p = dptr[z0+1]; /* XOR parity */42 q = dptr[z0+2]; /* RS syndrome */43 44 kernel_fpu_begin();45 46 asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));47 asm volatile("vpxor %ymm3,%ymm3,%ymm3"); /* Zero temp */48 49 for (d = 0; d < bytes; d += 32) {50 asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));51 asm volatile("vmovdqa %0,%%ymm2" : : "m" (dptr[z0][d]));/* P[0] */52 asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));53 asm volatile("vmovdqa %ymm2,%ymm4");/* Q[0] */54 asm volatile("vmovdqa %0,%%ymm6" : : "m" (dptr[z0-1][d]));55 for (z = z0-2; z >= 0; z--) {56 asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));57 asm volatile("vpcmpgtb %ymm4,%ymm3,%ymm5");58 asm volatile("vpaddb %ymm4,%ymm4,%ymm4");59 asm volatile("vpand %ymm0,%ymm5,%ymm5");60 asm volatile("vpxor %ymm5,%ymm4,%ymm4");61 asm volatile("vpxor %ymm6,%ymm2,%ymm2");62 asm volatile("vpxor %ymm6,%ymm4,%ymm4");63 asm volatile("vmovdqa %0,%%ymm6" : : "m" (dptr[z][d]));64 }65 asm volatile("vpcmpgtb %ymm4,%ymm3,%ymm5");66 asm volatile("vpaddb %ymm4,%ymm4,%ymm4");67 asm volatile("vpand %ymm0,%ymm5,%ymm5");68 asm volatile("vpxor %ymm5,%ymm4,%ymm4");69 asm volatile("vpxor %ymm6,%ymm2,%ymm2");70 asm volatile("vpxor %ymm6,%ymm4,%ymm4");71 72 asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));73 asm volatile("vpxor %ymm2,%ymm2,%ymm2");74 asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));75 asm volatile("vpxor %ymm4,%ymm4,%ymm4");76 }77 78 asm volatile("sfence" : : : "memory");79 kernel_fpu_end();80}81 82static void raid6_avx21_xor_syndrome(int disks, int start, int stop,83 size_t bytes, void **ptrs)84{85 u8 **dptr = (u8 **)ptrs;86 u8 *p, *q;87 int d, z, z0;88 89 z0 = stop; /* P/Q right side optimization */90 p = dptr[disks-2]; /* XOR parity */91 q = dptr[disks-1]; /* RS syndrome */92 93 kernel_fpu_begin();94 95 asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));96 97 for (d = 0 ; d < bytes ; d += 32) {98 asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));99 asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));100 asm volatile("vpxor %ymm4,%ymm2,%ymm2");101 /* P/Q data pages */102 for (z = z0-1 ; z >= start ; z--) {103 asm volatile("vpxor %ymm5,%ymm5,%ymm5");104 asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");105 asm volatile("vpaddb %ymm4,%ymm4,%ymm4");106 asm volatile("vpand %ymm0,%ymm5,%ymm5");107 asm volatile("vpxor %ymm5,%ymm4,%ymm4");108 asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));109 asm volatile("vpxor %ymm5,%ymm2,%ymm2");110 asm volatile("vpxor %ymm5,%ymm4,%ymm4");111 }112 /* P/Q left side optimization */113 for (z = start-1 ; z >= 0 ; z--) {114 asm volatile("vpxor %ymm5,%ymm5,%ymm5");115 asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");116 asm volatile("vpaddb %ymm4,%ymm4,%ymm4");117 asm volatile("vpand %ymm0,%ymm5,%ymm5");118 asm volatile("vpxor %ymm5,%ymm4,%ymm4");119 }120 asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));121 /* Don't use movntdq for r/w memory area < cache line */122 asm volatile("vmovdqa %%ymm4,%0" : "=m" (q[d]));123 asm volatile("vmovdqa %%ymm2,%0" : "=m" (p[d]));124 }125 126 asm volatile("sfence" : : : "memory");127 kernel_fpu_end();128}129 130const struct raid6_calls raid6_avx2x1 = {131 raid6_avx21_gen_syndrome,132 raid6_avx21_xor_syndrome,133 raid6_have_avx2,134 "avx2x1",135 .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */136};137 138/*139 * Unrolled-by-2 AVX2 implementation140 */141static void raid6_avx22_gen_syndrome(int disks, size_t bytes, void **ptrs)142{143 u8 **dptr = (u8 **)ptrs;144 u8 *p, *q;145 int d, z, z0;146 147 z0 = disks - 3; /* Highest data disk */148 p = dptr[z0+1]; /* XOR parity */149 q = dptr[z0+2]; /* RS syndrome */150 151 kernel_fpu_begin();152 153 asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));154 asm volatile("vpxor %ymm1,%ymm1,%ymm1"); /* Zero temp */155 156 /* We uniformly assume a single prefetch covers at least 32 bytes */157 for (d = 0; d < bytes; d += 64) {158 asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));159 asm volatile("prefetchnta %0" : : "m" (dptr[z0][d+32]));160 asm volatile("vmovdqa %0,%%ymm2" : : "m" (dptr[z0][d]));/* P[0] */161 asm volatile("vmovdqa %0,%%ymm3" : : "m" (dptr[z0][d+32]));/* P[1] */162 asm volatile("vmovdqa %ymm2,%ymm4"); /* Q[0] */163 asm volatile("vmovdqa %ymm3,%ymm6"); /* Q[1] */164 for (z = z0-1; z >= 0; z--) {165 asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));166 asm volatile("prefetchnta %0" : : "m" (dptr[z][d+32]));167 asm volatile("vpcmpgtb %ymm4,%ymm1,%ymm5");168 asm volatile("vpcmpgtb %ymm6,%ymm1,%ymm7");169 asm volatile("vpaddb %ymm4,%ymm4,%ymm4");170 asm volatile("vpaddb %ymm6,%ymm6,%ymm6");171 asm volatile("vpand %ymm0,%ymm5,%ymm5");172 asm volatile("vpand %ymm0,%ymm7,%ymm7");173 asm volatile("vpxor %ymm5,%ymm4,%ymm4");174 asm volatile("vpxor %ymm7,%ymm6,%ymm6");175 asm volatile("vmovdqa %0,%%ymm5" : : "m" (dptr[z][d]));176 asm volatile("vmovdqa %0,%%ymm7" : : "m" (dptr[z][d+32]));177 asm volatile("vpxor %ymm5,%ymm2,%ymm2");178 asm volatile("vpxor %ymm7,%ymm3,%ymm3");179 asm volatile("vpxor %ymm5,%ymm4,%ymm4");180 asm volatile("vpxor %ymm7,%ymm6,%ymm6");181 }182 asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));183 asm volatile("vmovntdq %%ymm3,%0" : "=m" (p[d+32]));184 asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));185 asm volatile("vmovntdq %%ymm6,%0" : "=m" (q[d+32]));186 }187 188 asm volatile("sfence" : : : "memory");189 kernel_fpu_end();190}191 192static void raid6_avx22_xor_syndrome(int disks, int start, int stop,193 size_t bytes, void **ptrs)194{195 u8 **dptr = (u8 **)ptrs;196 u8 *p, *q;197 int d, z, z0;198 199 z0 = stop; /* P/Q right side optimization */200 p = dptr[disks-2]; /* XOR parity */201 q = dptr[disks-1]; /* RS syndrome */202 203 kernel_fpu_begin();204 205 asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));206 207 for (d = 0 ; d < bytes ; d += 64) {208 asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));209 asm volatile("vmovdqa %0,%%ymm6" :: "m" (dptr[z0][d+32]));210 asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));211 asm volatile("vmovdqa %0,%%ymm3" : : "m" (p[d+32]));212 asm volatile("vpxor %ymm4,%ymm2,%ymm2");213 asm volatile("vpxor %ymm6,%ymm3,%ymm3");214 /* P/Q data pages */215 for (z = z0-1 ; z >= start ; z--) {216 asm volatile("vpxor %ymm5,%ymm5,%ymm5");217 asm volatile("vpxor %ymm7,%ymm7,%ymm7");218 asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");219 asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");220 asm volatile("vpaddb %ymm4,%ymm4,%ymm4");221 asm volatile("vpaddb %ymm6,%ymm6,%ymm6");222 asm volatile("vpand %ymm0,%ymm5,%ymm5");223 asm volatile("vpand %ymm0,%ymm7,%ymm7");224 asm volatile("vpxor %ymm5,%ymm4,%ymm4");225 asm volatile("vpxor %ymm7,%ymm6,%ymm6");226 asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));227 asm volatile("vmovdqa %0,%%ymm7"228 :: "m" (dptr[z][d+32]));229 asm volatile("vpxor %ymm5,%ymm2,%ymm2");230 asm volatile("vpxor %ymm7,%ymm3,%ymm3");231 asm volatile("vpxor %ymm5,%ymm4,%ymm4");232 asm volatile("vpxor %ymm7,%ymm6,%ymm6");233 }234 /* P/Q left side optimization */235 for (z = start-1 ; z >= 0 ; z--) {236 asm volatile("vpxor %ymm5,%ymm5,%ymm5");237 asm volatile("vpxor %ymm7,%ymm7,%ymm7");238 asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");239 asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");240 asm volatile("vpaddb %ymm4,%ymm4,%ymm4");241 asm volatile("vpaddb %ymm6,%ymm6,%ymm6");242 asm volatile("vpand %ymm0,%ymm5,%ymm5");243 asm volatile("vpand %ymm0,%ymm7,%ymm7");244 asm volatile("vpxor %ymm5,%ymm4,%ymm4");245 asm volatile("vpxor %ymm7,%ymm6,%ymm6");246 }247 asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));248 asm volatile("vpxor %0,%%ymm6,%%ymm6" : : "m" (q[d+32]));249 /* Don't use movntdq for r/w memory area < cache line */250 asm volatile("vmovdqa %%ymm4,%0" : "=m" (q[d]));251 asm volatile("vmovdqa %%ymm6,%0" : "=m" (q[d+32]));252 asm volatile("vmovdqa %%ymm2,%0" : "=m" (p[d]));253 asm volatile("vmovdqa %%ymm3,%0" : "=m" (p[d+32]));254 }255 256 asm volatile("sfence" : : : "memory");257 kernel_fpu_end();258}259 260const struct raid6_calls raid6_avx2x2 = {261 raid6_avx22_gen_syndrome,262 raid6_avx22_xor_syndrome,263 raid6_have_avx2,264 "avx2x2",265 .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */266};267 268#ifdef CONFIG_X86_64269 270/*271 * Unrolled-by-4 AVX2 implementation272 */273static void raid6_avx24_gen_syndrome(int disks, size_t bytes, void **ptrs)274{275 u8 **dptr = (u8 **)ptrs;276 u8 *p, *q;277 int d, z, z0;278 279 z0 = disks - 3; /* Highest data disk */280 p = dptr[z0+1]; /* XOR parity */281 q = dptr[z0+2]; /* RS syndrome */282 283 kernel_fpu_begin();284 285 asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));286 asm volatile("vpxor %ymm1,%ymm1,%ymm1"); /* Zero temp */287 asm volatile("vpxor %ymm2,%ymm2,%ymm2"); /* P[0] */288 asm volatile("vpxor %ymm3,%ymm3,%ymm3"); /* P[1] */289 asm volatile("vpxor %ymm4,%ymm4,%ymm4"); /* Q[0] */290 asm volatile("vpxor %ymm6,%ymm6,%ymm6"); /* Q[1] */291 asm volatile("vpxor %ymm10,%ymm10,%ymm10"); /* P[2] */292 asm volatile("vpxor %ymm11,%ymm11,%ymm11"); /* P[3] */293 asm volatile("vpxor %ymm12,%ymm12,%ymm12"); /* Q[2] */294 asm volatile("vpxor %ymm14,%ymm14,%ymm14"); /* Q[3] */295 296 for (d = 0; d < bytes; d += 128) {297 for (z = z0; z >= 0; z--) {298 asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));299 asm volatile("prefetchnta %0" : : "m" (dptr[z][d+32]));300 asm volatile("prefetchnta %0" : : "m" (dptr[z][d+64]));301 asm volatile("prefetchnta %0" : : "m" (dptr[z][d+96]));302 asm volatile("vpcmpgtb %ymm4,%ymm1,%ymm5");303 asm volatile("vpcmpgtb %ymm6,%ymm1,%ymm7");304 asm volatile("vpcmpgtb %ymm12,%ymm1,%ymm13");305 asm volatile("vpcmpgtb %ymm14,%ymm1,%ymm15");306 asm volatile("vpaddb %ymm4,%ymm4,%ymm4");307 asm volatile("vpaddb %ymm6,%ymm6,%ymm6");308 asm volatile("vpaddb %ymm12,%ymm12,%ymm12");309 asm volatile("vpaddb %ymm14,%ymm14,%ymm14");310 asm volatile("vpand %ymm0,%ymm5,%ymm5");311 asm volatile("vpand %ymm0,%ymm7,%ymm7");312 asm volatile("vpand %ymm0,%ymm13,%ymm13");313 asm volatile("vpand %ymm0,%ymm15,%ymm15");314 asm volatile("vpxor %ymm5,%ymm4,%ymm4");315 asm volatile("vpxor %ymm7,%ymm6,%ymm6");316 asm volatile("vpxor %ymm13,%ymm12,%ymm12");317 asm volatile("vpxor %ymm15,%ymm14,%ymm14");318 asm volatile("vmovdqa %0,%%ymm5" : : "m" (dptr[z][d]));319 asm volatile("vmovdqa %0,%%ymm7" : : "m" (dptr[z][d+32]));320 asm volatile("vmovdqa %0,%%ymm13" : : "m" (dptr[z][d+64]));321 asm volatile("vmovdqa %0,%%ymm15" : : "m" (dptr[z][d+96]));322 asm volatile("vpxor %ymm5,%ymm2,%ymm2");323 asm volatile("vpxor %ymm7,%ymm3,%ymm3");324 asm volatile("vpxor %ymm13,%ymm10,%ymm10");325 asm volatile("vpxor %ymm15,%ymm11,%ymm11");326 asm volatile("vpxor %ymm5,%ymm4,%ymm4");327 asm volatile("vpxor %ymm7,%ymm6,%ymm6");328 asm volatile("vpxor %ymm13,%ymm12,%ymm12");329 asm volatile("vpxor %ymm15,%ymm14,%ymm14");330 }331 asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));332 asm volatile("vpxor %ymm2,%ymm2,%ymm2");333 asm volatile("vmovntdq %%ymm3,%0" : "=m" (p[d+32]));334 asm volatile("vpxor %ymm3,%ymm3,%ymm3");335 asm volatile("vmovntdq %%ymm10,%0" : "=m" (p[d+64]));336 asm volatile("vpxor %ymm10,%ymm10,%ymm10");337 asm volatile("vmovntdq %%ymm11,%0" : "=m" (p[d+96]));338 asm volatile("vpxor %ymm11,%ymm11,%ymm11");339 asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));340 asm volatile("vpxor %ymm4,%ymm4,%ymm4");341 asm volatile("vmovntdq %%ymm6,%0" : "=m" (q[d+32]));342 asm volatile("vpxor %ymm6,%ymm6,%ymm6");343 asm volatile("vmovntdq %%ymm12,%0" : "=m" (q[d+64]));344 asm volatile("vpxor %ymm12,%ymm12,%ymm12");345 asm volatile("vmovntdq %%ymm14,%0" : "=m" (q[d+96]));346 asm volatile("vpxor %ymm14,%ymm14,%ymm14");347 }348 349 asm volatile("sfence" : : : "memory");350 kernel_fpu_end();351}352 353static void raid6_avx24_xor_syndrome(int disks, int start, int stop,354 size_t bytes, void **ptrs)355{356 u8 **dptr = (u8 **)ptrs;357 u8 *p, *q;358 int d, z, z0;359 360 z0 = stop; /* P/Q right side optimization */361 p = dptr[disks-2]; /* XOR parity */362 q = dptr[disks-1]; /* RS syndrome */363 364 kernel_fpu_begin();365 366 asm volatile("vmovdqa %0,%%ymm0" :: "m" (raid6_avx2_constants.x1d[0]));367 368 for (d = 0 ; d < bytes ; d += 128) {369 asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));370 asm volatile("vmovdqa %0,%%ymm6" :: "m" (dptr[z0][d+32]));371 asm volatile("vmovdqa %0,%%ymm12" :: "m" (dptr[z0][d+64]));372 asm volatile("vmovdqa %0,%%ymm14" :: "m" (dptr[z0][d+96]));373 asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));374 asm volatile("vmovdqa %0,%%ymm3" : : "m" (p[d+32]));375 asm volatile("vmovdqa %0,%%ymm10" : : "m" (p[d+64]));376 asm volatile("vmovdqa %0,%%ymm11" : : "m" (p[d+96]));377 asm volatile("vpxor %ymm4,%ymm2,%ymm2");378 asm volatile("vpxor %ymm6,%ymm3,%ymm3");379 asm volatile("vpxor %ymm12,%ymm10,%ymm10");380 asm volatile("vpxor %ymm14,%ymm11,%ymm11");381 /* P/Q data pages */382 for (z = z0-1 ; z >= start ; z--) {383 asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));384 asm volatile("prefetchnta %0" :: "m" (dptr[z][d+64]));385 asm volatile("vpxor %ymm5,%ymm5,%ymm5");386 asm volatile("vpxor %ymm7,%ymm7,%ymm7");387 asm volatile("vpxor %ymm13,%ymm13,%ymm13");388 asm volatile("vpxor %ymm15,%ymm15,%ymm15");389 asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");390 asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");391 asm volatile("vpcmpgtb %ymm12,%ymm13,%ymm13");392 asm volatile("vpcmpgtb %ymm14,%ymm15,%ymm15");393 asm volatile("vpaddb %ymm4,%ymm4,%ymm4");394 asm volatile("vpaddb %ymm6,%ymm6,%ymm6");395 asm volatile("vpaddb %ymm12,%ymm12,%ymm12");396 asm volatile("vpaddb %ymm14,%ymm14,%ymm14");397 asm volatile("vpand %ymm0,%ymm5,%ymm5");398 asm volatile("vpand %ymm0,%ymm7,%ymm7");399 asm volatile("vpand %ymm0,%ymm13,%ymm13");400 asm volatile("vpand %ymm0,%ymm15,%ymm15");401 asm volatile("vpxor %ymm5,%ymm4,%ymm4");402 asm volatile("vpxor %ymm7,%ymm6,%ymm6");403 asm volatile("vpxor %ymm13,%ymm12,%ymm12");404 asm volatile("vpxor %ymm15,%ymm14,%ymm14");405 asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));406 asm volatile("vmovdqa %0,%%ymm7"407 :: "m" (dptr[z][d+32]));408 asm volatile("vmovdqa %0,%%ymm13"409 :: "m" (dptr[z][d+64]));410 asm volatile("vmovdqa %0,%%ymm15"411 :: "m" (dptr[z][d+96]));412 asm volatile("vpxor %ymm5,%ymm2,%ymm2");413 asm volatile("vpxor %ymm7,%ymm3,%ymm3");414 asm volatile("vpxor %ymm13,%ymm10,%ymm10");415 asm volatile("vpxor %ymm15,%ymm11,%ymm11");416 asm volatile("vpxor %ymm5,%ymm4,%ymm4");417 asm volatile("vpxor %ymm7,%ymm6,%ymm6");418 asm volatile("vpxor %ymm13,%ymm12,%ymm12");419 asm volatile("vpxor %ymm15,%ymm14,%ymm14");420 }421 asm volatile("prefetchnta %0" :: "m" (q[d]));422 asm volatile("prefetchnta %0" :: "m" (q[d+64]));423 /* P/Q left side optimization */424 for (z = start-1 ; z >= 0 ; z--) {425 asm volatile("vpxor %ymm5,%ymm5,%ymm5");426 asm volatile("vpxor %ymm7,%ymm7,%ymm7");427 asm volatile("vpxor %ymm13,%ymm13,%ymm13");428 asm volatile("vpxor %ymm15,%ymm15,%ymm15");429 asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");430 asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");431 asm volatile("vpcmpgtb %ymm12,%ymm13,%ymm13");432 asm volatile("vpcmpgtb %ymm14,%ymm15,%ymm15");433 asm volatile("vpaddb %ymm4,%ymm4,%ymm4");434 asm volatile("vpaddb %ymm6,%ymm6,%ymm6");435 asm volatile("vpaddb %ymm12,%ymm12,%ymm12");436 asm volatile("vpaddb %ymm14,%ymm14,%ymm14");437 asm volatile("vpand %ymm0,%ymm5,%ymm5");438 asm volatile("vpand %ymm0,%ymm7,%ymm7");439 asm volatile("vpand %ymm0,%ymm13,%ymm13");440 asm volatile("vpand %ymm0,%ymm15,%ymm15");441 asm volatile("vpxor %ymm5,%ymm4,%ymm4");442 asm volatile("vpxor %ymm7,%ymm6,%ymm6");443 asm volatile("vpxor %ymm13,%ymm12,%ymm12");444 asm volatile("vpxor %ymm15,%ymm14,%ymm14");445 }446 asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));447 asm volatile("vmovntdq %%ymm3,%0" : "=m" (p[d+32]));448 asm volatile("vmovntdq %%ymm10,%0" : "=m" (p[d+64]));449 asm volatile("vmovntdq %%ymm11,%0" : "=m" (p[d+96]));450 asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));451 asm volatile("vpxor %0,%%ymm6,%%ymm6" : : "m" (q[d+32]));452 asm volatile("vpxor %0,%%ymm12,%%ymm12" : : "m" (q[d+64]));453 asm volatile("vpxor %0,%%ymm14,%%ymm14" : : "m" (q[d+96]));454 asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));455 asm volatile("vmovntdq %%ymm6,%0" : "=m" (q[d+32]));456 asm volatile("vmovntdq %%ymm12,%0" : "=m" (q[d+64]));457 asm volatile("vmovntdq %%ymm14,%0" : "=m" (q[d+96]));458 }459 asm volatile("sfence" : : : "memory");460 kernel_fpu_end();461}462 463const struct raid6_calls raid6_avx2x4 = {464 raid6_avx24_gen_syndrome,465 raid6_avx24_xor_syndrome,466 raid6_have_avx2,467 "avx2x4",468 .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */469};470#endif /* CONFIG_X86_64 */471