703 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 22; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=alderlake -verify-machineinstrs| FileCheck %s --check-prefixes=AVX,ADL3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=sapphirerapids -verify-machineinstrs | FileCheck %s --check-prefixes=AVX,SPR4; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=icelake-server -verify-machineinstrs | FileCheck %s --check-prefixes=AVX5125 6define <2 x i64> @foo_reg_128(<2 x i64> %0, <2 x i64> %1, <2 x i64> %2, <2 x i64> %3, <2 x i64> %4, <2 x i64> %5) {7; AVX-LABEL: foo_reg_128:8; AVX: # %bb.0:9; AVX-NEXT: {vex} vpdpwssd %xmm2, %xmm1, %xmm010; AVX-NEXT: vpmaddwd %xmm3, %xmm1, %xmm211; AVX-NEXT: vpaddd %xmm2, %xmm0, %xmm012; AVX-NEXT: vpmaddwd %xmm4, %xmm1, %xmm213; AVX-NEXT: vpaddd %xmm2, %xmm0, %xmm014; AVX-NEXT: vpmaddwd %xmm5, %xmm1, %xmm115; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm016; AVX-NEXT: retq17;18; AVX512-LABEL: foo_reg_128:19; AVX512: # %bb.0:20; AVX512-NEXT: vpdpwssd %xmm2, %xmm1, %xmm021; AVX512-NEXT: vpmaddwd %xmm3, %xmm1, %xmm222; AVX512-NEXT: vpaddd %xmm2, %xmm0, %xmm023; AVX512-NEXT: vpmaddwd %xmm4, %xmm1, %xmm224; AVX512-NEXT: vpaddd %xmm2, %xmm0, %xmm025; AVX512-NEXT: vpmaddwd %xmm5, %xmm1, %xmm126; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm027; AVX512-NEXT: retq28 %7 = bitcast <2 x i64> %0 to <4 x i32>29 %8 = bitcast <2 x i64> %1 to <4 x i32>30 %9 = bitcast <2 x i64> %2 to <4 x i32>31 %10 = tail call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %7, <4 x i32> %8, <4 x i32> %9)32 %11 = bitcast <2 x i64> %3 to <4 x i32>33 %12 = tail call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %10, <4 x i32> %8, <4 x i32> %11)34 %13 = bitcast <2 x i64> %4 to <4 x i32>35 %14 = tail call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %12, <4 x i32> %8, <4 x i32> %13)36 %15 = bitcast <2 x i64> %5 to <4 x i32>37 %16 = tail call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %14, <4 x i32> %8, <4 x i32> %15)38 %17 = bitcast <4 x i32> %16 to <2 x i64>39 ret <2 x i64> %1740}41 42declare <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32>, <4 x i32>, <4 x i32>) #143 44define <2 x i64> @foo_128(i32 %0, <2 x i64> %1, <2 x i64> %2, ptr %3) {45; AVX-LABEL: foo_128:46; AVX: # %bb.0:47; AVX-NEXT: testl %edi, %edi48; AVX-NEXT: jle .LBB1_649; AVX-NEXT: # %bb.1:50; AVX-NEXT: movl %edi, %edx51; AVX-NEXT: movl %edx, %eax52; AVX-NEXT: andl $3, %eax53; AVX-NEXT: cmpl $4, %edi54; AVX-NEXT: jae .LBB1_755; AVX-NEXT: # %bb.2:56; AVX-NEXT: xorl %ecx, %ecx57; AVX-NEXT: jmp .LBB1_358; AVX-NEXT: .LBB1_7:59; AVX-NEXT: andl $-4, %edx60; AVX-NEXT: leaq 48(%rsi), %rdi61; AVX-NEXT: xorl %ecx, %ecx62; AVX-NEXT: .p2align 463; AVX-NEXT: .LBB1_8: # =>This Inner Loop Header: Depth=164; AVX-NEXT: {vex} vpdpwssd -48(%rdi), %xmm1, %xmm065; AVX-NEXT: vpmaddwd -32(%rdi), %xmm1, %xmm266; AVX-NEXT: vpaddd %xmm2, %xmm0, %xmm067; AVX-NEXT: vpmaddwd -16(%rdi), %xmm1, %xmm268; AVX-NEXT: vpaddd %xmm2, %xmm0, %xmm069; AVX-NEXT: vpmaddwd (%rdi), %xmm1, %xmm270; AVX-NEXT: vpaddd %xmm2, %xmm0, %xmm071; AVX-NEXT: addq $4, %rcx72; AVX-NEXT: addq $64, %rdi73; AVX-NEXT: cmpq %rcx, %rdx74; AVX-NEXT: jne .LBB1_875; AVX-NEXT: .LBB1_3:76; AVX-NEXT: testq %rax, %rax77; AVX-NEXT: je .LBB1_678; AVX-NEXT: # %bb.4: # %.preheader79; AVX-NEXT: shlq $4, %rcx80; AVX-NEXT: addq %rcx, %rsi81; AVX-NEXT: shll $4, %eax82; AVX-NEXT: xorl %ecx, %ecx83; AVX-NEXT: .p2align 484; AVX-NEXT: .LBB1_5: # =>This Inner Loop Header: Depth=185; AVX-NEXT: {vex} vpdpwssd (%rsi,%rcx), %xmm1, %xmm086; AVX-NEXT: addq $16, %rcx87; AVX-NEXT: cmpq %rcx, %rax88; AVX-NEXT: jne .LBB1_589; AVX-NEXT: .LBB1_6:90; AVX-NEXT: retq91;92; AVX512-LABEL: foo_128:93; AVX512: # %bb.0:94; AVX512-NEXT: testl %edi, %edi95; AVX512-NEXT: jle .LBB1_696; AVX512-NEXT: # %bb.1:97; AVX512-NEXT: movl %edi, %edx98; AVX512-NEXT: movl %edx, %eax99; AVX512-NEXT: andl $3, %eax100; AVX512-NEXT: cmpl $4, %edi101; AVX512-NEXT: jae .LBB1_7102; AVX512-NEXT: # %bb.2:103; AVX512-NEXT: xorl %ecx, %ecx104; AVX512-NEXT: jmp .LBB1_3105; AVX512-NEXT: .LBB1_7:106; AVX512-NEXT: andl $-4, %edx107; AVX512-NEXT: leaq 48(%rsi), %rdi108; AVX512-NEXT: xorl %ecx, %ecx109; AVX512-NEXT: .p2align 4110; AVX512-NEXT: .LBB1_8: # =>This Inner Loop Header: Depth=1111; AVX512-NEXT: vpdpwssd -48(%rdi), %xmm1, %xmm0112; AVX512-NEXT: vpmaddwd -32(%rdi), %xmm1, %xmm2113; AVX512-NEXT: vpaddd %xmm2, %xmm0, %xmm0114; AVX512-NEXT: vpmaddwd -16(%rdi), %xmm1, %xmm2115; AVX512-NEXT: vpaddd %xmm2, %xmm0, %xmm0116; AVX512-NEXT: vpmaddwd (%rdi), %xmm1, %xmm2117; AVX512-NEXT: vpaddd %xmm2, %xmm0, %xmm0118; AVX512-NEXT: addq $4, %rcx119; AVX512-NEXT: addq $64, %rdi120; AVX512-NEXT: cmpq %rcx, %rdx121; AVX512-NEXT: jne .LBB1_8122; AVX512-NEXT: .LBB1_3:123; AVX512-NEXT: testq %rax, %rax124; AVX512-NEXT: je .LBB1_6125; AVX512-NEXT: # %bb.4: # %.preheader126; AVX512-NEXT: shlq $4, %rcx127; AVX512-NEXT: addq %rcx, %rsi128; AVX512-NEXT: shll $4, %eax129; AVX512-NEXT: xorl %ecx, %ecx130; AVX512-NEXT: .p2align 4131; AVX512-NEXT: .LBB1_5: # =>This Inner Loop Header: Depth=1132; AVX512-NEXT: vpdpwssd (%rsi,%rcx), %xmm1, %xmm0133; AVX512-NEXT: addq $16, %rcx134; AVX512-NEXT: cmpq %rcx, %rax135; AVX512-NEXT: jne .LBB1_5136; AVX512-NEXT: .LBB1_6:137; AVX512-NEXT: retq138 %5 = icmp sgt i32 %0, 0139 br i1 %5, label %6, label %33140 1416: ; preds = %4142 %7 = bitcast <2 x i64> %2 to <8 x i16>143 %8 = bitcast <2 x i64> %1 to <4 x i32>144 %9 = zext i32 %0 to i64145 %10 = and i64 %9, 3146 %11 = icmp ult i32 %0, 4147 br i1 %11, label %14, label %12148 14912: ; preds = %6150 %13 = and i64 %9, 4294967292151 br label %35152 15314: ; preds = %35, %6154 %15 = phi <4 x i32> [ undef, %6 ], [ %57, %35 ]155 %16 = phi i64 [ 0, %6 ], [ %58, %35 ]156 %17 = phi <4 x i32> [ %8, %6 ], [ %57, %35 ]157 %18 = icmp eq i64 %10, 0158 br i1 %18, label %30, label %19159 16019: ; preds = %14, %19161 %20 = phi i64 [ %27, %19 ], [ %16, %14 ]162 %21 = phi <4 x i32> [ %26, %19 ], [ %17, %14 ]163 %22 = phi i64 [ %28, %19 ], [ 0, %14 ]164 %23 = getelementptr inbounds <2 x i64>, ptr %3, i64 %20165 %24 = load <8 x i16>, ptr %23, align 16166 %25 = tail call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %7, <8 x i16> %24)167 %26 = add <4 x i32> %25, %21168 %27 = add nuw nsw i64 %20, 1169 %28 = add i64 %22, 1170 %29 = icmp eq i64 %28, %10171 br i1 %29, label %30, label %19172 17330: ; preds = %19, %14174 %31 = phi <4 x i32> [ %15, %14 ], [ %26, %19 ]175 %32 = bitcast <4 x i32> %31 to <2 x i64>176 br label %33177 17833: ; preds = %30, %4179 %34 = phi <2 x i64> [ %32, %30 ], [ %1, %4 ]180 ret <2 x i64> %34181 18235: ; preds = %35, %12183 %36 = phi i64 [ 0, %12 ], [ %58, %35 ]184 %37 = phi <4 x i32> [ %8, %12 ], [ %57, %35 ]185 %38 = phi i64 [ 0, %12 ], [ %59, %35 ]186 %39 = getelementptr inbounds <2 x i64>, ptr %3, i64 %36187 %40 = load <8 x i16>, ptr %39, align 16188 %41 = tail call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %7, <8 x i16> %40)189 %42 = add <4 x i32> %41, %37190 %43 = or disjoint i64 %36, 1191 %44 = getelementptr inbounds <2 x i64>, ptr %3, i64 %43192 %45 = load <8 x i16>, ptr %44, align 16193 %46 = tail call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %7, <8 x i16> %45)194 %47 = add <4 x i32> %46, %42195 %48 = or disjoint i64 %36, 2196 %49 = getelementptr inbounds <2 x i64>, ptr %3, i64 %48197 %50 = load <8 x i16>, ptr %49, align 16198 %51 = tail call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %7, <8 x i16> %50)199 %52 = add <4 x i32> %51, %47200 %53 = or disjoint i64 %36, 3201 %54 = getelementptr inbounds <2 x i64>, ptr %3, i64 %53202 %55 = load <8 x i16>, ptr %54, align 16203 %56 = tail call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %7, <8 x i16> %55)204 %57 = add <4 x i32> %56, %52205 %58 = add nuw nsw i64 %36, 4206 %59 = add i64 %38, 4207 %60 = icmp eq i64 %59, %13208 br i1 %60, label %14, label %35209}210 211define void @bar_128(i32 %0, ptr %1, <2 x i64> %2, ptr %3) {212; AVX-LABEL: bar_128:213; AVX: # %bb.0:214; AVX-NEXT: testl %edi, %edi215; AVX-NEXT: jle .LBB2_5216; AVX-NEXT: # %bb.1:217; AVX-NEXT: movl %edi, %eax218; AVX-NEXT: cmpl $1, %edi219; AVX-NEXT: jne .LBB2_6220; AVX-NEXT: # %bb.2:221; AVX-NEXT: xorl %ecx, %ecx222; AVX-NEXT: jmp .LBB2_3223; AVX-NEXT: .LBB2_6:224; AVX-NEXT: movl %eax, %edi225; AVX-NEXT: andl $-2, %edi226; AVX-NEXT: movl $16, %r8d227; AVX-NEXT: xorl %ecx, %ecx228; AVX-NEXT: .p2align 4229; AVX-NEXT: .LBB2_7: # =>This Inner Loop Header: Depth=1230; AVX-NEXT: vmovdqa (%rsi,%r8), %xmm1231; AVX-NEXT: vpmaddwd -16(%rdx,%r8), %xmm0, %xmm2232; AVX-NEXT: vpaddd -16(%rsi,%r8), %xmm2, %xmm2233; AVX-NEXT: vmovdqa %xmm2, -16(%rsi,%r8)234; AVX-NEXT: vpmaddwd (%rdx,%r8), %xmm0, %xmm2235; AVX-NEXT: vpaddd %xmm2, %xmm1, %xmm1236; AVX-NEXT: vmovdqa %xmm1, (%rsi,%r8)237; AVX-NEXT: addq $2, %rcx238; AVX-NEXT: addq $32, %r8239; AVX-NEXT: cmpq %rcx, %rdi240; AVX-NEXT: jne .LBB2_7241; AVX-NEXT: .LBB2_3:242; AVX-NEXT: testb $1, %al243; AVX-NEXT: je .LBB2_5244; AVX-NEXT: # %bb.4:245; AVX-NEXT: shlq $4, %rcx246; AVX-NEXT: vmovdqa (%rsi,%rcx), %xmm1247; AVX-NEXT: {vex} vpdpwssd (%rdx,%rcx), %xmm0, %xmm1248; AVX-NEXT: vmovdqa %xmm1, (%rsi,%rcx)249; AVX-NEXT: .LBB2_5:250; AVX-NEXT: retq251;252; AVX512-LABEL: bar_128:253; AVX512: # %bb.0:254; AVX512-NEXT: testl %edi, %edi255; AVX512-NEXT: jle .LBB2_5256; AVX512-NEXT: # %bb.1:257; AVX512-NEXT: movl %edi, %eax258; AVX512-NEXT: cmpl $1, %edi259; AVX512-NEXT: jne .LBB2_6260; AVX512-NEXT: # %bb.2:261; AVX512-NEXT: xorl %ecx, %ecx262; AVX512-NEXT: jmp .LBB2_3263; AVX512-NEXT: .LBB2_6:264; AVX512-NEXT: movl %eax, %edi265; AVX512-NEXT: andl $-2, %edi266; AVX512-NEXT: movl $16, %r8d267; AVX512-NEXT: xorl %ecx, %ecx268; AVX512-NEXT: .p2align 4269; AVX512-NEXT: .LBB2_7: # =>This Inner Loop Header: Depth=1270; AVX512-NEXT: vmovdqa (%rsi,%r8), %xmm1271; AVX512-NEXT: vpmaddwd -16(%rdx,%r8), %xmm0, %xmm2272; AVX512-NEXT: vpaddd -16(%rsi,%r8), %xmm2, %xmm2273; AVX512-NEXT: vmovdqa %xmm2, -16(%rsi,%r8)274; AVX512-NEXT: vpmaddwd (%rdx,%r8), %xmm0, %xmm2275; AVX512-NEXT: vpaddd %xmm2, %xmm1, %xmm1276; AVX512-NEXT: vmovdqa %xmm1, (%rsi,%r8)277; AVX512-NEXT: addq $2, %rcx278; AVX512-NEXT: addq $32, %r8279; AVX512-NEXT: cmpq %rcx, %rdi280; AVX512-NEXT: jne .LBB2_7281; AVX512-NEXT: .LBB2_3:282; AVX512-NEXT: testb $1, %al283; AVX512-NEXT: je .LBB2_5284; AVX512-NEXT: # %bb.4:285; AVX512-NEXT: shlq $4, %rcx286; AVX512-NEXT: vpmaddwd (%rdx,%rcx), %xmm0, %xmm0287; AVX512-NEXT: vpaddd (%rsi,%rcx), %xmm0, %xmm0288; AVX512-NEXT: vmovdqa %xmm0, (%rsi,%rcx)289; AVX512-NEXT: .LBB2_5:290; AVX512-NEXT: retq291 %5 = icmp sgt i32 %0, 0292 br i1 %5, label %6, label %22293 2946: ; preds = %4295 %7 = bitcast <2 x i64> %2 to <4 x i32>296 %8 = zext i32 %0 to i64297 %9 = and i64 %8, 1298 %10 = icmp eq i32 %0, 1299 br i1 %10, label %13, label %11300 30111: ; preds = %6302 %12 = and i64 %8, 4294967294303 br label %23304 30513: ; preds = %23, %6306 %14 = phi i64 [ 0, %6 ], [ %37, %23 ]307 %15 = icmp eq i64 %9, 0308 br i1 %15, label %22, label %16309 31016: ; preds = %13311 %17 = getelementptr inbounds <2 x i64>, ptr %3, i64 %14312 %18 = load <4 x i32>, ptr %17, align 16313 %19 = getelementptr inbounds <2 x i64>, ptr %1, i64 %14314 %20 = load <4 x i32>, ptr %19, align 16315 %21 = tail call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %20, <4 x i32> %7, <4 x i32> %18)316 store <4 x i32> %21, ptr %19, align 16317 br label %22318 31922: ; preds = %16, %13, %4320 ret void321 32223: ; preds = %23, %11323 %24 = phi i64 [ 0, %11 ], [ %37, %23 ]324 %25 = phi i64 [ 0, %11 ], [ %38, %23 ]325 %26 = getelementptr inbounds <2 x i64>, ptr %3, i64 %24326 %27 = load <4 x i32>, ptr %26, align 16327 %28 = getelementptr inbounds <2 x i64>, ptr %1, i64 %24328 %29 = load <4 x i32>, ptr %28, align 16329 %30 = tail call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %29, <4 x i32> %7, <4 x i32> %27)330 store <4 x i32> %30, ptr %28, align 16331 %31 = or disjoint i64 %24, 1332 %32 = getelementptr inbounds <2 x i64>, ptr %3, i64 %31333 %33 = load <4 x i32>, ptr %32, align 16334 %34 = getelementptr inbounds <2 x i64>, ptr %1, i64 %31335 %35 = load <4 x i32>, ptr %34, align 16336 %36 = tail call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %35, <4 x i32> %7, <4 x i32> %33)337 store <4 x i32> %36, ptr %34, align 16338 %37 = add nuw nsw i64 %24, 2339 %38 = add i64 %25, 2340 %39 = icmp eq i64 %38, %12341 br i1 %39, label %13, label %23342}343 344declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>) #1345 346define <4 x i64> @foo_reg_256(<4 x i64> %0, <4 x i64> %1, <4 x i64> %2, <4 x i64> %3, <4 x i64> %4, <4 x i64> %5) {347; AVX-LABEL: foo_reg_256:348; AVX: # %bb.0:349; AVX-NEXT: {vex} vpdpwssd %ymm2, %ymm1, %ymm0350; AVX-NEXT: vpmaddwd %ymm3, %ymm1, %ymm2351; AVX-NEXT: vpaddd %ymm2, %ymm0, %ymm0352; AVX-NEXT: vpmaddwd %ymm4, %ymm1, %ymm2353; AVX-NEXT: vpaddd %ymm2, %ymm0, %ymm0354; AVX-NEXT: vpmaddwd %ymm5, %ymm1, %ymm1355; AVX-NEXT: vpaddd %ymm1, %ymm0, %ymm0356; AVX-NEXT: retq357;358; AVX512-LABEL: foo_reg_256:359; AVX512: # %bb.0:360; AVX512-NEXT: vpdpwssd %ymm2, %ymm1, %ymm0361; AVX512-NEXT: vpmaddwd %ymm3, %ymm1, %ymm2362; AVX512-NEXT: vpaddd %ymm2, %ymm0, %ymm0363; AVX512-NEXT: vpmaddwd %ymm4, %ymm1, %ymm2364; AVX512-NEXT: vpaddd %ymm2, %ymm0, %ymm0365; AVX512-NEXT: vpmaddwd %ymm5, %ymm1, %ymm1366; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0367; AVX512-NEXT: retq368 %7 = bitcast <4 x i64> %0 to <8 x i32>369 %8 = bitcast <4 x i64> %1 to <8 x i32>370 %9 = bitcast <4 x i64> %2 to <8 x i32>371 %10 = tail call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %7, <8 x i32> %8, <8 x i32> %9)372 %11 = bitcast <4 x i64> %3 to <8 x i32>373 %12 = tail call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %10, <8 x i32> %8, <8 x i32> %11)374 %13 = bitcast <4 x i64> %4 to <8 x i32>375 %14 = tail call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %12, <8 x i32> %8, <8 x i32> %13)376 %15 = bitcast <4 x i64> %5 to <8 x i32>377 %16 = tail call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %14, <8 x i32> %8, <8 x i32> %15)378 %17 = bitcast <8 x i32> %16 to <4 x i64>379 ret <4 x i64> %17380}381 382; __m256i foo(int cnt, __m256i c, __m256i b, __m256i *p) {383; for (int i = 0; i < cnt; ++i) {384; __m256i a = p[i];385; __m256i m = _mm256_madd_epi16 (b, a);386; c = _mm256_add_epi32(m, c);387; }388; return c;389; }390 391define <4 x i64> @foo_256(i32 %0, <4 x i64> %1, <4 x i64> %2, ptr %3) {392; AVX-LABEL: foo_256:393; AVX: # %bb.0:394; AVX-NEXT: testl %edi, %edi395; AVX-NEXT: jle .LBB4_6396; AVX-NEXT: # %bb.1:397; AVX-NEXT: movl %edi, %edx398; AVX-NEXT: movl %edx, %eax399; AVX-NEXT: andl $3, %eax400; AVX-NEXT: cmpl $4, %edi401; AVX-NEXT: jae .LBB4_7402; AVX-NEXT: # %bb.2:403; AVX-NEXT: xorl %ecx, %ecx404; AVX-NEXT: jmp .LBB4_3405; AVX-NEXT: .LBB4_7:406; AVX-NEXT: andl $-4, %edx407; AVX-NEXT: leaq 96(%rsi), %rdi408; AVX-NEXT: xorl %ecx, %ecx409; AVX-NEXT: .p2align 4410; AVX-NEXT: .LBB4_8: # =>This Inner Loop Header: Depth=1411; AVX-NEXT: {vex} vpdpwssd -96(%rdi), %ymm1, %ymm0412; AVX-NEXT: vpmaddwd -64(%rdi), %ymm1, %ymm2413; AVX-NEXT: vpaddd %ymm2, %ymm0, %ymm0414; AVX-NEXT: vpmaddwd -32(%rdi), %ymm1, %ymm2415; AVX-NEXT: vpaddd %ymm2, %ymm0, %ymm0416; AVX-NEXT: vpmaddwd (%rdi), %ymm1, %ymm2417; AVX-NEXT: vpaddd %ymm2, %ymm0, %ymm0418; AVX-NEXT: addq $4, %rcx419; AVX-NEXT: subq $-128, %rdi420; AVX-NEXT: cmpq %rcx, %rdx421; AVX-NEXT: jne .LBB4_8422; AVX-NEXT: .LBB4_3:423; AVX-NEXT: testq %rax, %rax424; AVX-NEXT: je .LBB4_6425; AVX-NEXT: # %bb.4: # %.preheader426; AVX-NEXT: shlq $5, %rcx427; AVX-NEXT: addq %rcx, %rsi428; AVX-NEXT: shll $5, %eax429; AVX-NEXT: xorl %ecx, %ecx430; AVX-NEXT: .p2align 4431; AVX-NEXT: .LBB4_5: # =>This Inner Loop Header: Depth=1432; AVX-NEXT: {vex} vpdpwssd (%rsi,%rcx), %ymm1, %ymm0433; AVX-NEXT: addq $32, %rcx434; AVX-NEXT: cmpq %rcx, %rax435; AVX-NEXT: jne .LBB4_5436; AVX-NEXT: .LBB4_6:437; AVX-NEXT: retq438;439; AVX512-LABEL: foo_256:440; AVX512: # %bb.0:441; AVX512-NEXT: testl %edi, %edi442; AVX512-NEXT: jle .LBB4_6443; AVX512-NEXT: # %bb.1:444; AVX512-NEXT: movl %edi, %edx445; AVX512-NEXT: movl %edx, %eax446; AVX512-NEXT: andl $3, %eax447; AVX512-NEXT: cmpl $4, %edi448; AVX512-NEXT: jae .LBB4_7449; AVX512-NEXT: # %bb.2:450; AVX512-NEXT: xorl %ecx, %ecx451; AVX512-NEXT: jmp .LBB4_3452; AVX512-NEXT: .LBB4_7:453; AVX512-NEXT: andl $-4, %edx454; AVX512-NEXT: leaq 96(%rsi), %rdi455; AVX512-NEXT: xorl %ecx, %ecx456; AVX512-NEXT: .p2align 4457; AVX512-NEXT: .LBB4_8: # =>This Inner Loop Header: Depth=1458; AVX512-NEXT: vpdpwssd -96(%rdi), %ymm1, %ymm0459; AVX512-NEXT: vpmaddwd -64(%rdi), %ymm1, %ymm2460; AVX512-NEXT: vpaddd %ymm2, %ymm0, %ymm0461; AVX512-NEXT: vpmaddwd -32(%rdi), %ymm1, %ymm2462; AVX512-NEXT: vpaddd %ymm2, %ymm0, %ymm0463; AVX512-NEXT: vpmaddwd (%rdi), %ymm1, %ymm2464; AVX512-NEXT: vpaddd %ymm2, %ymm0, %ymm0465; AVX512-NEXT: addq $4, %rcx466; AVX512-NEXT: subq $-128, %rdi467; AVX512-NEXT: cmpq %rcx, %rdx468; AVX512-NEXT: jne .LBB4_8469; AVX512-NEXT: .LBB4_3:470; AVX512-NEXT: testq %rax, %rax471; AVX512-NEXT: je .LBB4_6472; AVX512-NEXT: # %bb.4: # %.preheader473; AVX512-NEXT: shlq $5, %rcx474; AVX512-NEXT: addq %rcx, %rsi475; AVX512-NEXT: shll $5, %eax476; AVX512-NEXT: xorl %ecx, %ecx477; AVX512-NEXT: .p2align 4478; AVX512-NEXT: .LBB4_5: # =>This Inner Loop Header: Depth=1479; AVX512-NEXT: vpdpwssd (%rsi,%rcx), %ymm1, %ymm0480; AVX512-NEXT: addq $32, %rcx481; AVX512-NEXT: cmpq %rcx, %rax482; AVX512-NEXT: jne .LBB4_5483; AVX512-NEXT: .LBB4_6:484; AVX512-NEXT: retq485 %5 = icmp sgt i32 %0, 0486 br i1 %5, label %6, label %33487 4886: ; preds = %4489 %7 = bitcast <4 x i64> %2 to <16 x i16>490 %8 = bitcast <4 x i64> %1 to <8 x i32>491 %9 = zext i32 %0 to i64492 %10 = and i64 %9, 3493 %11 = icmp ult i32 %0, 4494 br i1 %11, label %14, label %12495 49612: ; preds = %6497 %13 = and i64 %9, 4294967292498 br label %35499 50014: ; preds = %35, %6501 %15 = phi <8 x i32> [ undef, %6 ], [ %57, %35 ]502 %16 = phi i64 [ 0, %6 ], [ %58, %35 ]503 %17 = phi <8 x i32> [ %8, %6 ], [ %57, %35 ]504 %18 = icmp eq i64 %10, 0505 br i1 %18, label %30, label %19506 50719: ; preds = %14, %19508 %20 = phi i64 [ %27, %19 ], [ %16, %14 ]509 %21 = phi <8 x i32> [ %26, %19 ], [ %17, %14 ]510 %22 = phi i64 [ %28, %19 ], [ 0, %14 ]511 %23 = getelementptr inbounds <4 x i64>, ptr %3, i64 %20512 %24 = load <16 x i16>, ptr %23, align 32513 %25 = tail call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %7, <16 x i16> %24)514 %26 = add <8 x i32> %25, %21515 %27 = add nuw nsw i64 %20, 1516 %28 = add i64 %22, 1517 %29 = icmp eq i64 %28, %10518 br i1 %29, label %30, label %19519 52030: ; preds = %19, %14521 %31 = phi <8 x i32> [ %15, %14 ], [ %26, %19 ]522 %32 = bitcast <8 x i32> %31 to <4 x i64>523 br label %33524 52533: ; preds = %30, %4526 %34 = phi <4 x i64> [ %32, %30 ], [ %1, %4 ]527 ret <4 x i64> %34528 52935: ; preds = %35, %12530 %36 = phi i64 [ 0, %12 ], [ %58, %35 ]531 %37 = phi <8 x i32> [ %8, %12 ], [ %57, %35 ]532 %38 = phi i64 [ 0, %12 ], [ %59, %35 ]533 %39 = getelementptr inbounds <4 x i64>, ptr %3, i64 %36534 %40 = load <16 x i16>, ptr %39, align 32535 %41 = tail call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %7, <16 x i16> %40)536 %42 = add <8 x i32> %41, %37537 %43 = or disjoint i64 %36, 1538 %44 = getelementptr inbounds <4 x i64>, ptr %3, i64 %43539 %45 = load <16 x i16>, ptr %44, align 32540 %46 = tail call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %7, <16 x i16> %45)541 %47 = add <8 x i32> %46, %42542 %48 = or disjoint i64 %36, 2543 %49 = getelementptr inbounds <4 x i64>, ptr %3, i64 %48544 %50 = load <16 x i16>, ptr %49, align 32545 %51 = tail call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %7, <16 x i16> %50)546 %52 = add <8 x i32> %51, %47547 %53 = or disjoint i64 %36, 3548 %54 = getelementptr inbounds <4 x i64>, ptr %3, i64 %53549 %55 = load <16 x i16>, ptr %54, align 32550 %56 = tail call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %7, <16 x i16> %55)551 %57 = add <8 x i32> %56, %52552 %58 = add nuw nsw i64 %36, 4553 %59 = add i64 %38, 4554 %60 = icmp eq i64 %59, %13555 br i1 %60, label %14, label %35556}557declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>)558 559; void bar(int cnt, __m256i *c, __m256i b, __m256i *p) {560; for (int i = 0; i < cnt; ++i) {561; __m256i a = p[i];562; c[i] = _mm256_dpwssd_epi32(c[i], b, a);563; }564; }565define void @bar_256(i32 %0, ptr %1, <4 x i64> %2, ptr %3) {566; AVX-LABEL: bar_256:567; AVX: # %bb.0:568; AVX-NEXT: testl %edi, %edi569; AVX-NEXT: jle .LBB5_5570; AVX-NEXT: # %bb.1:571; AVX-NEXT: movl %edi, %eax572; AVX-NEXT: cmpl $1, %edi573; AVX-NEXT: jne .LBB5_6574; AVX-NEXT: # %bb.2:575; AVX-NEXT: xorl %ecx, %ecx576; AVX-NEXT: jmp .LBB5_3577; AVX-NEXT: .LBB5_6:578; AVX-NEXT: movl %eax, %edi579; AVX-NEXT: andl $-2, %edi580; AVX-NEXT: movl $32, %r8d581; AVX-NEXT: xorl %ecx, %ecx582; AVX-NEXT: .p2align 4583; AVX-NEXT: .LBB5_7: # =>This Inner Loop Header: Depth=1584; AVX-NEXT: vmovdqa (%rsi,%r8), %ymm1585; AVX-NEXT: vpmaddwd -32(%rdx,%r8), %ymm0, %ymm2586; AVX-NEXT: vpaddd -32(%rsi,%r8), %ymm2, %ymm2587; AVX-NEXT: vmovdqa %ymm2, -32(%rsi,%r8)588; AVX-NEXT: vpmaddwd (%rdx,%r8), %ymm0, %ymm2589; AVX-NEXT: vpaddd %ymm2, %ymm1, %ymm1590; AVX-NEXT: vmovdqa %ymm1, (%rsi,%r8)591; AVX-NEXT: addq $2, %rcx592; AVX-NEXT: addq $64, %r8593; AVX-NEXT: cmpq %rcx, %rdi594; AVX-NEXT: jne .LBB5_7595; AVX-NEXT: .LBB5_3:596; AVX-NEXT: testb $1, %al597; AVX-NEXT: je .LBB5_5598; AVX-NEXT: # %bb.4:599; AVX-NEXT: shlq $5, %rcx600; AVX-NEXT: vmovdqa (%rsi,%rcx), %ymm1601; AVX-NEXT: {vex} vpdpwssd (%rdx,%rcx), %ymm0, %ymm1602; AVX-NEXT: vmovdqa %ymm1, (%rsi,%rcx)603; AVX-NEXT: .LBB5_5:604; AVX-NEXT: vzeroupper605; AVX-NEXT: retq606;607; AVX512-LABEL: bar_256:608; AVX512: # %bb.0:609; AVX512-NEXT: testl %edi, %edi610; AVX512-NEXT: jle .LBB5_5611; AVX512-NEXT: # %bb.1:612; AVX512-NEXT: movl %edi, %eax613; AVX512-NEXT: cmpl $1, %edi614; AVX512-NEXT: jne .LBB5_6615; AVX512-NEXT: # %bb.2:616; AVX512-NEXT: xorl %ecx, %ecx617; AVX512-NEXT: jmp .LBB5_3618; AVX512-NEXT: .LBB5_6:619; AVX512-NEXT: movl %eax, %edi620; AVX512-NEXT: andl $-2, %edi621; AVX512-NEXT: movl $32, %r8d622; AVX512-NEXT: xorl %ecx, %ecx623; AVX512-NEXT: .p2align 4624; AVX512-NEXT: .LBB5_7: # =>This Inner Loop Header: Depth=1625; AVX512-NEXT: vmovdqa (%rsi,%r8), %ymm1626; AVX512-NEXT: vpmaddwd -32(%rdx,%r8), %ymm0, %ymm2627; AVX512-NEXT: vpaddd -32(%rsi,%r8), %ymm2, %ymm2628; AVX512-NEXT: vmovdqa %ymm2, -32(%rsi,%r8)629; AVX512-NEXT: vpmaddwd (%rdx,%r8), %ymm0, %ymm2630; AVX512-NEXT: vpaddd %ymm2, %ymm1, %ymm1631; AVX512-NEXT: vmovdqa %ymm1, (%rsi,%r8)632; AVX512-NEXT: addq $2, %rcx633; AVX512-NEXT: addq $64, %r8634; AVX512-NEXT: cmpq %rcx, %rdi635; AVX512-NEXT: jne .LBB5_7636; AVX512-NEXT: .LBB5_3:637; AVX512-NEXT: testb $1, %al638; AVX512-NEXT: je .LBB5_5639; AVX512-NEXT: # %bb.4:640; AVX512-NEXT: shlq $5, %rcx641; AVX512-NEXT: vpmaddwd (%rdx,%rcx), %ymm0, %ymm0642; AVX512-NEXT: vpaddd (%rsi,%rcx), %ymm0, %ymm0643; AVX512-NEXT: vmovdqa %ymm0, (%rsi,%rcx)644; AVX512-NEXT: .LBB5_5:645; AVX512-NEXT: vzeroupper646; AVX512-NEXT: retq647 %5 = icmp sgt i32 %0, 0648 br i1 %5, label %6, label %22649 6506: ; preds = %4651 %7 = bitcast <4 x i64> %2 to <8 x i32>652 %8 = zext i32 %0 to i64653 %9 = and i64 %8, 1654 %10 = icmp eq i32 %0, 1655 br i1 %10, label %13, label %11656 65711: ; preds = %6658 %12 = and i64 %8, 4294967294659 br label %23660 66113: ; preds = %23, %6662 %14 = phi i64 [ 0, %6 ], [ %37, %23 ]663 %15 = icmp eq i64 %9, 0664 br i1 %15, label %22, label %16665 66616: ; preds = %13667 %17 = getelementptr inbounds <4 x i64>, ptr %3, i64 %14668 %18 = load <8 x i32>, ptr %17, align 32669 %19 = getelementptr inbounds <4 x i64>, ptr %1, i64 %14670 %20 = load <8 x i32>, ptr %19, align 32671 %21 = tail call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %20, <8 x i32> %7, <8 x i32> %18)672 store <8 x i32> %21, ptr %19, align 32673 br label %22674 67522: ; preds = %16, %13, %4676 ret void677 67823: ; preds = %23, %11679 %24 = phi i64 [ 0, %11 ], [ %37, %23 ]680 %25 = phi i64 [ 0, %11 ], [ %38, %23 ]681 %26 = getelementptr inbounds <4 x i64>, ptr %3, i64 %24682 %27 = load <8 x i32>, ptr %26, align 32683 %28 = getelementptr inbounds <4 x i64>, ptr %1, i64 %24684 %29 = load <8 x i32>, ptr %28, align 32685 %30 = tail call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %29, <8 x i32> %7, <8 x i32> %27)686 store <8 x i32> %30, ptr %28, align 32687 %31 = or disjoint i64 %24, 1688 %32 = getelementptr inbounds <4 x i64>, ptr %3, i64 %31689 %33 = load <8 x i32>, ptr %32, align 32690 %34 = getelementptr inbounds <4 x i64>, ptr %1, i64 %31691 %35 = load <8 x i32>, ptr %34, align 32692 %36 = tail call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %35, <8 x i32> %7, <8 x i32> %33)693 store <8 x i32> %36, ptr %34, align 32694 %37 = add nuw nsw i64 %24, 2695 %38 = add i64 %25, 2696 %39 = icmp eq i64 %38, %12697 br i1 %39, label %13, label %23698}699declare <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32>, <8 x i32>, <8 x i32>)700;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:701; ADL: {{.*}}702; SPR: {{.*}}703