4385 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=avx512vl | FileCheck %s3 4; Test that we can unfold constant pool loads when we're using avx512's5; ability to fold a broadcast load into an operation.6 7define void @bcast_unfold_add_v16i32(ptr %arg) {8; CHECK-LABEL: bcast_unfold_add_v16i32:9; CHECK: # %bb.0: # %bb10; CHECK-NEXT: movq $-4096, %rax # imm = 0xF00011; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]12; CHECK-NEXT: .p2align 413; CHECK-NEXT: .LBB0_1: # %bb214; CHECK-NEXT: # =>This Inner Loop Header: Depth=115; CHECK-NEXT: vpaddd 4096(%rdi,%rax), %zmm0, %zmm116; CHECK-NEXT: vmovdqu64 %zmm1, 4096(%rdi,%rax)17; CHECK-NEXT: addq $64, %rax18; CHECK-NEXT: jne .LBB0_119; CHECK-NEXT: # %bb.2: # %bb1020; CHECK-NEXT: vzeroupper21; CHECK-NEXT: retq22bb:23 br label %bb224 25bb2: ; preds = %bb2, %bb26 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]27 %tmp3 = getelementptr inbounds i32, ptr %arg, i64 %tmp28 %tmp5 = load <16 x i32>, ptr %tmp3, align 429 %tmp6 = add nsw <16 x i32> %tmp5, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>30 store <16 x i32> %tmp6, ptr %tmp3, align 431 %tmp8 = add i64 %tmp, 1632 %tmp9 = icmp eq i64 %tmp8, 102433 br i1 %tmp9, label %bb10, label %bb234 35bb10: ; preds = %bb236 ret void37}38 39define void @bcast_unfold_add_v8i32(ptr %arg) {40; CHECK-LABEL: bcast_unfold_add_v8i32:41; CHECK: # %bb.0: # %bb42; CHECK-NEXT: movq $-4096, %rax # imm = 0xF00043; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]44; CHECK-NEXT: .p2align 445; CHECK-NEXT: .LBB1_1: # %bb246; CHECK-NEXT: # =>This Inner Loop Header: Depth=147; CHECK-NEXT: vpaddd 4096(%rdi,%rax), %ymm0, %ymm148; CHECK-NEXT: vmovdqu %ymm1, 4096(%rdi,%rax)49; CHECK-NEXT: addq $32, %rax50; CHECK-NEXT: jne .LBB1_151; CHECK-NEXT: # %bb.2: # %bb1052; CHECK-NEXT: vzeroupper53; CHECK-NEXT: retq54bb:55 br label %bb256 57bb2: ; preds = %bb2, %bb58 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]59 %tmp3 = getelementptr inbounds i32, ptr %arg, i64 %tmp60 %tmp5 = load <8 x i32>, ptr %tmp3, align 461 %tmp6 = add nsw <8 x i32> %tmp5, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>62 store <8 x i32> %tmp6, ptr %tmp3, align 463 %tmp8 = add i64 %tmp, 864 %tmp9 = icmp eq i64 %tmp8, 102465 br i1 %tmp9, label %bb10, label %bb266 67bb10: ; preds = %bb268 ret void69}70 71define void @bcast_unfold_add_v4i32(ptr %arg) {72; CHECK-LABEL: bcast_unfold_add_v4i32:73; CHECK: # %bb.0: # %bb74; CHECK-NEXT: movq $-4096, %rax # imm = 0xF00075; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]76; CHECK-NEXT: .p2align 477; CHECK-NEXT: .LBB2_1: # %bb278; CHECK-NEXT: # =>This Inner Loop Header: Depth=179; CHECK-NEXT: vpaddd 4096(%rdi,%rax), %xmm0, %xmm180; CHECK-NEXT: vmovdqu %xmm1, 4096(%rdi,%rax)81; CHECK-NEXT: addq $16, %rax82; CHECK-NEXT: jne .LBB2_183; CHECK-NEXT: # %bb.2: # %bb1084; CHECK-NEXT: retq85bb:86 br label %bb287 88bb2: ; preds = %bb2, %bb89 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]90 %tmp3 = getelementptr inbounds i32, ptr %arg, i64 %tmp91 %tmp5 = load <4 x i32>, ptr %tmp3, align 492 %tmp6 = add nsw <4 x i32> %tmp5, <i32 2, i32 2, i32 2, i32 2>93 store <4 x i32> %tmp6, ptr %tmp3, align 494 %tmp8 = add i64 %tmp, 495 %tmp9 = icmp eq i64 %tmp8, 102496 br i1 %tmp9, label %bb10, label %bb297 98bb10: ; preds = %bb299 ret void100}101 102define void @bcast_unfold_add_v8i64(ptr %arg) {103; CHECK-LABEL: bcast_unfold_add_v8i64:104; CHECK: # %bb.0: # %bb105; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000106; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]107; CHECK-NEXT: .p2align 4108; CHECK-NEXT: .LBB3_1: # %bb2109; CHECK-NEXT: # =>This Inner Loop Header: Depth=1110; CHECK-NEXT: vpaddq 8192(%rdi,%rax), %zmm0, %zmm1111; CHECK-NEXT: vmovdqu64 %zmm1, 8192(%rdi,%rax)112; CHECK-NEXT: addq $64, %rax113; CHECK-NEXT: jne .LBB3_1114; CHECK-NEXT: # %bb.2: # %bb10115; CHECK-NEXT: vzeroupper116; CHECK-NEXT: retq117bb:118 br label %bb2119 120bb2: ; preds = %bb2, %bb121 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]122 %tmp3 = getelementptr inbounds i64, ptr %arg, i64 %tmp123 %tmp5 = load <8 x i64>, ptr %tmp3, align 8124 %tmp6 = add nsw <8 x i64> %tmp5, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>125 store <8 x i64> %tmp6, ptr %tmp3, align 8126 %tmp8 = add i64 %tmp, 8127 %tmp9 = icmp eq i64 %tmp8, 1024128 br i1 %tmp9, label %bb10, label %bb2129 130bb10: ; preds = %bb2131 ret void132}133 134define void @bcast_unfold_add_v4i64(ptr %arg) {135; CHECK-LABEL: bcast_unfold_add_v4i64:136; CHECK: # %bb.0: # %bb137; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000138; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]139; CHECK-NEXT: .p2align 4140; CHECK-NEXT: .LBB4_1: # %bb2141; CHECK-NEXT: # =>This Inner Loop Header: Depth=1142; CHECK-NEXT: vpaddq 8192(%rdi,%rax), %ymm0, %ymm1143; CHECK-NEXT: vmovdqu %ymm1, 8192(%rdi,%rax)144; CHECK-NEXT: addq $32, %rax145; CHECK-NEXT: jne .LBB4_1146; CHECK-NEXT: # %bb.2: # %bb10147; CHECK-NEXT: vzeroupper148; CHECK-NEXT: retq149bb:150 br label %bb2151 152bb2: ; preds = %bb2, %bb153 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]154 %tmp3 = getelementptr inbounds i64, ptr %arg, i64 %tmp155 %tmp5 = load <4 x i64>, ptr %tmp3, align 8156 %tmp6 = add nsw <4 x i64> %tmp5, <i64 2, i64 2, i64 2, i64 2>157 store <4 x i64> %tmp6, ptr %tmp3, align 8158 %tmp8 = add i64 %tmp, 4159 %tmp9 = icmp eq i64 %tmp8, 1024160 br i1 %tmp9, label %bb10, label %bb2161 162bb10: ; preds = %bb2163 ret void164}165 166define void @bcast_unfold_add_v2i64(ptr %arg) {167; CHECK-LABEL: bcast_unfold_add_v2i64:168; CHECK: # %bb.0: # %bb169; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000170; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm0 = [2,2]171; CHECK-NEXT: .p2align 4172; CHECK-NEXT: .LBB5_1: # %bb2173; CHECK-NEXT: # =>This Inner Loop Header: Depth=1174; CHECK-NEXT: vpaddq 8192(%rdi,%rax), %xmm0, %xmm1175; CHECK-NEXT: vmovdqu %xmm1, 8192(%rdi,%rax)176; CHECK-NEXT: addq $16, %rax177; CHECK-NEXT: jne .LBB5_1178; CHECK-NEXT: # %bb.2: # %bb10179; CHECK-NEXT: retq180bb:181 br label %bb2182 183bb2: ; preds = %bb2, %bb184 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]185 %tmp3 = getelementptr inbounds i64, ptr %arg, i64 %tmp186 %tmp5 = load <2 x i64>, ptr %tmp3, align 8187 %tmp6 = add nsw <2 x i64> %tmp5, <i64 2, i64 2>188 store <2 x i64> %tmp6, ptr %tmp3, align 8189 %tmp8 = add i64 %tmp, 2190 %tmp9 = icmp eq i64 %tmp8, 1024191 br i1 %tmp9, label %bb10, label %bb2192 193bb10: ; preds = %bb2194 ret void195}196 197define void @bcast_unfold_mul_v16i32(ptr %arg) {198; CHECK-LABEL: bcast_unfold_mul_v16i32:199; CHECK: # %bb.0: # %bb200; CHECK-NEXT: movq $-4096, %rax # imm = 0xF000201; CHECK-NEXT: .p2align 4202; CHECK-NEXT: .LBB6_1: # %bb2203; CHECK-NEXT: # =>This Inner Loop Header: Depth=1204; CHECK-NEXT: vmovdqu64 4096(%rdi,%rax), %zmm0205; CHECK-NEXT: vpaddd %zmm0, %zmm0, %zmm1206; CHECK-NEXT: vpaddd %zmm1, %zmm0, %zmm0207; CHECK-NEXT: vmovdqu64 %zmm0, 4096(%rdi,%rax)208; CHECK-NEXT: addq $64, %rax209; CHECK-NEXT: jne .LBB6_1210; CHECK-NEXT: # %bb.2: # %bb10211; CHECK-NEXT: vzeroupper212; CHECK-NEXT: retq213bb:214 br label %bb2215 216bb2: ; preds = %bb2, %bb217 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]218 %tmp3 = getelementptr inbounds i32, ptr %arg, i64 %tmp219 %tmp5 = load <16 x i32>, ptr %tmp3, align 4220 %tmp6 = mul nsw <16 x i32> %tmp5, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>221 store <16 x i32> %tmp6, ptr %tmp3, align 4222 %tmp8 = add i64 %tmp, 16223 %tmp9 = icmp eq i64 %tmp8, 1024224 br i1 %tmp9, label %bb10, label %bb2225 226bb10: ; preds = %bb2227 ret void228}229 230define void @bcast_unfold_mul_v8i32(ptr %arg) {231; CHECK-LABEL: bcast_unfold_mul_v8i32:232; CHECK: # %bb.0: # %bb233; CHECK-NEXT: movq $-4096, %rax # imm = 0xF000234; CHECK-NEXT: .p2align 4235; CHECK-NEXT: .LBB7_1: # %bb2236; CHECK-NEXT: # =>This Inner Loop Header: Depth=1237; CHECK-NEXT: vmovdqu 4096(%rdi,%rax), %ymm0238; CHECK-NEXT: vpaddd %ymm0, %ymm0, %ymm1239; CHECK-NEXT: vpaddd %ymm1, %ymm0, %ymm0240; CHECK-NEXT: vmovdqu %ymm0, 4096(%rdi,%rax)241; CHECK-NEXT: addq $32, %rax242; CHECK-NEXT: jne .LBB7_1243; CHECK-NEXT: # %bb.2: # %bb10244; CHECK-NEXT: vzeroupper245; CHECK-NEXT: retq246bb:247 br label %bb2248 249bb2: ; preds = %bb2, %bb250 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]251 %tmp3 = getelementptr inbounds i32, ptr %arg, i64 %tmp252 %tmp5 = load <8 x i32>, ptr %tmp3, align 4253 %tmp6 = mul nsw <8 x i32> %tmp5, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>254 store <8 x i32> %tmp6, ptr %tmp3, align 4255 %tmp8 = add i64 %tmp, 8256 %tmp9 = icmp eq i64 %tmp8, 1024257 br i1 %tmp9, label %bb10, label %bb2258 259bb10: ; preds = %bb2260 ret void261}262 263define void @bcast_unfold_mul_v4i32(ptr %arg) {264; CHECK-LABEL: bcast_unfold_mul_v4i32:265; CHECK: # %bb.0: # %bb266; CHECK-NEXT: movq $-4096, %rax # imm = 0xF000267; CHECK-NEXT: .p2align 4268; CHECK-NEXT: .LBB8_1: # %bb2269; CHECK-NEXT: # =>This Inner Loop Header: Depth=1270; CHECK-NEXT: vmovdqu 4096(%rdi,%rax), %xmm0271; CHECK-NEXT: vpaddd %xmm0, %xmm0, %xmm1272; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0273; CHECK-NEXT: vmovdqu %xmm0, 4096(%rdi,%rax)274; CHECK-NEXT: addq $16, %rax275; CHECK-NEXT: jne .LBB8_1276; CHECK-NEXT: # %bb.2: # %bb10277; CHECK-NEXT: retq278bb:279 br label %bb2280 281bb2: ; preds = %bb2, %bb282 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]283 %tmp3 = getelementptr inbounds i32, ptr %arg, i64 %tmp284 %tmp5 = load <4 x i32>, ptr %tmp3, align 4285 %tmp6 = mul nsw <4 x i32> %tmp5, <i32 3, i32 3, i32 3, i32 3>286 store <4 x i32> %tmp6, ptr %tmp3, align 4287 %tmp8 = add i64 %tmp, 4288 %tmp9 = icmp eq i64 %tmp8, 1024289 br i1 %tmp9, label %bb10, label %bb2290 291bb10: ; preds = %bb2292 ret void293}294 295define void @bcast_unfold_mul_v8i64(ptr %arg) {296; CHECK-LABEL: bcast_unfold_mul_v8i64:297; CHECK: # %bb.0: # %bb298; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000299; CHECK-NEXT: .p2align 4300; CHECK-NEXT: .LBB9_1: # %bb2301; CHECK-NEXT: # =>This Inner Loop Header: Depth=1302; CHECK-NEXT: vmovdqu64 8192(%rdi,%rax), %zmm0303; CHECK-NEXT: vpaddq %zmm0, %zmm0, %zmm1304; CHECK-NEXT: vpaddq %zmm0, %zmm1, %zmm0305; CHECK-NEXT: vmovdqu64 %zmm0, 8192(%rdi,%rax)306; CHECK-NEXT: addq $64, %rax307; CHECK-NEXT: jne .LBB9_1308; CHECK-NEXT: # %bb.2: # %bb10309; CHECK-NEXT: vzeroupper310; CHECK-NEXT: retq311bb:312 br label %bb2313 314bb2: ; preds = %bb2, %bb315 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]316 %tmp3 = getelementptr inbounds i64, ptr %arg, i64 %tmp317 %tmp5 = load <8 x i64>, ptr %tmp3, align 8318 %tmp6 = mul nsw <8 x i64> %tmp5, <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>319 store <8 x i64> %tmp6, ptr %tmp3, align 8320 %tmp8 = add i64 %tmp, 8321 %tmp9 = icmp eq i64 %tmp8, 1024322 br i1 %tmp9, label %bb10, label %bb2323 324bb10: ; preds = %bb2325 ret void326}327 328define void @bcast_unfold_mul_v4i64(ptr %arg) {329; CHECK-LABEL: bcast_unfold_mul_v4i64:330; CHECK: # %bb.0: # %bb331; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000332; CHECK-NEXT: .p2align 4333; CHECK-NEXT: .LBB10_1: # %bb2334; CHECK-NEXT: # =>This Inner Loop Header: Depth=1335; CHECK-NEXT: vmovdqu 8192(%rdi,%rax), %ymm0336; CHECK-NEXT: vpaddq %ymm0, %ymm0, %ymm1337; CHECK-NEXT: vpaddq %ymm0, %ymm1, %ymm0338; CHECK-NEXT: vmovdqu %ymm0, 8192(%rdi,%rax)339; CHECK-NEXT: addq $32, %rax340; CHECK-NEXT: jne .LBB10_1341; CHECK-NEXT: # %bb.2: # %bb10342; CHECK-NEXT: vzeroupper343; CHECK-NEXT: retq344bb:345 br label %bb2346 347bb2: ; preds = %bb2, %bb348 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]349 %tmp3 = getelementptr inbounds i64, ptr %arg, i64 %tmp350 %tmp5 = load <4 x i64>, ptr %tmp3, align 8351 %tmp6 = mul nsw <4 x i64> %tmp5, <i64 3, i64 3, i64 3, i64 3>352 store <4 x i64> %tmp6, ptr %tmp3, align 8353 %tmp8 = add i64 %tmp, 4354 %tmp9 = icmp eq i64 %tmp8, 1024355 br i1 %tmp9, label %bb10, label %bb2356 357bb10: ; preds = %bb2358 ret void359}360 361define void @bcast_unfold_mul_v2i64(ptr %arg) {362; CHECK-LABEL: bcast_unfold_mul_v2i64:363; CHECK: # %bb.0: # %bb364; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000365; CHECK-NEXT: .p2align 4366; CHECK-NEXT: .LBB11_1: # %bb2367; CHECK-NEXT: # =>This Inner Loop Header: Depth=1368; CHECK-NEXT: vmovdqu 8192(%rdi,%rax), %xmm0369; CHECK-NEXT: vpaddq %xmm0, %xmm0, %xmm1370; CHECK-NEXT: vpaddq %xmm0, %xmm1, %xmm0371; CHECK-NEXT: vmovdqu %xmm0, 8192(%rdi,%rax)372; CHECK-NEXT: addq $16, %rax373; CHECK-NEXT: jne .LBB11_1374; CHECK-NEXT: # %bb.2: # %bb10375; CHECK-NEXT: retq376bb:377 br label %bb2378 379bb2: ; preds = %bb2, %bb380 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]381 %tmp3 = getelementptr inbounds i64, ptr %arg, i64 %tmp382 %tmp5 = load <2 x i64>, ptr %tmp3, align 8383 %tmp6 = mul nsw <2 x i64> %tmp5, <i64 3, i64 3>384 store <2 x i64> %tmp6, ptr %tmp3, align 8385 %tmp8 = add i64 %tmp, 2386 %tmp9 = icmp eq i64 %tmp8, 1024387 br i1 %tmp9, label %bb10, label %bb2388 389bb10: ; preds = %bb2390 ret void391}392 393define void @bcast_unfold_or_v16i32(ptr %arg) {394; CHECK-LABEL: bcast_unfold_or_v16i32:395; CHECK: # %bb.0: # %bb396; CHECK-NEXT: movq $-4096, %rax # imm = 0xF000397; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm0 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]398; CHECK-NEXT: .p2align 4399; CHECK-NEXT: .LBB12_1: # %bb2400; CHECK-NEXT: # =>This Inner Loop Header: Depth=1401; CHECK-NEXT: vpord 4096(%rdi,%rax), %zmm0, %zmm1402; CHECK-NEXT: vmovdqu64 %zmm1, 4096(%rdi,%rax)403; CHECK-NEXT: addq $64, %rax404; CHECK-NEXT: jne .LBB12_1405; CHECK-NEXT: # %bb.2: # %bb10406; CHECK-NEXT: vzeroupper407; CHECK-NEXT: retq408bb:409 br label %bb2410 411bb2: ; preds = %bb2, %bb412 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]413 %tmp3 = getelementptr inbounds i32, ptr %arg, i64 %tmp414 %tmp5 = load <16 x i32>, ptr %tmp3, align 4415 %tmp6 = or <16 x i32> %tmp5, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>416 store <16 x i32> %tmp6, ptr %tmp3, align 4417 %tmp8 = add i64 %tmp, 16418 %tmp9 = icmp eq i64 %tmp8, 1024419 br i1 %tmp9, label %bb10, label %bb2420 421bb10: ; preds = %bb2422 ret void423}424 425define void @bcast_unfold_or_v8i32(ptr %arg) {426; CHECK-LABEL: bcast_unfold_or_v8i32:427; CHECK: # %bb.0: # %bb428; CHECK-NEXT: movq $-4096, %rax # imm = 0xF000429; CHECK-NEXT: vbroadcastss {{.*#+}} ymm0 = [3,3,3,3,3,3,3,3]430; CHECK-NEXT: .p2align 4431; CHECK-NEXT: .LBB13_1: # %bb2432; CHECK-NEXT: # =>This Inner Loop Header: Depth=1433; CHECK-NEXT: vorps 4096(%rdi,%rax), %ymm0, %ymm1434; CHECK-NEXT: vmovups %ymm1, 4096(%rdi,%rax)435; CHECK-NEXT: addq $32, %rax436; CHECK-NEXT: jne .LBB13_1437; CHECK-NEXT: # %bb.2: # %bb10438; CHECK-NEXT: vzeroupper439; CHECK-NEXT: retq440bb:441 br label %bb2442 443bb2: ; preds = %bb2, %bb444 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]445 %tmp3 = getelementptr inbounds i32, ptr %arg, i64 %tmp446 %tmp5 = load <8 x i32>, ptr %tmp3, align 4447 %tmp6 = or <8 x i32> %tmp5, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>448 store <8 x i32> %tmp6, ptr %tmp3, align 4449 %tmp8 = add i64 %tmp, 8450 %tmp9 = icmp eq i64 %tmp8, 1024451 br i1 %tmp9, label %bb10, label %bb2452 453bb10: ; preds = %bb2454 ret void455}456 457define void @bcast_unfold_or_v4i32(ptr %arg) {458; CHECK-LABEL: bcast_unfold_or_v4i32:459; CHECK: # %bb.0: # %bb460; CHECK-NEXT: movq $-4096, %rax # imm = 0xF000461; CHECK-NEXT: vbroadcastss {{.*#+}} xmm0 = [3,3,3,3]462; CHECK-NEXT: .p2align 4463; CHECK-NEXT: .LBB14_1: # %bb2464; CHECK-NEXT: # =>This Inner Loop Header: Depth=1465; CHECK-NEXT: vorps 4096(%rdi,%rax), %xmm0, %xmm1466; CHECK-NEXT: vmovups %xmm1, 4096(%rdi,%rax)467; CHECK-NEXT: addq $16, %rax468; CHECK-NEXT: jne .LBB14_1469; CHECK-NEXT: # %bb.2: # %bb10470; CHECK-NEXT: retq471bb:472 br label %bb2473 474bb2: ; preds = %bb2, %bb475 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]476 %tmp3 = getelementptr inbounds i32, ptr %arg, i64 %tmp477 %tmp5 = load <4 x i32>, ptr %tmp3, align 4478 %tmp6 = or <4 x i32> %tmp5, <i32 3, i32 3, i32 3, i32 3>479 store <4 x i32> %tmp6, ptr %tmp3, align 4480 %tmp8 = add i64 %tmp, 4481 %tmp9 = icmp eq i64 %tmp8, 1024482 br i1 %tmp9, label %bb10, label %bb2483 484bb10: ; preds = %bb2485 ret void486}487 488define void @bcast_unfold_or_v8i64(ptr %arg) {489; CHECK-LABEL: bcast_unfold_or_v8i64:490; CHECK: # %bb.0: # %bb491; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000492; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm0 = [3,3,3,3,3,3,3,3]493; CHECK-NEXT: .p2align 4494; CHECK-NEXT: .LBB15_1: # %bb2495; CHECK-NEXT: # =>This Inner Loop Header: Depth=1496; CHECK-NEXT: vporq 8192(%rdi,%rax), %zmm0, %zmm1497; CHECK-NEXT: vmovdqu64 %zmm1, 8192(%rdi,%rax)498; CHECK-NEXT: addq $64, %rax499; CHECK-NEXT: jne .LBB15_1500; CHECK-NEXT: # %bb.2: # %bb10501; CHECK-NEXT: vzeroupper502; CHECK-NEXT: retq503bb:504 br label %bb2505 506bb2: ; preds = %bb2, %bb507 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]508 %tmp3 = getelementptr inbounds i64, ptr %arg, i64 %tmp509 %tmp5 = load <8 x i64>, ptr %tmp3, align 8510 %tmp6 = or <8 x i64> %tmp5, <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>511 store <8 x i64> %tmp6, ptr %tmp3, align 8512 %tmp8 = add i64 %tmp, 8513 %tmp9 = icmp eq i64 %tmp8, 1024514 br i1 %tmp9, label %bb10, label %bb2515 516bb10: ; preds = %bb2517 ret void518}519 520define void @bcast_unfold_or_v4i64(ptr %arg) {521; CHECK-LABEL: bcast_unfold_or_v4i64:522; CHECK: # %bb.0: # %bb523; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000524; CHECK-NEXT: vbroadcastsd {{.*#+}} ymm0 = [3,3,3,3]525; CHECK-NEXT: .p2align 4526; CHECK-NEXT: .LBB16_1: # %bb2527; CHECK-NEXT: # =>This Inner Loop Header: Depth=1528; CHECK-NEXT: vorps 8192(%rdi,%rax), %ymm0, %ymm1529; CHECK-NEXT: vmovups %ymm1, 8192(%rdi,%rax)530; CHECK-NEXT: addq $32, %rax531; CHECK-NEXT: jne .LBB16_1532; CHECK-NEXT: # %bb.2: # %bb10533; CHECK-NEXT: vzeroupper534; CHECK-NEXT: retq535bb:536 br label %bb2537 538bb2: ; preds = %bb2, %bb539 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]540 %tmp3 = getelementptr inbounds i64, ptr %arg, i64 %tmp541 %tmp5 = load <4 x i64>, ptr %tmp3, align 8542 %tmp6 = or <4 x i64> %tmp5, <i64 3, i64 3, i64 3, i64 3>543 store <4 x i64> %tmp6, ptr %tmp3, align 8544 %tmp8 = add i64 %tmp, 4545 %tmp9 = icmp eq i64 %tmp8, 1024546 br i1 %tmp9, label %bb10, label %bb2547 548bb10: ; preds = %bb2549 ret void550}551 552define void @bcast_unfold_or_v2i64(ptr %arg) {553; CHECK-LABEL: bcast_unfold_or_v2i64:554; CHECK: # %bb.0: # %bb555; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000556; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = [3,3]557; CHECK-NEXT: # xmm0 = mem[0,0]558; CHECK-NEXT: .p2align 4559; CHECK-NEXT: .LBB17_1: # %bb2560; CHECK-NEXT: # =>This Inner Loop Header: Depth=1561; CHECK-NEXT: vorps 8192(%rdi,%rax), %xmm0, %xmm1562; CHECK-NEXT: vmovups %xmm1, 8192(%rdi,%rax)563; CHECK-NEXT: addq $16, %rax564; CHECK-NEXT: jne .LBB17_1565; CHECK-NEXT: # %bb.2: # %bb10566; CHECK-NEXT: retq567bb:568 br label %bb2569 570bb2: ; preds = %bb2, %bb571 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb2 ]572 %tmp3 = getelementptr inbounds i64, ptr %arg, i64 %tmp573 %tmp5 = load <2 x i64>, ptr %tmp3, align 8574 %tmp6 = or <2 x i64> %tmp5, <i64 3, i64 3>575 store <2 x i64> %tmp6, ptr %tmp3, align 8576 %tmp8 = add i64 %tmp, 2577 %tmp9 = icmp eq i64 %tmp8, 1024578 br i1 %tmp9, label %bb10, label %bb2579 580bb10: ; preds = %bb2581 ret void582}583 584define void @bcast_unfold_fneg_v16f32(ptr %arg) {585; CHECK-LABEL: bcast_unfold_fneg_v16f32:586; CHECK: # %bb.0: # %bb587; CHECK-NEXT: movq $-4096, %rax # imm = 0xF000588; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]589; CHECK-NEXT: .p2align 4590; CHECK-NEXT: .LBB18_1: # %bb1591; CHECK-NEXT: # =>This Inner Loop Header: Depth=1592; CHECK-NEXT: vpxord 4096(%rdi,%rax), %zmm0, %zmm1593; CHECK-NEXT: vmovdqu64 %zmm1, 4096(%rdi,%rax)594; CHECK-NEXT: addq $64, %rax595; CHECK-NEXT: jne .LBB18_1596; CHECK-NEXT: # %bb.2: # %bb9597; CHECK-NEXT: vzeroupper598; CHECK-NEXT: retq599bb:600 br label %bb1601 602bb1: ; preds = %bb1, %bb603 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]604 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp605 %tmp4 = load <16 x float>, ptr %tmp2, align 4606 %tmp5 = fneg <16 x float> %tmp4607 store <16 x float> %tmp5, ptr %tmp2, align 4608 %tmp7 = add i64 %tmp, 16609 %tmp8 = icmp eq i64 %tmp7, 1024610 br i1 %tmp8, label %bb9, label %bb1611 612bb9: ; preds = %bb1613 ret void614}615 616define void @bcast_unfold_fneg_v8f32(ptr %arg) {617; CHECK-LABEL: bcast_unfold_fneg_v8f32:618; CHECK: # %bb.0: # %bb619; CHECK-NEXT: movq $-4096, %rax # imm = 0xF000620; CHECK-NEXT: vbroadcastss {{.*#+}} ymm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]621; CHECK-NEXT: .p2align 4622; CHECK-NEXT: .LBB19_1: # %bb1623; CHECK-NEXT: # =>This Inner Loop Header: Depth=1624; CHECK-NEXT: vxorps 4096(%rdi,%rax), %ymm0, %ymm1625; CHECK-NEXT: vmovups %ymm1, 4096(%rdi,%rax)626; CHECK-NEXT: addq $32, %rax627; CHECK-NEXT: jne .LBB19_1628; CHECK-NEXT: # %bb.2: # %bb9629; CHECK-NEXT: vzeroupper630; CHECK-NEXT: retq631bb:632 br label %bb1633 634bb1: ; preds = %bb1, %bb635 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]636 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp637 %tmp4 = load <8 x float>, ptr %tmp2, align 4638 %tmp5 = fneg <8 x float> %tmp4639 store <8 x float> %tmp5, ptr %tmp2, align 4640 %tmp7 = add i64 %tmp, 8641 %tmp8 = icmp eq i64 %tmp7, 1024642 br i1 %tmp8, label %bb9, label %bb1643 644bb9: ; preds = %bb1645 ret void646}647 648define void @bcast_unfold_fneg_v4f32(ptr %arg) {649; CHECK-LABEL: bcast_unfold_fneg_v4f32:650; CHECK: # %bb.0: # %bb651; CHECK-NEXT: movq $-4096, %rax # imm = 0xF000652; CHECK-NEXT: vbroadcastss {{.*#+}} xmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]653; CHECK-NEXT: .p2align 4654; CHECK-NEXT: .LBB20_1: # %bb1655; CHECK-NEXT: # =>This Inner Loop Header: Depth=1656; CHECK-NEXT: vxorps 4096(%rdi,%rax), %xmm0, %xmm1657; CHECK-NEXT: vmovups %xmm1, 4096(%rdi,%rax)658; CHECK-NEXT: addq $16, %rax659; CHECK-NEXT: jne .LBB20_1660; CHECK-NEXT: # %bb.2: # %bb9661; CHECK-NEXT: retq662bb:663 br label %bb1664 665bb1: ; preds = %bb1, %bb666 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]667 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp668 %tmp4 = load <4 x float>, ptr %tmp2, align 4669 %tmp5 = fneg <4 x float> %tmp4670 store <4 x float> %tmp5, ptr %tmp2, align 4671 %tmp7 = add i64 %tmp, 4672 %tmp8 = icmp eq i64 %tmp7, 1024673 br i1 %tmp8, label %bb9, label %bb1674 675bb9: ; preds = %bb1676 ret void677}678 679define void @bcast_unfold_fneg_v8f64(ptr %arg) {680; CHECK-LABEL: bcast_unfold_fneg_v8f64:681; CHECK: # %bb.0: # %bb682; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000683; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]684; CHECK-NEXT: .p2align 4685; CHECK-NEXT: .LBB21_1: # %bb1686; CHECK-NEXT: # =>This Inner Loop Header: Depth=1687; CHECK-NEXT: vpxorq 8192(%rdi,%rax), %zmm0, %zmm1688; CHECK-NEXT: vmovdqu64 %zmm1, 8192(%rdi,%rax)689; CHECK-NEXT: addq $64, %rax690; CHECK-NEXT: jne .LBB21_1691; CHECK-NEXT: # %bb.2: # %bb9692; CHECK-NEXT: vzeroupper693; CHECK-NEXT: retq694bb:695 br label %bb1696 697bb1: ; preds = %bb1, %bb698 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]699 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp700 %tmp4 = load <8 x double>, ptr %tmp2, align 8701 %tmp5 = fneg <8 x double> %tmp4702 store <8 x double> %tmp5, ptr %tmp2, align 8703 %tmp7 = add i64 %tmp, 8704 %tmp8 = icmp eq i64 %tmp7, 1024705 br i1 %tmp8, label %bb9, label %bb1706 707bb9: ; preds = %bb1708 ret void709}710 711define void @bcast_unfold_fneg_v4f64(ptr %arg) {712; CHECK-LABEL: bcast_unfold_fneg_v4f64:713; CHECK: # %bb.0: # %bb714; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000715; CHECK-NEXT: vbroadcastsd {{.*#+}} ymm0 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]716; CHECK-NEXT: .p2align 4717; CHECK-NEXT: .LBB22_1: # %bb1718; CHECK-NEXT: # =>This Inner Loop Header: Depth=1719; CHECK-NEXT: vxorps 8192(%rdi,%rax), %ymm0, %ymm1720; CHECK-NEXT: vmovups %ymm1, 8192(%rdi,%rax)721; CHECK-NEXT: addq $32, %rax722; CHECK-NEXT: jne .LBB22_1723; CHECK-NEXT: # %bb.2: # %bb9724; CHECK-NEXT: vzeroupper725; CHECK-NEXT: retq726bb:727 br label %bb1728 729bb1: ; preds = %bb1, %bb730 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]731 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp732 %tmp4 = load <4 x double>, ptr %tmp2, align 8733 %tmp5 = fneg <4 x double> %tmp4734 store <4 x double> %tmp5, ptr %tmp2, align 8735 %tmp7 = add i64 %tmp, 4736 %tmp8 = icmp eq i64 %tmp7, 1024737 br i1 %tmp8, label %bb9, label %bb1738 739bb9: ; preds = %bb1740 ret void741}742 743define void @bcast_unfold_fneg_v2f64(ptr %arg) {744; CHECK-LABEL: bcast_unfold_fneg_v2f64:745; CHECK: # %bb.0: # %bb746; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000747; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = [-0.0E+0,-0.0E+0]748; CHECK-NEXT: # xmm0 = mem[0,0]749; CHECK-NEXT: .p2align 4750; CHECK-NEXT: .LBB23_1: # %bb1751; CHECK-NEXT: # =>This Inner Loop Header: Depth=1752; CHECK-NEXT: vxorps 8192(%rdi,%rax), %xmm0, %xmm1753; CHECK-NEXT: vmovups %xmm1, 8192(%rdi,%rax)754; CHECK-NEXT: addq $16, %rax755; CHECK-NEXT: jne .LBB23_1756; CHECK-NEXT: # %bb.2: # %bb9757; CHECK-NEXT: retq758bb:759 br label %bb1760 761bb1: ; preds = %bb1, %bb762 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]763 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp764 %tmp4 = load <2 x double>, ptr %tmp2, align 8765 %tmp5 = fneg <2 x double> %tmp4766 store <2 x double> %tmp5, ptr %tmp2, align 8767 %tmp7 = add i64 %tmp, 2768 %tmp8 = icmp eq i64 %tmp7, 1024769 br i1 %tmp8, label %bb9, label %bb1770 771bb9: ; preds = %bb1772 ret void773}774 775define void @bcast_unfold_fabs_v16f32(ptr %arg) {776; CHECK-LABEL: bcast_unfold_fabs_v16f32:777; CHECK: # %bb.0: # %bb778; CHECK-NEXT: movq $-4096, %rax # imm = 0xF000779; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]780; CHECK-NEXT: .p2align 4781; CHECK-NEXT: .LBB24_1: # %bb1782; CHECK-NEXT: # =>This Inner Loop Header: Depth=1783; CHECK-NEXT: vpandd 4096(%rdi,%rax), %zmm0, %zmm1784; CHECK-NEXT: vmovdqu64 %zmm1, 4096(%rdi,%rax)785; CHECK-NEXT: addq $64, %rax786; CHECK-NEXT: jne .LBB24_1787; CHECK-NEXT: # %bb.2: # %bb9788; CHECK-NEXT: vzeroupper789; CHECK-NEXT: retq790bb:791 br label %bb1792 793bb1: ; preds = %bb1, %bb794 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]795 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp796 %tmp4 = load <16 x float>, ptr %tmp2, align 4797 %tmp5 = call <16 x float> @llvm.fabs.v16f32(<16 x float> %tmp4)798 store <16 x float> %tmp5, ptr %tmp2, align 4799 %tmp7 = add i64 %tmp, 16800 %tmp8 = icmp eq i64 %tmp7, 1024801 br i1 %tmp8, label %bb9, label %bb1802 803bb9: ; preds = %bb1804 ret void805}806 807; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)808declare <16 x float> @llvm.fabs.v16f32(<16 x float>) #0809 810define void @bcast_unfold_fabs_v8f32(ptr %arg) {811; CHECK-LABEL: bcast_unfold_fabs_v8f32:812; CHECK: # %bb.0: # %bb813; CHECK-NEXT: movq $-4096, %rax # imm = 0xF000814; CHECK-NEXT: vbroadcastss {{.*#+}} ymm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]815; CHECK-NEXT: .p2align 4816; CHECK-NEXT: .LBB25_1: # %bb1817; CHECK-NEXT: # =>This Inner Loop Header: Depth=1818; CHECK-NEXT: vandps 4096(%rdi,%rax), %ymm0, %ymm1819; CHECK-NEXT: vmovups %ymm1, 4096(%rdi,%rax)820; CHECK-NEXT: addq $32, %rax821; CHECK-NEXT: jne .LBB25_1822; CHECK-NEXT: # %bb.2: # %bb9823; CHECK-NEXT: vzeroupper824; CHECK-NEXT: retq825bb:826 br label %bb1827 828bb1: ; preds = %bb1, %bb829 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]830 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp831 %tmp4 = load <8 x float>, ptr %tmp2, align 4832 %tmp5 = call <8 x float> @llvm.fabs.v8f32(<8 x float> %tmp4)833 store <8 x float> %tmp5, ptr %tmp2, align 4834 %tmp7 = add i64 %tmp, 8835 %tmp8 = icmp eq i64 %tmp7, 1024836 br i1 %tmp8, label %bb9, label %bb1837 838bb9: ; preds = %bb1839 ret void840}841 842; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)843declare <8 x float> @llvm.fabs.v8f32(<8 x float>) #0844 845define void @bcast_unfold_fabs_v4f32(ptr %arg) {846; CHECK-LABEL: bcast_unfold_fabs_v4f32:847; CHECK: # %bb.0: # %bb848; CHECK-NEXT: movq $-4096, %rax # imm = 0xF000849; CHECK-NEXT: vbroadcastss {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN]850; CHECK-NEXT: .p2align 4851; CHECK-NEXT: .LBB26_1: # %bb1852; CHECK-NEXT: # =>This Inner Loop Header: Depth=1853; CHECK-NEXT: vandps 4096(%rdi,%rax), %xmm0, %xmm1854; CHECK-NEXT: vmovups %xmm1, 4096(%rdi,%rax)855; CHECK-NEXT: addq $16, %rax856; CHECK-NEXT: jne .LBB26_1857; CHECK-NEXT: # %bb.2: # %bb9858; CHECK-NEXT: retq859bb:860 br label %bb1861 862bb1: ; preds = %bb1, %bb863 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]864 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp865 %tmp4 = load <4 x float>, ptr %tmp2, align 4866 %tmp5 = call <4 x float> @llvm.fabs.v4f32(<4 x float> %tmp4)867 store <4 x float> %tmp5, ptr %tmp2, align 4868 %tmp7 = add i64 %tmp, 4869 %tmp8 = icmp eq i64 %tmp7, 1024870 br i1 %tmp8, label %bb9, label %bb1871 872bb9: ; preds = %bb1873 ret void874}875 876; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)877declare <4 x float> @llvm.fabs.v4f32(<4 x float>) #0878 879define void @bcast_unfold_fabs_v8f64(ptr %arg) {880; CHECK-LABEL: bcast_unfold_fabs_v8f64:881; CHECK: # %bb.0: # %bb882; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000883; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]884; CHECK-NEXT: .p2align 4885; CHECK-NEXT: .LBB27_1: # %bb1886; CHECK-NEXT: # =>This Inner Loop Header: Depth=1887; CHECK-NEXT: vpandq 8192(%rdi,%rax), %zmm0, %zmm1888; CHECK-NEXT: vmovdqu64 %zmm1, 8192(%rdi,%rax)889; CHECK-NEXT: addq $64, %rax890; CHECK-NEXT: jne .LBB27_1891; CHECK-NEXT: # %bb.2: # %bb9892; CHECK-NEXT: vzeroupper893; CHECK-NEXT: retq894bb:895 br label %bb1896 897bb1: ; preds = %bb1, %bb898 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]899 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp900 %tmp4 = load <8 x double>, ptr %tmp2, align 8901 %tmp5 = call <8 x double> @llvm.fabs.v8f64(<8 x double> %tmp4)902 store <8 x double> %tmp5, ptr %tmp2, align 8903 %tmp7 = add i64 %tmp, 8904 %tmp8 = icmp eq i64 %tmp7, 1024905 br i1 %tmp8, label %bb9, label %bb1906 907bb9: ; preds = %bb1908 ret void909}910 911; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)912declare <8 x double> @llvm.fabs.v8f64(<8 x double>) #0913 914define void @bcast_unfold_fabs_v4f64(ptr %arg) {915; CHECK-LABEL: bcast_unfold_fabs_v4f64:916; CHECK: # %bb.0: # %bb917; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000918; CHECK-NEXT: vbroadcastsd {{.*#+}} ymm0 = [NaN,NaN,NaN,NaN]919; CHECK-NEXT: .p2align 4920; CHECK-NEXT: .LBB28_1: # %bb1921; CHECK-NEXT: # =>This Inner Loop Header: Depth=1922; CHECK-NEXT: vandps 8192(%rdi,%rax), %ymm0, %ymm1923; CHECK-NEXT: vmovups %ymm1, 8192(%rdi,%rax)924; CHECK-NEXT: addq $32, %rax925; CHECK-NEXT: jne .LBB28_1926; CHECK-NEXT: # %bb.2: # %bb9927; CHECK-NEXT: vzeroupper928; CHECK-NEXT: retq929bb:930 br label %bb1931 932bb1: ; preds = %bb1, %bb933 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]934 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp935 %tmp4 = load <4 x double>, ptr %tmp2, align 8936 %tmp5 = call <4 x double> @llvm.fabs.v4f64(<4 x double> %tmp4)937 store <4 x double> %tmp5, ptr %tmp2, align 8938 %tmp7 = add i64 %tmp, 4939 %tmp8 = icmp eq i64 %tmp7, 1024940 br i1 %tmp8, label %bb9, label %bb1941 942bb9: ; preds = %bb1943 ret void944}945 946; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)947declare <4 x double> @llvm.fabs.v4f64(<4 x double>) #0948 949define void @bcast_unfold_fabs_v2f64(ptr %arg) {950; CHECK-LABEL: bcast_unfold_fabs_v2f64:951; CHECK: # %bb.0: # %bb952; CHECK-NEXT: movq $-8192, %rax # imm = 0xE000953; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = [NaN,NaN]954; CHECK-NEXT: # xmm0 = mem[0,0]955; CHECK-NEXT: .p2align 4956; CHECK-NEXT: .LBB29_1: # %bb1957; CHECK-NEXT: # =>This Inner Loop Header: Depth=1958; CHECK-NEXT: vandps 8192(%rdi,%rax), %xmm0, %xmm1959; CHECK-NEXT: vmovups %xmm1, 8192(%rdi,%rax)960; CHECK-NEXT: addq $16, %rax961; CHECK-NEXT: jne .LBB29_1962; CHECK-NEXT: # %bb.2: # %bb9963; CHECK-NEXT: retq964bb:965 br label %bb1966 967bb1: ; preds = %bb1, %bb968 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]969 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp970 %tmp4 = load <2 x double>, ptr %tmp2, align 8971 %tmp5 = call <2 x double> @llvm.fabs.v2f64(<2 x double> %tmp4)972 store <2 x double> %tmp5, ptr %tmp2, align 8973 %tmp7 = add i64 %tmp, 2974 %tmp8 = icmp eq i64 %tmp7, 1024975 br i1 %tmp8, label %bb9, label %bb1976 977bb9: ; preds = %bb1978 ret void979}980 981; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)982declare <2 x double> @llvm.fabs.v2f64(<2 x double>) #0983 984define void @bcast_unfold_fadd_v16f32(ptr nocapture %arg) {985; CHECK-LABEL: bcast_unfold_fadd_v16f32:986; CHECK: # %bb.0: # %bb987; CHECK-NEXT: movq $-4096, %rax # imm = 0xF000988; CHECK-NEXT: vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]989; CHECK-NEXT: .p2align 4990; CHECK-NEXT: .LBB30_1: # %bb1991; CHECK-NEXT: # =>This Inner Loop Header: Depth=1992; CHECK-NEXT: vaddps 4096(%rdi,%rax), %zmm0, %zmm1993; CHECK-NEXT: vmovups %zmm1, 4096(%rdi,%rax)994; CHECK-NEXT: addq $64, %rax995; CHECK-NEXT: jne .LBB30_1996; CHECK-NEXT: # %bb.2: # %bb9997; CHECK-NEXT: vzeroupper998; CHECK-NEXT: retq999bb:1000 br label %bb11001 1002bb1: ; preds = %bb1, %bb1003 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1004 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp1005 %tmp4 = load <16 x float>, ptr %tmp2, align 41006 %tmp5 = fadd <16 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>1007 store <16 x float> %tmp5, ptr %tmp2, align 41008 %tmp7 = add i64 %tmp, 161009 %tmp8 = icmp eq i64 %tmp7, 10241010 br i1 %tmp8, label %bb9, label %bb11011 1012bb9: ; preds = %bb11013 ret void1014}1015 1016define void @bcast_unfold_fadd_v8f32(ptr nocapture %arg) {1017; CHECK-LABEL: bcast_unfold_fadd_v8f32:1018; CHECK: # %bb.0: # %bb1019; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001020; CHECK-NEXT: vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]1021; CHECK-NEXT: .p2align 41022; CHECK-NEXT: .LBB31_1: # %bb11023; CHECK-NEXT: # =>This Inner Loop Header: Depth=11024; CHECK-NEXT: vaddps 4096(%rdi,%rax), %ymm0, %ymm11025; CHECK-NEXT: vmovups %ymm1, 4096(%rdi,%rax)1026; CHECK-NEXT: addq $32, %rax1027; CHECK-NEXT: jne .LBB31_11028; CHECK-NEXT: # %bb.2: # %bb91029; CHECK-NEXT: vzeroupper1030; CHECK-NEXT: retq1031bb:1032 br label %bb11033 1034bb1: ; preds = %bb1, %bb1035 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1036 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp1037 %tmp4 = load <8 x float>, ptr %tmp2, align 41038 %tmp5 = fadd <8 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>1039 store <8 x float> %tmp5, ptr %tmp2, align 41040 %tmp7 = add i64 %tmp, 81041 %tmp8 = icmp eq i64 %tmp7, 10241042 br i1 %tmp8, label %bb9, label %bb11043 1044bb9: ; preds = %bb11045 ret void1046}1047 1048define void @bcast_unfold_fadd_v4f32(ptr nocapture %arg) {1049; CHECK-LABEL: bcast_unfold_fadd_v4f32:1050; CHECK: # %bb.0: # %bb1051; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001052; CHECK-NEXT: vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]1053; CHECK-NEXT: .p2align 41054; CHECK-NEXT: .LBB32_1: # %bb11055; CHECK-NEXT: # =>This Inner Loop Header: Depth=11056; CHECK-NEXT: vaddps 4096(%rdi,%rax), %xmm0, %xmm11057; CHECK-NEXT: vmovups %xmm1, 4096(%rdi,%rax)1058; CHECK-NEXT: addq $16, %rax1059; CHECK-NEXT: jne .LBB32_11060; CHECK-NEXT: # %bb.2: # %bb91061; CHECK-NEXT: retq1062bb:1063 br label %bb11064 1065bb1: ; preds = %bb1, %bb1066 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1067 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp1068 %tmp4 = load <4 x float>, ptr %tmp2, align 41069 %tmp5 = fadd <4 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>1070 store <4 x float> %tmp5, ptr %tmp2, align 41071 %tmp7 = add i64 %tmp, 41072 %tmp8 = icmp eq i64 %tmp7, 10241073 br i1 %tmp8, label %bb9, label %bb11074 1075bb9: ; preds = %bb11076 ret void1077}1078 1079define void @bcast_unfold_fadd_v8f64(ptr nocapture %arg) {1080; CHECK-LABEL: bcast_unfold_fadd_v8f64:1081; CHECK: # %bb.0: # %bb1082; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001083; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]1084; CHECK-NEXT: .p2align 41085; CHECK-NEXT: .LBB33_1: # %bb11086; CHECK-NEXT: # =>This Inner Loop Header: Depth=11087; CHECK-NEXT: vaddpd 8192(%rdi,%rax), %zmm0, %zmm11088; CHECK-NEXT: vmovupd %zmm1, 8192(%rdi,%rax)1089; CHECK-NEXT: addq $64, %rax1090; CHECK-NEXT: jne .LBB33_11091; CHECK-NEXT: # %bb.2: # %bb91092; CHECK-NEXT: vzeroupper1093; CHECK-NEXT: retq1094bb:1095 br label %bb11096 1097bb1: ; preds = %bb1, %bb1098 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1099 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp1100 %tmp4 = load <8 x double>, ptr %tmp2, align 81101 %tmp5 = fadd <8 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>1102 store <8 x double> %tmp5, ptr %tmp2, align 81103 %tmp7 = add i64 %tmp, 81104 %tmp8 = icmp eq i64 %tmp7, 10241105 br i1 %tmp8, label %bb9, label %bb11106 1107bb9: ; preds = %bb11108 ret void1109}1110 1111define void @bcast_unfold_fadd_v4f64(ptr nocapture %arg) {1112; CHECK-LABEL: bcast_unfold_fadd_v4f64:1113; CHECK: # %bb.0: # %bb1114; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001115; CHECK-NEXT: vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]1116; CHECK-NEXT: .p2align 41117; CHECK-NEXT: .LBB34_1: # %bb11118; CHECK-NEXT: # =>This Inner Loop Header: Depth=11119; CHECK-NEXT: vaddpd 8192(%rdi,%rax), %ymm0, %ymm11120; CHECK-NEXT: vmovupd %ymm1, 8192(%rdi,%rax)1121; CHECK-NEXT: addq $32, %rax1122; CHECK-NEXT: jne .LBB34_11123; CHECK-NEXT: # %bb.2: # %bb91124; CHECK-NEXT: vzeroupper1125; CHECK-NEXT: retq1126bb:1127 br label %bb11128 1129bb1: ; preds = %bb1, %bb1130 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1131 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp1132 %tmp4 = load <4 x double>, ptr %tmp2, align 81133 %tmp5 = fadd <4 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>1134 store <4 x double> %tmp5, ptr %tmp2, align 81135 %tmp7 = add i64 %tmp, 41136 %tmp8 = icmp eq i64 %tmp7, 10241137 br i1 %tmp8, label %bb9, label %bb11138 1139bb9: ; preds = %bb11140 ret void1141}1142 1143define void @bcast_unfold_fadd_v2f64(ptr nocapture %arg) {1144; CHECK-LABEL: bcast_unfold_fadd_v2f64:1145; CHECK: # %bb.0: # %bb1146; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001147; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = [2.0E+0,2.0E+0]1148; CHECK-NEXT: # xmm0 = mem[0,0]1149; CHECK-NEXT: .p2align 41150; CHECK-NEXT: .LBB35_1: # %bb11151; CHECK-NEXT: # =>This Inner Loop Header: Depth=11152; CHECK-NEXT: vaddpd 8192(%rdi,%rax), %xmm0, %xmm11153; CHECK-NEXT: vmovupd %xmm1, 8192(%rdi,%rax)1154; CHECK-NEXT: addq $16, %rax1155; CHECK-NEXT: jne .LBB35_11156; CHECK-NEXT: # %bb.2: # %bb91157; CHECK-NEXT: retq1158bb:1159 br label %bb11160 1161bb1: ; preds = %bb1, %bb1162 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1163 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp1164 %tmp4 = load <2 x double>, ptr %tmp2, align 81165 %tmp5 = fadd <2 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00>1166 store <2 x double> %tmp5, ptr %tmp2, align 81167 %tmp7 = add i64 %tmp, 21168 %tmp8 = icmp eq i64 %tmp7, 10241169 br i1 %tmp8, label %bb9, label %bb11170 1171bb9: ; preds = %bb11172 ret void1173}1174 1175define void @bcast_unfold_fmul_v16f32(ptr nocapture %arg) {1176; CHECK-LABEL: bcast_unfold_fmul_v16f32:1177; CHECK: # %bb.0: # %bb1178; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001179; CHECK-NEXT: vbroadcastss {{.*#+}} zmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]1180; CHECK-NEXT: .p2align 41181; CHECK-NEXT: .LBB36_1: # %bb11182; CHECK-NEXT: # =>This Inner Loop Header: Depth=11183; CHECK-NEXT: vmulps 4096(%rdi,%rax), %zmm0, %zmm11184; CHECK-NEXT: vmovups %zmm1, 4096(%rdi,%rax)1185; CHECK-NEXT: addq $64, %rax1186; CHECK-NEXT: jne .LBB36_11187; CHECK-NEXT: # %bb.2: # %bb91188; CHECK-NEXT: vzeroupper1189; CHECK-NEXT: retq1190bb:1191 br label %bb11192 1193bb1: ; preds = %bb1, %bb1194 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1195 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp1196 %tmp4 = load <16 x float>, ptr %tmp2, align 41197 %tmp5 = fmul <16 x float> %tmp4, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>1198 store <16 x float> %tmp5, ptr %tmp2, align 41199 %tmp7 = add i64 %tmp, 161200 %tmp8 = icmp eq i64 %tmp7, 10241201 br i1 %tmp8, label %bb9, label %bb11202 1203bb9: ; preds = %bb11204 ret void1205}1206 1207define void @bcast_unfold_fmul_v8f32(ptr nocapture %arg) {1208; CHECK-LABEL: bcast_unfold_fmul_v8f32:1209; CHECK: # %bb.0: # %bb1210; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001211; CHECK-NEXT: vbroadcastss {{.*#+}} ymm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]1212; CHECK-NEXT: .p2align 41213; CHECK-NEXT: .LBB37_1: # %bb11214; CHECK-NEXT: # =>This Inner Loop Header: Depth=11215; CHECK-NEXT: vmulps 4096(%rdi,%rax), %ymm0, %ymm11216; CHECK-NEXT: vmovups %ymm1, 4096(%rdi,%rax)1217; CHECK-NEXT: addq $32, %rax1218; CHECK-NEXT: jne .LBB37_11219; CHECK-NEXT: # %bb.2: # %bb91220; CHECK-NEXT: vzeroupper1221; CHECK-NEXT: retq1222bb:1223 br label %bb11224 1225bb1: ; preds = %bb1, %bb1226 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1227 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp1228 %tmp4 = load <8 x float>, ptr %tmp2, align 41229 %tmp5 = fmul <8 x float> %tmp4, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>1230 store <8 x float> %tmp5, ptr %tmp2, align 41231 %tmp7 = add i64 %tmp, 81232 %tmp8 = icmp eq i64 %tmp7, 10241233 br i1 %tmp8, label %bb9, label %bb11234 1235bb9: ; preds = %bb11236 ret void1237}1238 1239define void @bcast_unfold_fmul_v4f32(ptr nocapture %arg) {1240; CHECK-LABEL: bcast_unfold_fmul_v4f32:1241; CHECK: # %bb.0: # %bb1242; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001243; CHECK-NEXT: vbroadcastss {{.*#+}} xmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]1244; CHECK-NEXT: .p2align 41245; CHECK-NEXT: .LBB38_1: # %bb11246; CHECK-NEXT: # =>This Inner Loop Header: Depth=11247; CHECK-NEXT: vmulps 4096(%rdi,%rax), %xmm0, %xmm11248; CHECK-NEXT: vmovups %xmm1, 4096(%rdi,%rax)1249; CHECK-NEXT: addq $16, %rax1250; CHECK-NEXT: jne .LBB38_11251; CHECK-NEXT: # %bb.2: # %bb91252; CHECK-NEXT: retq1253bb:1254 br label %bb11255 1256bb1: ; preds = %bb1, %bb1257 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1258 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp1259 %tmp4 = load <4 x float>, ptr %tmp2, align 41260 %tmp5 = fmul <4 x float> %tmp4, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>1261 store <4 x float> %tmp5, ptr %tmp2, align 41262 %tmp7 = add i64 %tmp, 41263 %tmp8 = icmp eq i64 %tmp7, 10241264 br i1 %tmp8, label %bb9, label %bb11265 1266bb9: ; preds = %bb11267 ret void1268}1269 1270define void @bcast_unfold_fmul_v8f64(ptr nocapture %arg) {1271; CHECK-LABEL: bcast_unfold_fmul_v8f64:1272; CHECK: # %bb.0: # %bb1273; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001274; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]1275; CHECK-NEXT: .p2align 41276; CHECK-NEXT: .LBB39_1: # %bb11277; CHECK-NEXT: # =>This Inner Loop Header: Depth=11278; CHECK-NEXT: vmulpd 8192(%rdi,%rax), %zmm0, %zmm11279; CHECK-NEXT: vmovupd %zmm1, 8192(%rdi,%rax)1280; CHECK-NEXT: addq $64, %rax1281; CHECK-NEXT: jne .LBB39_11282; CHECK-NEXT: # %bb.2: # %bb91283; CHECK-NEXT: vzeroupper1284; CHECK-NEXT: retq1285bb:1286 br label %bb11287 1288bb1: ; preds = %bb1, %bb1289 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1290 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp1291 %tmp4 = load <8 x double>, ptr %tmp2, align 81292 %tmp5 = fmul <8 x double> %tmp4, <double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00>1293 store <8 x double> %tmp5, ptr %tmp2, align 81294 %tmp7 = add i64 %tmp, 81295 %tmp8 = icmp eq i64 %tmp7, 10241296 br i1 %tmp8, label %bb9, label %bb11297 1298bb9: ; preds = %bb11299 ret void1300}1301 1302define void @bcast_unfold_fmul_v4f64(ptr nocapture %arg) {1303; CHECK-LABEL: bcast_unfold_fmul_v4f64:1304; CHECK: # %bb.0: # %bb1305; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001306; CHECK-NEXT: vbroadcastsd {{.*#+}} ymm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]1307; CHECK-NEXT: .p2align 41308; CHECK-NEXT: .LBB40_1: # %bb11309; CHECK-NEXT: # =>This Inner Loop Header: Depth=11310; CHECK-NEXT: vmulpd 8192(%rdi,%rax), %ymm0, %ymm11311; CHECK-NEXT: vmovupd %ymm1, 8192(%rdi,%rax)1312; CHECK-NEXT: addq $32, %rax1313; CHECK-NEXT: jne .LBB40_11314; CHECK-NEXT: # %bb.2: # %bb91315; CHECK-NEXT: vzeroupper1316; CHECK-NEXT: retq1317bb:1318 br label %bb11319 1320bb1: ; preds = %bb1, %bb1321 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1322 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp1323 %tmp4 = load <4 x double>, ptr %tmp2, align 81324 %tmp5 = fmul <4 x double> %tmp4, <double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00>1325 store <4 x double> %tmp5, ptr %tmp2, align 81326 %tmp7 = add i64 %tmp, 41327 %tmp8 = icmp eq i64 %tmp7, 10241328 br i1 %tmp8, label %bb9, label %bb11329 1330bb9: ; preds = %bb11331 ret void1332}1333 1334define void @bcast_unfold_fmul_v2f64(ptr nocapture %arg) {1335; CHECK-LABEL: bcast_unfold_fmul_v2f64:1336; CHECK: # %bb.0: # %bb1337; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001338; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = [3.0E+0,3.0E+0]1339; CHECK-NEXT: # xmm0 = mem[0,0]1340; CHECK-NEXT: .p2align 41341; CHECK-NEXT: .LBB41_1: # %bb11342; CHECK-NEXT: # =>This Inner Loop Header: Depth=11343; CHECK-NEXT: vmulpd 8192(%rdi,%rax), %xmm0, %xmm11344; CHECK-NEXT: vmovupd %xmm1, 8192(%rdi,%rax)1345; CHECK-NEXT: addq $16, %rax1346; CHECK-NEXT: jne .LBB41_11347; CHECK-NEXT: # %bb.2: # %bb91348; CHECK-NEXT: retq1349bb:1350 br label %bb11351 1352bb1: ; preds = %bb1, %bb1353 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1354 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp1355 %tmp4 = load <2 x double>, ptr %tmp2, align 81356 %tmp5 = fmul <2 x double> %tmp4, <double 3.000000e+00, double 3.000000e+00>1357 store <2 x double> %tmp5, ptr %tmp2, align 81358 %tmp7 = add i64 %tmp, 21359 %tmp8 = icmp eq i64 %tmp7, 10241360 br i1 %tmp8, label %bb9, label %bb11361 1362bb9: ; preds = %bb11363 ret void1364}1365 1366define void @bcast_unfold_fdiv_v16f32(ptr nocapture %arg) {1367; CHECK-LABEL: bcast_unfold_fdiv_v16f32:1368; CHECK: # %bb.0: # %bb1369; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001370; CHECK-NEXT: vbroadcastss {{.*#+}} zmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]1371; CHECK-NEXT: .p2align 41372; CHECK-NEXT: .LBB42_1: # %bb11373; CHECK-NEXT: # =>This Inner Loop Header: Depth=11374; CHECK-NEXT: vmovups 4096(%rdi,%rax), %zmm11375; CHECK-NEXT: vdivps %zmm0, %zmm1, %zmm11376; CHECK-NEXT: vmovups %zmm1, 4096(%rdi,%rax)1377; CHECK-NEXT: addq $64, %rax1378; CHECK-NEXT: jne .LBB42_11379; CHECK-NEXT: # %bb.2: # %bb91380; CHECK-NEXT: vzeroupper1381; CHECK-NEXT: retq1382bb:1383 br label %bb11384 1385bb1: ; preds = %bb1, %bb1386 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1387 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp1388 %tmp4 = load <16 x float>, ptr %tmp2, align 41389 %tmp5 = fdiv <16 x float> %tmp4, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>1390 store <16 x float> %tmp5, ptr %tmp2, align 41391 %tmp7 = add i64 %tmp, 161392 %tmp8 = icmp eq i64 %tmp7, 10241393 br i1 %tmp8, label %bb9, label %bb11394 1395bb9: ; preds = %bb11396 ret void1397}1398 1399define void @bcast_unfold_fdiv_v8f32(ptr nocapture %arg) {1400; CHECK-LABEL: bcast_unfold_fdiv_v8f32:1401; CHECK: # %bb.0: # %bb1402; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001403; CHECK-NEXT: vbroadcastss {{.*#+}} ymm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]1404; CHECK-NEXT: .p2align 41405; CHECK-NEXT: .LBB43_1: # %bb11406; CHECK-NEXT: # =>This Inner Loop Header: Depth=11407; CHECK-NEXT: vmovups 4096(%rdi,%rax), %ymm11408; CHECK-NEXT: vdivps %ymm0, %ymm1, %ymm11409; CHECK-NEXT: vmovups %ymm1, 4096(%rdi,%rax)1410; CHECK-NEXT: addq $32, %rax1411; CHECK-NEXT: jne .LBB43_11412; CHECK-NEXT: # %bb.2: # %bb91413; CHECK-NEXT: vzeroupper1414; CHECK-NEXT: retq1415bb:1416 br label %bb11417 1418bb1: ; preds = %bb1, %bb1419 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1420 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp1421 %tmp4 = load <8 x float>, ptr %tmp2, align 41422 %tmp5 = fdiv <8 x float> %tmp4, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>1423 store <8 x float> %tmp5, ptr %tmp2, align 41424 %tmp7 = add i64 %tmp, 81425 %tmp8 = icmp eq i64 %tmp7, 10241426 br i1 %tmp8, label %bb9, label %bb11427 1428bb9: ; preds = %bb11429 ret void1430}1431 1432define void @bcast_unfold_fdiv_v4f32(ptr nocapture %arg) {1433; CHECK-LABEL: bcast_unfold_fdiv_v4f32:1434; CHECK: # %bb.0: # %bb1435; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001436; CHECK-NEXT: vbroadcastss {{.*#+}} xmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]1437; CHECK-NEXT: .p2align 41438; CHECK-NEXT: .LBB44_1: # %bb11439; CHECK-NEXT: # =>This Inner Loop Header: Depth=11440; CHECK-NEXT: vmovups 4096(%rdi,%rax), %xmm11441; CHECK-NEXT: vdivps %xmm0, %xmm1, %xmm11442; CHECK-NEXT: vmovups %xmm1, 4096(%rdi,%rax)1443; CHECK-NEXT: addq $16, %rax1444; CHECK-NEXT: jne .LBB44_11445; CHECK-NEXT: # %bb.2: # %bb91446; CHECK-NEXT: retq1447bb:1448 br label %bb11449 1450bb1: ; preds = %bb1, %bb1451 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1452 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp1453 %tmp4 = load <4 x float>, ptr %tmp2, align 41454 %tmp5 = fdiv <4 x float> %tmp4, <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>1455 store <4 x float> %tmp5, ptr %tmp2, align 41456 %tmp7 = add i64 %tmp, 41457 %tmp8 = icmp eq i64 %tmp7, 10241458 br i1 %tmp8, label %bb9, label %bb11459 1460bb9: ; preds = %bb11461 ret void1462}1463 1464define void @bcast_unfold_fdiv_v8f64(ptr nocapture %arg) {1465; CHECK-LABEL: bcast_unfold_fdiv_v8f64:1466; CHECK: # %bb.0: # %bb1467; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001468; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]1469; CHECK-NEXT: .p2align 41470; CHECK-NEXT: .LBB45_1: # %bb11471; CHECK-NEXT: # =>This Inner Loop Header: Depth=11472; CHECK-NEXT: vmovupd 8192(%rdi,%rax), %zmm11473; CHECK-NEXT: vdivpd %zmm0, %zmm1, %zmm11474; CHECK-NEXT: vmovupd %zmm1, 8192(%rdi,%rax)1475; CHECK-NEXT: addq $64, %rax1476; CHECK-NEXT: jne .LBB45_11477; CHECK-NEXT: # %bb.2: # %bb91478; CHECK-NEXT: vzeroupper1479; CHECK-NEXT: retq1480bb:1481 br label %bb11482 1483bb1: ; preds = %bb1, %bb1484 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1485 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp1486 %tmp4 = load <8 x double>, ptr %tmp2, align 81487 %tmp5 = fdiv <8 x double> %tmp4, <double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00>1488 store <8 x double> %tmp5, ptr %tmp2, align 81489 %tmp7 = add i64 %tmp, 81490 %tmp8 = icmp eq i64 %tmp7, 10241491 br i1 %tmp8, label %bb9, label %bb11492 1493bb9: ; preds = %bb11494 ret void1495}1496 1497define void @bcast_unfold_fdiv_v4f64(ptr nocapture %arg) {1498; CHECK-LABEL: bcast_unfold_fdiv_v4f64:1499; CHECK: # %bb.0: # %bb1500; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001501; CHECK-NEXT: vbroadcastsd {{.*#+}} ymm0 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]1502; CHECK-NEXT: .p2align 41503; CHECK-NEXT: .LBB46_1: # %bb11504; CHECK-NEXT: # =>This Inner Loop Header: Depth=11505; CHECK-NEXT: vmovupd 8192(%rdi,%rax), %ymm11506; CHECK-NEXT: vdivpd %ymm0, %ymm1, %ymm11507; CHECK-NEXT: vmovupd %ymm1, 8192(%rdi,%rax)1508; CHECK-NEXT: addq $32, %rax1509; CHECK-NEXT: jne .LBB46_11510; CHECK-NEXT: # %bb.2: # %bb91511; CHECK-NEXT: vzeroupper1512; CHECK-NEXT: retq1513bb:1514 br label %bb11515 1516bb1: ; preds = %bb1, %bb1517 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1518 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp1519 %tmp4 = load <4 x double>, ptr %tmp2, align 81520 %tmp5 = fdiv <4 x double> %tmp4, <double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00>1521 store <4 x double> %tmp5, ptr %tmp2, align 81522 %tmp7 = add i64 %tmp, 41523 %tmp8 = icmp eq i64 %tmp7, 10241524 br i1 %tmp8, label %bb9, label %bb11525 1526bb9: ; preds = %bb11527 ret void1528}1529 1530define void @bcast_unfold_fdiv_v2f64(ptr nocapture %arg) {1531; CHECK-LABEL: bcast_unfold_fdiv_v2f64:1532; CHECK: # %bb.0: # %bb1533; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001534; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = [3.0E+0,3.0E+0]1535; CHECK-NEXT: # xmm0 = mem[0,0]1536; CHECK-NEXT: .p2align 41537; CHECK-NEXT: .LBB47_1: # %bb11538; CHECK-NEXT: # =>This Inner Loop Header: Depth=11539; CHECK-NEXT: vmovupd 8192(%rdi,%rax), %xmm11540; CHECK-NEXT: vdivpd %xmm0, %xmm1, %xmm11541; CHECK-NEXT: vmovupd %xmm1, 8192(%rdi,%rax)1542; CHECK-NEXT: addq $16, %rax1543; CHECK-NEXT: jne .LBB47_11544; CHECK-NEXT: # %bb.2: # %bb91545; CHECK-NEXT: retq1546bb:1547 br label %bb11548 1549bb1: ; preds = %bb1, %bb1550 %tmp = phi i64 [ 0, %bb ], [ %tmp7, %bb1 ]1551 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp1552 %tmp4 = load <2 x double>, ptr %tmp2, align 81553 %tmp5 = fdiv <2 x double> %tmp4, <double 3.000000e+00, double 3.000000e+00>1554 store <2 x double> %tmp5, ptr %tmp2, align 81555 %tmp7 = add i64 %tmp, 21556 %tmp8 = icmp eq i64 %tmp7, 10241557 br i1 %tmp8, label %bb9, label %bb11558 1559bb9: ; preds = %bb11560 ret void1561}1562 1563define void @bcast_unfold_fma213_v4f32(ptr %arg) {1564; CHECK-LABEL: bcast_unfold_fma213_v4f32:1565; CHECK: # %bb.0: # %bb1566; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001567; CHECK-NEXT: vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]1568; CHECK-NEXT: .p2align 41569; CHECK-NEXT: .LBB48_1: # %bb21570; CHECK-NEXT: # =>This Inner Loop Header: Depth=11571; CHECK-NEXT: vmovups 4096(%rdi,%rax), %xmm11572; CHECK-NEXT: vfmadd213ps {{.*#+}} xmm1 = (xmm1 * xmm1) + xmm01573; CHECK-NEXT: vmovups %xmm1, 4096(%rdi,%rax)1574; CHECK-NEXT: addq $16, %rax1575; CHECK-NEXT: jne .LBB48_11576; CHECK-NEXT: # %bb.2: # %bb111577; CHECK-NEXT: retq1578bb:1579 br label %bb21580 1581bb2: ; preds = %bb2, %bb1582 %tmp = phi i64 [ 0, %bb ], [ %tmp9, %bb2 ]1583 %tmp3 = getelementptr inbounds float, ptr %arg, i64 %tmp1584 %tmp5 = load <4 x float>, ptr %tmp3, align 41585 %tmp6 = fmul contract <4 x float> %tmp5, %tmp51586 %tmp7 = fadd contract <4 x float> %tmp6, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>1587 store <4 x float> %tmp7, ptr %tmp3, align 41588 %tmp9 = add i64 %tmp, 41589 %tmp10 = icmp eq i64 %tmp9, 10241590 br i1 %tmp10, label %bb11, label %bb21591 1592bb11: ; preds = %bb21593 ret void1594}1595 1596define void @bcast_unfold_fma231_v4f32(ptr %arg) {1597; CHECK-LABEL: bcast_unfold_fma231_v4f32:1598; CHECK: # %bb.0: # %bb1599; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001600; CHECK-NEXT: vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]1601; CHECK-NEXT: .p2align 41602; CHECK-NEXT: .LBB49_1: # %bb11603; CHECK-NEXT: # =>This Inner Loop Header: Depth=11604; CHECK-NEXT: vmovups 4096(%rdi,%rax), %xmm11605; CHECK-NEXT: vfmadd231ps {{.*#+}} xmm1 = (xmm1 * xmm0) + xmm11606; CHECK-NEXT: vmovups %xmm1, 4096(%rdi,%rax)1607; CHECK-NEXT: addq $16, %rax1608; CHECK-NEXT: jne .LBB49_11609; CHECK-NEXT: # %bb.2: # %bb101610; CHECK-NEXT: retq1611bb:1612 br label %bb11613 1614bb1: ; preds = %bb1, %bb1615 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]1616 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp1617 %tmp4 = load <4 x float>, ptr %tmp2, align 41618 %tmp5 = fmul contract <4 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>1619 %tmp6 = fadd contract <4 x float> %tmp4, %tmp51620 store <4 x float> %tmp6, ptr %tmp2, align 41621 %tmp8 = add i64 %tmp, 41622 %tmp9 = icmp eq i64 %tmp8, 10241623 br i1 %tmp9, label %bb10, label %bb11624 1625bb10: ; preds = %bb11626 ret void1627}1628 1629define void @bcast_unfold_fma213_v8f32(ptr %arg) {1630; CHECK-LABEL: bcast_unfold_fma213_v8f32:1631; CHECK: # %bb.0: # %bb1632; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001633; CHECK-NEXT: vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]1634; CHECK-NEXT: .p2align 41635; CHECK-NEXT: .LBB50_1: # %bb21636; CHECK-NEXT: # =>This Inner Loop Header: Depth=11637; CHECK-NEXT: vmovups 4096(%rdi,%rax), %ymm11638; CHECK-NEXT: vfmadd213ps {{.*#+}} ymm1 = (ymm1 * ymm1) + ymm01639; CHECK-NEXT: vmovups %ymm1, 4096(%rdi,%rax)1640; CHECK-NEXT: addq $32, %rax1641; CHECK-NEXT: jne .LBB50_11642; CHECK-NEXT: # %bb.2: # %bb111643; CHECK-NEXT: vzeroupper1644; CHECK-NEXT: retq1645bb:1646 br label %bb21647 1648bb2: ; preds = %bb2, %bb1649 %tmp = phi i64 [ 0, %bb ], [ %tmp9, %bb2 ]1650 %tmp3 = getelementptr inbounds float, ptr %arg, i64 %tmp1651 %tmp5 = load <8 x float>, ptr %tmp3, align 41652 %tmp6 = fmul contract <8 x float> %tmp5, %tmp51653 %tmp7 = fadd contract <8 x float> %tmp6, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>1654 store <8 x float> %tmp7, ptr %tmp3, align 41655 %tmp9 = add i64 %tmp, 81656 %tmp10 = icmp eq i64 %tmp9, 10241657 br i1 %tmp10, label %bb11, label %bb21658 1659bb11: ; preds = %bb21660 ret void1661}1662 1663define void @bcast_unfold_fma231_v8f32(ptr %arg) {1664; CHECK-LABEL: bcast_unfold_fma231_v8f32:1665; CHECK: # %bb.0: # %bb1666; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001667; CHECK-NEXT: vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]1668; CHECK-NEXT: .p2align 41669; CHECK-NEXT: .LBB51_1: # %bb11670; CHECK-NEXT: # =>This Inner Loop Header: Depth=11671; CHECK-NEXT: vmovups 4096(%rdi,%rax), %ymm11672; CHECK-NEXT: vfmadd231ps {{.*#+}} ymm1 = (ymm1 * ymm0) + ymm11673; CHECK-NEXT: vmovups %ymm1, 4096(%rdi,%rax)1674; CHECK-NEXT: addq $32, %rax1675; CHECK-NEXT: jne .LBB51_11676; CHECK-NEXT: # %bb.2: # %bb101677; CHECK-NEXT: vzeroupper1678; CHECK-NEXT: retq1679bb:1680 br label %bb11681 1682bb1: ; preds = %bb1, %bb1683 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]1684 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp1685 %tmp4 = load <8 x float>, ptr %tmp2, align 41686 %tmp5 = fmul contract <8 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>1687 %tmp6 = fadd contract <8 x float> %tmp4, %tmp51688 store <8 x float> %tmp6, ptr %tmp2, align 41689 %tmp8 = add i64 %tmp, 81690 %tmp9 = icmp eq i64 %tmp8, 10241691 br i1 %tmp9, label %bb10, label %bb11692 1693bb10: ; preds = %bb11694 ret void1695}1696 1697define void @bcast_unfold_fma213_v16f32(ptr %arg) {1698; CHECK-LABEL: bcast_unfold_fma213_v16f32:1699; CHECK: # %bb.0: # %bb1700; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001701; CHECK-NEXT: vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]1702; CHECK-NEXT: .p2align 41703; CHECK-NEXT: .LBB52_1: # %bb21704; CHECK-NEXT: # =>This Inner Loop Header: Depth=11705; CHECK-NEXT: vmovups 4096(%rdi,%rax), %zmm11706; CHECK-NEXT: vfmadd213ps {{.*#+}} zmm1 = (zmm1 * zmm1) + zmm01707; CHECK-NEXT: vmovups %zmm1, 4096(%rdi,%rax)1708; CHECK-NEXT: addq $64, %rax1709; CHECK-NEXT: jne .LBB52_11710; CHECK-NEXT: # %bb.2: # %bb111711; CHECK-NEXT: vzeroupper1712; CHECK-NEXT: retq1713bb:1714 br label %bb21715 1716bb2: ; preds = %bb2, %bb1717 %tmp = phi i64 [ 0, %bb ], [ %tmp9, %bb2 ]1718 %tmp3 = getelementptr inbounds float, ptr %arg, i64 %tmp1719 %tmp5 = load <16 x float>, ptr %tmp3, align 41720 %tmp6 = fmul contract <16 x float> %tmp5, %tmp51721 %tmp7 = fadd contract <16 x float> %tmp6, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>1722 store <16 x float> %tmp7, ptr %tmp3, align 41723 %tmp9 = add i64 %tmp, 161724 %tmp10 = icmp eq i64 %tmp9, 10241725 br i1 %tmp10, label %bb11, label %bb21726 1727bb11: ; preds = %bb21728 ret void1729}1730 1731define void @bcast_unfold_fma231_v16f32(ptr %arg) {1732; CHECK-LABEL: bcast_unfold_fma231_v16f32:1733; CHECK: # %bb.0: # %bb1734; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001735; CHECK-NEXT: vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]1736; CHECK-NEXT: .p2align 41737; CHECK-NEXT: .LBB53_1: # %bb11738; CHECK-NEXT: # =>This Inner Loop Header: Depth=11739; CHECK-NEXT: vmovups 4096(%rdi,%rax), %zmm11740; CHECK-NEXT: vfmadd231ps {{.*#+}} zmm1 = (zmm1 * zmm0) + zmm11741; CHECK-NEXT: vmovups %zmm1, 4096(%rdi,%rax)1742; CHECK-NEXT: addq $64, %rax1743; CHECK-NEXT: jne .LBB53_11744; CHECK-NEXT: # %bb.2: # %bb101745; CHECK-NEXT: vzeroupper1746; CHECK-NEXT: retq1747bb:1748 br label %bb11749 1750bb1: ; preds = %bb1, %bb1751 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]1752 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp1753 %tmp4 = load <16 x float>, ptr %tmp2, align 41754 %tmp5 = fmul contract <16 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>1755 %tmp6 = fadd contract <16 x float> %tmp4, %tmp51756 store <16 x float> %tmp6, ptr %tmp2, align 41757 %tmp8 = add i64 %tmp, 161758 %tmp9 = icmp eq i64 %tmp8, 10241759 br i1 %tmp9, label %bb10, label %bb11760 1761bb10: ; preds = %bb11762 ret void1763}1764 1765define void @bcast_unfold_fma213_v2f64(ptr %arg) {1766; CHECK-LABEL: bcast_unfold_fma213_v2f64:1767; CHECK: # %bb.0: # %bb1768; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001769; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = [2.0E+0,2.0E+0]1770; CHECK-NEXT: # xmm0 = mem[0,0]1771; CHECK-NEXT: .p2align 41772; CHECK-NEXT: .LBB54_1: # %bb21773; CHECK-NEXT: # =>This Inner Loop Header: Depth=11774; CHECK-NEXT: vmovupd 8192(%rdi,%rax), %xmm11775; CHECK-NEXT: vfmadd213pd {{.*#+}} xmm1 = (xmm1 * xmm1) + xmm01776; CHECK-NEXT: vmovupd %xmm1, 8192(%rdi,%rax)1777; CHECK-NEXT: addq $16, %rax1778; CHECK-NEXT: jne .LBB54_11779; CHECK-NEXT: # %bb.2: # %bb111780; CHECK-NEXT: retq1781bb:1782 br label %bb21783 1784bb2: ; preds = %bb2, %bb1785 %tmp = phi i64 [ 0, %bb ], [ %tmp9, %bb2 ]1786 %tmp3 = getelementptr inbounds double, ptr %arg, i64 %tmp1787 %tmp5 = load <2 x double>, ptr %tmp3, align 41788 %tmp6 = fmul contract <2 x double> %tmp5, %tmp51789 %tmp7 = fadd contract <2 x double> %tmp6, <double 2.000000e+00, double 2.000000e+00>1790 store <2 x double> %tmp7, ptr %tmp3, align 81791 %tmp9 = add i64 %tmp, 21792 %tmp10 = icmp eq i64 %tmp9, 10241793 br i1 %tmp10, label %bb11, label %bb21794 1795bb11: ; preds = %bb21796 ret void1797}1798 1799define void @bcast_unfold_fma231_v2f64(ptr %arg) {1800; CHECK-LABEL: bcast_unfold_fma231_v2f64:1801; CHECK: # %bb.0: # %bb1802; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001803; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = [2.0E+0,2.0E+0]1804; CHECK-NEXT: # xmm0 = mem[0,0]1805; CHECK-NEXT: .p2align 41806; CHECK-NEXT: .LBB55_1: # %bb11807; CHECK-NEXT: # =>This Inner Loop Header: Depth=11808; CHECK-NEXT: vmovupd 8192(%rdi,%rax), %xmm11809; CHECK-NEXT: vfmadd231pd {{.*#+}} xmm1 = (xmm1 * xmm0) + xmm11810; CHECK-NEXT: vmovupd %xmm1, 8192(%rdi,%rax)1811; CHECK-NEXT: addq $16, %rax1812; CHECK-NEXT: jne .LBB55_11813; CHECK-NEXT: # %bb.2: # %bb101814; CHECK-NEXT: retq1815bb:1816 br label %bb11817 1818bb1: ; preds = %bb1, %bb1819 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]1820 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp1821 %tmp4 = load <2 x double>, ptr %tmp2, align 81822 %tmp5 = fmul contract <2 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00>1823 %tmp6 = fadd contract <2 x double> %tmp4, %tmp51824 store <2 x double> %tmp6, ptr %tmp2, align 81825 %tmp8 = add i64 %tmp, 21826 %tmp9 = icmp eq i64 %tmp8, 10241827 br i1 %tmp9, label %bb10, label %bb11828 1829bb10: ; preds = %bb11830 ret void1831}1832 1833define void @bcast_unfold_fma213_v4f64(ptr %arg) {1834; CHECK-LABEL: bcast_unfold_fma213_v4f64:1835; CHECK: # %bb.0: # %bb1836; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001837; CHECK-NEXT: vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]1838; CHECK-NEXT: .p2align 41839; CHECK-NEXT: .LBB56_1: # %bb21840; CHECK-NEXT: # =>This Inner Loop Header: Depth=11841; CHECK-NEXT: vmovupd 8192(%rdi,%rax), %ymm11842; CHECK-NEXT: vfmadd213pd {{.*#+}} ymm1 = (ymm1 * ymm1) + ymm01843; CHECK-NEXT: vmovupd %ymm1, 8192(%rdi,%rax)1844; CHECK-NEXT: addq $32, %rax1845; CHECK-NEXT: jne .LBB56_11846; CHECK-NEXT: # %bb.2: # %bb111847; CHECK-NEXT: vzeroupper1848; CHECK-NEXT: retq1849bb:1850 br label %bb21851 1852bb2: ; preds = %bb2, %bb1853 %tmp = phi i64 [ 0, %bb ], [ %tmp9, %bb2 ]1854 %tmp3 = getelementptr inbounds double, ptr %arg, i64 %tmp1855 %tmp5 = load <4 x double>, ptr %tmp3, align 81856 %tmp6 = fmul contract <4 x double> %tmp5, %tmp51857 %tmp7 = fadd contract <4 x double> %tmp6, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>1858 store <4 x double> %tmp7, ptr %tmp3, align 81859 %tmp9 = add i64 %tmp, 41860 %tmp10 = icmp eq i64 %tmp9, 10241861 br i1 %tmp10, label %bb11, label %bb21862 1863bb11: ; preds = %bb21864 ret void1865}1866 1867define void @bcast_unfold_fma231_v4f64(ptr %arg) {1868; CHECK-LABEL: bcast_unfold_fma231_v4f64:1869; CHECK: # %bb.0: # %bb1870; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001871; CHECK-NEXT: vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]1872; CHECK-NEXT: .p2align 41873; CHECK-NEXT: .LBB57_1: # %bb11874; CHECK-NEXT: # =>This Inner Loop Header: Depth=11875; CHECK-NEXT: vmovupd 8192(%rdi,%rax), %ymm11876; CHECK-NEXT: vfmadd231pd {{.*#+}} ymm1 = (ymm1 * ymm0) + ymm11877; CHECK-NEXT: vmovupd %ymm1, 8192(%rdi,%rax)1878; CHECK-NEXT: addq $32, %rax1879; CHECK-NEXT: jne .LBB57_11880; CHECK-NEXT: # %bb.2: # %bb101881; CHECK-NEXT: vzeroupper1882; CHECK-NEXT: retq1883bb:1884 br label %bb11885 1886bb1: ; preds = %bb1, %bb1887 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]1888 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp1889 %tmp4 = load <4 x double>, ptr %tmp2, align 81890 %tmp5 = fmul contract <4 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>1891 %tmp6 = fadd contract <4 x double> %tmp4, %tmp51892 store <4 x double> %tmp6, ptr %tmp2, align 81893 %tmp8 = add i64 %tmp, 41894 %tmp9 = icmp eq i64 %tmp8, 10241895 br i1 %tmp9, label %bb10, label %bb11896 1897bb10: ; preds = %bb11898 ret void1899}1900 1901define void @bcast_unfold_fma213_v8f64(ptr %arg) {1902; CHECK-LABEL: bcast_unfold_fma213_v8f64:1903; CHECK: # %bb.0: # %bb1904; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001905; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]1906; CHECK-NEXT: .p2align 41907; CHECK-NEXT: .LBB58_1: # %bb21908; CHECK-NEXT: # =>This Inner Loop Header: Depth=11909; CHECK-NEXT: vmovupd 8192(%rdi,%rax), %zmm11910; CHECK-NEXT: vfmadd213pd {{.*#+}} zmm1 = (zmm1 * zmm1) + zmm01911; CHECK-NEXT: vmovupd %zmm1, 8192(%rdi,%rax)1912; CHECK-NEXT: addq $64, %rax1913; CHECK-NEXT: jne .LBB58_11914; CHECK-NEXT: # %bb.2: # %bb111915; CHECK-NEXT: vzeroupper1916; CHECK-NEXT: retq1917bb:1918 br label %bb21919 1920bb2: ; preds = %bb2, %bb1921 %tmp = phi i64 [ 0, %bb ], [ %tmp9, %bb2 ]1922 %tmp3 = getelementptr inbounds double, ptr %arg, i64 %tmp1923 %tmp5 = load <8 x double>, ptr %tmp3, align 81924 %tmp6 = fmul contract <8 x double> %tmp5, %tmp51925 %tmp7 = fadd contract <8 x double> %tmp6, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>1926 store <8 x double> %tmp7, ptr %tmp3, align 81927 %tmp9 = add i64 %tmp, 81928 %tmp10 = icmp eq i64 %tmp9, 10241929 br i1 %tmp10, label %bb11, label %bb21930 1931bb11: ; preds = %bb21932 ret void1933}1934 1935define void @bcast_unfold_fma231_v8f64(ptr %arg) {1936; CHECK-LABEL: bcast_unfold_fma231_v8f64:1937; CHECK: # %bb.0: # %bb1938; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0001939; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]1940; CHECK-NEXT: .p2align 41941; CHECK-NEXT: .LBB59_1: # %bb11942; CHECK-NEXT: # =>This Inner Loop Header: Depth=11943; CHECK-NEXT: vmovupd 8192(%rdi,%rax), %zmm11944; CHECK-NEXT: vfmadd231pd {{.*#+}} zmm1 = (zmm1 * zmm0) + zmm11945; CHECK-NEXT: vmovupd %zmm1, 8192(%rdi,%rax)1946; CHECK-NEXT: addq $64, %rax1947; CHECK-NEXT: jne .LBB59_11948; CHECK-NEXT: # %bb.2: # %bb101949; CHECK-NEXT: vzeroupper1950; CHECK-NEXT: retq1951bb:1952 br label %bb11953 1954bb1: ; preds = %bb1, %bb1955 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]1956 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp1957 %tmp4 = load <8 x double>, ptr %tmp2, align 81958 %tmp5 = fmul contract <8 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>1959 %tmp6 = fadd contract <8 x double> %tmp4, %tmp51960 store <8 x double> %tmp6, ptr %tmp2, align 81961 %tmp8 = add i64 %tmp, 81962 %tmp9 = icmp eq i64 %tmp8, 10241963 br i1 %tmp9, label %bb10, label %bb11964 1965bb10: ; preds = %bb11966 ret void1967}1968 1969define void @bcast_unfold_fmax_v4f32(ptr %arg) {1970; CHECK-LABEL: bcast_unfold_fmax_v4f32:1971; CHECK: # %bb.0: # %bb1972; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0001973; CHECK-NEXT: vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]1974; CHECK-NEXT: .p2align 41975; CHECK-NEXT: .LBB60_1: # %bb11976; CHECK-NEXT: # =>This Inner Loop Header: Depth=11977; CHECK-NEXT: vcmpnltps 4096(%rdi,%rax), %xmm0, %k11978; CHECK-NEXT: vmovups %xmm0, 4096(%rdi,%rax) {%k1}1979; CHECK-NEXT: addq $16, %rax1980; CHECK-NEXT: jne .LBB60_11981; CHECK-NEXT: # %bb.2: # %bb101982; CHECK-NEXT: retq1983bb:1984 br label %bb11985 1986bb1: ; preds = %bb1, %bb1987 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]1988 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp1989 %tmp4 = load <4 x float>, ptr %tmp2, align 41990 %tmp5 = fcmp ogt <4 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>1991 %tmp6 = select <4 x i1> %tmp5, <4 x float> %tmp4, <4 x float> <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>1992 store <4 x float> %tmp6, ptr %tmp2, align 41993 %tmp8 = add i64 %tmp, 41994 %tmp9 = icmp eq i64 %tmp8, 10241995 br i1 %tmp9, label %bb10, label %bb11996 1997bb10: ; preds = %bb11998 ret void1999}2000 2001define void @bcast_unfold_fmax_v8f32(ptr %arg) {2002; CHECK-LABEL: bcast_unfold_fmax_v8f32:2003; CHECK: # %bb.0: # %bb2004; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002005; CHECK-NEXT: vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]2006; CHECK-NEXT: .p2align 42007; CHECK-NEXT: .LBB61_1: # %bb12008; CHECK-NEXT: # =>This Inner Loop Header: Depth=12009; CHECK-NEXT: vcmpnltps 4096(%rdi,%rax), %ymm0, %k12010; CHECK-NEXT: vmovups %ymm0, 4096(%rdi,%rax) {%k1}2011; CHECK-NEXT: addq $32, %rax2012; CHECK-NEXT: jne .LBB61_12013; CHECK-NEXT: # %bb.2: # %bb102014; CHECK-NEXT: vzeroupper2015; CHECK-NEXT: retq2016bb:2017 br label %bb12018 2019bb1: ; preds = %bb1, %bb2020 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2021 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp2022 %tmp4 = load <8 x float>, ptr %tmp2, align 42023 %tmp5 = fcmp ogt <8 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>2024 %tmp6 = select <8 x i1> %tmp5, <8 x float> %tmp4, <8 x float> <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>2025 store <8 x float> %tmp6, ptr %tmp2, align 42026 %tmp8 = add i64 %tmp, 82027 %tmp9 = icmp eq i64 %tmp8, 10242028 br i1 %tmp9, label %bb10, label %bb12029 2030bb10: ; preds = %bb12031 ret void2032}2033 2034define void @bcast_unfold_fmax_v16f32(ptr %arg) {2035; CHECK-LABEL: bcast_unfold_fmax_v16f32:2036; CHECK: # %bb.0: # %bb2037; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002038; CHECK-NEXT: vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]2039; CHECK-NEXT: .p2align 42040; CHECK-NEXT: .LBB62_1: # %bb12041; CHECK-NEXT: # =>This Inner Loop Header: Depth=12042; CHECK-NEXT: vcmpnltps 4096(%rdi,%rax), %zmm0, %k12043; CHECK-NEXT: vmovups %zmm0, 4096(%rdi,%rax) {%k1}2044; CHECK-NEXT: addq $64, %rax2045; CHECK-NEXT: jne .LBB62_12046; CHECK-NEXT: # %bb.2: # %bb102047; CHECK-NEXT: vzeroupper2048; CHECK-NEXT: retq2049bb:2050 br label %bb12051 2052bb1: ; preds = %bb1, %bb2053 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2054 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp2055 %tmp4 = load <16 x float>, ptr %tmp2, align 42056 %tmp5 = fcmp ogt <16 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>2057 %tmp6 = select <16 x i1> %tmp5, <16 x float> %tmp4, <16 x float> <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>2058 store <16 x float> %tmp6, ptr %tmp2, align 42059 %tmp8 = add i64 %tmp, 162060 %tmp9 = icmp eq i64 %tmp8, 10242061 br i1 %tmp9, label %bb10, label %bb12062 2063bb10: ; preds = %bb12064 ret void2065}2066 2067define void @bcast_unfold_fmax_v2f64(ptr %arg) {2068; CHECK-LABEL: bcast_unfold_fmax_v2f64:2069; CHECK: # %bb.0: # %bb2070; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002071; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = [2.0E+0,2.0E+0]2072; CHECK-NEXT: # xmm0 = mem[0,0]2073; CHECK-NEXT: .p2align 42074; CHECK-NEXT: .LBB63_1: # %bb12075; CHECK-NEXT: # =>This Inner Loop Header: Depth=12076; CHECK-NEXT: vcmpnltpd 8192(%rdi,%rax), %xmm0, %k12077; CHECK-NEXT: vmovupd %xmm0, 8192(%rdi,%rax) {%k1}2078; CHECK-NEXT: addq $16, %rax2079; CHECK-NEXT: jne .LBB63_12080; CHECK-NEXT: # %bb.2: # %bb102081; CHECK-NEXT: retq2082bb:2083 br label %bb12084 2085bb1: ; preds = %bb1, %bb2086 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2087 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp2088 %tmp4 = load <2 x double>, ptr %tmp2, align 82089 %tmp5 = fcmp ogt <2 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00>2090 %tmp6 = select <2 x i1> %tmp5, <2 x double> %tmp4, <2 x double> <double 2.000000e+00, double 2.000000e+00>2091 store <2 x double> %tmp6, ptr %tmp2, align 82092 %tmp8 = add i64 %tmp, 22093 %tmp9 = icmp eq i64 %tmp8, 10242094 br i1 %tmp9, label %bb10, label %bb12095 2096bb10: ; preds = %bb12097 ret void2098}2099 2100define void @bcast_unfold_fmax_v4f64(ptr %arg) {2101; CHECK-LABEL: bcast_unfold_fmax_v4f64:2102; CHECK: # %bb.0: # %bb2103; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002104; CHECK-NEXT: vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]2105; CHECK-NEXT: .p2align 42106; CHECK-NEXT: .LBB64_1: # %bb12107; CHECK-NEXT: # =>This Inner Loop Header: Depth=12108; CHECK-NEXT: vcmpnltpd 8192(%rdi,%rax), %ymm0, %k12109; CHECK-NEXT: vmovupd %ymm0, 8192(%rdi,%rax) {%k1}2110; CHECK-NEXT: addq $32, %rax2111; CHECK-NEXT: jne .LBB64_12112; CHECK-NEXT: # %bb.2: # %bb102113; CHECK-NEXT: vzeroupper2114; CHECK-NEXT: retq2115bb:2116 br label %bb12117 2118bb1: ; preds = %bb1, %bb2119 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2120 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp2121 %tmp4 = load <4 x double>, ptr %tmp2, align 82122 %tmp5 = fcmp ogt <4 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>2123 %tmp6 = select <4 x i1> %tmp5, <4 x double> %tmp4, <4 x double> <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>2124 store <4 x double> %tmp6, ptr %tmp2, align 82125 %tmp8 = add i64 %tmp, 42126 %tmp9 = icmp eq i64 %tmp8, 10242127 br i1 %tmp9, label %bb10, label %bb12128 2129bb10: ; preds = %bb12130 ret void2131}2132 2133define void @bcast_unfold_fmax_v8f64(ptr %arg) {2134; CHECK-LABEL: bcast_unfold_fmax_v8f64:2135; CHECK: # %bb.0: # %bb2136; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002137; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]2138; CHECK-NEXT: .p2align 42139; CHECK-NEXT: .LBB65_1: # %bb12140; CHECK-NEXT: # =>This Inner Loop Header: Depth=12141; CHECK-NEXT: vcmpnltpd 8192(%rdi,%rax), %zmm0, %k12142; CHECK-NEXT: vmovupd %zmm0, 8192(%rdi,%rax) {%k1}2143; CHECK-NEXT: addq $64, %rax2144; CHECK-NEXT: jne .LBB65_12145; CHECK-NEXT: # %bb.2: # %bb102146; CHECK-NEXT: vzeroupper2147; CHECK-NEXT: retq2148bb:2149 br label %bb12150 2151bb1: ; preds = %bb1, %bb2152 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2153 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp2154 %tmp4 = load <8 x double>, ptr %tmp2, align 82155 %tmp5 = fcmp ogt <8 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>2156 %tmp6 = select <8 x i1> %tmp5, <8 x double> %tmp4, <8 x double> <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>2157 store <8 x double> %tmp6, ptr %tmp2, align 82158 %tmp8 = add i64 %tmp, 82159 %tmp9 = icmp eq i64 %tmp8, 10242160 br i1 %tmp9, label %bb10, label %bb12161 2162bb10: ; preds = %bb12163 ret void2164}2165 2166define void @bcast_unfold_fmin_v4f32(ptr %arg) {2167; CHECK-LABEL: bcast_unfold_fmin_v4f32:2168; CHECK: # %bb.0: # %bb2169; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002170; CHECK-NEXT: vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]2171; CHECK-NEXT: .p2align 42172; CHECK-NEXT: .LBB66_1: # %bb12173; CHECK-NEXT: # =>This Inner Loop Header: Depth=12174; CHECK-NEXT: vcmpngtps 4096(%rdi,%rax), %xmm0, %k12175; CHECK-NEXT: vmovups %xmm0, 4096(%rdi,%rax) {%k1}2176; CHECK-NEXT: addq $16, %rax2177; CHECK-NEXT: jne .LBB66_12178; CHECK-NEXT: # %bb.2: # %bb102179; CHECK-NEXT: retq2180bb:2181 br label %bb12182 2183bb1: ; preds = %bb1, %bb2184 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2185 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp2186 %tmp4 = load <4 x float>, ptr %tmp2, align 42187 %tmp5 = fcmp olt <4 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>2188 %tmp6 = select <4 x i1> %tmp5, <4 x float> %tmp4, <4 x float> <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>2189 store <4 x float> %tmp6, ptr %tmp2, align 42190 %tmp8 = add i64 %tmp, 42191 %tmp9 = icmp eq i64 %tmp8, 10242192 br i1 %tmp9, label %bb10, label %bb12193 2194bb10: ; preds = %bb12195 ret void2196}2197 2198define void @bcast_unfold_fmin_v8f32(ptr %arg) {2199; CHECK-LABEL: bcast_unfold_fmin_v8f32:2200; CHECK: # %bb.0: # %bb2201; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002202; CHECK-NEXT: vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]2203; CHECK-NEXT: .p2align 42204; CHECK-NEXT: .LBB67_1: # %bb12205; CHECK-NEXT: # =>This Inner Loop Header: Depth=12206; CHECK-NEXT: vcmpngtps 4096(%rdi,%rax), %ymm0, %k12207; CHECK-NEXT: vmovups %ymm0, 4096(%rdi,%rax) {%k1}2208; CHECK-NEXT: addq $32, %rax2209; CHECK-NEXT: jne .LBB67_12210; CHECK-NEXT: # %bb.2: # %bb102211; CHECK-NEXT: vzeroupper2212; CHECK-NEXT: retq2213bb:2214 br label %bb12215 2216bb1: ; preds = %bb1, %bb2217 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2218 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp2219 %tmp4 = load <8 x float>, ptr %tmp2, align 42220 %tmp5 = fcmp olt <8 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>2221 %tmp6 = select <8 x i1> %tmp5, <8 x float> %tmp4, <8 x float> <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>2222 store <8 x float> %tmp6, ptr %tmp2, align 42223 %tmp8 = add i64 %tmp, 82224 %tmp9 = icmp eq i64 %tmp8, 10242225 br i1 %tmp9, label %bb10, label %bb12226 2227bb10: ; preds = %bb12228 ret void2229}2230 2231define void @bcast_unfold_fmin_v16f32(ptr %arg) {2232; CHECK-LABEL: bcast_unfold_fmin_v16f32:2233; CHECK: # %bb.0: # %bb2234; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002235; CHECK-NEXT: vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]2236; CHECK-NEXT: .p2align 42237; CHECK-NEXT: .LBB68_1: # %bb12238; CHECK-NEXT: # =>This Inner Loop Header: Depth=12239; CHECK-NEXT: vcmpngtps 4096(%rdi,%rax), %zmm0, %k12240; CHECK-NEXT: vmovups %zmm0, 4096(%rdi,%rax) {%k1}2241; CHECK-NEXT: addq $64, %rax2242; CHECK-NEXT: jne .LBB68_12243; CHECK-NEXT: # %bb.2: # %bb102244; CHECK-NEXT: vzeroupper2245; CHECK-NEXT: retq2246bb:2247 br label %bb12248 2249bb1: ; preds = %bb1, %bb2250 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2251 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp2252 %tmp4 = load <16 x float>, ptr %tmp2, align 42253 %tmp5 = fcmp olt <16 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>2254 %tmp6 = select <16 x i1> %tmp5, <16 x float> %tmp4, <16 x float> <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>2255 store <16 x float> %tmp6, ptr %tmp2, align 42256 %tmp8 = add i64 %tmp, 162257 %tmp9 = icmp eq i64 %tmp8, 10242258 br i1 %tmp9, label %bb10, label %bb12259 2260bb10: ; preds = %bb12261 ret void2262}2263 2264define void @bcast_unfold_fmin_v2f64(ptr %arg) {2265; CHECK-LABEL: bcast_unfold_fmin_v2f64:2266; CHECK: # %bb.0: # %bb2267; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002268; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = [2.0E+0,2.0E+0]2269; CHECK-NEXT: # xmm0 = mem[0,0]2270; CHECK-NEXT: .p2align 42271; CHECK-NEXT: .LBB69_1: # %bb12272; CHECK-NEXT: # =>This Inner Loop Header: Depth=12273; CHECK-NEXT: vcmpngtpd 8192(%rdi,%rax), %xmm0, %k12274; CHECK-NEXT: vmovupd %xmm0, 8192(%rdi,%rax) {%k1}2275; CHECK-NEXT: addq $16, %rax2276; CHECK-NEXT: jne .LBB69_12277; CHECK-NEXT: # %bb.2: # %bb102278; CHECK-NEXT: retq2279bb:2280 br label %bb12281 2282bb1: ; preds = %bb1, %bb2283 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2284 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp2285 %tmp4 = load <2 x double>, ptr %tmp2, align 82286 %tmp5 = fcmp olt <2 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00>2287 %tmp6 = select <2 x i1> %tmp5, <2 x double> %tmp4, <2 x double> <double 2.000000e+00, double 2.000000e+00>2288 store <2 x double> %tmp6, ptr %tmp2, align 82289 %tmp8 = add i64 %tmp, 22290 %tmp9 = icmp eq i64 %tmp8, 10242291 br i1 %tmp9, label %bb10, label %bb12292 2293bb10: ; preds = %bb12294 ret void2295}2296 2297define void @bcast_unfold_fmin_v4f64(ptr %arg) {2298; CHECK-LABEL: bcast_unfold_fmin_v4f64:2299; CHECK: # %bb.0: # %bb2300; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002301; CHECK-NEXT: vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]2302; CHECK-NEXT: .p2align 42303; CHECK-NEXT: .LBB70_1: # %bb12304; CHECK-NEXT: # =>This Inner Loop Header: Depth=12305; CHECK-NEXT: vcmpngtpd 8192(%rdi,%rax), %ymm0, %k12306; CHECK-NEXT: vmovupd %ymm0, 8192(%rdi,%rax) {%k1}2307; CHECK-NEXT: addq $32, %rax2308; CHECK-NEXT: jne .LBB70_12309; CHECK-NEXT: # %bb.2: # %bb102310; CHECK-NEXT: vzeroupper2311; CHECK-NEXT: retq2312bb:2313 br label %bb12314 2315bb1: ; preds = %bb1, %bb2316 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2317 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp2318 %tmp4 = load <4 x double>, ptr %tmp2, align 82319 %tmp5 = fcmp olt <4 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>2320 %tmp6 = select <4 x i1> %tmp5, <4 x double> %tmp4, <4 x double> <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>2321 store <4 x double> %tmp6, ptr %tmp2, align 82322 %tmp8 = add i64 %tmp, 42323 %tmp9 = icmp eq i64 %tmp8, 10242324 br i1 %tmp9, label %bb10, label %bb12325 2326bb10: ; preds = %bb12327 ret void2328}2329 2330define void @bcast_unfold_fmin_v8f64(ptr %arg) {2331; CHECK-LABEL: bcast_unfold_fmin_v8f64:2332; CHECK: # %bb.0: # %bb2333; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002334; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]2335; CHECK-NEXT: .p2align 42336; CHECK-NEXT: .LBB71_1: # %bb12337; CHECK-NEXT: # =>This Inner Loop Header: Depth=12338; CHECK-NEXT: vcmpngtpd 8192(%rdi,%rax), %zmm0, %k12339; CHECK-NEXT: vmovupd %zmm0, 8192(%rdi,%rax) {%k1}2340; CHECK-NEXT: addq $64, %rax2341; CHECK-NEXT: jne .LBB71_12342; CHECK-NEXT: # %bb.2: # %bb102343; CHECK-NEXT: vzeroupper2344; CHECK-NEXT: retq2345bb:2346 br label %bb12347 2348bb1: ; preds = %bb1, %bb2349 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2350 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp2351 %tmp4 = load <8 x double>, ptr %tmp2, align 82352 %tmp5 = fcmp olt <8 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>2353 %tmp6 = select <8 x i1> %tmp5, <8 x double> %tmp4, <8 x double> <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>2354 store <8 x double> %tmp6, ptr %tmp2, align 82355 %tmp8 = add i64 %tmp, 82356 %tmp9 = icmp eq i64 %tmp8, 10242357 br i1 %tmp9, label %bb10, label %bb12358 2359bb10: ; preds = %bb12360 ret void2361}2362 2363define void @bcast_unfold_smin_v4i32(ptr %arg) {2364; CHECK-LABEL: bcast_unfold_smin_v4i32:2365; CHECK: # %bb.0: # %bb2366; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002367; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]2368; CHECK-NEXT: .p2align 42369; CHECK-NEXT: .LBB72_1: # %bb12370; CHECK-NEXT: # =>This Inner Loop Header: Depth=12371; CHECK-NEXT: vpminsd 4096(%rdi,%rax), %xmm0, %xmm12372; CHECK-NEXT: vmovdqu %xmm1, 4096(%rdi,%rax)2373; CHECK-NEXT: addq $16, %rax2374; CHECK-NEXT: jne .LBB72_12375; CHECK-NEXT: # %bb.2: # %bb102376; CHECK-NEXT: retq2377bb:2378 br label %bb12379 2380bb1: ; preds = %bb1, %bb2381 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2382 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp2383 %tmp4 = load <4 x i32>, ptr %tmp2, align 42384 %tmp5 = icmp slt <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>2385 %tmp6 = select <4 x i1> %tmp5, <4 x i32> %tmp4, <4 x i32> <i32 2, i32 2, i32 2, i32 2>2386 store <4 x i32> %tmp6, ptr %tmp2, align 42387 %tmp8 = add i64 %tmp, 42388 %tmp9 = icmp eq i64 %tmp8, 10242389 br i1 %tmp9, label %bb10, label %bb12390 2391bb10: ; preds = %bb12392 ret void2393}2394 2395define void @bcast_unfold_smin_v8i32(ptr %arg) {2396; CHECK-LABEL: bcast_unfold_smin_v8i32:2397; CHECK: # %bb.0: # %bb2398; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002399; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]2400; CHECK-NEXT: .p2align 42401; CHECK-NEXT: .LBB73_1: # %bb12402; CHECK-NEXT: # =>This Inner Loop Header: Depth=12403; CHECK-NEXT: vpminsd 4096(%rdi,%rax), %ymm0, %ymm12404; CHECK-NEXT: vmovdqu %ymm1, 4096(%rdi,%rax)2405; CHECK-NEXT: addq $32, %rax2406; CHECK-NEXT: jne .LBB73_12407; CHECK-NEXT: # %bb.2: # %bb102408; CHECK-NEXT: vzeroupper2409; CHECK-NEXT: retq2410bb:2411 br label %bb12412 2413bb1: ; preds = %bb1, %bb2414 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2415 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp2416 %tmp4 = load <8 x i32>, ptr %tmp2, align 42417 %tmp5 = icmp slt <8 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>2418 %tmp6 = select <8 x i1> %tmp5, <8 x i32> %tmp4, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>2419 store <8 x i32> %tmp6, ptr %tmp2, align 42420 %tmp8 = add i64 %tmp, 82421 %tmp9 = icmp eq i64 %tmp8, 10242422 br i1 %tmp9, label %bb10, label %bb12423 2424bb10: ; preds = %bb12425 ret void2426}2427 2428define void @bcast_unfold_smin_v16i32(ptr %arg) {2429; CHECK-LABEL: bcast_unfold_smin_v16i32:2430; CHECK: # %bb.0: # %bb2431; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002432; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]2433; CHECK-NEXT: .p2align 42434; CHECK-NEXT: .LBB74_1: # %bb12435; CHECK-NEXT: # =>This Inner Loop Header: Depth=12436; CHECK-NEXT: vpminsd 4096(%rdi,%rax), %zmm0, %zmm12437; CHECK-NEXT: vmovdqu64 %zmm1, 4096(%rdi,%rax)2438; CHECK-NEXT: addq $64, %rax2439; CHECK-NEXT: jne .LBB74_12440; CHECK-NEXT: # %bb.2: # %bb102441; CHECK-NEXT: vzeroupper2442; CHECK-NEXT: retq2443bb:2444 br label %bb12445 2446bb1: ; preds = %bb1, %bb2447 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2448 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp2449 %tmp4 = load <16 x i32>, ptr %tmp2, align 42450 %tmp5 = icmp slt <16 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>2451 %tmp6 = select <16 x i1> %tmp5, <16 x i32> %tmp4, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>2452 store <16 x i32> %tmp6, ptr %tmp2, align 42453 %tmp8 = add i64 %tmp, 162454 %tmp9 = icmp eq i64 %tmp8, 10242455 br i1 %tmp9, label %bb10, label %bb12456 2457bb10: ; preds = %bb12458 ret void2459}2460 2461define void @bcast_unfold_smin_v2i64(ptr %arg) {2462; CHECK-LABEL: bcast_unfold_smin_v2i64:2463; CHECK: # %bb.0: # %bb2464; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002465; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm0 = [2,2]2466; CHECK-NEXT: .p2align 42467; CHECK-NEXT: .LBB75_1: # %bb12468; CHECK-NEXT: # =>This Inner Loop Header: Depth=12469; CHECK-NEXT: vpminsq 8192(%rdi,%rax), %xmm0, %xmm12470; CHECK-NEXT: vmovdqu %xmm1, 8192(%rdi,%rax)2471; CHECK-NEXT: addq $16, %rax2472; CHECK-NEXT: jne .LBB75_12473; CHECK-NEXT: # %bb.2: # %bb102474; CHECK-NEXT: retq2475bb:2476 br label %bb12477 2478bb1: ; preds = %bb1, %bb2479 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2480 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp2481 %tmp4 = load <2 x i64>, ptr %tmp2, align 82482 %tmp5 = icmp slt <2 x i64> %tmp4, <i64 2, i64 2>2483 %tmp6 = select <2 x i1> %tmp5, <2 x i64> %tmp4, <2 x i64> <i64 2, i64 2>2484 store <2 x i64> %tmp6, ptr %tmp2, align 82485 %tmp8 = add i64 %tmp, 22486 %tmp9 = icmp eq i64 %tmp8, 10242487 br i1 %tmp9, label %bb10, label %bb12488 2489bb10: ; preds = %bb12490 ret void2491}2492 2493define void @bcast_unfold_smin_v4i64(ptr %arg) {2494; CHECK-LABEL: bcast_unfold_smin_v4i64:2495; CHECK: # %bb.0: # %bb2496; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002497; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]2498; CHECK-NEXT: .p2align 42499; CHECK-NEXT: .LBB76_1: # %bb12500; CHECK-NEXT: # =>This Inner Loop Header: Depth=12501; CHECK-NEXT: vpminsq 8192(%rdi,%rax), %ymm0, %ymm12502; CHECK-NEXT: vmovdqu %ymm1, 8192(%rdi,%rax)2503; CHECK-NEXT: addq $32, %rax2504; CHECK-NEXT: jne .LBB76_12505; CHECK-NEXT: # %bb.2: # %bb102506; CHECK-NEXT: vzeroupper2507; CHECK-NEXT: retq2508bb:2509 br label %bb12510 2511bb1: ; preds = %bb1, %bb2512 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2513 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp2514 %tmp4 = load <4 x i64>, ptr %tmp2, align 82515 %tmp5 = icmp slt <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>2516 %tmp6 = select <4 x i1> %tmp5, <4 x i64> %tmp4, <4 x i64> <i64 2, i64 2, i64 2, i64 2>2517 store <4 x i64> %tmp6, ptr %tmp2, align 82518 %tmp8 = add i64 %tmp, 42519 %tmp9 = icmp eq i64 %tmp8, 10242520 br i1 %tmp9, label %bb10, label %bb12521 2522bb10: ; preds = %bb12523 ret void2524}2525 2526define void @bcast_unfold_smin_v8i64(ptr %arg) {2527; CHECK-LABEL: bcast_unfold_smin_v8i64:2528; CHECK: # %bb.0: # %bb2529; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002530; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]2531; CHECK-NEXT: .p2align 42532; CHECK-NEXT: .LBB77_1: # %bb12533; CHECK-NEXT: # =>This Inner Loop Header: Depth=12534; CHECK-NEXT: vpminsq 8192(%rdi,%rax), %zmm0, %zmm12535; CHECK-NEXT: vmovdqu64 %zmm1, 8192(%rdi,%rax)2536; CHECK-NEXT: addq $64, %rax2537; CHECK-NEXT: jne .LBB77_12538; CHECK-NEXT: # %bb.2: # %bb102539; CHECK-NEXT: vzeroupper2540; CHECK-NEXT: retq2541bb:2542 br label %bb12543 2544bb1: ; preds = %bb1, %bb2545 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2546 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp2547 %tmp4 = load <8 x i64>, ptr %tmp2, align 82548 %tmp5 = icmp slt <8 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>2549 %tmp6 = select <8 x i1> %tmp5, <8 x i64> %tmp4, <8 x i64> <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>2550 store <8 x i64> %tmp6, ptr %tmp2, align 82551 %tmp8 = add i64 %tmp, 82552 %tmp9 = icmp eq i64 %tmp8, 10242553 br i1 %tmp9, label %bb10, label %bb12554 2555bb10: ; preds = %bb12556 ret void2557}2558 2559define void @bcast_unfold_smax_v4i32(ptr %arg) {2560; CHECK-LABEL: bcast_unfold_smax_v4i32:2561; CHECK: # %bb.0: # %bb2562; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002563; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]2564; CHECK-NEXT: .p2align 42565; CHECK-NEXT: .LBB78_1: # %bb12566; CHECK-NEXT: # =>This Inner Loop Header: Depth=12567; CHECK-NEXT: vpmaxsd 4096(%rdi,%rax), %xmm0, %xmm12568; CHECK-NEXT: vmovdqu %xmm1, 4096(%rdi,%rax)2569; CHECK-NEXT: addq $16, %rax2570; CHECK-NEXT: jne .LBB78_12571; CHECK-NEXT: # %bb.2: # %bb102572; CHECK-NEXT: retq2573bb:2574 br label %bb12575 2576bb1: ; preds = %bb1, %bb2577 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2578 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp2579 %tmp4 = load <4 x i32>, ptr %tmp2, align 42580 %tmp5 = icmp sgt <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>2581 %tmp6 = select <4 x i1> %tmp5, <4 x i32> %tmp4, <4 x i32> <i32 2, i32 2, i32 2, i32 2>2582 store <4 x i32> %tmp6, ptr %tmp2, align 42583 %tmp8 = add i64 %tmp, 42584 %tmp9 = icmp eq i64 %tmp8, 10242585 br i1 %tmp9, label %bb10, label %bb12586 2587bb10: ; preds = %bb12588 ret void2589}2590 2591define void @bcast_unfold_smax_v8i32(ptr %arg) {2592; CHECK-LABEL: bcast_unfold_smax_v8i32:2593; CHECK: # %bb.0: # %bb2594; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002595; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]2596; CHECK-NEXT: .p2align 42597; CHECK-NEXT: .LBB79_1: # %bb12598; CHECK-NEXT: # =>This Inner Loop Header: Depth=12599; CHECK-NEXT: vpmaxsd 4096(%rdi,%rax), %ymm0, %ymm12600; CHECK-NEXT: vmovdqu %ymm1, 4096(%rdi,%rax)2601; CHECK-NEXT: addq $32, %rax2602; CHECK-NEXT: jne .LBB79_12603; CHECK-NEXT: # %bb.2: # %bb102604; CHECK-NEXT: vzeroupper2605; CHECK-NEXT: retq2606bb:2607 br label %bb12608 2609bb1: ; preds = %bb1, %bb2610 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2611 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp2612 %tmp4 = load <8 x i32>, ptr %tmp2, align 42613 %tmp5 = icmp sgt <8 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>2614 %tmp6 = select <8 x i1> %tmp5, <8 x i32> %tmp4, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>2615 store <8 x i32> %tmp6, ptr %tmp2, align 42616 %tmp8 = add i64 %tmp, 82617 %tmp9 = icmp eq i64 %tmp8, 10242618 br i1 %tmp9, label %bb10, label %bb12619 2620bb10: ; preds = %bb12621 ret void2622}2623 2624define void @bcast_unfold_smax_v16i32(ptr %arg) {2625; CHECK-LABEL: bcast_unfold_smax_v16i32:2626; CHECK: # %bb.0: # %bb2627; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002628; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]2629; CHECK-NEXT: .p2align 42630; CHECK-NEXT: .LBB80_1: # %bb12631; CHECK-NEXT: # =>This Inner Loop Header: Depth=12632; CHECK-NEXT: vpmaxsd 4096(%rdi,%rax), %zmm0, %zmm12633; CHECK-NEXT: vmovdqu64 %zmm1, 4096(%rdi,%rax)2634; CHECK-NEXT: addq $64, %rax2635; CHECK-NEXT: jne .LBB80_12636; CHECK-NEXT: # %bb.2: # %bb102637; CHECK-NEXT: vzeroupper2638; CHECK-NEXT: retq2639bb:2640 br label %bb12641 2642bb1: ; preds = %bb1, %bb2643 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2644 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp2645 %tmp4 = load <16 x i32>, ptr %tmp2, align 42646 %tmp5 = icmp sgt <16 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>2647 %tmp6 = select <16 x i1> %tmp5, <16 x i32> %tmp4, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>2648 store <16 x i32> %tmp6, ptr %tmp2, align 42649 %tmp8 = add i64 %tmp, 162650 %tmp9 = icmp eq i64 %tmp8, 10242651 br i1 %tmp9, label %bb10, label %bb12652 2653bb10: ; preds = %bb12654 ret void2655}2656 2657define void @bcast_unfold_smax_v2i64(ptr %arg) {2658; CHECK-LABEL: bcast_unfold_smax_v2i64:2659; CHECK: # %bb.0: # %bb2660; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002661; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm0 = [2,2]2662; CHECK-NEXT: .p2align 42663; CHECK-NEXT: .LBB81_1: # %bb12664; CHECK-NEXT: # =>This Inner Loop Header: Depth=12665; CHECK-NEXT: vpmaxsq 8192(%rdi,%rax), %xmm0, %xmm12666; CHECK-NEXT: vmovdqu %xmm1, 8192(%rdi,%rax)2667; CHECK-NEXT: addq $16, %rax2668; CHECK-NEXT: jne .LBB81_12669; CHECK-NEXT: # %bb.2: # %bb102670; CHECK-NEXT: retq2671bb:2672 br label %bb12673 2674bb1: ; preds = %bb1, %bb2675 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2676 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp2677 %tmp4 = load <2 x i64>, ptr %tmp2, align 82678 %tmp5 = icmp sgt <2 x i64> %tmp4, <i64 2, i64 2>2679 %tmp6 = select <2 x i1> %tmp5, <2 x i64> %tmp4, <2 x i64> <i64 2, i64 2>2680 store <2 x i64> %tmp6, ptr %tmp2, align 82681 %tmp8 = add i64 %tmp, 22682 %tmp9 = icmp eq i64 %tmp8, 10242683 br i1 %tmp9, label %bb10, label %bb12684 2685bb10: ; preds = %bb12686 ret void2687}2688 2689define void @bcast_unfold_smax_v4i64(ptr %arg) {2690; CHECK-LABEL: bcast_unfold_smax_v4i64:2691; CHECK: # %bb.0: # %bb2692; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002693; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]2694; CHECK-NEXT: .p2align 42695; CHECK-NEXT: .LBB82_1: # %bb12696; CHECK-NEXT: # =>This Inner Loop Header: Depth=12697; CHECK-NEXT: vpmaxsq 8192(%rdi,%rax), %ymm0, %ymm12698; CHECK-NEXT: vmovdqu %ymm1, 8192(%rdi,%rax)2699; CHECK-NEXT: addq $32, %rax2700; CHECK-NEXT: jne .LBB82_12701; CHECK-NEXT: # %bb.2: # %bb102702; CHECK-NEXT: vzeroupper2703; CHECK-NEXT: retq2704bb:2705 br label %bb12706 2707bb1: ; preds = %bb1, %bb2708 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2709 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp2710 %tmp4 = load <4 x i64>, ptr %tmp2, align 82711 %tmp5 = icmp sgt <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>2712 %tmp6 = select <4 x i1> %tmp5, <4 x i64> %tmp4, <4 x i64> <i64 2, i64 2, i64 2, i64 2>2713 store <4 x i64> %tmp6, ptr %tmp2, align 82714 %tmp8 = add i64 %tmp, 42715 %tmp9 = icmp eq i64 %tmp8, 10242716 br i1 %tmp9, label %bb10, label %bb12717 2718bb10: ; preds = %bb12719 ret void2720}2721 2722define void @bcast_unfold_smax_v8i64(ptr %arg) {2723; CHECK-LABEL: bcast_unfold_smax_v8i64:2724; CHECK: # %bb.0: # %bb2725; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002726; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]2727; CHECK-NEXT: .p2align 42728; CHECK-NEXT: .LBB83_1: # %bb12729; CHECK-NEXT: # =>This Inner Loop Header: Depth=12730; CHECK-NEXT: vpmaxsq 8192(%rdi,%rax), %zmm0, %zmm12731; CHECK-NEXT: vmovdqu64 %zmm1, 8192(%rdi,%rax)2732; CHECK-NEXT: addq $64, %rax2733; CHECK-NEXT: jne .LBB83_12734; CHECK-NEXT: # %bb.2: # %bb102735; CHECK-NEXT: vzeroupper2736; CHECK-NEXT: retq2737bb:2738 br label %bb12739 2740bb1: ; preds = %bb1, %bb2741 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2742 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp2743 %tmp4 = load <8 x i64>, ptr %tmp2, align 82744 %tmp5 = icmp sgt <8 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>2745 %tmp6 = select <8 x i1> %tmp5, <8 x i64> %tmp4, <8 x i64> <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>2746 store <8 x i64> %tmp6, ptr %tmp2, align 82747 %tmp8 = add i64 %tmp, 82748 %tmp9 = icmp eq i64 %tmp8, 10242749 br i1 %tmp9, label %bb10, label %bb12750 2751bb10: ; preds = %bb12752 ret void2753}2754 2755define void @bcast_unfold_umin_v4i32(ptr %arg) {2756; CHECK-LABEL: bcast_unfold_umin_v4i32:2757; CHECK: # %bb.0: # %bb2758; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002759; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]2760; CHECK-NEXT: .p2align 42761; CHECK-NEXT: .LBB84_1: # %bb12762; CHECK-NEXT: # =>This Inner Loop Header: Depth=12763; CHECK-NEXT: vpminud 4096(%rdi,%rax), %xmm0, %xmm12764; CHECK-NEXT: vmovdqu %xmm1, 4096(%rdi,%rax)2765; CHECK-NEXT: addq $16, %rax2766; CHECK-NEXT: jne .LBB84_12767; CHECK-NEXT: # %bb.2: # %bb102768; CHECK-NEXT: retq2769bb:2770 br label %bb12771 2772bb1: ; preds = %bb1, %bb2773 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2774 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp2775 %tmp4 = load <4 x i32>, ptr %tmp2, align 42776 %tmp5 = icmp ult <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>2777 %tmp6 = select <4 x i1> %tmp5, <4 x i32> %tmp4, <4 x i32> <i32 2, i32 2, i32 2, i32 2>2778 store <4 x i32> %tmp6, ptr %tmp2, align 42779 %tmp8 = add i64 %tmp, 42780 %tmp9 = icmp eq i64 %tmp8, 10242781 br i1 %tmp9, label %bb10, label %bb12782 2783bb10: ; preds = %bb12784 ret void2785}2786 2787define void @bcast_unfold_umin_v8i32(ptr %arg) {2788; CHECK-LABEL: bcast_unfold_umin_v8i32:2789; CHECK: # %bb.0: # %bb2790; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002791; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]2792; CHECK-NEXT: .p2align 42793; CHECK-NEXT: .LBB85_1: # %bb12794; CHECK-NEXT: # =>This Inner Loop Header: Depth=12795; CHECK-NEXT: vpminud 4096(%rdi,%rax), %ymm0, %ymm12796; CHECK-NEXT: vmovdqu %ymm1, 4096(%rdi,%rax)2797; CHECK-NEXT: addq $32, %rax2798; CHECK-NEXT: jne .LBB85_12799; CHECK-NEXT: # %bb.2: # %bb102800; CHECK-NEXT: vzeroupper2801; CHECK-NEXT: retq2802bb:2803 br label %bb12804 2805bb1: ; preds = %bb1, %bb2806 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2807 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp2808 %tmp4 = load <8 x i32>, ptr %tmp2, align 42809 %tmp5 = icmp ult <8 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>2810 %tmp6 = select <8 x i1> %tmp5, <8 x i32> %tmp4, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>2811 store <8 x i32> %tmp6, ptr %tmp2, align 42812 %tmp8 = add i64 %tmp, 82813 %tmp9 = icmp eq i64 %tmp8, 10242814 br i1 %tmp9, label %bb10, label %bb12815 2816bb10: ; preds = %bb12817 ret void2818}2819 2820define void @bcast_unfold_umin_v16i32(ptr %arg) {2821; CHECK-LABEL: bcast_unfold_umin_v16i32:2822; CHECK: # %bb.0: # %bb2823; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002824; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]2825; CHECK-NEXT: .p2align 42826; CHECK-NEXT: .LBB86_1: # %bb12827; CHECK-NEXT: # =>This Inner Loop Header: Depth=12828; CHECK-NEXT: vpminud 4096(%rdi,%rax), %zmm0, %zmm12829; CHECK-NEXT: vmovdqu64 %zmm1, 4096(%rdi,%rax)2830; CHECK-NEXT: addq $64, %rax2831; CHECK-NEXT: jne .LBB86_12832; CHECK-NEXT: # %bb.2: # %bb102833; CHECK-NEXT: vzeroupper2834; CHECK-NEXT: retq2835bb:2836 br label %bb12837 2838bb1: ; preds = %bb1, %bb2839 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2840 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp2841 %tmp4 = load <16 x i32>, ptr %tmp2, align 42842 %tmp5 = icmp ult <16 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>2843 %tmp6 = select <16 x i1> %tmp5, <16 x i32> %tmp4, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>2844 store <16 x i32> %tmp6, ptr %tmp2, align 42845 %tmp8 = add i64 %tmp, 162846 %tmp9 = icmp eq i64 %tmp8, 10242847 br i1 %tmp9, label %bb10, label %bb12848 2849bb10: ; preds = %bb12850 ret void2851}2852 2853define void @bcast_unfold_umin_v2i64(ptr %arg) {2854; CHECK-LABEL: bcast_unfold_umin_v2i64:2855; CHECK: # %bb.0: # %bb2856; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002857; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm0 = [2,2]2858; CHECK-NEXT: .p2align 42859; CHECK-NEXT: .LBB87_1: # %bb12860; CHECK-NEXT: # =>This Inner Loop Header: Depth=12861; CHECK-NEXT: vpminuq 8192(%rdi,%rax), %xmm0, %xmm12862; CHECK-NEXT: vmovdqu %xmm1, 8192(%rdi,%rax)2863; CHECK-NEXT: addq $16, %rax2864; CHECK-NEXT: jne .LBB87_12865; CHECK-NEXT: # %bb.2: # %bb102866; CHECK-NEXT: retq2867bb:2868 br label %bb12869 2870bb1: ; preds = %bb1, %bb2871 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2872 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp2873 %tmp4 = load <2 x i64>, ptr %tmp2, align 82874 %tmp5 = icmp ult <2 x i64> %tmp4, <i64 2, i64 2>2875 %tmp6 = select <2 x i1> %tmp5, <2 x i64> %tmp4, <2 x i64> <i64 2, i64 2>2876 store <2 x i64> %tmp6, ptr %tmp2, align 82877 %tmp8 = add i64 %tmp, 22878 %tmp9 = icmp eq i64 %tmp8, 10242879 br i1 %tmp9, label %bb10, label %bb12880 2881bb10: ; preds = %bb12882 ret void2883}2884 2885define void @bcast_unfold_umin_v4i64(ptr %arg) {2886; CHECK-LABEL: bcast_unfold_umin_v4i64:2887; CHECK: # %bb.0: # %bb2888; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002889; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]2890; CHECK-NEXT: .p2align 42891; CHECK-NEXT: .LBB88_1: # %bb12892; CHECK-NEXT: # =>This Inner Loop Header: Depth=12893; CHECK-NEXT: vpminuq 8192(%rdi,%rax), %ymm0, %ymm12894; CHECK-NEXT: vmovdqu %ymm1, 8192(%rdi,%rax)2895; CHECK-NEXT: addq $32, %rax2896; CHECK-NEXT: jne .LBB88_12897; CHECK-NEXT: # %bb.2: # %bb102898; CHECK-NEXT: vzeroupper2899; CHECK-NEXT: retq2900bb:2901 br label %bb12902 2903bb1: ; preds = %bb1, %bb2904 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2905 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp2906 %tmp4 = load <4 x i64>, ptr %tmp2, align 82907 %tmp5 = icmp ult <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>2908 %tmp6 = select <4 x i1> %tmp5, <4 x i64> %tmp4, <4 x i64> <i64 2, i64 2, i64 2, i64 2>2909 store <4 x i64> %tmp6, ptr %tmp2, align 82910 %tmp8 = add i64 %tmp, 42911 %tmp9 = icmp eq i64 %tmp8, 10242912 br i1 %tmp9, label %bb10, label %bb12913 2914bb10: ; preds = %bb12915 ret void2916}2917 2918define void @bcast_unfold_umin_v8i64(ptr %arg) {2919; CHECK-LABEL: bcast_unfold_umin_v8i64:2920; CHECK: # %bb.0: # %bb2921; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0002922; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]2923; CHECK-NEXT: .p2align 42924; CHECK-NEXT: .LBB89_1: # %bb12925; CHECK-NEXT: # =>This Inner Loop Header: Depth=12926; CHECK-NEXT: vpminuq 8192(%rdi,%rax), %zmm0, %zmm12927; CHECK-NEXT: vmovdqu64 %zmm1, 8192(%rdi,%rax)2928; CHECK-NEXT: addq $64, %rax2929; CHECK-NEXT: jne .LBB89_12930; CHECK-NEXT: # %bb.2: # %bb102931; CHECK-NEXT: vzeroupper2932; CHECK-NEXT: retq2933bb:2934 br label %bb12935 2936bb1: ; preds = %bb1, %bb2937 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2938 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp2939 %tmp4 = load <8 x i64>, ptr %tmp2, align 82940 %tmp5 = icmp ult <8 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>2941 %tmp6 = select <8 x i1> %tmp5, <8 x i64> %tmp4, <8 x i64> <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>2942 store <8 x i64> %tmp6, ptr %tmp2, align 82943 %tmp8 = add i64 %tmp, 82944 %tmp9 = icmp eq i64 %tmp8, 10242945 br i1 %tmp9, label %bb10, label %bb12946 2947bb10: ; preds = %bb12948 ret void2949}2950 2951define void @bcast_unfold_umax_v4i32(ptr %arg) {2952; CHECK-LABEL: bcast_unfold_umax_v4i32:2953; CHECK: # %bb.0: # %bb2954; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002955; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]2956; CHECK-NEXT: .p2align 42957; CHECK-NEXT: .LBB90_1: # %bb12958; CHECK-NEXT: # =>This Inner Loop Header: Depth=12959; CHECK-NEXT: vpmaxud 4096(%rdi,%rax), %xmm0, %xmm12960; CHECK-NEXT: vmovdqu %xmm1, 4096(%rdi,%rax)2961; CHECK-NEXT: addq $16, %rax2962; CHECK-NEXT: jne .LBB90_12963; CHECK-NEXT: # %bb.2: # %bb102964; CHECK-NEXT: retq2965bb:2966 br label %bb12967 2968bb1: ; preds = %bb1, %bb2969 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]2970 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp2971 %tmp4 = load <4 x i32>, ptr %tmp2, align 42972 %tmp5 = icmp ugt <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>2973 %tmp6 = select <4 x i1> %tmp5, <4 x i32> %tmp4, <4 x i32> <i32 2, i32 2, i32 2, i32 2>2974 store <4 x i32> %tmp6, ptr %tmp2, align 42975 %tmp8 = add i64 %tmp, 42976 %tmp9 = icmp eq i64 %tmp8, 10242977 br i1 %tmp9, label %bb10, label %bb12978 2979bb10: ; preds = %bb12980 ret void2981}2982 2983define void @bcast_unfold_umax_v8i32(ptr %arg) {2984; CHECK-LABEL: bcast_unfold_umax_v8i32:2985; CHECK: # %bb.0: # %bb2986; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0002987; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]2988; CHECK-NEXT: .p2align 42989; CHECK-NEXT: .LBB91_1: # %bb12990; CHECK-NEXT: # =>This Inner Loop Header: Depth=12991; CHECK-NEXT: vpmaxud 4096(%rdi,%rax), %ymm0, %ymm12992; CHECK-NEXT: vmovdqu %ymm1, 4096(%rdi,%rax)2993; CHECK-NEXT: addq $32, %rax2994; CHECK-NEXT: jne .LBB91_12995; CHECK-NEXT: # %bb.2: # %bb102996; CHECK-NEXT: vzeroupper2997; CHECK-NEXT: retq2998bb:2999 br label %bb13000 3001bb1: ; preds = %bb1, %bb3002 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3003 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp3004 %tmp4 = load <8 x i32>, ptr %tmp2, align 43005 %tmp5 = icmp ugt <8 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>3006 %tmp6 = select <8 x i1> %tmp5, <8 x i32> %tmp4, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>3007 store <8 x i32> %tmp6, ptr %tmp2, align 43008 %tmp8 = add i64 %tmp, 83009 %tmp9 = icmp eq i64 %tmp8, 10243010 br i1 %tmp9, label %bb10, label %bb13011 3012bb10: ; preds = %bb13013 ret void3014}3015 3016define void @bcast_unfold_umax_v16i32(ptr %arg) {3017; CHECK-LABEL: bcast_unfold_umax_v16i32:3018; CHECK: # %bb.0: # %bb3019; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0003020; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]3021; CHECK-NEXT: .p2align 43022; CHECK-NEXT: .LBB92_1: # %bb13023; CHECK-NEXT: # =>This Inner Loop Header: Depth=13024; CHECK-NEXT: vpmaxud 4096(%rdi,%rax), %zmm0, %zmm13025; CHECK-NEXT: vmovdqu64 %zmm1, 4096(%rdi,%rax)3026; CHECK-NEXT: addq $64, %rax3027; CHECK-NEXT: jne .LBB92_13028; CHECK-NEXT: # %bb.2: # %bb103029; CHECK-NEXT: vzeroupper3030; CHECK-NEXT: retq3031bb:3032 br label %bb13033 3034bb1: ; preds = %bb1, %bb3035 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3036 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp3037 %tmp4 = load <16 x i32>, ptr %tmp2, align 43038 %tmp5 = icmp ugt <16 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>3039 %tmp6 = select <16 x i1> %tmp5, <16 x i32> %tmp4, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>3040 store <16 x i32> %tmp6, ptr %tmp2, align 43041 %tmp8 = add i64 %tmp, 163042 %tmp9 = icmp eq i64 %tmp8, 10243043 br i1 %tmp9, label %bb10, label %bb13044 3045bb10: ; preds = %bb13046 ret void3047}3048 3049define void @bcast_unfold_umax_v2i64(ptr %arg) {3050; CHECK-LABEL: bcast_unfold_umax_v2i64:3051; CHECK: # %bb.0: # %bb3052; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0003053; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm0 = [2,2]3054; CHECK-NEXT: .p2align 43055; CHECK-NEXT: .LBB93_1: # %bb13056; CHECK-NEXT: # =>This Inner Loop Header: Depth=13057; CHECK-NEXT: vpmaxuq 8192(%rdi,%rax), %xmm0, %xmm13058; CHECK-NEXT: vmovdqu %xmm1, 8192(%rdi,%rax)3059; CHECK-NEXT: addq $16, %rax3060; CHECK-NEXT: jne .LBB93_13061; CHECK-NEXT: # %bb.2: # %bb103062; CHECK-NEXT: retq3063bb:3064 br label %bb13065 3066bb1: ; preds = %bb1, %bb3067 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3068 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3069 %tmp4 = load <2 x i64>, ptr %tmp2, align 83070 %tmp5 = icmp ugt <2 x i64> %tmp4, <i64 2, i64 2>3071 %tmp6 = select <2 x i1> %tmp5, <2 x i64> %tmp4, <2 x i64> <i64 2, i64 2>3072 store <2 x i64> %tmp6, ptr %tmp2, align 83073 %tmp8 = add i64 %tmp, 23074 %tmp9 = icmp eq i64 %tmp8, 10243075 br i1 %tmp9, label %bb10, label %bb13076 3077bb10: ; preds = %bb13078 ret void3079}3080 3081define void @bcast_unfold_umax_v4i64(ptr %arg) {3082; CHECK-LABEL: bcast_unfold_umax_v4i64:3083; CHECK: # %bb.0: # %bb3084; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0003085; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]3086; CHECK-NEXT: .p2align 43087; CHECK-NEXT: .LBB94_1: # %bb13088; CHECK-NEXT: # =>This Inner Loop Header: Depth=13089; CHECK-NEXT: vpmaxuq 8192(%rdi,%rax), %ymm0, %ymm13090; CHECK-NEXT: vmovdqu %ymm1, 8192(%rdi,%rax)3091; CHECK-NEXT: addq $32, %rax3092; CHECK-NEXT: jne .LBB94_13093; CHECK-NEXT: # %bb.2: # %bb103094; CHECK-NEXT: vzeroupper3095; CHECK-NEXT: retq3096bb:3097 br label %bb13098 3099bb1: ; preds = %bb1, %bb3100 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3101 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3102 %tmp4 = load <4 x i64>, ptr %tmp2, align 83103 %tmp5 = icmp ugt <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>3104 %tmp6 = select <4 x i1> %tmp5, <4 x i64> %tmp4, <4 x i64> <i64 2, i64 2, i64 2, i64 2>3105 store <4 x i64> %tmp6, ptr %tmp2, align 83106 %tmp8 = add i64 %tmp, 43107 %tmp9 = icmp eq i64 %tmp8, 10243108 br i1 %tmp9, label %bb10, label %bb13109 3110bb10: ; preds = %bb13111 ret void3112}3113 3114define void @bcast_unfold_umax_v8i64(ptr %arg) {3115; CHECK-LABEL: bcast_unfold_umax_v8i64:3116; CHECK: # %bb.0: # %bb3117; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0003118; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]3119; CHECK-NEXT: .p2align 43120; CHECK-NEXT: .LBB95_1: # %bb13121; CHECK-NEXT: # =>This Inner Loop Header: Depth=13122; CHECK-NEXT: vpmaxuq 8192(%rdi,%rax), %zmm0, %zmm13123; CHECK-NEXT: vmovdqu64 %zmm1, 8192(%rdi,%rax)3124; CHECK-NEXT: addq $64, %rax3125; CHECK-NEXT: jne .LBB95_13126; CHECK-NEXT: # %bb.2: # %bb103127; CHECK-NEXT: vzeroupper3128; CHECK-NEXT: retq3129bb:3130 br label %bb13131 3132bb1: ; preds = %bb1, %bb3133 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3134 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3135 %tmp4 = load <8 x i64>, ptr %tmp2, align 83136 %tmp5 = icmp ugt <8 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>3137 %tmp6 = select <8 x i1> %tmp5, <8 x i64> %tmp4, <8 x i64> <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>3138 store <8 x i64> %tmp6, ptr %tmp2, align 83139 %tmp8 = add i64 %tmp, 83140 %tmp9 = icmp eq i64 %tmp8, 10243141 br i1 %tmp9, label %bb10, label %bb13142 3143bb10: ; preds = %bb13144 ret void3145}3146 3147define void @bcast_unfold_pcmpgt_v4i32(ptr %arg) {3148; CHECK-LABEL: bcast_unfold_pcmpgt_v4i32:3149; CHECK: # %bb.0: # %bb3150; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0003151; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm0 = [1,1,1,1]3152; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm1 = [3,3,3,3]3153; CHECK-NEXT: .p2align 43154; CHECK-NEXT: .LBB96_1: # %bb13155; CHECK-NEXT: # =>This Inner Loop Header: Depth=13156; CHECK-NEXT: vpcmpltd 4096(%rdi,%rax), %xmm0, %k13157; CHECK-NEXT: vmovdqu32 %xmm1, 4096(%rdi,%rax) {%k1}3158; CHECK-NEXT: addq $16, %rax3159; CHECK-NEXT: jne .LBB96_13160; CHECK-NEXT: # %bb.2: # %bb103161; CHECK-NEXT: retq3162bb:3163 br label %bb13164 3165bb1: ; preds = %bb1, %bb3166 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3167 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp3168 %tmp4 = load <4 x i32>, ptr %tmp2, align 43169 %tmp5 = icmp sgt <4 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1>3170 %tmp6 = select <4 x i1> %tmp5, <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> %tmp43171 store <4 x i32> %tmp6, ptr %tmp2, align 43172 %tmp8 = add i64 %tmp, 43173 %tmp9 = icmp eq i64 %tmp8, 10243174 br i1 %tmp9, label %bb10, label %bb13175 3176bb10: ; preds = %bb13177 ret void3178}3179 3180define void @bcast_unfold_pcmpgt_v8i32(ptr %arg) {3181; CHECK-LABEL: bcast_unfold_pcmpgt_v8i32:3182; CHECK: # %bb.0: # %bb3183; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0003184; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm0 = [1,1,1,1,1,1,1,1]3185; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,3,3,3,3,3,3,3]3186; CHECK-NEXT: .p2align 43187; CHECK-NEXT: .LBB97_1: # %bb13188; CHECK-NEXT: # =>This Inner Loop Header: Depth=13189; CHECK-NEXT: vpcmpltd 4096(%rdi,%rax), %ymm0, %k13190; CHECK-NEXT: vmovdqu32 %ymm1, 4096(%rdi,%rax) {%k1}3191; CHECK-NEXT: addq $32, %rax3192; CHECK-NEXT: jne .LBB97_13193; CHECK-NEXT: # %bb.2: # %bb103194; CHECK-NEXT: vzeroupper3195; CHECK-NEXT: retq3196bb:3197 br label %bb13198 3199bb1: ; preds = %bb1, %bb3200 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3201 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp3202 %tmp4 = load <8 x i32>, ptr %tmp2, align 43203 %tmp5 = icmp sgt <8 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>3204 %tmp6 = select <8 x i1> %tmp5, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <8 x i32> %tmp43205 store <8 x i32> %tmp6, ptr %tmp2, align 43206 %tmp8 = add i64 %tmp, 83207 %tmp9 = icmp eq i64 %tmp8, 10243208 br i1 %tmp9, label %bb10, label %bb13209 3210bb10: ; preds = %bb13211 ret void3212}3213 3214define void @bcast_unfold_pcmpgt_v16i32(ptr %arg) {3215; CHECK-LABEL: bcast_unfold_pcmpgt_v16i32:3216; CHECK: # %bb.0: # %bb3217; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0003218; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]3219; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm1 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]3220; CHECK-NEXT: .p2align 43221; CHECK-NEXT: .LBB98_1: # %bb13222; CHECK-NEXT: # =>This Inner Loop Header: Depth=13223; CHECK-NEXT: vpcmpltd 4096(%rdi,%rax), %zmm0, %k13224; CHECK-NEXT: vmovdqu32 %zmm1, 4096(%rdi,%rax) {%k1}3225; CHECK-NEXT: addq $64, %rax3226; CHECK-NEXT: jne .LBB98_13227; CHECK-NEXT: # %bb.2: # %bb103228; CHECK-NEXT: vzeroupper3229; CHECK-NEXT: retq3230bb:3231 br label %bb13232 3233bb1: ; preds = %bb1, %bb3234 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3235 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp3236 %tmp4 = load <16 x i32>, ptr %tmp2, align 43237 %tmp5 = icmp sgt <16 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>3238 %tmp6 = select <16 x i1> %tmp5, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <16 x i32> %tmp43239 store <16 x i32> %tmp6, ptr %tmp2, align 43240 %tmp8 = add i64 %tmp, 163241 %tmp9 = icmp eq i64 %tmp8, 10243242 br i1 %tmp9, label %bb10, label %bb13243 3244bb10: ; preds = %bb13245 ret void3246}3247 3248define void @bcast_unfold_pcmpgt_v2i64(ptr %arg) {3249; CHECK-LABEL: bcast_unfold_pcmpgt_v2i64:3250; CHECK: # %bb.0: # %bb3251; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0003252; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]3253; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm1 = [3,3]3254; CHECK-NEXT: .p2align 43255; CHECK-NEXT: .LBB99_1: # %bb13256; CHECK-NEXT: # =>This Inner Loop Header: Depth=13257; CHECK-NEXT: vpcmpltq 8192(%rdi,%rax), %xmm0, %k13258; CHECK-NEXT: vmovdqu64 %xmm1, 8192(%rdi,%rax) {%k1}3259; CHECK-NEXT: addq $16, %rax3260; CHECK-NEXT: jne .LBB99_13261; CHECK-NEXT: # %bb.2: # %bb103262; CHECK-NEXT: retq3263bb:3264 br label %bb13265 3266bb1: ; preds = %bb1, %bb3267 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3268 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3269 %tmp4 = load <2 x i64>, ptr %tmp2, align 43270 %tmp5 = icmp sgt <2 x i64> %tmp4, <i64 1, i64 1>3271 %tmp6 = select <2 x i1> %tmp5, <2 x i64> <i64 3, i64 3>, <2 x i64> %tmp43272 store <2 x i64> %tmp6, ptr %tmp2, align 43273 %tmp8 = add i64 %tmp, 23274 %tmp9 = icmp eq i64 %tmp8, 10243275 br i1 %tmp9, label %bb10, label %bb13276 3277bb10: ; preds = %bb13278 ret void3279}3280 3281define void @bcast_unfold_pcmpgt_v4i64(ptr %arg) {3282; CHECK-LABEL: bcast_unfold_pcmpgt_v4i64:3283; CHECK: # %bb.0: # %bb3284; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0003285; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm0 = [1,1,1,1]3286; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm1 = [3,3,3,3]3287; CHECK-NEXT: .p2align 43288; CHECK-NEXT: .LBB100_1: # %bb13289; CHECK-NEXT: # =>This Inner Loop Header: Depth=13290; CHECK-NEXT: vpcmpltq 8192(%rdi,%rax), %ymm0, %k13291; CHECK-NEXT: vmovdqu64 %ymm1, 8192(%rdi,%rax) {%k1}3292; CHECK-NEXT: addq $32, %rax3293; CHECK-NEXT: jne .LBB100_13294; CHECK-NEXT: # %bb.2: # %bb103295; CHECK-NEXT: vzeroupper3296; CHECK-NEXT: retq3297bb:3298 br label %bb13299 3300bb1: ; preds = %bb1, %bb3301 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3302 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3303 %tmp4 = load <4 x i64>, ptr %tmp2, align 43304 %tmp5 = icmp sgt <4 x i64> %tmp4, <i64 1, i64 1, i64 1, i64 1>3305 %tmp6 = select <4 x i1> %tmp5, <4 x i64> <i64 3, i64 3, i64 3, i64 3>, <4 x i64> %tmp43306 store <4 x i64> %tmp6, ptr %tmp2, align 43307 %tmp8 = add i64 %tmp, 43308 %tmp9 = icmp eq i64 %tmp8, 10243309 br i1 %tmp9, label %bb10, label %bb13310 3311bb10: ; preds = %bb13312 ret void3313}3314 3315define void @bcast_unfold_pcmpgt_v8i64(ptr %arg) {3316; CHECK-LABEL: bcast_unfold_pcmpgt_v8i64:3317; CHECK: # %bb.0: # %bb3318; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0003319; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1]3320; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm1 = [3,3,3,3,3,3,3,3]3321; CHECK-NEXT: .p2align 43322; CHECK-NEXT: .LBB101_1: # %bb13323; CHECK-NEXT: # =>This Inner Loop Header: Depth=13324; CHECK-NEXT: vpcmpltq 8192(%rdi,%rax), %zmm0, %k13325; CHECK-NEXT: vmovdqu64 %zmm1, 8192(%rdi,%rax) {%k1}3326; CHECK-NEXT: addq $64, %rax3327; CHECK-NEXT: jne .LBB101_13328; CHECK-NEXT: # %bb.2: # %bb103329; CHECK-NEXT: vzeroupper3330; CHECK-NEXT: retq3331bb:3332 br label %bb13333 3334bb1: ; preds = %bb1, %bb3335 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3336 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3337 %tmp4 = load <8 x i64>, ptr %tmp2, align 43338 %tmp5 = icmp sgt <8 x i64> %tmp4, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>3339 %tmp6 = select <8 x i1> %tmp5, <8 x i64> <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>, <8 x i64> %tmp43340 store <8 x i64> %tmp6, ptr %tmp2, align 43341 %tmp8 = add i64 %tmp, 83342 %tmp9 = icmp eq i64 %tmp8, 10243343 br i1 %tmp9, label %bb10, label %bb13344 3345bb10: ; preds = %bb13346 ret void3347}3348 3349define void @bcast_unfold_pcmpeq_v4i32(ptr %arg) {3350; CHECK-LABEL: bcast_unfold_pcmpeq_v4i32:3351; CHECK: # %bb.0: # %bb3352; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0003353; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm0 = [1,1,1,1]3354; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm1 = [3,3,3,3]3355; CHECK-NEXT: .p2align 43356; CHECK-NEXT: .LBB102_1: # %bb13357; CHECK-NEXT: # =>This Inner Loop Header: Depth=13358; CHECK-NEXT: vpcmpeqd 4096(%rdi,%rax), %xmm0, %k13359; CHECK-NEXT: vmovdqu32 %xmm1, 4096(%rdi,%rax) {%k1}3360; CHECK-NEXT: addq $16, %rax3361; CHECK-NEXT: jne .LBB102_13362; CHECK-NEXT: # %bb.2: # %bb103363; CHECK-NEXT: retq3364bb:3365 br label %bb13366 3367bb1: ; preds = %bb1, %bb3368 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3369 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp3370 %tmp4 = load <4 x i32>, ptr %tmp2, align 43371 %tmp5 = icmp eq <4 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1>3372 %tmp6 = select <4 x i1> %tmp5, <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> %tmp43373 store <4 x i32> %tmp6, ptr %tmp2, align 43374 %tmp8 = add i64 %tmp, 43375 %tmp9 = icmp eq i64 %tmp8, 10243376 br i1 %tmp9, label %bb10, label %bb13377 3378bb10: ; preds = %bb13379 ret void3380}3381 3382define void @bcast_unfold_pcmpeq_v8i32(ptr %arg) {3383; CHECK-LABEL: bcast_unfold_pcmpeq_v8i32:3384; CHECK: # %bb.0: # %bb3385; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0003386; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm0 = [1,1,1,1,1,1,1,1]3387; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,3,3,3,3,3,3,3]3388; CHECK-NEXT: .p2align 43389; CHECK-NEXT: .LBB103_1: # %bb13390; CHECK-NEXT: # =>This Inner Loop Header: Depth=13391; CHECK-NEXT: vpcmpeqd 4096(%rdi,%rax), %ymm0, %k13392; CHECK-NEXT: vmovdqu32 %ymm1, 4096(%rdi,%rax) {%k1}3393; CHECK-NEXT: addq $32, %rax3394; CHECK-NEXT: jne .LBB103_13395; CHECK-NEXT: # %bb.2: # %bb103396; CHECK-NEXT: vzeroupper3397; CHECK-NEXT: retq3398bb:3399 br label %bb13400 3401bb1: ; preds = %bb1, %bb3402 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3403 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp3404 %tmp4 = load <8 x i32>, ptr %tmp2, align 43405 %tmp5 = icmp eq <8 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>3406 %tmp6 = select <8 x i1> %tmp5, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <8 x i32> %tmp43407 store <8 x i32> %tmp6, ptr %tmp2, align 43408 %tmp8 = add i64 %tmp, 83409 %tmp9 = icmp eq i64 %tmp8, 10243410 br i1 %tmp9, label %bb10, label %bb13411 3412bb10: ; preds = %bb13413 ret void3414}3415 3416define void @bcast_unfold_pcmpeq_v16i32(ptr %arg) {3417; CHECK-LABEL: bcast_unfold_pcmpeq_v16i32:3418; CHECK: # %bb.0: # %bb3419; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0003420; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]3421; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm1 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]3422; CHECK-NEXT: .p2align 43423; CHECK-NEXT: .LBB104_1: # %bb13424; CHECK-NEXT: # =>This Inner Loop Header: Depth=13425; CHECK-NEXT: vpcmpeqd 4096(%rdi,%rax), %zmm0, %k13426; CHECK-NEXT: vmovdqu32 %zmm1, 4096(%rdi,%rax) {%k1}3427; CHECK-NEXT: addq $64, %rax3428; CHECK-NEXT: jne .LBB104_13429; CHECK-NEXT: # %bb.2: # %bb103430; CHECK-NEXT: vzeroupper3431; CHECK-NEXT: retq3432bb:3433 br label %bb13434 3435bb1: ; preds = %bb1, %bb3436 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3437 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp3438 %tmp4 = load <16 x i32>, ptr %tmp2, align 43439 %tmp5 = icmp eq <16 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>3440 %tmp6 = select <16 x i1> %tmp5, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <16 x i32> %tmp43441 store <16 x i32> %tmp6, ptr %tmp2, align 43442 %tmp8 = add i64 %tmp, 163443 %tmp9 = icmp eq i64 %tmp8, 10243444 br i1 %tmp9, label %bb10, label %bb13445 3446bb10: ; preds = %bb13447 ret void3448}3449 3450define void @bcast_unfold_pcmpeq_v2i64(ptr %arg) {3451; CHECK-LABEL: bcast_unfold_pcmpeq_v2i64:3452; CHECK: # %bb.0: # %bb3453; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0003454; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]3455; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm1 = [3,3]3456; CHECK-NEXT: .p2align 43457; CHECK-NEXT: .LBB105_1: # %bb13458; CHECK-NEXT: # =>This Inner Loop Header: Depth=13459; CHECK-NEXT: vpcmpeqq 8192(%rdi,%rax), %xmm0, %k13460; CHECK-NEXT: vmovdqu64 %xmm1, 8192(%rdi,%rax) {%k1}3461; CHECK-NEXT: addq $16, %rax3462; CHECK-NEXT: jne .LBB105_13463; CHECK-NEXT: # %bb.2: # %bb103464; CHECK-NEXT: retq3465bb:3466 br label %bb13467 3468bb1: ; preds = %bb1, %bb3469 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3470 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3471 %tmp4 = load <2 x i64>, ptr %tmp2, align 43472 %tmp5 = icmp eq <2 x i64> %tmp4, <i64 1, i64 1>3473 %tmp6 = select <2 x i1> %tmp5, <2 x i64> <i64 3, i64 3>, <2 x i64> %tmp43474 store <2 x i64> %tmp6, ptr %tmp2, align 43475 %tmp8 = add i64 %tmp, 23476 %tmp9 = icmp eq i64 %tmp8, 10243477 br i1 %tmp9, label %bb10, label %bb13478 3479bb10: ; preds = %bb13480 ret void3481}3482 3483define void @bcast_unfold_pcmpeq_v4i64(ptr %arg) {3484; CHECK-LABEL: bcast_unfold_pcmpeq_v4i64:3485; CHECK: # %bb.0: # %bb3486; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0003487; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm0 = [1,1,1,1]3488; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm1 = [3,3,3,3]3489; CHECK-NEXT: .p2align 43490; CHECK-NEXT: .LBB106_1: # %bb13491; CHECK-NEXT: # =>This Inner Loop Header: Depth=13492; CHECK-NEXT: vpcmpeqq 8192(%rdi,%rax), %ymm0, %k13493; CHECK-NEXT: vmovdqu64 %ymm1, 8192(%rdi,%rax) {%k1}3494; CHECK-NEXT: addq $32, %rax3495; CHECK-NEXT: jne .LBB106_13496; CHECK-NEXT: # %bb.2: # %bb103497; CHECK-NEXT: vzeroupper3498; CHECK-NEXT: retq3499bb:3500 br label %bb13501 3502bb1: ; preds = %bb1, %bb3503 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3504 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3505 %tmp4 = load <4 x i64>, ptr %tmp2, align 43506 %tmp5 = icmp eq <4 x i64> %tmp4, <i64 1, i64 1, i64 1, i64 1>3507 %tmp6 = select <4 x i1> %tmp5, <4 x i64> <i64 3, i64 3, i64 3, i64 3>, <4 x i64> %tmp43508 store <4 x i64> %tmp6, ptr %tmp2, align 43509 %tmp8 = add i64 %tmp, 43510 %tmp9 = icmp eq i64 %tmp8, 10243511 br i1 %tmp9, label %bb10, label %bb13512 3513bb10: ; preds = %bb13514 ret void3515}3516 3517define void @bcast_unfold_pcmpeq_v8i64(ptr %arg) {3518; CHECK-LABEL: bcast_unfold_pcmpeq_v8i64:3519; CHECK: # %bb.0: # %bb3520; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0003521; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1]3522; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm1 = [3,3,3,3,3,3,3,3]3523; CHECK-NEXT: .p2align 43524; CHECK-NEXT: .LBB107_1: # %bb13525; CHECK-NEXT: # =>This Inner Loop Header: Depth=13526; CHECK-NEXT: vpcmpeqq 8192(%rdi,%rax), %zmm0, %k13527; CHECK-NEXT: vmovdqu64 %zmm1, 8192(%rdi,%rax) {%k1}3528; CHECK-NEXT: addq $64, %rax3529; CHECK-NEXT: jne .LBB107_13530; CHECK-NEXT: # %bb.2: # %bb103531; CHECK-NEXT: vzeroupper3532; CHECK-NEXT: retq3533bb:3534 br label %bb13535 3536bb1: ; preds = %bb1, %bb3537 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3538 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3539 %tmp4 = load <8 x i64>, ptr %tmp2, align 43540 %tmp5 = icmp eq <8 x i64> %tmp4, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>3541 %tmp6 = select <8 x i1> %tmp5, <8 x i64> <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>, <8 x i64> %tmp43542 store <8 x i64> %tmp6, ptr %tmp2, align 43543 %tmp8 = add i64 %tmp, 83544 %tmp9 = icmp eq i64 %tmp8, 10243545 br i1 %tmp9, label %bb10, label %bb13546 3547bb10: ; preds = %bb13548 ret void3549}3550 3551define void @bcast_unfold_pcmp_v4i32(ptr %arg) {3552; CHECK-LABEL: bcast_unfold_pcmp_v4i32:3553; CHECK: # %bb.0: # %bb3554; CHECK-NEXT: xorl %eax, %eax3555; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm0 = [1,1,1,1]3556; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm1 = [3,3,3,3]3557; CHECK-NEXT: .p2align 43558; CHECK-NEXT: .LBB108_1: # %bb13559; CHECK-NEXT: # =>This Inner Loop Header: Depth=13560; CHECK-NEXT: vpcmpgtd (%rdi,%rax,4), %xmm0, %k13561; CHECK-NEXT: vmovdqu32 %xmm1, (%rdi,%rax,4) {%k1}3562; CHECK-NEXT: addq $4, %rax3563; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF3564; CHECK-NEXT: jg .LBB108_13565; CHECK-NEXT: # %bb.2: # %bb103566; CHECK-NEXT: retq3567bb:3568 br label %bb13569 3570bb1: ; preds = %bb1, %bb3571 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3572 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp3573 %tmp4 = load <4 x i32>, ptr %tmp2, align 43574 %tmp5 = icmp slt <4 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1>3575 %tmp6 = select <4 x i1> %tmp5, <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> %tmp43576 store <4 x i32> %tmp6, ptr %tmp2, align 43577 %tmp8 = add i64 %tmp, 43578 %tmp9 = icmp slt i64 %tmp8, 10243579 br i1 %tmp9, label %bb10, label %bb13580 3581bb10: ; preds = %bb13582 ret void3583}3584 3585define void @bcast_unfold_pcmp_v8i32(ptr %arg) {3586; CHECK-LABEL: bcast_unfold_pcmp_v8i32:3587; CHECK: # %bb.0: # %bb3588; CHECK-NEXT: xorl %eax, %eax3589; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm0 = [1,1,1,1,1,1,1,1]3590; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,3,3,3,3,3,3,3]3591; CHECK-NEXT: .p2align 43592; CHECK-NEXT: .LBB109_1: # %bb13593; CHECK-NEXT: # =>This Inner Loop Header: Depth=13594; CHECK-NEXT: vpcmpgtd (%rdi,%rax,4), %ymm0, %k13595; CHECK-NEXT: vmovdqu32 %ymm1, (%rdi,%rax,4) {%k1}3596; CHECK-NEXT: addq $8, %rax3597; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF3598; CHECK-NEXT: jg .LBB109_13599; CHECK-NEXT: # %bb.2: # %bb103600; CHECK-NEXT: vzeroupper3601; CHECK-NEXT: retq3602bb:3603 br label %bb13604 3605bb1: ; preds = %bb1, %bb3606 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3607 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp3608 %tmp4 = load <8 x i32>, ptr %tmp2, align 43609 %tmp5 = icmp slt <8 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>3610 %tmp6 = select <8 x i1> %tmp5, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <8 x i32> %tmp43611 store <8 x i32> %tmp6, ptr %tmp2, align 43612 %tmp8 = add i64 %tmp, 83613 %tmp9 = icmp slt i64 %tmp8, 10243614 br i1 %tmp9, label %bb10, label %bb13615 3616bb10: ; preds = %bb13617 ret void3618}3619 3620define void @bcast_unfold_pcmp_v16i32(ptr %arg) {3621; CHECK-LABEL: bcast_unfold_pcmp_v16i32:3622; CHECK: # %bb.0: # %bb3623; CHECK-NEXT: xorl %eax, %eax3624; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]3625; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm1 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]3626; CHECK-NEXT: .p2align 43627; CHECK-NEXT: .LBB110_1: # %bb13628; CHECK-NEXT: # =>This Inner Loop Header: Depth=13629; CHECK-NEXT: vpcmpgtd (%rdi,%rax,4), %zmm0, %k13630; CHECK-NEXT: vmovdqu32 %zmm1, (%rdi,%rax,4) {%k1}3631; CHECK-NEXT: addq $16, %rax3632; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF3633; CHECK-NEXT: jg .LBB110_13634; CHECK-NEXT: # %bb.2: # %bb103635; CHECK-NEXT: vzeroupper3636; CHECK-NEXT: retq3637bb:3638 br label %bb13639 3640bb1: ; preds = %bb1, %bb3641 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3642 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp3643 %tmp4 = load <16 x i32>, ptr %tmp2, align 43644 %tmp5 = icmp slt <16 x i32> %tmp4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>3645 %tmp6 = select <16 x i1> %tmp5, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <16 x i32> %tmp43646 store <16 x i32> %tmp6, ptr %tmp2, align 43647 %tmp8 = add i64 %tmp, 163648 %tmp9 = icmp slt i64 %tmp8, 10243649 br i1 %tmp9, label %bb10, label %bb13650 3651bb10: ; preds = %bb13652 ret void3653}3654 3655define void @bcast_unfold_pcmp_v2i64(ptr %arg) {3656; CHECK-LABEL: bcast_unfold_pcmp_v2i64:3657; CHECK: # %bb.0: # %bb3658; CHECK-NEXT: xorl %eax, %eax3659; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,1]3660; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm1 = [3,3]3661; CHECK-NEXT: .p2align 43662; CHECK-NEXT: .LBB111_1: # %bb13663; CHECK-NEXT: # =>This Inner Loop Header: Depth=13664; CHECK-NEXT: vpcmpgtq (%rdi,%rax,8), %xmm0, %k13665; CHECK-NEXT: vmovdqu64 %xmm1, (%rdi,%rax,8) {%k1}3666; CHECK-NEXT: addq $2, %rax3667; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF3668; CHECK-NEXT: jg .LBB111_13669; CHECK-NEXT: # %bb.2: # %bb103670; CHECK-NEXT: retq3671bb:3672 br label %bb13673 3674bb1: ; preds = %bb1, %bb3675 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3676 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3677 %tmp4 = load <2 x i64>, ptr %tmp2, align 43678 %tmp5 = icmp slt <2 x i64> %tmp4, <i64 1, i64 1>3679 %tmp6 = select <2 x i1> %tmp5, <2 x i64> <i64 3, i64 3>, <2 x i64> %tmp43680 store <2 x i64> %tmp6, ptr %tmp2, align 43681 %tmp8 = add i64 %tmp, 23682 %tmp9 = icmp slt i64 %tmp8, 10243683 br i1 %tmp9, label %bb10, label %bb13684 3685bb10: ; preds = %bb13686 ret void3687}3688 3689define void @bcast_unfold_pcmp_v4i64(ptr %arg) {3690; CHECK-LABEL: bcast_unfold_pcmp_v4i64:3691; CHECK: # %bb.0: # %bb3692; CHECK-NEXT: xorl %eax, %eax3693; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm0 = [1,1,1,1]3694; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm1 = [3,3,3,3]3695; CHECK-NEXT: .p2align 43696; CHECK-NEXT: .LBB112_1: # %bb13697; CHECK-NEXT: # =>This Inner Loop Header: Depth=13698; CHECK-NEXT: vpcmpgtq (%rdi,%rax,8), %ymm0, %k13699; CHECK-NEXT: vmovdqu64 %ymm1, (%rdi,%rax,8) {%k1}3700; CHECK-NEXT: addq $4, %rax3701; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF3702; CHECK-NEXT: jg .LBB112_13703; CHECK-NEXT: # %bb.2: # %bb103704; CHECK-NEXT: vzeroupper3705; CHECK-NEXT: retq3706bb:3707 br label %bb13708 3709bb1: ; preds = %bb1, %bb3710 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3711 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3712 %tmp4 = load <4 x i64>, ptr %tmp2, align 43713 %tmp5 = icmp slt <4 x i64> %tmp4, <i64 1, i64 1, i64 1, i64 1>3714 %tmp6 = select <4 x i1> %tmp5, <4 x i64> <i64 3, i64 3, i64 3, i64 3>, <4 x i64> %tmp43715 store <4 x i64> %tmp6, ptr %tmp2, align 43716 %tmp8 = add i64 %tmp, 43717 %tmp9 = icmp slt i64 %tmp8, 10243718 br i1 %tmp9, label %bb10, label %bb13719 3720bb10: ; preds = %bb13721 ret void3722}3723 3724define void @bcast_unfold_pcmp_v8i64(ptr %arg) {3725; CHECK-LABEL: bcast_unfold_pcmp_v8i64:3726; CHECK: # %bb.0: # %bb3727; CHECK-NEXT: xorl %eax, %eax3728; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1]3729; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm1 = [3,3,3,3,3,3,3,3]3730; CHECK-NEXT: .p2align 43731; CHECK-NEXT: .LBB113_1: # %bb13732; CHECK-NEXT: # =>This Inner Loop Header: Depth=13733; CHECK-NEXT: vpcmpgtq (%rdi,%rax,8), %zmm0, %k13734; CHECK-NEXT: vmovdqu64 %zmm1, (%rdi,%rax,8) {%k1}3735; CHECK-NEXT: addq $8, %rax3736; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF3737; CHECK-NEXT: jg .LBB113_13738; CHECK-NEXT: # %bb.2: # %bb103739; CHECK-NEXT: vzeroupper3740; CHECK-NEXT: retq3741bb:3742 br label %bb13743 3744bb1: ; preds = %bb1, %bb3745 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3746 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3747 %tmp4 = load <8 x i64>, ptr %tmp2, align 43748 %tmp5 = icmp slt <8 x i64> %tmp4, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>3749 %tmp6 = select <8 x i1> %tmp5, <8 x i64> <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>, <8 x i64> %tmp43750 store <8 x i64> %tmp6, ptr %tmp2, align 43751 %tmp8 = add i64 %tmp, 83752 %tmp9 = icmp slt i64 %tmp8, 10243753 br i1 %tmp9, label %bb10, label %bb13754 3755bb10: ; preds = %bb13756 ret void3757}3758 3759define void @bcast_unfold_pcmpu_v4i32(ptr %arg) {3760; CHECK-LABEL: bcast_unfold_pcmpu_v4i32:3761; CHECK: # %bb.0: # %bb3762; CHECK-NEXT: xorl %eax, %eax3763; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]3764; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm1 = [3,3,3,3]3765; CHECK-NEXT: .p2align 43766; CHECK-NEXT: .LBB114_1: # %bb13767; CHECK-NEXT: # =>This Inner Loop Header: Depth=13768; CHECK-NEXT: vpcmpnleud (%rdi,%rax,4), %xmm0, %k13769; CHECK-NEXT: vmovdqu32 %xmm1, (%rdi,%rax,4) {%k1}3770; CHECK-NEXT: addq $4, %rax3771; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF3772; CHECK-NEXT: ja .LBB114_13773; CHECK-NEXT: # %bb.2: # %bb103774; CHECK-NEXT: retq3775bb:3776 br label %bb13777 3778bb1: ; preds = %bb1, %bb3779 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3780 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp3781 %tmp4 = load <4 x i32>, ptr %tmp2, align 43782 %tmp5 = icmp ult <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>3783 %tmp6 = select <4 x i1> %tmp5, <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> %tmp43784 store <4 x i32> %tmp6, ptr %tmp2, align 43785 %tmp8 = add i64 %tmp, 43786 %tmp9 = icmp ult i64 %tmp8, 10243787 br i1 %tmp9, label %bb10, label %bb13788 3789bb10: ; preds = %bb13790 ret void3791}3792 3793define void @bcast_unfold_pcmpu_v8i32(ptr %arg) {3794; CHECK-LABEL: bcast_unfold_pcmpu_v8i32:3795; CHECK: # %bb.0: # %bb3796; CHECK-NEXT: xorl %eax, %eax3797; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm0 = [2,2,2,2,2,2,2,2]3798; CHECK-NEXT: vpbroadcastd {{.*#+}} ymm1 = [3,3,3,3,3,3,3,3]3799; CHECK-NEXT: .p2align 43800; CHECK-NEXT: .LBB115_1: # %bb13801; CHECK-NEXT: # =>This Inner Loop Header: Depth=13802; CHECK-NEXT: vpcmpnleud (%rdi,%rax,4), %ymm0, %k13803; CHECK-NEXT: vmovdqu32 %ymm1, (%rdi,%rax,4) {%k1}3804; CHECK-NEXT: addq $8, %rax3805; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF3806; CHECK-NEXT: ja .LBB115_13807; CHECK-NEXT: # %bb.2: # %bb103808; CHECK-NEXT: vzeroupper3809; CHECK-NEXT: retq3810bb:3811 br label %bb13812 3813bb1: ; preds = %bb1, %bb3814 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3815 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp3816 %tmp4 = load <8 x i32>, ptr %tmp2, align 43817 %tmp5 = icmp ult <8 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>3818 %tmp6 = select <8 x i1> %tmp5, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <8 x i32> %tmp43819 store <8 x i32> %tmp6, ptr %tmp2, align 43820 %tmp8 = add i64 %tmp, 83821 %tmp9 = icmp ult i64 %tmp8, 10243822 br i1 %tmp9, label %bb10, label %bb13823 3824bb10: ; preds = %bb13825 ret void3826}3827 3828define void @bcast_unfold_pcmpu_v16i32(ptr %arg) {3829; CHECK-LABEL: bcast_unfold_pcmpu_v16i32:3830; CHECK: # %bb.0: # %bb3831; CHECK-NEXT: xorl %eax, %eax3832; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]3833; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm1 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]3834; CHECK-NEXT: .p2align 43835; CHECK-NEXT: .LBB116_1: # %bb13836; CHECK-NEXT: # =>This Inner Loop Header: Depth=13837; CHECK-NEXT: vpcmpnleud (%rdi,%rax,4), %zmm0, %k13838; CHECK-NEXT: vmovdqu32 %zmm1, (%rdi,%rax,4) {%k1}3839; CHECK-NEXT: addq $16, %rax3840; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF3841; CHECK-NEXT: ja .LBB116_13842; CHECK-NEXT: # %bb.2: # %bb103843; CHECK-NEXT: vzeroupper3844; CHECK-NEXT: retq3845bb:3846 br label %bb13847 3848bb1: ; preds = %bb1, %bb3849 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3850 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp3851 %tmp4 = load <16 x i32>, ptr %tmp2, align 43852 %tmp5 = icmp ult <16 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>3853 %tmp6 = select <16 x i1> %tmp5, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <16 x i32> %tmp43854 store <16 x i32> %tmp6, ptr %tmp2, align 43855 %tmp8 = add i64 %tmp, 163856 %tmp9 = icmp ult i64 %tmp8, 10243857 br i1 %tmp9, label %bb10, label %bb13858 3859bb10: ; preds = %bb13860 ret void3861}3862 3863define void @bcast_unfold_pcmpu_v2i64(ptr %arg) {3864; CHECK-LABEL: bcast_unfold_pcmpu_v2i64:3865; CHECK: # %bb.0: # %bb3866; CHECK-NEXT: xorl %eax, %eax3867; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm0 = [2,2]3868; CHECK-NEXT: vpbroadcastq {{.*#+}} xmm1 = [3,3]3869; CHECK-NEXT: .p2align 43870; CHECK-NEXT: .LBB117_1: # %bb13871; CHECK-NEXT: # =>This Inner Loop Header: Depth=13872; CHECK-NEXT: vpcmpnleuq (%rdi,%rax,8), %xmm0, %k13873; CHECK-NEXT: vmovdqu64 %xmm1, (%rdi,%rax,8) {%k1}3874; CHECK-NEXT: addq $2, %rax3875; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF3876; CHECK-NEXT: ja .LBB117_13877; CHECK-NEXT: # %bb.2: # %bb103878; CHECK-NEXT: retq3879bb:3880 br label %bb13881 3882bb1: ; preds = %bb1, %bb3883 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3884 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3885 %tmp4 = load <2 x i64>, ptr %tmp2, align 43886 %tmp5 = icmp ult <2 x i64> %tmp4, <i64 2, i64 2>3887 %tmp6 = select <2 x i1> %tmp5, <2 x i64> <i64 3, i64 3>, <2 x i64> %tmp43888 store <2 x i64> %tmp6, ptr %tmp2, align 43889 %tmp8 = add i64 %tmp, 23890 %tmp9 = icmp ult i64 %tmp8, 10243891 br i1 %tmp9, label %bb10, label %bb13892 3893bb10: ; preds = %bb13894 ret void3895}3896 3897define void @bcast_unfold_pcmpu_v4i64(ptr %arg) {3898; CHECK-LABEL: bcast_unfold_pcmpu_v4i64:3899; CHECK: # %bb.0: # %bb3900; CHECK-NEXT: xorl %eax, %eax3901; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]3902; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm1 = [3,3,3,3]3903; CHECK-NEXT: .p2align 43904; CHECK-NEXT: .LBB118_1: # %bb13905; CHECK-NEXT: # =>This Inner Loop Header: Depth=13906; CHECK-NEXT: vpcmpnleuq (%rdi,%rax,8), %ymm0, %k13907; CHECK-NEXT: vmovdqu64 %ymm1, (%rdi,%rax,8) {%k1}3908; CHECK-NEXT: addq $4, %rax3909; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF3910; CHECK-NEXT: ja .LBB118_13911; CHECK-NEXT: # %bb.2: # %bb103912; CHECK-NEXT: vzeroupper3913; CHECK-NEXT: retq3914bb:3915 br label %bb13916 3917bb1: ; preds = %bb1, %bb3918 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3919 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3920 %tmp4 = load <4 x i64>, ptr %tmp2, align 43921 %tmp5 = icmp ult <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>3922 %tmp6 = select <4 x i1> %tmp5, <4 x i64> <i64 3, i64 3, i64 3, i64 3>, <4 x i64> %tmp43923 store <4 x i64> %tmp6, ptr %tmp2, align 43924 %tmp8 = add i64 %tmp, 43925 %tmp9 = icmp ult i64 %tmp8, 10243926 br i1 %tmp9, label %bb10, label %bb13927 3928bb10: ; preds = %bb13929 ret void3930}3931 3932define void @bcast_unfold_pcmpu_v8i64(ptr %arg) {3933; CHECK-LABEL: bcast_unfold_pcmpu_v8i64:3934; CHECK: # %bb.0: # %bb3935; CHECK-NEXT: xorl %eax, %eax3936; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm0 = [2,2,2,2,2,2,2,2]3937; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm1 = [3,3,3,3,3,3,3,3]3938; CHECK-NEXT: .p2align 43939; CHECK-NEXT: .LBB119_1: # %bb13940; CHECK-NEXT: # =>This Inner Loop Header: Depth=13941; CHECK-NEXT: vpcmpnleuq (%rdi,%rax,8), %zmm0, %k13942; CHECK-NEXT: vmovdqu64 %zmm1, (%rdi,%rax,8) {%k1}3943; CHECK-NEXT: addq $8, %rax3944; CHECK-NEXT: cmpq $1023, %rax # imm = 0x3FF3945; CHECK-NEXT: ja .LBB119_13946; CHECK-NEXT: # %bb.2: # %bb103947; CHECK-NEXT: vzeroupper3948; CHECK-NEXT: retq3949bb:3950 br label %bb13951 3952bb1: ; preds = %bb1, %bb3953 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3954 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp3955 %tmp4 = load <8 x i64>, ptr %tmp2, align 43956 %tmp5 = icmp ult <8 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>3957 %tmp6 = select <8 x i1> %tmp5, <8 x i64> <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>, <8 x i64> %tmp43958 store <8 x i64> %tmp6, ptr %tmp2, align 43959 %tmp8 = add i64 %tmp, 83960 %tmp9 = icmp ult i64 %tmp8, 10243961 br i1 %tmp9, label %bb10, label %bb13962 3963bb10: ; preds = %bb13964 ret void3965}3966 3967define void @bcast_unfold_cmp_v4f32(ptr %arg) {3968; CHECK-LABEL: bcast_unfold_cmp_v4f32:3969; CHECK: # %bb.0: # %bb3970; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0003971; CHECK-NEXT: vbroadcastss {{.*#+}} xmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]3972; CHECK-NEXT: vbroadcastss {{.*#+}} xmm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]3973; CHECK-NEXT: .p2align 43974; CHECK-NEXT: .LBB120_1: # %bb13975; CHECK-NEXT: # =>This Inner Loop Header: Depth=13976; CHECK-NEXT: vcmpngtps 4096(%rdi,%rax), %xmm0, %k13977; CHECK-NEXT: vmovups %xmm1, 4096(%rdi,%rax) {%k1}3978; CHECK-NEXT: addq $16, %rax3979; CHECK-NEXT: jne .LBB120_13980; CHECK-NEXT: # %bb.2: # %bb103981; CHECK-NEXT: retq3982bb:3983 br label %bb13984 3985bb1: ; preds = %bb1, %bb3986 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]3987 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp3988 %tmp4 = load <4 x float>, ptr %tmp2, align 43989 %tmp5 = fcmp olt <4 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>3990 %tmp6 = select <4 x i1> %tmp5, <4 x float> %tmp4, <4 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>3991 store <4 x float> %tmp6, ptr %tmp2, align 43992 %tmp8 = add i64 %tmp, 43993 %tmp9 = icmp eq i64 %tmp8, 10243994 br i1 %tmp9, label %bb10, label %bb13995 3996bb10: ; preds = %bb13997 ret void3998}3999 4000define void @bcast_unfold_cmp_v8f32(ptr %arg) {4001; CHECK-LABEL: bcast_unfold_cmp_v8f32:4002; CHECK: # %bb.0: # %bb4003; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0004004; CHECK-NEXT: vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]4005; CHECK-NEXT: vbroadcastss {{.*#+}} ymm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]4006; CHECK-NEXT: .p2align 44007; CHECK-NEXT: .LBB121_1: # %bb14008; CHECK-NEXT: # =>This Inner Loop Header: Depth=14009; CHECK-NEXT: vcmpngtps 4096(%rdi,%rax), %ymm0, %k14010; CHECK-NEXT: vmovups %ymm1, 4096(%rdi,%rax) {%k1}4011; CHECK-NEXT: addq $32, %rax4012; CHECK-NEXT: jne .LBB121_14013; CHECK-NEXT: # %bb.2: # %bb104014; CHECK-NEXT: vzeroupper4015; CHECK-NEXT: retq4016bb:4017 br label %bb14018 4019bb1: ; preds = %bb1, %bb4020 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]4021 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp4022 %tmp4 = load <8 x float>, ptr %tmp2, align 44023 %tmp5 = fcmp olt <8 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>4024 %tmp6 = select <8 x i1> %tmp5, <8 x float> %tmp4, <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>4025 store <8 x float> %tmp6, ptr %tmp2, align 44026 %tmp8 = add i64 %tmp, 84027 %tmp9 = icmp eq i64 %tmp8, 10244028 br i1 %tmp9, label %bb10, label %bb14029 4030bb10: ; preds = %bb14031 ret void4032}4033 4034define void @bcast_unfold_cmp_v16f32(ptr %arg) {4035; CHECK-LABEL: bcast_unfold_cmp_v16f32:4036; CHECK: # %bb.0: # %bb4037; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0004038; CHECK-NEXT: vbroadcastss {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]4039; CHECK-NEXT: vbroadcastss {{.*#+}} zmm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]4040; CHECK-NEXT: .p2align 44041; CHECK-NEXT: .LBB122_1: # %bb14042; CHECK-NEXT: # =>This Inner Loop Header: Depth=14043; CHECK-NEXT: vcmpngtps 4096(%rdi,%rax), %zmm0, %k14044; CHECK-NEXT: vmovups %zmm1, 4096(%rdi,%rax) {%k1}4045; CHECK-NEXT: addq $64, %rax4046; CHECK-NEXT: jne .LBB122_14047; CHECK-NEXT: # %bb.2: # %bb104048; CHECK-NEXT: vzeroupper4049; CHECK-NEXT: retq4050bb:4051 br label %bb14052 4053bb1: ; preds = %bb1, %bb4054 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]4055 %tmp2 = getelementptr inbounds float, ptr %arg, i64 %tmp4056 %tmp4 = load <16 x float>, ptr %tmp2, align 44057 %tmp5 = fcmp olt <16 x float> %tmp4, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>4058 %tmp6 = select <16 x i1> %tmp5, <16 x float> %tmp4, <16 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>4059 store <16 x float> %tmp6, ptr %tmp2, align 44060 %tmp8 = add i64 %tmp, 164061 %tmp9 = icmp eq i64 %tmp8, 10244062 br i1 %tmp9, label %bb10, label %bb14063 4064bb10: ; preds = %bb14065 ret void4066}4067 4068define void @bcast_unfold_cmp_v2f64(ptr %arg) {4069; CHECK-LABEL: bcast_unfold_cmp_v2f64:4070; CHECK: # %bb.0: # %bb4071; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0004072; CHECK-NEXT: vmovddup {{.*#+}} xmm0 = [2.0E+0,2.0E+0]4073; CHECK-NEXT: # xmm0 = mem[0,0]4074; CHECK-NEXT: vmovddup {{.*#+}} xmm1 = [3.0E+0,3.0E+0]4075; CHECK-NEXT: # xmm1 = mem[0,0]4076; CHECK-NEXT: .p2align 44077; CHECK-NEXT: .LBB123_1: # %bb14078; CHECK-NEXT: # =>This Inner Loop Header: Depth=14079; CHECK-NEXT: vcmpngtpd 8192(%rdi,%rax), %xmm0, %k14080; CHECK-NEXT: vmovupd %xmm1, 8192(%rdi,%rax) {%k1}4081; CHECK-NEXT: addq $16, %rax4082; CHECK-NEXT: jne .LBB123_14083; CHECK-NEXT: # %bb.2: # %bb104084; CHECK-NEXT: retq4085bb:4086 br label %bb14087 4088bb1: ; preds = %bb1, %bb4089 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]4090 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp4091 %tmp4 = load <2 x double>, ptr %tmp2, align 84092 %tmp5 = fcmp olt <2 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00>4093 %tmp6 = select <2 x i1> %tmp5, <2 x double> %tmp4, <2 x double> <double 3.000000e+00, double 3.000000e+00>4094 store <2 x double> %tmp6, ptr %tmp2, align 84095 %tmp8 = add i64 %tmp, 24096 %tmp9 = icmp eq i64 %tmp8, 10244097 br i1 %tmp9, label %bb10, label %bb14098 4099bb10: ; preds = %bb14100 ret void4101}4102 4103define void @bcast_unfold_cmp_v4f64(ptr %arg) {4104; CHECK-LABEL: bcast_unfold_cmp_v4f64:4105; CHECK: # %bb.0: # %bb4106; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0004107; CHECK-NEXT: vbroadcastsd {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0]4108; CHECK-NEXT: vbroadcastsd {{.*#+}} ymm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0]4109; CHECK-NEXT: .p2align 44110; CHECK-NEXT: .LBB124_1: # %bb14111; CHECK-NEXT: # =>This Inner Loop Header: Depth=14112; CHECK-NEXT: vcmpngtpd 8192(%rdi,%rax), %ymm0, %k14113; CHECK-NEXT: vmovupd %ymm1, 8192(%rdi,%rax) {%k1}4114; CHECK-NEXT: addq $32, %rax4115; CHECK-NEXT: jne .LBB124_14116; CHECK-NEXT: # %bb.2: # %bb104117; CHECK-NEXT: vzeroupper4118; CHECK-NEXT: retq4119bb:4120 br label %bb14121 4122bb1: ; preds = %bb1, %bb4123 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]4124 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp4125 %tmp4 = load <4 x double>, ptr %tmp2, align 84126 %tmp5 = fcmp olt <4 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>4127 %tmp6 = select <4 x i1> %tmp5, <4 x double> %tmp4, <4 x double> <double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00>4128 store <4 x double> %tmp6, ptr %tmp2, align 84129 %tmp8 = add i64 %tmp, 44130 %tmp9 = icmp eq i64 %tmp8, 10244131 br i1 %tmp9, label %bb10, label %bb14132 4133bb10: ; preds = %bb14134 ret void4135}4136 4137define void @bcast_unfold_cmp_v8f64(ptr %arg) {4138; CHECK-LABEL: bcast_unfold_cmp_v8f64:4139; CHECK: # %bb.0: # %bb4140; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0004141; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]4142; CHECK-NEXT: vbroadcastsd {{.*#+}} zmm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]4143; CHECK-NEXT: .p2align 44144; CHECK-NEXT: .LBB125_1: # %bb14145; CHECK-NEXT: # =>This Inner Loop Header: Depth=14146; CHECK-NEXT: vcmpngtpd 8192(%rdi,%rax), %zmm0, %k14147; CHECK-NEXT: vmovupd %zmm1, 8192(%rdi,%rax) {%k1}4148; CHECK-NEXT: addq $64, %rax4149; CHECK-NEXT: jne .LBB125_14150; CHECK-NEXT: # %bb.2: # %bb104151; CHECK-NEXT: vzeroupper4152; CHECK-NEXT: retq4153bb:4154 br label %bb14155 4156bb1: ; preds = %bb1, %bb4157 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]4158 %tmp2 = getelementptr inbounds double, ptr %arg, i64 %tmp4159 %tmp4 = load <8 x double>, ptr %tmp2, align 84160 %tmp5 = fcmp olt <8 x double> %tmp4, <double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00, double 2.000000e+00>4161 %tmp6 = select <8 x i1> %tmp5, <8 x double> %tmp4, <8 x double> <double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00, double 3.000000e+00>4162 store <8 x double> %tmp6, ptr %tmp2, align 84163 %tmp8 = add i64 %tmp, 84164 %tmp9 = icmp eq i64 %tmp8, 10244165 br i1 %tmp9, label %bb10, label %bb14166 4167bb10: ; preds = %bb14168 ret void4169}4170 4171define void @bcast_unfold_cmp_v8f32_refold(ptr nocapture %0) {4172; CHECK-LABEL: bcast_unfold_cmp_v8f32_refold:4173; CHECK: # %bb.0:4174; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0004175; CHECK-NEXT: vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]4176; CHECK-NEXT: vbroadcastss {{.*#+}} ymm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]4177; CHECK-NEXT: .p2align 44178; CHECK-NEXT: .LBB126_1: # =>This Inner Loop Header: Depth=14179; CHECK-NEXT: vcmpgtps 4096(%rdi,%rax), %ymm0, %k14180; CHECK-NEXT: vblendmps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm2 {%k1}4181; CHECK-NEXT: vmovups %ymm2, 4096(%rdi,%rax)4182; CHECK-NEXT: addq $32, %rax4183; CHECK-NEXT: jne .LBB126_14184; CHECK-NEXT: # %bb.2:4185; CHECK-NEXT: vzeroupper4186; CHECK-NEXT: retq4187 br label %24188 41892: ; preds = %2, %14190 %3 = phi i64 [ 0, %1 ], [ %8, %2 ]4191 %4 = getelementptr inbounds float, ptr %0, i64 %34192 %5 = load <8 x float>, ptr %4, align 44193 %6 = fcmp olt <8 x float> %5, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>4194 %7 = select <8 x i1> %6, <8 x float> <float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00>, <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>4195 store <8 x float> %7, ptr %4, align 44196 %8 = add i64 %3, 84197 %9 = icmp eq i64 %8, 10244198 br i1 %9, label %10, label %24199 420010: ; preds = %24201 ret void4202}4203 4204define void @bcast_unfold_ptestm_v4i32(ptr %arg) {4205; CHECK-LABEL: bcast_unfold_ptestm_v4i32:4206; CHECK: # %bb.0: # %bb4207; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0004208; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]4209; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm1 = [3,3,3,3]4210; CHECK-NEXT: .p2align 44211; CHECK-NEXT: .LBB127_1: # %bb14212; CHECK-NEXT: # =>This Inner Loop Header: Depth=14213; CHECK-NEXT: vptestmd 4096(%rdi,%rax), %xmm0, %k14214; CHECK-NEXT: vmovdqu32 %xmm1, 4096(%rdi,%rax) {%k1}4215; CHECK-NEXT: addq $16, %rax4216; CHECK-NEXT: jne .LBB127_14217; CHECK-NEXT: # %bb.2: # %bb104218; CHECK-NEXT: retq4219bb:4220 br label %bb14221 4222bb1: ; preds = %bb1, %bb4223 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]4224 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp4225 %tmp4 = load <4 x i32>, ptr %tmp2, align 44226 %tmp4b = and <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>4227 %tmp5 = icmp ne <4 x i32> %tmp4b, zeroinitializer4228 %tmp6 = select <4 x i1> %tmp5, <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> %tmp44229 store <4 x i32> %tmp6, ptr %tmp2, align 44230 %tmp8 = add i64 %tmp, 44231 %tmp9 = icmp eq i64 %tmp8, 10244232 br i1 %tmp9, label %bb10, label %bb14233 4234bb10: ; preds = %bb14235 ret void4236}4237 4238define void @bcast_unfold_ptestnm_v4i32(ptr %arg) {4239; CHECK-LABEL: bcast_unfold_ptestnm_v4i32:4240; CHECK: # %bb.0: # %bb4241; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0004242; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]4243; CHECK-NEXT: vpbroadcastd {{.*#+}} xmm1 = [3,3,3,3]4244; CHECK-NEXT: .p2align 44245; CHECK-NEXT: .LBB128_1: # %bb14246; CHECK-NEXT: # =>This Inner Loop Header: Depth=14247; CHECK-NEXT: vptestnmd 4096(%rdi,%rax), %xmm0, %k14248; CHECK-NEXT: vmovdqu32 %xmm1, 4096(%rdi,%rax) {%k1}4249; CHECK-NEXT: addq $16, %rax4250; CHECK-NEXT: jne .LBB128_14251; CHECK-NEXT: # %bb.2: # %bb104252; CHECK-NEXT: retq4253bb:4254 br label %bb14255 4256bb1: ; preds = %bb1, %bb4257 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]4258 %tmp2 = getelementptr inbounds i32, ptr %arg, i64 %tmp4259 %tmp4 = load <4 x i32>, ptr %tmp2, align 44260 %tmp4b = and <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>4261 %tmp5 = icmp eq <4 x i32> %tmp4b, zeroinitializer4262 %tmp6 = select <4 x i1> %tmp5, <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> %tmp44263 store <4 x i32> %tmp6, ptr %tmp2, align 44264 %tmp8 = add i64 %tmp, 44265 %tmp9 = icmp eq i64 %tmp8, 10244266 br i1 %tmp9, label %bb10, label %bb14267 4268bb10: ; preds = %bb14269 ret void4270}4271 4272define void @bcast_unfold_ptestm_v4i64(ptr %arg) {4273; CHECK-LABEL: bcast_unfold_ptestm_v4i64:4274; CHECK: # %bb.0: # %bb4275; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0004276; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]4277; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm1 = [3,3,3,3]4278; CHECK-NEXT: .p2align 44279; CHECK-NEXT: .LBB129_1: # %bb14280; CHECK-NEXT: # =>This Inner Loop Header: Depth=14281; CHECK-NEXT: vptestmq 8192(%rdi,%rax), %ymm0, %k14282; CHECK-NEXT: vmovdqu64 %ymm1, 8192(%rdi,%rax) {%k1}4283; CHECK-NEXT: addq $32, %rax4284; CHECK-NEXT: jne .LBB129_14285; CHECK-NEXT: # %bb.2: # %bb104286; CHECK-NEXT: vzeroupper4287; CHECK-NEXT: retq4288bb:4289 br label %bb14290 4291bb1: ; preds = %bb1, %bb4292 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]4293 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp4294 %tmp4 = load <4 x i64>, ptr %tmp2, align 84295 %tmp4b = and <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>4296 %tmp5 = icmp ne <4 x i64> %tmp4b, zeroinitializer4297 %tmp6 = select <4 x i1> %tmp5, <4 x i64> <i64 3, i64 3, i64 3, i64 3>, <4 x i64> %tmp44298 store <4 x i64> %tmp6, ptr %tmp2, align 84299 %tmp8 = add i64 %tmp, 44300 %tmp9 = icmp eq i64 %tmp8, 10244301 br i1 %tmp9, label %bb10, label %bb14302 4303bb10: ; preds = %bb14304 ret void4305}4306 4307define void @bcast_unfold_ptestnm_v4i64(ptr %arg) {4308; CHECK-LABEL: bcast_unfold_ptestnm_v4i64:4309; CHECK: # %bb.0: # %bb4310; CHECK-NEXT: movq $-8192, %rax # imm = 0xE0004311; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]4312; CHECK-NEXT: vpbroadcastq {{.*#+}} ymm1 = [3,3,3,3]4313; CHECK-NEXT: .p2align 44314; CHECK-NEXT: .LBB130_1: # %bb14315; CHECK-NEXT: # =>This Inner Loop Header: Depth=14316; CHECK-NEXT: vptestnmq 8192(%rdi,%rax), %ymm0, %k14317; CHECK-NEXT: vmovdqu64 %ymm1, 8192(%rdi,%rax) {%k1}4318; CHECK-NEXT: addq $32, %rax4319; CHECK-NEXT: jne .LBB130_14320; CHECK-NEXT: # %bb.2: # %bb104321; CHECK-NEXT: vzeroupper4322; CHECK-NEXT: retq4323bb:4324 br label %bb14325 4326bb1: ; preds = %bb1, %bb4327 %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]4328 %tmp2 = getelementptr inbounds i64, ptr %arg, i64 %tmp4329 %tmp4 = load <4 x i64>, ptr %tmp2, align 84330 %tmp4b = and <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>4331 %tmp5 = icmp eq <4 x i64> %tmp4b, zeroinitializer4332 %tmp6 = select <4 x i1> %tmp5, <4 x i64> <i64 3, i64 3, i64 3, i64 3>, <4 x i64> %tmp44333 store <4 x i64> %tmp6, ptr %tmp2, align 84334 %tmp8 = add i64 %tmp, 44335 %tmp9 = icmp eq i64 %tmp8, 10244336 br i1 %tmp9, label %bb10, label %bb14337 4338bb10: ; preds = %bb14339 ret void4340}4341 4342define void @bcast_unfold_vpternlog_v16i32(ptr %arg, ptr %arg1) {4343; CHECK-LABEL: bcast_unfold_vpternlog_v16i32:4344; CHECK: # %bb.0: # %bb4345; CHECK-NEXT: movq $-4096, %rax # imm = 0xF0004346; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm0 = [32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767,32767]4347; CHECK-NEXT: .p2align 44348; CHECK-NEXT: .LBB131_1: # %bb24349; CHECK-NEXT: # =>This Inner Loop Header: Depth=14350; CHECK-NEXT: vmovdqu64 4096(%rdi,%rax), %zmm14351; CHECK-NEXT: vmovdqu64 4096(%rsi,%rax), %zmm24352; CHECK-NEXT: vpmulld %zmm2, %zmm1, %zmm34353; CHECK-NEXT: vpternlogd {{.*#+}} zmm2 = zmm2 ^ (zmm0 & (zmm2 ^ zmm1))4354; CHECK-NEXT: vpmulld %zmm3, %zmm2, %zmm14355; CHECK-NEXT: vmovdqu64 %zmm1, 4096(%rdi,%rax)4356; CHECK-NEXT: addq $64, %rax4357; CHECK-NEXT: jne .LBB131_14358; CHECK-NEXT: # %bb.2: # %bb204359; CHECK-NEXT: vzeroupper4360; CHECK-NEXT: retq4361bb:4362 br label %bb24363 4364bb2: ; preds = %bb2, %bb4365 %tmp = phi i64 [ 0, %bb ], [ %tmp18, %bb2 ]4366 %tmp3 = getelementptr inbounds i32, ptr %arg, i64 %tmp4367 %tmp5 = load <16 x i32>, ptr %tmp3, align 44368 %tmp6 = getelementptr inbounds i32, ptr %arg1, i64 %tmp4369 %tmp11 = load <16 x i32>, ptr %tmp6, align 44370 %tmp12 = and <16 x i32> %tmp5, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>4371 %tmp13 = and <16 x i32> %tmp11, <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>4372 %tmp14 = or <16 x i32> %tmp12, %tmp134373 %tmp15 = mul <16 x i32> %tmp14, %tmp54374 %tmp16 = mul <16 x i32> %tmp15, %tmp114375 store <16 x i32> %tmp16, ptr %tmp3, align 44376 %tmp18 = add i64 %tmp, 164377 %tmp19 = icmp eq i64 %tmp18, 10244378 br i1 %tmp19, label %bb20, label %bb24379 4380bb20: ; preds = %bb24381 ret void4382}4383 4384attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }4385