1089 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+zvl256b | FileCheck %s --check-prefixes=CHECK,V3; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+f,+zve32f,+zvl256b | FileCheck %s --check-prefixes=CHECK,ZVE32F4; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+v,+optimized-zero-stride-load,+zvl256b | FileCheck %s --check-prefixes=CHECK,OPTIMIZED,OPTZVE32F5; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+f,+zve32f,+optimized-zero-stride-load,+zvl256b | FileCheck %s --check-prefixes=CHECK,OPTIMIZED,OPTV6 7%struct.foo = type { i32, i32, i32, i32 }8 9; void gather(signed char * __restrict A, signed char * __restrict B) {10; for (int i = 0; i != 1024; ++i)11; A[i] += B[i * 5];12; }13define void @gather(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {14; CHECK-LABEL: gather:15; CHECK: # %bb.0: # %entry16; CHECK-NEXT: addi a2, a0, 102417; CHECK-NEXT: li a4, 3218; CHECK-NEXT: li a3, 519; CHECK-NEXT: vsetvli zero, a4, e8, m1, ta, ma20; CHECK-NEXT: .LBB0_1: # %vector.body21; CHECK-NEXT: # =>This Inner Loop Header: Depth=122; CHECK-NEXT: vlse8.v v8, (a1), a323; CHECK-NEXT: vle8.v v9, (a0)24; CHECK-NEXT: vadd.vv v8, v9, v825; CHECK-NEXT: vse8.v v8, (a0)26; CHECK-NEXT: addi a0, a0, 3227; CHECK-NEXT: addi a1, a1, 16028; CHECK-NEXT: bne a0, a2, .LBB0_129; CHECK-NEXT: # %bb.2: # %for.cond.cleanup30; CHECK-NEXT: ret31entry:32 br label %vector.body33 34vector.body: ; preds = %vector.body, %entry35 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]36 %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]37 %i = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)38 %i1 = getelementptr inbounds i8, ptr %B, <32 x i64> %i39 %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i1, i32 1, <32 x i1> splat (i1 true), <32 x i8> poison)40 %i2 = getelementptr inbounds i8, ptr %A, i64 %index41 %wide.load = load <32 x i8>, ptr %i2, align 142 %i4 = add <32 x i8> %wide.load, %wide.masked.gather43 store <32 x i8> %i4, ptr %i2, align 144 %index.next = add nuw i64 %index, 3245 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)46 %i6 = icmp eq i64 %index.next, 102447 br i1 %i6, label %for.cond.cleanup, label %vector.body48 49for.cond.cleanup: ; preds = %vector.body50 ret void51}52 53define void @gather_masked(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, <32 x i8> %maskedoff) {54; CHECK-LABEL: gather_masked:55; CHECK: # %bb.0: # %entry56; CHECK-NEXT: addi a2, a0, 102457; CHECK-NEXT: lui a4, 98376558; CHECK-NEXT: li a3, 3259; CHECK-NEXT: addi a4, a4, 87360; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma61; CHECK-NEXT: vmv.s.x v0, a462; CHECK-NEXT: li a4, 563; CHECK-NEXT: .LBB1_1: # %vector.body64; CHECK-NEXT: # =>This Inner Loop Header: Depth=165; CHECK-NEXT: vsetvli zero, a3, e8, m1, ta, mu66; CHECK-NEXT: vmv1r.v v9, v867; CHECK-NEXT: vlse8.v v9, (a1), a4, v0.t68; CHECK-NEXT: vle8.v v10, (a0)69; CHECK-NEXT: vadd.vv v9, v10, v970; CHECK-NEXT: vse8.v v9, (a0)71; CHECK-NEXT: addi a0, a0, 3272; CHECK-NEXT: addi a1, a1, 16073; CHECK-NEXT: bne a0, a2, .LBB1_174; CHECK-NEXT: # %bb.2: # %for.cond.cleanup75; CHECK-NEXT: ret76entry:77 br label %vector.body78 79vector.body: ; preds = %vector.body, %entry80 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]81 %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]82 %i = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)83 %i1 = getelementptr inbounds i8, ptr %B, <32 x i64> %i84 %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i1, i32 1, <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, <32 x i8> %maskedoff)85 %i2 = getelementptr inbounds i8, ptr %A, i64 %index86 %wide.load = load <32 x i8>, ptr %i2, align 187 %i4 = add <32 x i8> %wide.load, %wide.masked.gather88 store <32 x i8> %i4, ptr %i2, align 189 %index.next = add nuw i64 %index, 3290 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)91 %i6 = icmp eq i64 %index.next, 102492 br i1 %i6, label %for.cond.cleanup, label %vector.body93 94for.cond.cleanup: ; preds = %vector.body95 ret void96}97 98define void @gather_negative_stride(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {99; CHECK-LABEL: gather_negative_stride:100; CHECK: # %bb.0: # %entry101; CHECK-NEXT: addi a1, a1, 155102; CHECK-NEXT: addi a2, a0, 1024103; CHECK-NEXT: li a4, 32104; CHECK-NEXT: li a3, -5105; CHECK-NEXT: vsetvli zero, a4, e8, m1, ta, ma106; CHECK-NEXT: .LBB2_1: # %vector.body107; CHECK-NEXT: # =>This Inner Loop Header: Depth=1108; CHECK-NEXT: vlse8.v v8, (a1), a3109; CHECK-NEXT: vle8.v v9, (a0)110; CHECK-NEXT: vadd.vv v8, v9, v8111; CHECK-NEXT: vse8.v v8, (a0)112; CHECK-NEXT: addi a0, a0, 32113; CHECK-NEXT: addi a1, a1, 160114; CHECK-NEXT: bne a0, a2, .LBB2_1115; CHECK-NEXT: # %bb.2: # %for.cond.cleanup116; CHECK-NEXT: ret117entry:118 br label %vector.body119 120vector.body: ; preds = %vector.body, %entry121 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]122 %vec.ind = phi <32 x i64> [ <i64 31, i64 30, i64 29, i64 28, i64 27, i64 26, i64 25, i64 24, i64 23, i64 22, i64 21, i64 20, i64 19, i64 18, i64 17, i64 16, i64 15, i64 14, i64 13, i64 12, i64 11, i64 10, i64 9, i64 8, i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>, %entry ], [ %vec.ind.next, %vector.body ]123 %i = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)124 %i1 = getelementptr inbounds i8, ptr %B, <32 x i64> %i125 %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i1, i32 1, <32 x i1> splat (i1 true), <32 x i8> poison)126 %i2 = getelementptr inbounds i8, ptr %A, i64 %index127 %wide.load = load <32 x i8>, ptr %i2, align 1128 %i4 = add <32 x i8> %wide.load, %wide.masked.gather129 store <32 x i8> %i4, ptr %i2, align 1130 %index.next = add nuw i64 %index, 32131 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)132 %i6 = icmp eq i64 %index.next, 1024133 br i1 %i6, label %for.cond.cleanup, label %vector.body134 135for.cond.cleanup: ; preds = %vector.body136 ret void137}138 139define void @gather_zero_stride(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {140; CHECK-LABEL: gather_zero_stride:141; CHECK: # %bb.0: # %entry142; CHECK-NEXT: addi a2, a0, 1024143; CHECK-NEXT: li a3, 32144; CHECK-NEXT: vsetvli zero, a3, e8, m1, ta, ma145; CHECK-NEXT: .LBB3_1: # %vector.body146; CHECK-NEXT: # =>This Inner Loop Header: Depth=1147; CHECK-NEXT: lbu a3, 0(a1)148; CHECK-NEXT: vle8.v v8, (a0)149; CHECK-NEXT: vadd.vx v8, v8, a3150; CHECK-NEXT: vse8.v v8, (a0)151; CHECK-NEXT: addi a0, a0, 32152; CHECK-NEXT: addi a1, a1, 160153; CHECK-NEXT: bne a0, a2, .LBB3_1154; CHECK-NEXT: # %bb.2: # %for.cond.cleanup155; CHECK-NEXT: ret156entry:157 br label %vector.body158 159vector.body: ; preds = %vector.body, %entry160 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]161 %vec.ind = phi <32 x i64> [ zeroinitializer, %entry ], [ %vec.ind.next, %vector.body ]162 %i = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)163 %i1 = getelementptr inbounds i8, ptr %B, <32 x i64> %i164 %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i1, i32 1, <32 x i1> splat (i1 true), <32 x i8> poison)165 %i2 = getelementptr inbounds i8, ptr %A, i64 %index166 %wide.load = load <32 x i8>, ptr %i2, align 1167 %i4 = add <32 x i8> %wide.load, %wide.masked.gather168 store <32 x i8> %i4, ptr %i2, align 1169 %index.next = add nuw i64 %index, 32170 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)171 %i6 = icmp eq i64 %index.next, 1024172 br i1 %i6, label %for.cond.cleanup, label %vector.body173 174for.cond.cleanup: ; preds = %vector.body175 ret void176}177 178define void @gather_zero_stride_i32(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {179; CHECK-LABEL: gather_zero_stride_i32:180; CHECK: # %bb.0: # %entry181; CHECK-NEXT: addi a2, a0, 1024182; CHECK-NEXT: vsetivli zero, 8, e32, m1, ta, ma183; CHECK-NEXT: .LBB4_1: # %vector.body184; CHECK-NEXT: # =>This Inner Loop Header: Depth=1185; CHECK-NEXT: lw a3, 0(a1)186; CHECK-NEXT: vle32.v v8, (a0)187; CHECK-NEXT: vadd.vx v8, v8, a3188; CHECK-NEXT: vse32.v v8, (a0)189; CHECK-NEXT: addi a0, a0, 8190; CHECK-NEXT: addi a1, a1, 160191; CHECK-NEXT: bne a0, a2, .LBB4_1192; CHECK-NEXT: # %bb.2: # %for.cond.cleanup193; CHECK-NEXT: ret194entry:195 br label %vector.body196 197vector.body: ; preds = %vector.body, %entry198 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]199 %vec.ind = phi <8 x i64> [ zeroinitializer, %entry ], [ %vec.ind.next, %vector.body ]200 %i = mul nuw nsw <8 x i64> %vec.ind, splat (i64 5)201 %i1 = getelementptr inbounds i8, ptr %B, <8 x i64> %i202 %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i1, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)203 %i2 = getelementptr inbounds i8, ptr %A, i64 %index204 %wide.load = load <8 x i32>, ptr %i2, align 4205 %i4 = add <8 x i32> %wide.load, %wide.masked.gather206 store <8 x i32> %i4, ptr %i2, align 4207 %index.next = add nuw i64 %index, 8208 %vec.ind.next = add <8 x i64> %vec.ind, splat (i64 32)209 %i6 = icmp eq i64 %index.next, 1024210 br i1 %i6, label %for.cond.cleanup, label %vector.body211 212for.cond.cleanup: ; preds = %vector.body213 ret void214}215 216define void @gather_zero_stride_unfold(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {217; V-LABEL: gather_zero_stride_unfold:218; V: # %bb.0: # %entry219; V-NEXT: addi a2, a0, 1024220; V-NEXT: li a3, 32221; V-NEXT: vsetvli zero, a3, e8, m1, ta, ma222; V-NEXT: .LBB5_1: # %vector.body223; V-NEXT: # =>This Inner Loop Header: Depth=1224; V-NEXT: lbu a3, 0(a1)225; V-NEXT: vle8.v v8, (a0)226; V-NEXT: vmv.v.x v9, a3227; V-NEXT: vdivu.vv v8, v9, v8228; V-NEXT: vse8.v v8, (a0)229; V-NEXT: addi a0, a0, 32230; V-NEXT: addi a1, a1, 160231; V-NEXT: bne a0, a2, .LBB5_1232; V-NEXT: # %bb.2: # %for.cond.cleanup233; V-NEXT: ret234;235; ZVE32F-LABEL: gather_zero_stride_unfold:236; ZVE32F: # %bb.0: # %entry237; ZVE32F-NEXT: addi a2, a0, 1024238; ZVE32F-NEXT: li a3, 32239; ZVE32F-NEXT: vsetvli zero, a3, e8, m1, ta, ma240; ZVE32F-NEXT: .LBB5_1: # %vector.body241; ZVE32F-NEXT: # =>This Inner Loop Header: Depth=1242; ZVE32F-NEXT: lbu a3, 0(a1)243; ZVE32F-NEXT: vle8.v v8, (a0)244; ZVE32F-NEXT: vmv.v.x v9, a3245; ZVE32F-NEXT: vdivu.vv v8, v9, v8246; ZVE32F-NEXT: vse8.v v8, (a0)247; ZVE32F-NEXT: addi a0, a0, 32248; ZVE32F-NEXT: addi a1, a1, 160249; ZVE32F-NEXT: bne a0, a2, .LBB5_1250; ZVE32F-NEXT: # %bb.2: # %for.cond.cleanup251; ZVE32F-NEXT: ret252;253; OPTIMIZED-LABEL: gather_zero_stride_unfold:254; OPTIMIZED: # %bb.0: # %entry255; OPTIMIZED-NEXT: addi a2, a0, 1024256; OPTIMIZED-NEXT: li a3, 32257; OPTIMIZED-NEXT: vsetvli zero, a3, e8, m1, ta, ma258; OPTIMIZED-NEXT: .LBB5_1: # %vector.body259; OPTIMIZED-NEXT: # =>This Inner Loop Header: Depth=1260; OPTIMIZED-NEXT: vlse8.v v8, (a1), zero261; OPTIMIZED-NEXT: vle8.v v9, (a0)262; OPTIMIZED-NEXT: vdivu.vv v8, v8, v9263; OPTIMIZED-NEXT: vse8.v v8, (a0)264; OPTIMIZED-NEXT: addi a0, a0, 32265; OPTIMIZED-NEXT: addi a1, a1, 160266; OPTIMIZED-NEXT: bne a0, a2, .LBB5_1267; OPTIMIZED-NEXT: # %bb.2: # %for.cond.cleanup268; OPTIMIZED-NEXT: ret269entry:270 br label %vector.body271 272vector.body: ; preds = %vector.body, %entry273 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]274 %vec.ind = phi <32 x i64> [ zeroinitializer, %entry ], [ %vec.ind.next, %vector.body ]275 %i = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)276 %i1 = getelementptr inbounds i8, ptr %B, <32 x i64> %i277 %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i1, i32 1, <32 x i1> splat (i1 true), <32 x i8> poison)278 %i2 = getelementptr inbounds i8, ptr %A, i64 %index279 %wide.load = load <32 x i8>, ptr %i2, align 1280 %i4 = udiv <32 x i8> %wide.masked.gather, %wide.load281 store <32 x i8> %i4, ptr %i2, align 1282 %index.next = add nuw i64 %index, 32283 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)284 %i6 = icmp eq i64 %index.next, 1024285 br i1 %i6, label %for.cond.cleanup, label %vector.body286 287for.cond.cleanup: ; preds = %vector.body288 ret void289}290 291;void scatter(signed char * __restrict A, signed char * __restrict B) {292; for (int i = 0; i < 1024; ++i)293; A[i * 5] += B[i];294;}295define void @scatter(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {296; CHECK-LABEL: scatter:297; CHECK: # %bb.0: # %entry298; CHECK-NEXT: addi a2, a1, 1024299; CHECK-NEXT: li a4, 32300; CHECK-NEXT: li a3, 5301; CHECK-NEXT: vsetvli zero, a4, e8, m1, ta, ma302; CHECK-NEXT: .LBB6_1: # %vector.body303; CHECK-NEXT: # =>This Inner Loop Header: Depth=1304; CHECK-NEXT: vle8.v v8, (a1)305; CHECK-NEXT: vlse8.v v9, (a0), a3306; CHECK-NEXT: addi a1, a1, 32307; CHECK-NEXT: vadd.vv v8, v9, v8308; CHECK-NEXT: vsse8.v v8, (a0), a3309; CHECK-NEXT: addi a0, a0, 160310; CHECK-NEXT: bne a1, a2, .LBB6_1311; CHECK-NEXT: # %bb.2: # %for.cond.cleanup312; CHECK-NEXT: ret313entry:314 br label %vector.body315 316vector.body: ; preds = %vector.body, %entry317 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]318 %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]319 %i = getelementptr inbounds i8, ptr %B, i64 %index320 %wide.load = load <32 x i8>, ptr %i, align 1321 %i2 = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)322 %i3 = getelementptr inbounds i8, ptr %A, <32 x i64> %i2323 %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i3, i32 1, <32 x i1> splat (i1 true), <32 x i8> poison)324 %i4 = add <32 x i8> %wide.masked.gather, %wide.load325 call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> %i4, <32 x ptr> %i3, i32 1, <32 x i1> splat (i1 true))326 %index.next = add nuw i64 %index, 32327 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)328 %i5 = icmp eq i64 %index.next, 1024329 br i1 %i5, label %for.cond.cleanup, label %vector.body330 331for.cond.cleanup: ; preds = %vector.body332 ret void333}334 335define void @scatter_masked(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, <32 x i8> %maskedoff) {336; CHECK-LABEL: scatter_masked:337; CHECK: # %bb.0: # %entry338; CHECK-NEXT: addi a2, a1, 1024339; CHECK-NEXT: li a3, 32340; CHECK-NEXT: lui a4, 983765341; CHECK-NEXT: addi a4, a4, 873342; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma343; CHECK-NEXT: vmv.s.x v0, a4344; CHECK-NEXT: li a4, 5345; CHECK-NEXT: .LBB7_1: # %vector.body346; CHECK-NEXT: # =>This Inner Loop Header: Depth=1347; CHECK-NEXT: vsetvli zero, a3, e8, m1, ta, mu348; CHECK-NEXT: vle8.v v9, (a1)349; CHECK-NEXT: vmv1r.v v10, v8350; CHECK-NEXT: vlse8.v v10, (a0), a4, v0.t351; CHECK-NEXT: addi a1, a1, 32352; CHECK-NEXT: vadd.vv v9, v10, v9353; CHECK-NEXT: vsse8.v v9, (a0), a4, v0.t354; CHECK-NEXT: addi a0, a0, 160355; CHECK-NEXT: bne a1, a2, .LBB7_1356; CHECK-NEXT: # %bb.2: # %for.cond.cleanup357; CHECK-NEXT: ret358entry:359 br label %vector.body360 361vector.body: ; preds = %vector.body, %entry362 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]363 %vec.ind = phi <32 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>, %entry ], [ %vec.ind.next, %vector.body ]364 %i = getelementptr inbounds i8, ptr %B, i64 %index365 %wide.load = load <32 x i8>, ptr %i, align 1366 %i2 = mul nuw nsw <32 x i64> %vec.ind, splat (i64 5)367 %i3 = getelementptr inbounds i8, ptr %A, <32 x i64> %i2368 %wide.masked.gather = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i3, i32 1, <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, <32 x i8> %maskedoff)369 %i4 = add <32 x i8> %wide.masked.gather, %wide.load370 call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> %i4, <32 x ptr> %i3, i32 1, <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>)371 %index.next = add nuw i64 %index, 32372 %vec.ind.next = add <32 x i64> %vec.ind, splat (i64 32)373 %i5 = icmp eq i64 %index.next, 1024374 br i1 %i5, label %for.cond.cleanup, label %vector.body375 376for.cond.cleanup: ; preds = %vector.body377 ret void378}379 380; void gather_pow2(signed char * __restrict A, signed char * __restrict B) {381; for (int i = 0; i != 1024; ++i)382; A[i] += B[i * 4];383; }384define void @gather_pow2(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {385; CHECK-LABEL: gather_pow2:386; CHECK: # %bb.0: # %entry387; CHECK-NEXT: lui a3, 1388; CHECK-NEXT: li a2, 16389; CHECK-NEXT: add a3, a0, a3390; CHECK-NEXT: li a4, 32391; CHECK-NEXT: .LBB8_1: # %vector.body392; CHECK-NEXT: # =>This Inner Loop Header: Depth=1393; CHECK-NEXT: vsetivli zero, 8, e32, m1, ta, ma394; CHECK-NEXT: vlse32.v v8, (a1), a2395; CHECK-NEXT: vsetvli zero, a4, e8, m1, ta, ma396; CHECK-NEXT: vle8.v v9, (a0)397; CHECK-NEXT: vsetivli zero, 8, e32, m1, ta, ma398; CHECK-NEXT: vadd.vv v8, v9, v8399; CHECK-NEXT: vsetvli zero, a4, e8, m1, ta, ma400; CHECK-NEXT: vse8.v v8, (a0)401; CHECK-NEXT: addi a0, a0, 32402; CHECK-NEXT: addi a1, a1, 128403; CHECK-NEXT: bne a0, a3, .LBB8_1404; CHECK-NEXT: # %bb.2: # %for.cond.cleanup405; CHECK-NEXT: ret406entry:407 br label %vector.body408 409vector.body: ; preds = %vector.body, %entry410 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]411 %vec.ind = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %entry ], [ %vec.ind.next, %vector.body ]412 %i = shl nsw <8 x i64> %vec.ind, splat (i64 2)413 %i1 = getelementptr inbounds i32, ptr %B, <8 x i64> %i414 %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i1, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)415 %i2 = getelementptr inbounds i32, ptr %A, i64 %index416 %wide.load = load <8 x i32>, ptr %i2, align 1417 %i4 = add <8 x i32> %wide.load, %wide.masked.gather418 store <8 x i32> %i4, ptr %i2, align 1419 %index.next = add nuw i64 %index, 8420 %vec.ind.next = add <8 x i64> %vec.ind, splat (i64 8)421 %i6 = icmp eq i64 %index.next, 1024422 br i1 %i6, label %for.cond.cleanup, label %vector.body423 424for.cond.cleanup: ; preds = %vector.body425 ret void426}427 428;void scatter_pow2(signed char * __restrict A, signed char * __restrict B) {429; for (int i = 0; i < 1024; ++i)430; A[i * 4] += B[i];431;}432define void @scatter_pow2(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {433; CHECK-LABEL: scatter_pow2:434; CHECK: # %bb.0: # %entry435; CHECK-NEXT: lui a3, 1436; CHECK-NEXT: li a2, 32437; CHECK-NEXT: add a3, a1, a3438; CHECK-NEXT: li a4, 16439; CHECK-NEXT: .LBB9_1: # %vector.body440; CHECK-NEXT: # =>This Inner Loop Header: Depth=1441; CHECK-NEXT: vsetvli zero, a2, e8, m1, ta, ma442; CHECK-NEXT: vle8.v v8, (a1)443; CHECK-NEXT: vsetivli zero, 8, e32, m1, ta, ma444; CHECK-NEXT: vlse32.v v9, (a0), a4445; CHECK-NEXT: addi a1, a1, 32446; CHECK-NEXT: vadd.vv v8, v9, v8447; CHECK-NEXT: vsse32.v v8, (a0), a4448; CHECK-NEXT: addi a0, a0, 128449; CHECK-NEXT: bne a1, a3, .LBB9_1450; CHECK-NEXT: # %bb.2: # %for.cond.cleanup451; CHECK-NEXT: ret452entry:453 br label %vector.body454 455vector.body: ; preds = %vector.body, %entry456 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]457 %vec.ind = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %entry ], [ %vec.ind.next, %vector.body ]458 %i = getelementptr inbounds i32, ptr %B, i64 %index459 %wide.load = load <8 x i32>, ptr %i, align 1460 %i2 = shl nuw nsw <8 x i64> %vec.ind, splat (i64 2)461 %i3 = getelementptr inbounds i32, ptr %A, <8 x i64> %i2462 %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i3, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)463 %i4 = add <8 x i32> %wide.masked.gather, %wide.load464 call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %i4, <8 x ptr> %i3, i32 4, <8 x i1> splat (i1 true))465 %index.next = add nuw i64 %index, 8466 %vec.ind.next = add <8 x i64> %vec.ind, splat (i64 8)467 %i5 = icmp eq i64 %index.next, 1024468 br i1 %i5, label %for.cond.cleanup, label %vector.body469 470for.cond.cleanup: ; preds = %vector.body471 ret void472}473 474;struct foo {475; int a, b, c, d;476;};477;478;void struct_gather(int * __restrict A, struct foo * __restrict B) {479; for (int i = 0; i < 1024; ++i)480; A[i] += B[i].b;481;}482define void @struct_gather(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {483; CHECK-LABEL: struct_gather:484; CHECK: # %bb.0: # %entry485; CHECK-NEXT: addi a1, a1, 132486; CHECK-NEXT: lui a2, 1487; CHECK-NEXT: add a2, a0, a2488; CHECK-NEXT: li a3, 16489; CHECK-NEXT: vsetivli zero, 8, e32, m1, ta, ma490; CHECK-NEXT: .LBB10_1: # %vector.body491; CHECK-NEXT: # =>This Inner Loop Header: Depth=1492; CHECK-NEXT: addi a4, a0, 32493; CHECK-NEXT: addi a5, a1, -128494; CHECK-NEXT: vlse32.v v8, (a1), a3495; CHECK-NEXT: vle32.v v9, (a0)496; CHECK-NEXT: vlse32.v v10, (a5), a3497; CHECK-NEXT: vle32.v v11, (a4)498; CHECK-NEXT: vadd.vv v9, v9, v10499; CHECK-NEXT: vadd.vv v8, v11, v8500; CHECK-NEXT: vse32.v v9, (a0)501; CHECK-NEXT: vse32.v v8, (a4)502; CHECK-NEXT: addi a0, a0, 64503; CHECK-NEXT: addi a1, a1, 256504; CHECK-NEXT: bne a0, a2, .LBB10_1505; CHECK-NEXT: # %bb.2: # %for.cond.cleanup506; CHECK-NEXT: ret507entry:508 br label %vector.body509 510vector.body: ; preds = %vector.body, %entry511 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]512 %vec.ind = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %entry ], [ %vec.ind.next, %vector.body ]513 %step.add = add <8 x i64> %vec.ind, splat (i64 8)514 %i = getelementptr inbounds %struct.foo, ptr %B, <8 x i64> %vec.ind, i32 1515 %i1 = getelementptr inbounds %struct.foo, ptr %B, <8 x i64> %step.add, i32 1516 %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)517 %wide.masked.gather9 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i1, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)518 %i2 = getelementptr inbounds i32, ptr %A, i64 %index519 %wide.load = load <8 x i32>, ptr %i2, align 4520 %i4 = getelementptr inbounds i32, ptr %i2, i64 8521 %wide.load10 = load <8 x i32>, ptr %i4, align 4522 %i6 = add nsw <8 x i32> %wide.load, %wide.masked.gather523 %i7 = add nsw <8 x i32> %wide.load10, %wide.masked.gather9524 store <8 x i32> %i6, ptr %i2, align 4525 store <8 x i32> %i7, ptr %i4, align 4526 %index.next = add nuw i64 %index, 16527 %vec.ind.next = add <8 x i64> %vec.ind, splat (i64 16)528 %i10 = icmp eq i64 %index.next, 1024529 br i1 %i10, label %for.cond.cleanup, label %vector.body530 531for.cond.cleanup: ; preds = %vector.body532 ret void533}534 535;void gather_unroll(int * __restrict A, int * __restrict B) {536; for (int i = 0; i < 1024; i+= 4 ) {537; A[i] += B[i * 4];538; A[i+1] += B[(i+1) * 4];539; A[i+2] += B[(i+2) * 4];540; A[i+3] += B[(i+3) * 4];541; }542;}543define void @gather_unroll(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {544; CHECK-LABEL: gather_unroll:545; CHECK: # %bb.0: # %entry546; CHECK-NEXT: li a2, 256547; CHECK-NEXT: li a3, 64548; CHECK-NEXT: li a4, 16549; CHECK-NEXT: vsetivli zero, 8, e32, m1, ta, ma550; CHECK-NEXT: .LBB11_1: # %vector.body551; CHECK-NEXT: # =>This Inner Loop Header: Depth=1552; CHECK-NEXT: vlse32.v v8, (a1), a3553; CHECK-NEXT: vlse32.v v9, (a0), a4554; CHECK-NEXT: addi a5, a1, 16555; CHECK-NEXT: vadd.vv v8, v9, v8556; CHECK-NEXT: vsse32.v v8, (a0), a4557; CHECK-NEXT: vlse32.v v8, (a5), a3558; CHECK-NEXT: addi a5, a0, 4559; CHECK-NEXT: vlse32.v v9, (a5), a4560; CHECK-NEXT: vadd.vv v8, v9, v8561; CHECK-NEXT: vsse32.v v8, (a5), a4562; CHECK-NEXT: addi a5, a1, 32563; CHECK-NEXT: vlse32.v v8, (a5), a3564; CHECK-NEXT: addi a5, a0, 8565; CHECK-NEXT: vlse32.v v9, (a5), a4566; CHECK-NEXT: vadd.vv v8, v9, v8567; CHECK-NEXT: vsse32.v v8, (a5), a4568; CHECK-NEXT: addi a5, a1, 48569; CHECK-NEXT: vlse32.v v8, (a5), a3570; CHECK-NEXT: addi a5, a0, 12571; CHECK-NEXT: vlse32.v v9, (a5), a4572; CHECK-NEXT: addi a2, a2, -8573; CHECK-NEXT: addi a1, a1, 512574; CHECK-NEXT: vadd.vv v8, v9, v8575; CHECK-NEXT: vsse32.v v8, (a5), a4576; CHECK-NEXT: addi a0, a0, 128577; CHECK-NEXT: bnez a2, .LBB11_1578; CHECK-NEXT: # %bb.2: # %for.cond.cleanup579; CHECK-NEXT: ret580entry:581 br label %vector.body582 583vector.body: ; preds = %vector.body, %entry584 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]585 %vec.ind = phi <8 x i64> [ <i64 0, i64 4, i64 8, i64 12, i64 16, i64 20, i64 24, i64 28>, %entry ], [ %vec.ind.next, %vector.body ]586 %i = shl nuw nsw <8 x i64> %vec.ind, splat (i64 2)587 %i1 = getelementptr inbounds i32, ptr %B, <8 x i64> %i588 %wide.masked.gather = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i1, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)589 %i2 = getelementptr inbounds i32, ptr %A, <8 x i64> %vec.ind590 %wide.masked.gather52 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i2, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)591 %i3 = add nsw <8 x i32> %wide.masked.gather52, %wide.masked.gather592 call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %i3, <8 x ptr> %i2, i32 4, <8 x i1> splat (i1 true))593 %i4 = or disjoint <8 x i64> %vec.ind, splat (i64 1)594 %i5 = shl nsw <8 x i64> %i4, splat (i64 2)595 %i6 = getelementptr inbounds i32, ptr %B, <8 x i64> %i5596 %wide.masked.gather53 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i6, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)597 %i7 = getelementptr inbounds i32, ptr %A, <8 x i64> %i4598 %wide.masked.gather54 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i7, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)599 %i8 = add nsw <8 x i32> %wide.masked.gather54, %wide.masked.gather53600 call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %i8, <8 x ptr> %i7, i32 4, <8 x i1> splat (i1 true))601 %i9 = or disjoint <8 x i64> %vec.ind, splat (i64 2)602 %i10 = shl nsw <8 x i64> %i9, splat (i64 2)603 %i11 = getelementptr inbounds i32, ptr %B, <8 x i64> %i10604 %wide.masked.gather55 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i11, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)605 %i12 = getelementptr inbounds i32, ptr %A, <8 x i64> %i9606 %wide.masked.gather56 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i12, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)607 %i13 = add nsw <8 x i32> %wide.masked.gather56, %wide.masked.gather55608 call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %i13, <8 x ptr> %i12, i32 4, <8 x i1> splat (i1 true))609 %i14 = or disjoint <8 x i64> %vec.ind, <i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3, i64 3>610 %i15 = shl nsw <8 x i64> %i14, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>611 %i16 = getelementptr inbounds i32, ptr %B, <8 x i64> %i15612 %wide.masked.gather57 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i16, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)613 %i17 = getelementptr inbounds i32, ptr %A, <8 x i64> %i14614 %wide.masked.gather58 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> %i17, i32 4, <8 x i1> splat (i1 true), <8 x i32> poison)615 %i18 = add nsw <8 x i32> %wide.masked.gather58, %wide.masked.gather57616 call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> %i18, <8 x ptr> %i17, i32 4, <8 x i1> splat (i1 true))617 %index.next = add nuw i64 %index, 8618 %vec.ind.next = add <8 x i64> %vec.ind, splat (i64 32)619 %i19 = icmp eq i64 %index.next, 256620 br i1 %i19, label %for.cond.cleanup, label %vector.body621 622for.cond.cleanup: ; preds = %vector.body623 ret void624}625 626; Make sure we don't crash in getTgtMemIntrinsic for a vector of pointers.627define void @gather_of_pointers(ptr noalias nocapture %arg, ptr noalias nocapture readonly %arg1) {628; V-LABEL: gather_of_pointers:629; V: # %bb.0: # %bb630; V-NEXT: lui a2, 2631; V-NEXT: add a2, a0, a2632; V-NEXT: li a3, 40633; V-NEXT: vsetivli zero, 2, e64, m1, ta, ma634; V-NEXT: .LBB12_1: # %bb2635; V-NEXT: # =>This Inner Loop Header: Depth=1636; V-NEXT: vlse64.v v8, (a1), a3637; V-NEXT: addi a4, a1, 80638; V-NEXT: vlse64.v v9, (a4), a3639; V-NEXT: addi a4, a0, 16640; V-NEXT: vse64.v v8, (a0)641; V-NEXT: addi a0, a0, 32642; V-NEXT: vse64.v v9, (a4)643; V-NEXT: addi a1, a1, 160644; V-NEXT: bne a0, a2, .LBB12_1645; V-NEXT: # %bb.2: # %bb18646; V-NEXT: ret647;648; ZVE32F-LABEL: gather_of_pointers:649; ZVE32F: # %bb.0: # %bb650; ZVE32F-NEXT: li a2, 0651; ZVE32F-NEXT: lui a3, 2652; ZVE32F-NEXT: add a3, a0, a3653; ZVE32F-NEXT: li a4, 1654; ZVE32F-NEXT: .LBB12_1: # %bb2655; ZVE32F-NEXT: # =>This Inner Loop Header: Depth=1656; ZVE32F-NEXT: slli a5, a4, 3657; ZVE32F-NEXT: slli a6, a4, 5658; ZVE32F-NEXT: slli a7, a2, 3659; ZVE32F-NEXT: slli t0, a2, 5660; ZVE32F-NEXT: addi a2, a2, 4661; ZVE32F-NEXT: add a5, a6, a5662; ZVE32F-NEXT: add a7, t0, a7663; ZVE32F-NEXT: add a5, a1, a5664; ZVE32F-NEXT: add a7, a1, a7665; ZVE32F-NEXT: ld a6, 0(a7)666; ZVE32F-NEXT: ld t0, 0(a5)667; ZVE32F-NEXT: ld a7, 80(a7)668; ZVE32F-NEXT: ld a5, 80(a5)669; ZVE32F-NEXT: sd a6, 0(a0)670; ZVE32F-NEXT: sd t0, 8(a0)671; ZVE32F-NEXT: sd a7, 16(a0)672; ZVE32F-NEXT: sd a5, 24(a0)673; ZVE32F-NEXT: addi a0, a0, 32674; ZVE32F-NEXT: addi a4, a4, 4675; ZVE32F-NEXT: bne a0, a3, .LBB12_1676; ZVE32F-NEXT: # %bb.2: # %bb18677; ZVE32F-NEXT: ret678;679; OPTZVE32F-LABEL: gather_of_pointers:680; OPTZVE32F: # %bb.0: # %bb681; OPTZVE32F-NEXT: lui a2, 2682; OPTZVE32F-NEXT: add a2, a0, a2683; OPTZVE32F-NEXT: li a3, 40684; OPTZVE32F-NEXT: vsetivli zero, 2, e64, m1, ta, ma685; OPTZVE32F-NEXT: .LBB12_1: # %bb2686; OPTZVE32F-NEXT: # =>This Inner Loop Header: Depth=1687; OPTZVE32F-NEXT: vlse64.v v8, (a1), a3688; OPTZVE32F-NEXT: addi a4, a1, 80689; OPTZVE32F-NEXT: vlse64.v v9, (a4), a3690; OPTZVE32F-NEXT: addi a4, a0, 16691; OPTZVE32F-NEXT: vse64.v v8, (a0)692; OPTZVE32F-NEXT: addi a0, a0, 32693; OPTZVE32F-NEXT: vse64.v v9, (a4)694; OPTZVE32F-NEXT: addi a1, a1, 160695; OPTZVE32F-NEXT: bne a0, a2, .LBB12_1696; OPTZVE32F-NEXT: # %bb.2: # %bb18697; OPTZVE32F-NEXT: ret698;699; OPTV-LABEL: gather_of_pointers:700; OPTV: # %bb.0: # %bb701; OPTV-NEXT: li a2, 0702; OPTV-NEXT: lui a3, 2703; OPTV-NEXT: add a3, a0, a3704; OPTV-NEXT: li a4, 1705; OPTV-NEXT: .LBB12_1: # %bb2706; OPTV-NEXT: # =>This Inner Loop Header: Depth=1707; OPTV-NEXT: slli a5, a4, 3708; OPTV-NEXT: slli a6, a4, 5709; OPTV-NEXT: slli a7, a2, 3710; OPTV-NEXT: slli t0, a2, 5711; OPTV-NEXT: addi a2, a2, 4712; OPTV-NEXT: add a5, a6, a5713; OPTV-NEXT: add a7, t0, a7714; OPTV-NEXT: add a5, a1, a5715; OPTV-NEXT: add a7, a1, a7716; OPTV-NEXT: ld a6, 0(a7)717; OPTV-NEXT: ld t0, 0(a5)718; OPTV-NEXT: ld a7, 80(a7)719; OPTV-NEXT: ld a5, 80(a5)720; OPTV-NEXT: sd a6, 0(a0)721; OPTV-NEXT: sd t0, 8(a0)722; OPTV-NEXT: sd a7, 16(a0)723; OPTV-NEXT: sd a5, 24(a0)724; OPTV-NEXT: addi a0, a0, 32725; OPTV-NEXT: addi a4, a4, 4726; OPTV-NEXT: bne a0, a3, .LBB12_1727; OPTV-NEXT: # %bb.2: # %bb18728; OPTV-NEXT: ret729bb:730 br label %bb2731 732bb2: ; preds = %bb2, %bb733 %i = phi i64 [ 0, %bb ], [ %i15, %bb2 ]734 %i3 = phi <2 x i64> [ <i64 0, i64 1>, %bb ], [ %i16, %bb2 ]735 %i4 = mul nuw nsw <2 x i64> %i3, splat (i64 5)736 %i5 = mul <2 x i64> %i3, splat (i64 5)737 %i6 = add <2 x i64> %i5, <i64 10, i64 10>738 %i7 = getelementptr inbounds ptr, ptr %arg1, <2 x i64> %i4739 %i8 = getelementptr inbounds ptr, ptr %arg1, <2 x i64> %i6740 %i9 = call <2 x ptr> @llvm.masked.gather.v2p0.v2p0(<2 x ptr> %i7, i32 8, <2 x i1> splat (i1 true), <2 x ptr> poison)741 %i10 = call <2 x ptr> @llvm.masked.gather.v2p0.v2p0(<2 x ptr> %i8, i32 8, <2 x i1> splat (i1 true), <2 x ptr> poison)742 %i11 = getelementptr inbounds ptr, ptr %arg, i64 %i743 store <2 x ptr> %i9, ptr %i11, align 8744 %i13 = getelementptr inbounds ptr, ptr %i11, i64 2745 store <2 x ptr> %i10, ptr %i13, align 8746 %i15 = add nuw i64 %i, 4747 %i16 = add <2 x i64> %i3, <i64 4, i64 4>748 %i17 = icmp eq i64 %i15, 1024749 br i1 %i17, label %bb18, label %bb2750 751bb18: ; preds = %bb2752 ret void753}754 755; Make sure we don't crash in getTgtMemIntrinsic for a vector of pointers.756define void @scatter_of_pointers(ptr noalias nocapture %arg, ptr noalias nocapture readonly %arg1) {757; V-LABEL: scatter_of_pointers:758; V: # %bb.0: # %bb759; V-NEXT: lui a2, 2760; V-NEXT: add a2, a1, a2761; V-NEXT: li a3, 40762; V-NEXT: vsetivli zero, 2, e64, m1, ta, ma763; V-NEXT: .LBB13_1: # %bb2764; V-NEXT: # =>This Inner Loop Header: Depth=1765; V-NEXT: addi a4, a1, 16766; V-NEXT: vle64.v v8, (a1)767; V-NEXT: vle64.v v9, (a4)768; V-NEXT: addi a4, a0, 80769; V-NEXT: addi a1, a1, 32770; V-NEXT: vsse64.v v8, (a0), a3771; V-NEXT: vsse64.v v9, (a4), a3772; V-NEXT: addi a0, a0, 160773; V-NEXT: bne a1, a2, .LBB13_1774; V-NEXT: # %bb.2: # %bb18775; V-NEXT: ret776;777; ZVE32F-LABEL: scatter_of_pointers:778; ZVE32F: # %bb.0: # %bb779; ZVE32F-NEXT: li a2, 0780; ZVE32F-NEXT: lui a3, 2781; ZVE32F-NEXT: add a3, a1, a3782; ZVE32F-NEXT: li a4, 1783; ZVE32F-NEXT: .LBB13_1: # %bb2784; ZVE32F-NEXT: # =>This Inner Loop Header: Depth=1785; ZVE32F-NEXT: ld a5, 0(a1)786; ZVE32F-NEXT: ld a6, 8(a1)787; ZVE32F-NEXT: ld a7, 16(a1)788; ZVE32F-NEXT: ld t0, 24(a1)789; ZVE32F-NEXT: slli t1, a4, 3790; ZVE32F-NEXT: slli t2, a4, 5791; ZVE32F-NEXT: slli t3, a2, 3792; ZVE32F-NEXT: add t1, t2, t1793; ZVE32F-NEXT: slli t2, a2, 5794; ZVE32F-NEXT: addi a2, a2, 4795; ZVE32F-NEXT: addi a1, a1, 32796; ZVE32F-NEXT: add t2, t2, t3797; ZVE32F-NEXT: add t1, a0, t1798; ZVE32F-NEXT: add t2, a0, t2799; ZVE32F-NEXT: sd a5, 0(t2)800; ZVE32F-NEXT: sd a6, 0(t1)801; ZVE32F-NEXT: sd a7, 80(t2)802; ZVE32F-NEXT: sd t0, 80(t1)803; ZVE32F-NEXT: addi a4, a4, 4804; ZVE32F-NEXT: bne a1, a3, .LBB13_1805; ZVE32F-NEXT: # %bb.2: # %bb18806; ZVE32F-NEXT: ret807;808; OPTZVE32F-LABEL: scatter_of_pointers:809; OPTZVE32F: # %bb.0: # %bb810; OPTZVE32F-NEXT: lui a2, 2811; OPTZVE32F-NEXT: add a2, a1, a2812; OPTZVE32F-NEXT: li a3, 40813; OPTZVE32F-NEXT: vsetivli zero, 2, e64, m1, ta, ma814; OPTZVE32F-NEXT: .LBB13_1: # %bb2815; OPTZVE32F-NEXT: # =>This Inner Loop Header: Depth=1816; OPTZVE32F-NEXT: addi a4, a1, 16817; OPTZVE32F-NEXT: vle64.v v8, (a1)818; OPTZVE32F-NEXT: vle64.v v9, (a4)819; OPTZVE32F-NEXT: addi a4, a0, 80820; OPTZVE32F-NEXT: addi a1, a1, 32821; OPTZVE32F-NEXT: vsse64.v v8, (a0), a3822; OPTZVE32F-NEXT: vsse64.v v9, (a4), a3823; OPTZVE32F-NEXT: addi a0, a0, 160824; OPTZVE32F-NEXT: bne a1, a2, .LBB13_1825; OPTZVE32F-NEXT: # %bb.2: # %bb18826; OPTZVE32F-NEXT: ret827;828; OPTV-LABEL: scatter_of_pointers:829; OPTV: # %bb.0: # %bb830; OPTV-NEXT: li a2, 0831; OPTV-NEXT: lui a3, 2832; OPTV-NEXT: add a3, a1, a3833; OPTV-NEXT: li a4, 1834; OPTV-NEXT: .LBB13_1: # %bb2835; OPTV-NEXT: # =>This Inner Loop Header: Depth=1836; OPTV-NEXT: ld a5, 0(a1)837; OPTV-NEXT: ld a6, 8(a1)838; OPTV-NEXT: ld a7, 16(a1)839; OPTV-NEXT: ld t0, 24(a1)840; OPTV-NEXT: slli t1, a4, 3841; OPTV-NEXT: slli t2, a4, 5842; OPTV-NEXT: slli t3, a2, 3843; OPTV-NEXT: add t1, t2, t1844; OPTV-NEXT: slli t2, a2, 5845; OPTV-NEXT: addi a2, a2, 4846; OPTV-NEXT: addi a1, a1, 32847; OPTV-NEXT: add t2, t2, t3848; OPTV-NEXT: add t1, a0, t1849; OPTV-NEXT: add t2, a0, t2850; OPTV-NEXT: sd a5, 0(t2)851; OPTV-NEXT: sd a6, 0(t1)852; OPTV-NEXT: sd a7, 80(t2)853; OPTV-NEXT: sd t0, 80(t1)854; OPTV-NEXT: addi a4, a4, 4855; OPTV-NEXT: bne a1, a3, .LBB13_1856; OPTV-NEXT: # %bb.2: # %bb18857; OPTV-NEXT: ret858bb:859 br label %bb2860 861bb2: ; preds = %bb2, %bb862 %i = phi i64 [ 0, %bb ], [ %i15, %bb2 ]863 %i3 = phi <2 x i64> [ <i64 0, i64 1>, %bb ], [ %i16, %bb2 ]864 %i4 = getelementptr inbounds ptr, ptr %arg1, i64 %i865 %i6 = load <2 x ptr>, ptr %i4, align 8866 %i7 = getelementptr inbounds ptr, ptr %i4, i64 2867 %i9 = load <2 x ptr>, ptr %i7, align 8868 %i10 = mul nuw nsw <2 x i64> %i3, splat (i64 5)869 %i11 = mul <2 x i64> %i3, splat (i64 5)870 %i12 = add <2 x i64> %i11, <i64 10, i64 10>871 %i13 = getelementptr inbounds ptr, ptr %arg, <2 x i64> %i10872 %i14 = getelementptr inbounds ptr, ptr %arg, <2 x i64> %i12873 call void @llvm.masked.scatter.v2p0.v2p0(<2 x ptr> %i6, <2 x ptr> %i13, i32 8, <2 x i1> splat (i1 true))874 call void @llvm.masked.scatter.v2p0.v2p0(<2 x ptr> %i9, <2 x ptr> %i14, i32 8, <2 x i1> splat (i1 true))875 %i15 = add nuw i64 %i, 4876 %i16 = add <2 x i64> %i3, <i64 4, i64 4>877 %i17 = icmp eq i64 %i15, 1024878 br i1 %i17, label %bb18, label %bb2879 880bb18: ; preds = %bb2881 ret void882}883 884define void @strided_load_startval_add_with_splat(ptr noalias nocapture %arg, ptr noalias nocapture readonly %arg1, i32 signext %arg2) {885; CHECK-LABEL: strided_load_startval_add_with_splat:886; CHECK: # %bb.0: # %bb887; CHECK-NEXT: li a3, 1024888; CHECK-NEXT: beq a2, a3, .LBB14_7889; CHECK-NEXT: # %bb.1: # %bb3890; CHECK-NEXT: li a3, 1023891; CHECK-NEXT: subw a5, a3, a2892; CHECK-NEXT: li a6, 31893; CHECK-NEXT: mv a4, a2894; CHECK-NEXT: bltu a5, a6, .LBB14_5895; CHECK-NEXT: # %bb.2: # %bb9896; CHECK-NEXT: slli a4, a5, 32897; CHECK-NEXT: slli t0, a2, 2898; CHECK-NEXT: add a5, a0, a2899; CHECK-NEXT: add a6, a1, a2900; CHECK-NEXT: li t2, 32901; CHECK-NEXT: srli a4, a4, 32902; CHECK-NEXT: add t0, a6, t0903; CHECK-NEXT: addi a6, a4, 1904; CHECK-NEXT: andi a7, a6, -32905; CHECK-NEXT: add a4, a7, a2906; CHECK-NEXT: add a2, a0, a4907; CHECK-NEXT: li t1, 5908; CHECK-NEXT: vsetvli zero, t2, e8, m1, ta, ma909; CHECK-NEXT: .LBB14_3: # %bb15910; CHECK-NEXT: # =>This Inner Loop Header: Depth=1911; CHECK-NEXT: vlse8.v v8, (t0), t1912; CHECK-NEXT: vle8.v v9, (a5)913; CHECK-NEXT: vadd.vv v8, v9, v8914; CHECK-NEXT: vse8.v v8, (a5)915; CHECK-NEXT: addi a5, a5, 32916; CHECK-NEXT: addi t0, t0, 160917; CHECK-NEXT: bne a5, a2, .LBB14_3918; CHECK-NEXT: # %bb.4: # %bb30919; CHECK-NEXT: beq a6, a7, .LBB14_7920; CHECK-NEXT: .LBB14_5: # %bb32921; CHECK-NEXT: add a2, a0, a4922; CHECK-NEXT: slli a5, a4, 2923; CHECK-NEXT: add a1, a1, a4924; CHECK-NEXT: sub a3, a3, a4925; CHECK-NEXT: add a1, a1, a5926; CHECK-NEXT: slli a3, a3, 32927; CHECK-NEXT: srli a3, a3, 32928; CHECK-NEXT: add a0, a0, a4929; CHECK-NEXT: add a0, a0, a3930; CHECK-NEXT: addi a0, a0, 1931; CHECK-NEXT: .LBB14_6: # %bb35932; CHECK-NEXT: # =>This Inner Loop Header: Depth=1933; CHECK-NEXT: lbu a3, 0(a1)934; CHECK-NEXT: lbu a4, 0(a2)935; CHECK-NEXT: add a3, a4, a3936; CHECK-NEXT: sb a3, 0(a2)937; CHECK-NEXT: addi a2, a2, 1938; CHECK-NEXT: addi a1, a1, 5939; CHECK-NEXT: bne a2, a0, .LBB14_6940; CHECK-NEXT: .LBB14_7: # %bb34941; CHECK-NEXT: ret942bb:943 %i = icmp eq i32 %arg2, 1024944 br i1 %i, label %bb34, label %bb3945 946bb3: ; preds = %bb947 %i4 = sext i32 %arg2 to i64948 %i5 = sub i32 1023, %arg2949 %i6 = zext i32 %i5 to i64950 %i7 = add nuw nsw i64 %i6, 1951 %i8 = icmp ult i32 %i5, 31952 br i1 %i8, label %bb32, label %bb9953 954bb9: ; preds = %bb3955 %i10 = and i64 %i7, 8589934560956 %i11 = add nsw i64 %i10, %i4957 %i12 = insertelement <32 x i64> poison, i64 %i4, i64 0958 %i13 = shufflevector <32 x i64> %i12, <32 x i64> poison, <32 x i32> zeroinitializer959 %i14 = add <32 x i64> %i13, <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31>960 br label %bb15961 962bb15: ; preds = %bb15, %bb9963 %i16 = phi i64 [ 0, %bb9 ], [ %i27, %bb15 ]964 %i17 = phi <32 x i64> [ %i14, %bb9 ], [ %i28, %bb15 ]965 %i18 = add i64 %i16, %i4966 %i19 = mul nsw <32 x i64> %i17, splat (i64 5)967 %i20 = getelementptr inbounds i8, ptr %arg1, <32 x i64> %i19968 %i21 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> %i20, i32 1, <32 x i1> splat (i1 true), <32 x i8> poison)969 %i22 = getelementptr inbounds i8, ptr %arg, i64 %i18970 %i24 = load <32 x i8>, ptr %i22, align 1971 %i25 = add <32 x i8> %i24, %i21972 store <32 x i8> %i25, ptr %i22, align 1973 %i27 = add nuw i64 %i16, 32974 %i28 = add <32 x i64> %i17, splat (i64 32)975 %i29 = icmp eq i64 %i27, %i10976 br i1 %i29, label %bb30, label %bb15977 978bb30: ; preds = %bb15979 %i31 = icmp eq i64 %i7, %i10980 br i1 %i31, label %bb34, label %bb32981 982bb32: ; preds = %bb30, %bb3983 %i33 = phi i64 [ %i4, %bb3 ], [ %i11, %bb30 ]984 br label %bb35985 986bb34: ; preds = %bb35, %bb30, %bb987 ret void988 989bb35: ; preds = %bb35, %bb32990 %i36 = phi i64 [ %i43, %bb35 ], [ %i33, %bb32 ]991 %i37 = mul nsw i64 %i36, 5992 %i38 = getelementptr inbounds i8, ptr %arg1, i64 %i37993 %i39 = load i8, ptr %i38, align 1994 %i40 = getelementptr inbounds i8, ptr %arg, i64 %i36995 %i41 = load i8, ptr %i40, align 1996 %i42 = add i8 %i41, %i39997 store i8 %i42, ptr %i40, align 1998 %i43 = add nsw i64 %i36, 1999 %i44 = trunc i64 %i43 to i321000 %i45 = icmp eq i32 %i44, 10241001 br i1 %i45, label %bb34, label %bb351002}1003 1004define void @gather_no_scalar_remainder(ptr noalias nocapture noundef %arg, ptr noalias nocapture noundef readonly %arg1, i64 noundef %arg2) {1005; CHECK-LABEL: gather_no_scalar_remainder:1006; CHECK: # %bb.0: # %bb1007; CHECK-NEXT: slli a2, a2, 41008; CHECK-NEXT: beqz a2, .LBB15_31009; CHECK-NEXT: # %bb.1: # %bb21010; CHECK-NEXT: add a2, a0, a21011; CHECK-NEXT: li a3, 51012; CHECK-NEXT: vsetivli zero, 16, e8, mf2, ta, ma1013; CHECK-NEXT: .LBB15_2: # %bb41014; CHECK-NEXT: # =>This Inner Loop Header: Depth=11015; CHECK-NEXT: vlse8.v v8, (a1), a31016; CHECK-NEXT: vle8.v v9, (a0)1017; CHECK-NEXT: vadd.vv v8, v9, v81018; CHECK-NEXT: vse8.v v8, (a0)1019; CHECK-NEXT: addi a0, a0, 161020; CHECK-NEXT: addi a1, a1, 801021; CHECK-NEXT: bne a0, a2, .LBB15_21022; CHECK-NEXT: .LBB15_3: # %bb161023; CHECK-NEXT: ret1024bb:1025 %i = shl i64 %arg2, 41026 %i3 = icmp eq i64 %i, 01027 br i1 %i3, label %bb16, label %bb21028 1029bb2: ; preds = %bb1030 br label %bb41031 1032bb4: ; preds = %bb4, %bb21033 %i5 = phi i64 [ %i13, %bb4 ], [ 0, %bb2 ]1034 %i6 = phi <16 x i64> [ %i14, %bb4 ], [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, %bb2 ]1035 %i7 = mul <16 x i64> %i6, splat (i64 5)1036 %i8 = getelementptr inbounds i8, ptr %arg1, <16 x i64> %i71037 %i9 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> %i8, i32 1, <16 x i1> splat (i1 true), <16 x i8> poison)1038 %i10 = getelementptr inbounds i8, ptr %arg, i64 %i51039 %i11 = load <16 x i8>, ptr %i10, align 11040 %i12 = add <16 x i8> %i11, %i91041 store <16 x i8> %i12, ptr %i10, align 11042 %i13 = add nuw i64 %i5, 161043 %i14 = add <16 x i64> %i6, splat (i64 16)1044 %i15 = icmp eq i64 %i13, %i1045 br i1 %i15, label %bb16, label %bb41046 1047bb16: ; preds = %bb4, %bb1048 ret void1049}1050 1051define void @gather_zero_stride_fp(ptr noalias nocapture %A, ptr noalias nocapture readonly %B) {1052; CHECK-LABEL: gather_zero_stride_fp:1053; CHECK: # %bb.0: # %entry1054; CHECK-NEXT: lui a2, 11055; CHECK-NEXT: add a2, a0, a21056; CHECK-NEXT: vsetivli zero, 8, e32, m1, ta, ma1057; CHECK-NEXT: .LBB16_1: # %vector.body1058; CHECK-NEXT: # =>This Inner Loop Header: Depth=11059; CHECK-NEXT: flw fa5, 0(a1)1060; CHECK-NEXT: vle32.v v8, (a0)1061; CHECK-NEXT: vfadd.vf v8, v8, fa51062; CHECK-NEXT: vse32.v v8, (a0)1063; CHECK-NEXT: addi a0, a0, 1281064; CHECK-NEXT: addi a1, a1, 6401065; CHECK-NEXT: bne a0, a2, .LBB16_11066; CHECK-NEXT: # %bb.2: # %for.cond.cleanup1067; CHECK-NEXT: ret1068entry:1069 br label %vector.body1070 1071vector.body: ; preds = %vector.body, %entry1072 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]1073 %vec.ind = phi <8 x i64> [ zeroinitializer, %entry ], [ %vec.ind.next, %vector.body ]1074 %i = mul nuw nsw <8 x i64> %vec.ind, splat (i64 5)1075 %i1 = getelementptr inbounds float, ptr %B, <8 x i64> %i1076 %wide.masked.gather = call <8 x float> @llvm.masked.gather.v8f32.v32p0(<8 x ptr> %i1, i32 4, <8 x i1> splat (i1 true), <8 x float> poison)1077 %i2 = getelementptr inbounds float, ptr %A, i64 %index1078 %wide.load = load <8 x float>, ptr %i2, align 41079 %i4 = fadd <8 x float> %wide.load, %wide.masked.gather1080 store <8 x float> %i4, ptr %i2, align 41081 %index.next = add nuw i64 %index, 321082 %vec.ind.next = add <8 x i64> %vec.ind, splat (i64 32)1083 %i6 = icmp eq i64 %index.next, 10241084 br i1 %i6, label %for.cond.cleanup, label %vector.body1085 1086for.cond.cleanup: ; preds = %vector.body1087 ret void1088}1089