122 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 52; RUN: opt -passes=loop-vectorize -scalable-vectorization=off -force-vector-width=4 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s | FileCheck %s3 4; NOTE: These tests aren't really target-specific, but it's convenient to target AArch645; so that TTI.isLegalMaskedLoad can return true.6 7target triple = "aarch64-linux-gnu"8 9; The original loop had an unconditional uniform load. Let's make sure10; we don't artificially create new predicated blocks for the load.11define void @uniform_load(ptr noalias %dst, ptr noalias readonly %src, i64 %n) #0 {12; CHECK-LABEL: define void @uniform_load(13; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {14; CHECK-NEXT: [[ENTRY:.*:]]15; CHECK-NEXT: br label %[[VECTOR_PH:.*]]16; CHECK: [[VECTOR_PH]]:17; CHECK-NEXT: [[N_MINUS_VF:%.*]] = sub i64 [[N]], 418; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i64 [[N]], 419; CHECK-NEXT: [[N2:%.*]] = select i1 [[CMP]], i64 [[N_MINUS_VF]], i64 020; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[N]])21; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]22; CHECK: [[VECTOR_BODY]]:23; CHECK-NEXT: [[IDX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]24; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[NEXT_ACTIVE_LANE_MASK:%.*]], %[[VECTOR_BODY]] ]25; CHECK-NEXT: [[LOAD_VAL:%.*]] = load i32, ptr [[SRC]], align 426; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> poison, i32 [[LOAD_VAL]], i64 027; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <4 x i32> zeroinitializer28; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IDX]]29; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP6]], <4 x i1> [[ACTIVE_LANE_MASK]])30; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[IDX]], 431; CHECK-NEXT: [[NEXT_ACTIVE_LANE_MASK]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[IDX]], i64 [[N2]])32; CHECK-NEXT: [[EXTRACT_FIRST_LANE_MASK:%.*]] = extractelement <4 x i1> [[NEXT_ACTIVE_LANE_MASK]], i32 033; CHECK-NEXT: [[FIRST_LANE_SET:%.*]] = xor i1 [[EXTRACT_FIRST_LANE_MASK]], true34; CHECK-NEXT: br i1 [[FIRST_LANE_SET]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]35; CHECK: [[MIDDLE_BLOCK]]:36; CHECK-NEXT: br label %[[FOR_END:.*]]37; CHECK: [[FOR_END]]:38; CHECK-NEXT: ret void39;40 41entry:42 br label %for.body43 44for.body: ; preds = %entry, %for.body45 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]46 %val = load i32, ptr %src, align 447 %arrayidx = getelementptr inbounds i32, ptr %dst, i64 %indvars.iv48 store i32 %val, ptr %arrayidx, align 449 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 150 %exitcond.not = icmp eq i64 %indvars.iv.next, %n51 br i1 %exitcond.not, label %for.end, label %for.body52 53for.end: ; preds = %for.body, %entry54 ret void55}56 57; The original loop had a conditional uniform load. In this case we actually58; do need to perform conditional loads and so we end up using a gather instead.59; However, we at least ensure the mask is the overlap of the loop predicate60; and the original condition.61define void @cond_uniform_load(ptr noalias nocapture %dst, ptr nocapture readonly %src, ptr nocapture readonly %cond, i64 %n) #0 {62; CHECK-LABEL: define void @cond_uniform_load(63; CHECK-SAME: ptr noalias captures(none) [[DST:%.*]], ptr readonly captures(none) [[SRC:%.*]], ptr readonly captures(none) [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {64; CHECK-NEXT: [[ENTRY:.*:]]65; CHECK-NEXT: br label %[[VECTOR_PH:.*]]66; CHECK: [[VECTOR_PH]]:67; CHECK-NEXT: [[TMP5:%.*]] = sub i64 [[N]], 468; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt i64 [[N]], 469; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], i64 [[TMP5]], i64 070; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[N]])71; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x ptr> poison, ptr [[SRC]], i64 072; CHECK-NEXT: [[SRC_SPLAT:%.*]] = shufflevector <4 x ptr> [[TMP1]], <4 x ptr> poison, <4 x i32> zeroinitializer73; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]74; CHECK: [[VECTOR_BODY]]:75; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]76; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]77; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX6]]78; CHECK-NEXT: [[COND_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP6]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)79; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <4 x i32> [[COND_LOAD]], zeroinitializer80; CHECK-NEXT: [[MASK:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer81; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[SRC_SPLAT]], <4 x i1> [[MASK]], <4 x i32> poison)82; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x i32> [[WIDE_MASKED_GATHER]], <4 x i32> zeroinitializer83; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX6]]84; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[PREDPHI]], ptr align 4 [[TMP7]], <4 x i1> [[ACTIVE_LANE_MASK]])85; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX6]], 486; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX6]], i64 [[TMP3]])87; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 088; CHECK-NEXT: [[TMP9:%.*]] = xor i1 [[TMP8]], true89; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]90; CHECK: [[MIDDLE_BLOCK]]:91; CHECK-NEXT: br label %[[FOR_END:.*]]92; CHECK: [[FOR_END]]:93; CHECK-NEXT: ret void94;95entry:96 br label %for.body97 98for.body: ; preds = %entry, %if.end99 %index = phi i64 [ %index.next, %if.end ], [ 0, %entry ]100 %arrayidx = getelementptr inbounds i32, ptr %cond, i64 %index101 %0 = load i32, ptr %arrayidx, align 4102 %tobool.not = icmp eq i32 %0, 0103 br i1 %tobool.not, label %if.end, label %if.then104 105if.then: ; preds = %for.body106 %1 = load i32, ptr %src, align 4107 br label %if.end108 109if.end: ; preds = %if.then, %for.body110 %val.0 = phi i32 [ %1, %if.then ], [ 0, %for.body ]111 %arrayidx1 = getelementptr inbounds i32, ptr %dst, i64 %index112 store i32 %val.0, ptr %arrayidx1, align 4113 %index.next = add nuw i64 %index, 1114 %exitcond.not = icmp eq i64 %index.next, %n115 br i1 %exitcond.not, label %for.end, label %for.body116 117for.end: ; preds = %for.inc, %entry118 ret void119}120 121attributes #0 = { "target-features"="+neon,+sve,+v8.1a" vscale_range(2, 0) }122