137 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 52; RUN: opt -mattr=+mve -passes=loop-vectorize < %s -S -o - | FileCheck %s3 4target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"5target triple = "thumbv8.1m.main-unknown-none-eabihf"6 7; Even though it has high register pressure, this example should still vectorise since the mul+add chains become VMLAs.8 9define void @fn(i32 noundef %n, ptr %in, ptr %out) #0 {10; CHECK-LABEL: define void @fn(11; CHECK-SAME: i32 noundef [[N:%.*]], ptr [[IN:%.*]], ptr [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {12; CHECK-NEXT: [[ENTRY:.*:]]13; CHECK-NEXT: [[CMP46_NOT:%.*]] = icmp eq i32 [[N]], 014; CHECK-NEXT: br i1 [[CMP46_NOT]], [[EXIT:label %.*]], label %[[FOR_BODY_PREHEADER:.*]]15; CHECK: [[FOR_BODY_PREHEADER]]:16; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]]17; CHECK: [[VECTOR_MEMCHECK]]:18; CHECK-NEXT: [[TMP0:%.*]] = mul i32 [[N]], 319; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OUT]], i32 [[TMP0]]20; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[IN]], i32 [[TMP0]]21; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[OUT]], [[SCEVGEP1]]22; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[IN]], [[SCEVGEP]]23; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]24; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]25; CHECK: [[VECTOR_PH]]:26; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 327; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 428; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]29; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]30; CHECK: [[VECTOR_BODY]]:31; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]32; CHECK-NEXT: [[POINTER_PHI2:%.*]] = phi ptr [ [[IN]], %[[VECTOR_PH]] ], [ [[PTR_IND3:%.*]], %[[VECTOR_BODY]] ]33; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[OUT]], %[[VECTOR_PH]] ], [ [[PTR_IND6:%.*]], %[[VECTOR_BODY]] ]34; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> <i32 0, i32 3, i32 6, i32 9>35; CHECK-NEXT: [[VECTOR_GEP4:%.*]] = getelementptr i8, ptr [[POINTER_PHI2]], <4 x i32> <i32 0, i32 3, i32 6, i32 9>36; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])37; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP4]], i32 138; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 [[VECTOR_GEP4]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison), !alias.scope [[META0:![0-9]+]]39; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP4]], i32 240; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 [[TMP1]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison), !alias.scope [[META0]]41; CHECK-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 [[TMP2]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison), !alias.scope [[META0]]42; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[WIDE_MASKED_GATHER]] to <4 x i32>43; CHECK-NEXT: [[TMP4:%.*]] = mul nuw nsw <4 x i32> [[TMP3]], splat (i32 19595)44; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[WIDE_MASKED_GATHER5]] to <4 x i32>45; CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw <4 x i32> [[TMP5]], splat (i32 38470)46; CHECK-NEXT: [[TMP7:%.*]] = zext <4 x i8> [[WIDE_MASKED_GATHER6]] to <4 x i32>47; CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw <4 x i32> [[TMP7]], splat (i32 7471)48; CHECK-NEXT: [[TMP9:%.*]] = add nuw nsw <4 x i32> [[TMP4]], splat (i32 32768)49; CHECK-NEXT: [[TMP10:%.*]] = add nuw nsw <4 x i32> [[TMP9]], [[TMP6]]50; CHECK-NEXT: [[TMP11:%.*]] = add nuw nsw <4 x i32> [[TMP10]], [[TMP8]]51; CHECK-NEXT: [[TMP12:%.*]] = lshr <4 x i32> [[TMP11]], splat (i32 16)52; CHECK-NEXT: [[TMP13:%.*]] = trunc nuw <4 x i32> [[TMP12]] to <4 x i8>53; CHECK-NEXT: [[TMP14:%.*]] = mul nuw nsw <4 x i32> [[TMP3]], splat (i32 32767)54; CHECK-NEXT: [[TMP15:%.*]] = mul nuw <4 x i32> [[TMP5]], splat (i32 16762097)55; CHECK-NEXT: [[TMP16:%.*]] = mul nuw <4 x i32> [[TMP7]], splat (i32 16759568)56; CHECK-NEXT: [[TMP17:%.*]] = add nuw nsw <4 x i32> [[TMP14]], splat (i32 32768)57; CHECK-NEXT: [[TMP18:%.*]] = add nuw <4 x i32> [[TMP17]], [[TMP15]]58; CHECK-NEXT: [[TMP19:%.*]] = add <4 x i32> [[TMP18]], [[TMP16]]59; CHECK-NEXT: [[TMP20:%.*]] = lshr <4 x i32> [[TMP19]], splat (i32 16)60; CHECK-NEXT: [[TMP21:%.*]] = trunc <4 x i32> [[TMP20]] to <4 x i8>61; CHECK-NEXT: [[TMP22:%.*]] = mul nuw nsw <4 x i32> [[TMP3]], splat (i32 13282)62; CHECK-NEXT: [[TMP23:%.*]] = mul nuw <4 x i32> [[TMP5]], splat (i32 16744449)63; CHECK-NEXT: [[TMP24:%.*]] = mul nuw nsw <4 x i32> [[TMP7]], splat (i32 19485)64; CHECK-NEXT: [[TMP25:%.*]] = add nuw nsw <4 x i32> [[TMP22]], splat (i32 32768)65; CHECK-NEXT: [[TMP26:%.*]] = add nuw <4 x i32> [[TMP25]], [[TMP23]]66; CHECK-NEXT: [[TMP27:%.*]] = add nuw <4 x i32> [[TMP26]], [[TMP24]]67; CHECK-NEXT: [[TMP28:%.*]] = lshr <4 x i32> [[TMP27]], splat (i32 16)68; CHECK-NEXT: [[TMP29:%.*]] = trunc <4 x i32> [[TMP28]] to <4 x i8>69; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP]], i32 170; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP13]], <4 x ptr> align 1 [[VECTOR_GEP]], <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]71; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP]], i32 272; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP21]], <4 x ptr> align 1 [[TMP30]], <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3]], !noalias [[META0]]73; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP29]], <4 x ptr> align 1 [[TMP31]], <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3]], !noalias [[META0]]74; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 475; CHECK-NEXT: [[PTR_IND3]] = getelementptr i8, ptr [[POINTER_PHI2]], i32 1276; CHECK-NEXT: [[PTR_IND6]] = getelementptr i8, ptr [[POINTER_PHI]], i32 1277; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]78; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]79; CHECK: [[MIDDLE_BLOCK]]:80; CHECK-NEXT: br [[EXIT_LOOPEXIT:label %.*]]81; CHECK: [[SCALAR_PH]]:82;83entry:84 %cmp46.not = icmp eq i32 %n, 085 br i1 %cmp46.not, label %exit, label %for.body86 87for.body: ; preds = %for.body.preheader, %for.body88 %ptr.iv.1 = phi ptr [ %in, %entry ], [ %ptr.iv.1.next, %for.body ]89 %ptr.iv.2 = phi ptr [ %out, %entry ], [ %ptr.iv.2.next, %for.body ]90 %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]91 %incdec.ptr = getelementptr inbounds nuw i8, ptr %ptr.iv.1, i32 192 %0 = load i8, ptr %ptr.iv.1, align 193 %incdec.ptr1 = getelementptr inbounds nuw i8, ptr %ptr.iv.1, i32 294 %1 = load i8, ptr %incdec.ptr, align 195 %ptr.iv.1.next = getelementptr inbounds nuw i8, ptr %ptr.iv.1, i32 396 %2 = load i8, ptr %incdec.ptr1, align 197 %conv = zext i8 %0 to i3298 %mul = mul nuw nsw i32 %conv, 1959599 %conv3 = zext i8 %1 to i32100 %mul4 = mul nuw nsw i32 %conv3, 38470101 %conv5 = zext i8 %2 to i32102 %mul6 = mul nuw nsw i32 %conv5, 7471103 %add = add nuw nsw i32 %mul, 32768104 %add7 = add nuw nsw i32 %add, %mul4105 %add8 = add nuw nsw i32 %add7, %mul6106 %shr = lshr i32 %add8, 16107 %conv9 = trunc nuw i32 %shr to i8108 %mul11 = mul nuw nsw i32 %conv, 32767109 %mul13 = mul nuw i32 %conv3, 16762097110 %mul16 = mul nuw i32 %conv5, 16759568111 %add14 = add nuw nsw i32 %mul11, 32768112 %add17 = add nuw i32 %add14, %mul13113 %add18 = add i32 %add17, %mul16114 %shr19 = lshr i32 %add18, 16115 %conv20 = trunc i32 %shr19 to i8116 %mul22 = mul nuw nsw i32 %conv, 13282117 %mul24 = mul nuw i32 %conv3, 16744449118 %mul27 = mul nuw nsw i32 %conv5, 19485119 %add25 = add nuw nsw i32 %mul22, 32768120 %add28 = add nuw i32 %add25, %mul24121 %add29 = add nuw i32 %add28, %mul27122 %shr30 = lshr i32 %add29, 16123 %conv31 = trunc i32 %shr30 to i8124 %incdec.ptr32 = getelementptr inbounds nuw i8, ptr %ptr.iv.2, i32 1125 store i8 %conv9, ptr %ptr.iv.2, align 1126 %incdec.ptr33 = getelementptr inbounds nuw i8, ptr %ptr.iv.2, i32 2127 store i8 %conv20, ptr %incdec.ptr32, align 1128 %ptr.iv.2.next = getelementptr inbounds nuw i8, ptr %ptr.iv.2, i32 3129 store i8 %conv31, ptr %incdec.ptr33, align 1130 %iv.next = add nuw i32 %iv, 1131 %exitcond.not = icmp eq i32 %iv.next, %n132 br i1 %exitcond.not, label %exit, label %for.body133 134exit: ; preds = %for.cond.cleanup.loopexit, %entry135 ret void136}137