863 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 22; RUN: opt -S < %s -passes=loop-vectorize -force-vector-interleave=1 2>&1 | FileCheck %s3 4target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"5target triple = "aarch64"6 7define void @add_a(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %len) #0 {8; CHECK-LABEL: define void @add_a9; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) {10; CHECK-NEXT: entry:11; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[LEN]], 012; CHECK-NEXT: br i1 [[CMP8]], label [[ITER_CHECK:%.*]], label [[FOR_COND_CLEANUP:%.*]]13; CHECK: iter.check:14; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[LEN]] to i6415; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 416; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]17; CHECK: vector.main.loop.iter.check:18; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 1619; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]20; CHECK: vector.ph:21; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 1622; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]23; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]24; CHECK: vector.body:25; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]26; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]27; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 128; CHECK-NEXT: [[TMP2:%.*]] = add <16 x i8> [[WIDE_LOAD]], splat (i8 2)29; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDEX]]30; CHECK-NEXT: store <16 x i8> [[TMP2]], ptr [[TMP3]], align 131; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1632; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]33; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]34; CHECK: middle.block:35; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]36; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]37; CHECK: vec.epilog.iter.check:38; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 439; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]40; CHECK: vec.epilog.ph:41; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]42; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[TMP0]], 443; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF2]]44; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]45; CHECK: vec.epilog.vector.body:46; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]47; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX4]]48; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP5]], align 149; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i8> [[WIDE_LOAD5]], splat (i8 2)50; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDEX4]]51; CHECK-NEXT: store <4 x i8> [[TMP6]], ptr [[TMP7]], align 152; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 453; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]54; CHECK-NEXT: br i1 [[TMP8]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]55; CHECK: vec.epilog.middle.block:56; CHECK-NEXT: [[CMP_N7:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]57; CHECK-NEXT: br i1 [[CMP_N7]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]58; CHECK: vec.epilog.scalar.ph:59; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]60; CHECK-NEXT: br label [[FOR_BODY:%.*]]61; CHECK: for.cond.cleanup.loopexit:62; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]63; CHECK: for.cond.cleanup:64; CHECK-NEXT: ret void65; CHECK: for.body:66; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]67; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDVARS_IV]]68; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[ARRAYIDX]], align 169; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP9]] to i3270; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[CONV]], 271; CHECK-NEXT: [[CONV1:%.*]] = trunc i32 [[ADD]] to i872; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDVARS_IV]]73; CHECK-NEXT: store i8 [[CONV1]], ptr [[ARRAYIDX3]], align 174; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 175; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i3276; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[LEN]]77; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]78;79entry:80 %cmp8 = icmp sgt i32 %len, 081 br i1 %cmp8, label %for.body, label %for.cond.cleanup82 83for.cond.cleanup: ; preds = %for.body, %entry84 ret void85 86for.body: ; preds = %entry, %for.body87 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]88 %arrayidx = getelementptr inbounds i8, ptr %p, i64 %indvars.iv89 %0 = load i8, ptr %arrayidx90 %conv = zext i8 %0 to i3291 %add = add nuw nsw i32 %conv, 292 %conv1 = trunc i32 %add to i893 %arrayidx3 = getelementptr inbounds i8, ptr %q, i64 %indvars.iv94 store i8 %conv1, ptr %arrayidx395 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 196 %lftr.wideiv = trunc i64 %indvars.iv.next to i3297 %exitcond = icmp eq i32 %lftr.wideiv, %len98 br i1 %exitcond, label %for.cond.cleanup, label %for.body99}100 101; Ensure that we preserve nuw/nsw if we're not shrinking the values we're102; working with.103define void @add_a1(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %len) #0 {104; CHECK-LABEL: define void @add_a1105; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) {106; CHECK-NEXT: entry:107; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[LEN]], 0108; CHECK-NEXT: br i1 [[CMP8]], label [[ITER_CHECK:%.*]], label [[FOR_COND_CLEANUP:%.*]]109; CHECK: iter.check:110; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[LEN]] to i64111; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4112; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]113; CHECK: vector.main.loop.iter.check:114; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 16115; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]116; CHECK: vector.ph:117; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 16118; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]119; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]120; CHECK: vector.body:121; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]122; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]123; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1124; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw <16 x i8> [[WIDE_LOAD]], splat (i8 2)125; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDEX]]126; CHECK-NEXT: store <16 x i8> [[TMP2]], ptr [[TMP3]], align 1127; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16128; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]129; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]130; CHECK: middle.block:131; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]132; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]133; CHECK: vec.epilog.iter.check:134; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4135; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]136; CHECK: vec.epilog.ph:137; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]138; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[TMP0]], 4139; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF2]]140; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]141; CHECK: vec.epilog.vector.body:142; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]143; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX4]]144; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP5]], align 1145; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw <4 x i8> [[WIDE_LOAD5]], splat (i8 2)146; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDEX4]]147; CHECK-NEXT: store <4 x i8> [[TMP6]], ptr [[TMP7]], align 1148; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 4149; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]150; CHECK-NEXT: br i1 [[TMP8]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]151; CHECK: vec.epilog.middle.block:152; CHECK-NEXT: [[CMP_N7:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]153; CHECK-NEXT: br i1 [[CMP_N7]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]154; CHECK: vec.epilog.scalar.ph:155; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]156; CHECK-NEXT: br label [[FOR_BODY:%.*]]157; CHECK: for.cond.cleanup.loopexit:158; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]159; CHECK: for.cond.cleanup:160; CHECK-NEXT: ret void161; CHECK: for.body:162; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]163; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDVARS_IV]]164; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[ARRAYIDX]], align 1165; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i8 [[TMP9]], 2166; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDVARS_IV]]167; CHECK-NEXT: store i8 [[ADD]], ptr [[ARRAYIDX3]], align 1168; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1169; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32170; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[LEN]]171; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]172;173entry:174 %cmp8 = icmp sgt i32 %len, 0175 br i1 %cmp8, label %for.body, label %for.cond.cleanup176 177for.cond.cleanup: ; preds = %for.body, %entry178 ret void179 180for.body: ; preds = %entry, %for.body181 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]182 %arrayidx = getelementptr inbounds i8, ptr %p, i64 %indvars.iv183 %0 = load i8, ptr %arrayidx184 %add = add nuw nsw i8 %0, 2185 %arrayidx3 = getelementptr inbounds i8, ptr %q, i64 %indvars.iv186 store i8 %add, ptr %arrayidx3187 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1188 %lftr.wideiv = trunc i64 %indvars.iv.next to i32189 %exitcond = icmp eq i32 %lftr.wideiv, %len190 br i1 %exitcond, label %for.cond.cleanup, label %for.body191}192 193define void @add_b(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %len) #0 {194; CHECK-LABEL: define void @add_b195; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) {196; CHECK-NEXT: entry:197; CHECK-NEXT: [[CMP9:%.*]] = icmp sgt i32 [[LEN]], 0198; CHECK-NEXT: br i1 [[CMP9]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]199; CHECK: for.body.preheader:200; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[LEN]] to i64201; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8202; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]203; CHECK: vector.ph:204; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8205; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]206; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]207; CHECK: vector.body:208; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]209; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[INDEX]]210; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2211; CHECK-NEXT: [[TMP2:%.*]] = add <8 x i16> [[WIDE_LOAD]], splat (i16 2)212; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]]213; CHECK-NEXT: store <8 x i16> [[TMP2]], ptr [[TMP3]], align 2214; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8215; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]216; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]217; CHECK: middle.block:218; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]219; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]220; CHECK: scalar.ph:221; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]222; CHECK-NEXT: br label [[FOR_BODY:%.*]]223; CHECK: for.cond.cleanup.loopexit:224; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]225; CHECK: for.cond.cleanup:226; CHECK-NEXT: ret void227; CHECK: for.body:228; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]229; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[INDVARS_IV]]230; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX]], align 2231; CHECK-NEXT: [[CONV8:%.*]] = zext i16 [[TMP5]] to i32232; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[CONV8]], 2233; CHECK-NEXT: [[CONV1:%.*]] = trunc i32 [[ADD]] to i16234; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDVARS_IV]]235; CHECK-NEXT: store i16 [[CONV1]], ptr [[ARRAYIDX3]], align 2236; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1237; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32238; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[LEN]]239; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]240;241entry:242 %cmp9 = icmp sgt i32 %len, 0243 br i1 %cmp9, label %for.body, label %for.cond.cleanup244 245for.cond.cleanup: ; preds = %for.body, %entry246 ret void247 248for.body: ; preds = %entry, %for.body249 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]250 %arrayidx = getelementptr inbounds i16, ptr %p, i64 %indvars.iv251 %0 = load i16, ptr %arrayidx252 %conv8 = zext i16 %0 to i32253 %add = add nuw nsw i32 %conv8, 2254 %conv1 = trunc i32 %add to i16255 %arrayidx3 = getelementptr inbounds i16, ptr %q, i64 %indvars.iv256 store i16 %conv1, ptr %arrayidx3257 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1258 %lftr.wideiv = trunc i64 %indvars.iv.next to i32259 %exitcond = icmp eq i32 %lftr.wideiv, %len260 br i1 %exitcond, label %for.cond.cleanup, label %for.body261}262 263define void @add_c(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %len) #0 {264; CHECK-LABEL: define void @add_c265; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) {266; CHECK-NEXT: entry:267; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[LEN]], 0268; CHECK-NEXT: br i1 [[CMP8]], label [[ITER_CHECK:%.*]], label [[FOR_COND_CLEANUP:%.*]]269; CHECK: iter.check:270; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[LEN]] to i64271; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4272; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]273; CHECK: vector.main.loop.iter.check:274; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 16275; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]276; CHECK: vector.ph:277; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 16278; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]279; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]280; CHECK: vector.body:281; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]282; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]283; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1284; CHECK-NEXT: [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i16>285; CHECK-NEXT: [[TMP3:%.*]] = add <16 x i16> [[TMP2]], splat (i16 2)286; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]]287; CHECK-NEXT: store <16 x i16> [[TMP3]], ptr [[TMP4]], align 2288; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16289; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]290; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]291; CHECK: middle.block:292; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]293; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]294; CHECK: vec.epilog.iter.check:295; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4296; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]297; CHECK: vec.epilog.ph:298; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]299; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[TMP0]], 4300; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF2]]301; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]302; CHECK: vec.epilog.vector.body:303; CHECK-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]304; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX4]]305; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP6]], align 1306; CHECK-NEXT: [[TMP7:%.*]] = zext <4 x i8> [[WIDE_LOAD5]] to <4 x i16>307; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i16> [[TMP7]], splat (i16 2)308; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX4]]309; CHECK-NEXT: store <4 x i16> [[TMP8]], ptr [[TMP9]], align 2310; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], 4311; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]312; CHECK-NEXT: br i1 [[TMP10]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]313; CHECK: vec.epilog.middle.block:314; CHECK-NEXT: [[CMP_N7:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]315; CHECK-NEXT: br i1 [[CMP_N7]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]316; CHECK: vec.epilog.scalar.ph:317; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]318; CHECK-NEXT: br label [[FOR_BODY:%.*]]319; CHECK: for.cond.cleanup.loopexit:320; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]321; CHECK: for.cond.cleanup:322; CHECK-NEXT: ret void323; CHECK: for.body:324; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]325; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDVARS_IV]]326; CHECK-NEXT: [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX]], align 1327; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP11]] to i32328; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[CONV]], 2329; CHECK-NEXT: [[CONV1:%.*]] = trunc i32 [[ADD]] to i16330; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDVARS_IV]]331; CHECK-NEXT: store i16 [[CONV1]], ptr [[ARRAYIDX3]], align 2332; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1333; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32334; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[LEN]]335; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]336;337entry:338 %cmp8 = icmp sgt i32 %len, 0339 br i1 %cmp8, label %for.body, label %for.cond.cleanup340 341for.cond.cleanup: ; preds = %for.body, %entry342 ret void343 344for.body: ; preds = %entry, %for.body345 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]346 %arrayidx = getelementptr inbounds i8, ptr %p, i64 %indvars.iv347 %0 = load i8, ptr %arrayidx348 %conv = zext i8 %0 to i32349 %add = add nuw nsw i32 %conv, 2350 %conv1 = trunc i32 %add to i16351 %arrayidx3 = getelementptr inbounds i16, ptr %q, i64 %indvars.iv352 store i16 %conv1, ptr %arrayidx3353 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1354 %lftr.wideiv = trunc i64 %indvars.iv.next to i32355 %exitcond = icmp eq i32 %lftr.wideiv, %len356 br i1 %exitcond, label %for.cond.cleanup, label %for.body357}358 359define void @add_d(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %len) #0 {360; CHECK-LABEL: define void @add_d361; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) {362; CHECK-NEXT: entry:363; CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i32 [[LEN]], 0364; CHECK-NEXT: br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]365; CHECK: for.body.preheader:366; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[LEN]] to i64367; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8368; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]369; CHECK: vector.ph:370; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8371; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]372; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]373; CHECK: vector.body:374; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]375; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[INDEX]]376; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP1]], align 2377; CHECK-NEXT: [[TMP2:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>378; CHECK-NEXT: [[TMP3:%.*]] = add nsw <8 x i32> [[TMP2]], splat (i32 2)379; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 [[INDEX]]380; CHECK-NEXT: store <8 x i32> [[TMP3]], ptr [[TMP4]], align 4381; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8382; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]383; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]384; CHECK: middle.block:385; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]386; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]387; CHECK: scalar.ph:388; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]389; CHECK-NEXT: br label [[FOR_BODY:%.*]]390; CHECK: for.cond.cleanup.loopexit:391; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]392; CHECK: for.cond.cleanup:393; CHECK-NEXT: ret void394; CHECK: for.body:395; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]396; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[INDVARS_IV]]397; CHECK-NEXT: [[TMP6:%.*]] = load i16, ptr [[ARRAYIDX]], align 2398; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP6]] to i32399; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV]], 2400; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 [[INDVARS_IV]]401; CHECK-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4402; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1403; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32404; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[LEN]]405; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]406;407entry:408 %cmp7 = icmp sgt i32 %len, 0409 br i1 %cmp7, label %for.body, label %for.cond.cleanup410 411for.cond.cleanup: ; preds = %for.body, %entry412 ret void413 414for.body: ; preds = %entry, %for.body415 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]416 %arrayidx = getelementptr inbounds i16, ptr %p, i64 %indvars.iv417 %0 = load i16, ptr %arrayidx418 %conv = sext i16 %0 to i32419 %add = add nsw i32 %conv, 2420 %arrayidx2 = getelementptr inbounds i32, ptr %q, i64 %indvars.iv421 store i32 %add, ptr %arrayidx2422 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1423 %lftr.wideiv = trunc i64 %indvars.iv.next to i32424 %exitcond = icmp eq i32 %lftr.wideiv, %len425 br i1 %exitcond, label %for.cond.cleanup, label %for.body426}427 428define void @add_e(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 %arg1, i8 %arg2, i32 %len) #0 {429; CHECK-LABEL: define void @add_e430; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 [[ARG1:%.*]], i8 [[ARG2:%.*]], i32 [[LEN:%.*]]) {431; CHECK-NEXT: entry:432; CHECK-NEXT: [[CMP_32:%.*]] = icmp sgt i32 [[LEN]], 0433; CHECK-NEXT: br i1 [[CMP_32]], label [[ITER_CHECK:%.*]], label [[FOR_COND_CLEANUP:%.*]]434; CHECK: iter.check:435; CHECK-NEXT: [[CONV11:%.*]] = zext i8 [[ARG2]] to i32436; CHECK-NEXT: [[CONV13:%.*]] = zext i8 [[ARG1]] to i32437; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[LEN]] to i64438; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4439; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]440; CHECK: vector.main.loop.iter.check:441; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 16442; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]443; CHECK: vector.ph:444; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 16445; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]446; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[ARG1]], i64 0447; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer448; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <16 x i8> poison, i8 [[ARG2]], i64 0449; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT2]], <16 x i8> poison, <16 x i32> zeroinitializer450; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]451; CHECK: vector.body:452; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]453; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]454; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1455; CHECK-NEXT: [[TMP2:%.*]] = shl <16 x i8> [[WIDE_LOAD]], splat (i8 4)456; CHECK-NEXT: [[TMP3:%.*]] = add <16 x i8> [[TMP2]], splat (i8 32)457; CHECK-NEXT: [[TMP4:%.*]] = or <16 x i8> [[WIDE_LOAD]], splat (i8 51)458; CHECK-NEXT: [[TMP5:%.*]] = mul <16 x i8> [[TMP4]], splat (i8 60)459; CHECK-NEXT: [[TMP6:%.*]] = and <16 x i8> [[TMP3]], [[BROADCAST_SPLAT]]460; CHECK-NEXT: [[TMP7:%.*]] = and <16 x i8> [[TMP5]], splat (i8 -4)461; CHECK-NEXT: [[TMP8:%.*]] = xor <16 x i8> [[TMP7]], [[BROADCAST_SPLAT3]]462; CHECK-NEXT: [[TMP9:%.*]] = mul <16 x i8> [[TMP8]], [[TMP6]]463; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDEX]]464; CHECK-NEXT: store <16 x i8> [[TMP9]], ptr [[TMP10]], align 1465; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16466; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]467; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]468; CHECK: middle.block:469; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]470; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]471; CHECK: vec.epilog.iter.check:472; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4473; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]474; CHECK: vec.epilog.ph:475; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]476; CHECK-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[TMP0]], 4477; CHECK-NEXT: [[N_VEC5:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF4]]478; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i8> poison, i8 [[ARG1]], i64 0479; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT6]], <4 x i8> poison, <4 x i32> zeroinitializer480; CHECK-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i8> poison, i8 [[ARG2]], i64 0481; CHECK-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT8]], <4 x i8> poison, <4 x i32> zeroinitializer482; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]483; CHECK: vec.epilog.vector.body:484; CHECK-NEXT: [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]485; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX10]]486; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x i8>, ptr [[TMP12]], align 1487; CHECK-NEXT: [[TMP13:%.*]] = shl <4 x i8> [[WIDE_LOAD11]], splat (i8 4)488; CHECK-NEXT: [[TMP14:%.*]] = add <4 x i8> [[TMP13]], splat (i8 32)489; CHECK-NEXT: [[TMP15:%.*]] = or <4 x i8> [[WIDE_LOAD11]], splat (i8 51)490; CHECK-NEXT: [[TMP16:%.*]] = mul <4 x i8> [[TMP15]], splat (i8 60)491; CHECK-NEXT: [[TMP17:%.*]] = and <4 x i8> [[TMP14]], [[BROADCAST_SPLAT7]]492; CHECK-NEXT: [[TMP18:%.*]] = and <4 x i8> [[TMP16]], splat (i8 -4)493; CHECK-NEXT: [[TMP19:%.*]] = xor <4 x i8> [[TMP18]], [[BROADCAST_SPLAT9]]494; CHECK-NEXT: [[TMP20:%.*]] = mul <4 x i8> [[TMP19]], [[TMP17]]495; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDEX10]]496; CHECK-NEXT: store <4 x i8> [[TMP20]], ptr [[TMP21]], align 1497; CHECK-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX10]], 4498; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT12]], [[N_VEC5]]499; CHECK-NEXT: br i1 [[TMP22]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]500; CHECK: vec.epilog.middle.block:501; CHECK-NEXT: [[CMP_N13:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC5]]502; CHECK-NEXT: br i1 [[CMP_N13]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]503; CHECK: vec.epilog.scalar.ph:504; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]505; CHECK-NEXT: br label [[FOR_BODY:%.*]]506; CHECK: for.cond.cleanup.loopexit:507; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]508; CHECK: for.cond.cleanup:509; CHECK-NEXT: ret void510; CHECK: for.body:511; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]512; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDVARS_IV]]513; CHECK-NEXT: [[TMP23:%.*]] = load i8, ptr [[ARRAYIDX]], align 1514; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP23]] to i32515; CHECK-NEXT: [[ADD:%.*]] = shl i32 [[CONV]], 4516; CHECK-NEXT: [[CONV2:%.*]] = add nuw nsw i32 [[ADD]], 32517; CHECK-NEXT: [[OR:%.*]] = or i32 [[CONV]], 51518; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[OR]], 60519; CHECK-NEXT: [[AND:%.*]] = and i32 [[CONV2]], [[CONV13]]520; CHECK-NEXT: [[MUL_MASKED:%.*]] = and i32 [[MUL]], 252521; CHECK-NEXT: [[CONV17:%.*]] = xor i32 [[MUL_MASKED]], [[CONV11]]522; CHECK-NEXT: [[MUL18:%.*]] = mul nuw nsw i32 [[CONV17]], [[AND]]523; CHECK-NEXT: [[CONV19:%.*]] = trunc i32 [[MUL18]] to i8524; CHECK-NEXT: [[ARRAYIDX21:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDVARS_IV]]525; CHECK-NEXT: store i8 [[CONV19]], ptr [[ARRAYIDX21]], align 1526; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1527; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32528; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[LEN]]529; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]530;531entry:532 %cmp.32 = icmp sgt i32 %len, 0533 br i1 %cmp.32, label %for.body.lr.ph, label %for.cond.cleanup534 535for.body.lr.ph: ; preds = %entry536 %conv11 = zext i8 %arg2 to i32537 %conv13 = zext i8 %arg1 to i32538 br label %for.body539 540for.cond.cleanup: ; preds = %for.body, %entry541 ret void542 543for.body: ; preds = %for.body, %for.body.lr.ph544 %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]545 %arrayidx = getelementptr inbounds i8, ptr %p, i64 %indvars.iv546 %0 = load i8, ptr %arrayidx547 %conv = zext i8 %0 to i32548 %add = shl i32 %conv, 4549 %conv2 = add nuw nsw i32 %add, 32550 %or = or i32 %conv, 51551 %mul = mul nuw nsw i32 %or, 60552 %and = and i32 %conv2, %conv13553 %mul.masked = and i32 %mul, 252554 %conv17 = xor i32 %mul.masked, %conv11555 %mul18 = mul nuw nsw i32 %conv17, %and556 %conv19 = trunc i32 %mul18 to i8557 %arrayidx21 = getelementptr inbounds i8, ptr %q, i64 %indvars.iv558 store i8 %conv19, ptr %arrayidx21559 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1560 %lftr.wideiv = trunc i64 %indvars.iv.next to i32561 %exitcond = icmp eq i32 %lftr.wideiv, %len562 br i1 %exitcond, label %for.cond.cleanup, label %for.body563}564 565define void @add_f(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 %arg1, i8 %arg2, i32 %len) #0 {566; CHECK-LABEL: define void @add_f567; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 [[ARG1:%.*]], i8 [[ARG2:%.*]], i32 [[LEN:%.*]]) {568; CHECK-NEXT: entry:569; CHECK-NEXT: [[CMP_32:%.*]] = icmp sgt i32 [[LEN]], 0570; CHECK-NEXT: br i1 [[CMP_32]], label [[ITER_CHECK:%.*]], label [[FOR_COND_CLEANUP:%.*]]571; CHECK: iter.check:572; CHECK-NEXT: [[CONV11:%.*]] = zext i8 [[ARG2]] to i32573; CHECK-NEXT: [[CONV13:%.*]] = zext i8 [[ARG1]] to i32574; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[LEN]] to i64575; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4576; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]577; CHECK: vector.main.loop.iter.check:578; CHECK-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP0]], 16579; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]580; CHECK: vector.ph:581; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 16582; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]583; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[ARG1]], i64 0584; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer585; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <16 x i8> poison, i8 [[ARG2]], i64 0586; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <16 x i8> [[BROADCAST_SPLATINSERT2]], <16 x i8> poison, <16 x i32> zeroinitializer587; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]588; CHECK: vector.body:589; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]590; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[INDEX]]591; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i16>, ptr [[TMP1]], align 2592; CHECK-NEXT: [[TMP2:%.*]] = trunc <16 x i16> [[WIDE_LOAD]] to <16 x i8>593; CHECK-NEXT: [[TMP3:%.*]] = shl <16 x i8> [[TMP2]], splat (i8 4)594; CHECK-NEXT: [[TMP4:%.*]] = add <16 x i8> [[TMP3]], splat (i8 32)595; CHECK-NEXT: [[TMP5:%.*]] = and <16 x i8> [[TMP2]], splat (i8 -52)596; CHECK-NEXT: [[TMP6:%.*]] = or <16 x i8> [[TMP5]], splat (i8 51)597; CHECK-NEXT: [[TMP7:%.*]] = mul <16 x i8> [[TMP6]], splat (i8 60)598; CHECK-NEXT: [[TMP8:%.*]] = and <16 x i8> [[TMP4]], [[BROADCAST_SPLAT]]599; CHECK-NEXT: [[TMP9:%.*]] = and <16 x i8> [[TMP7]], splat (i8 -4)600; CHECK-NEXT: [[TMP10:%.*]] = xor <16 x i8> [[TMP9]], [[BROADCAST_SPLAT3]]601; CHECK-NEXT: [[TMP11:%.*]] = mul <16 x i8> [[TMP10]], [[TMP8]]602; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDEX]]603; CHECK-NEXT: store <16 x i8> [[TMP11]], ptr [[TMP12]], align 1604; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16605; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]606; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]607; CHECK: middle.block:608; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]609; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]610; CHECK: vec.epilog.iter.check:611; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4612; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]613; CHECK: vec.epilog.ph:614; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]615; CHECK-NEXT: [[N_MOD_VF4:%.*]] = urem i64 [[TMP0]], 4616; CHECK-NEXT: [[N_VEC5:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF4]]617; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i8> poison, i8 [[ARG1]], i64 0618; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT6]], <4 x i8> poison, <4 x i32> zeroinitializer619; CHECK-NEXT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i8> poison, i8 [[ARG2]], i64 0620; CHECK-NEXT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT8]], <4 x i8> poison, <4 x i32> zeroinitializer621; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]622; CHECK: vec.epilog.vector.body:623; CHECK-NEXT: [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]624; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[INDEX10]]625; CHECK-NEXT: [[WIDE_LOAD11:%.*]] = load <4 x i16>, ptr [[TMP14]], align 2626; CHECK-NEXT: [[TMP15:%.*]] = trunc <4 x i16> [[WIDE_LOAD11]] to <4 x i8>627; CHECK-NEXT: [[TMP16:%.*]] = shl <4 x i8> [[TMP15]], splat (i8 4)628; CHECK-NEXT: [[TMP17:%.*]] = add <4 x i8> [[TMP16]], splat (i8 32)629; CHECK-NEXT: [[TMP18:%.*]] = and <4 x i8> [[TMP15]], splat (i8 -52)630; CHECK-NEXT: [[TMP19:%.*]] = or <4 x i8> [[TMP18]], splat (i8 51)631; CHECK-NEXT: [[TMP20:%.*]] = mul <4 x i8> [[TMP19]], splat (i8 60)632; CHECK-NEXT: [[TMP21:%.*]] = and <4 x i8> [[TMP17]], [[BROADCAST_SPLAT7]]633; CHECK-NEXT: [[TMP22:%.*]] = and <4 x i8> [[TMP20]], splat (i8 -4)634; CHECK-NEXT: [[TMP23:%.*]] = xor <4 x i8> [[TMP22]], [[BROADCAST_SPLAT9]]635; CHECK-NEXT: [[TMP24:%.*]] = mul <4 x i8> [[TMP23]], [[TMP21]]636; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDEX10]]637; CHECK-NEXT: store <4 x i8> [[TMP24]], ptr [[TMP25]], align 1638; CHECK-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX10]], 4639; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT12]], [[N_VEC5]]640; CHECK-NEXT: br i1 [[TMP26]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]641; CHECK: vec.epilog.middle.block:642; CHECK-NEXT: [[CMP_N13:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC5]]643; CHECK-NEXT: br i1 [[CMP_N13]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]644; CHECK: vec.epilog.scalar.ph:645; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]646; CHECK-NEXT: br label [[FOR_BODY:%.*]]647; CHECK: for.cond.cleanup.loopexit:648; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]649; CHECK: for.cond.cleanup:650; CHECK-NEXT: ret void651; CHECK: for.body:652; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]653; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[INDVARS_IV]]654; CHECK-NEXT: [[TMP27:%.*]] = load i16, ptr [[ARRAYIDX]], align 2655; CHECK-NEXT: [[CONV:%.*]] = sext i16 [[TMP27]] to i32656; CHECK-NEXT: [[ADD:%.*]] = shl i32 [[CONV]], 4657; CHECK-NEXT: [[CONV2:%.*]] = add nsw i32 [[ADD]], 32658; CHECK-NEXT: [[OR:%.*]] = and i32 [[CONV]], 204659; CHECK-NEXT: [[CONV8:%.*]] = or i32 [[OR]], 51660; CHECK-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV8]], 60661; CHECK-NEXT: [[AND:%.*]] = and i32 [[CONV2]], [[CONV13]]662; CHECK-NEXT: [[MUL_MASKED:%.*]] = and i32 [[MUL]], 252663; CHECK-NEXT: [[CONV17:%.*]] = xor i32 [[MUL_MASKED]], [[CONV11]]664; CHECK-NEXT: [[MUL18:%.*]] = mul nuw nsw i32 [[CONV17]], [[AND]]665; CHECK-NEXT: [[CONV19:%.*]] = trunc i32 [[MUL18]] to i8666; CHECK-NEXT: [[ARRAYIDX21:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDVARS_IV]]667; CHECK-NEXT: store i8 [[CONV19]], ptr [[ARRAYIDX21]], align 1668; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1669; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32670; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[LEN]]671; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]672;673entry:674 %cmp.32 = icmp sgt i32 %len, 0675 br i1 %cmp.32, label %for.body.lr.ph, label %for.cond.cleanup676 677for.body.lr.ph: ; preds = %entry678 %conv11 = zext i8 %arg2 to i32679 %conv13 = zext i8 %arg1 to i32680 br label %for.body681 682for.cond.cleanup: ; preds = %for.body, %entry683 ret void684 685for.body: ; preds = %for.body, %for.body.lr.ph686 %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ]687 %arrayidx = getelementptr inbounds i16, ptr %p, i64 %indvars.iv688 %0 = load i16, ptr %arrayidx689 %conv = sext i16 %0 to i32690 %add = shl i32 %conv, 4691 %conv2 = add nsw i32 %add, 32692 %or = and i32 %conv, 204693 %conv8 = or i32 %or, 51694 %mul = mul nuw nsw i32 %conv8, 60695 %and = and i32 %conv2, %conv13696 %mul.masked = and i32 %mul, 252697 %conv17 = xor i32 %mul.masked, %conv11698 %mul18 = mul nuw nsw i32 %conv17, %and699 %conv19 = trunc i32 %mul18 to i8700 %arrayidx21 = getelementptr inbounds i8, ptr %q, i64 %indvars.iv701 store i8 %conv19, ptr %arrayidx21702 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1703 %lftr.wideiv = trunc i64 %indvars.iv.next to i32704 %exitcond = icmp eq i32 %lftr.wideiv, %len705 br i1 %exitcond, label %for.cond.cleanup, label %for.body706}707 708define void @add_phifail(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %len) #0 {709; CHECK-LABEL: define void @add_phifail710; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) {711; CHECK-NEXT: entry:712; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[LEN]], 0713; CHECK-NEXT: br i1 [[CMP8]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]714; CHECK: for.body.preheader:715; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[LEN]] to i64716; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16717; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]718; CHECK: vector.ph:719; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 16720; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]721; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]722; CHECK: vector.body:723; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]724; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]725; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1726; CHECK-NEXT: [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>727; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw <16 x i32> [[TMP2]], splat (i32 2)728; CHECK-NEXT: [[TMP4:%.*]] = trunc <16 x i32> [[TMP3]] to <16 x i8>729; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDEX]]730; CHECK-NEXT: store <16 x i8> [[TMP4]], ptr [[TMP5]], align 1731; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16732; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]733; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]734; CHECK: middle.block:735; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i32> [[TMP2]], i32 15736; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]737; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]738; CHECK: scalar.ph:739; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]740; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]741; CHECK-NEXT: br label [[FOR_BODY:%.*]]742; CHECK: for.cond.cleanup.loopexit:743; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]744; CHECK: for.cond.cleanup:745; CHECK-NEXT: ret void746; CHECK: for.body:747; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]748; CHECK-NEXT: [[A_PHI:%.*]] = phi i32 [ [[CONV:%.*]], [[FOR_BODY]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ]749; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDVARS_IV]]750; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX]], align 1751; CHECK-NEXT: [[CONV]] = zext i8 [[TMP7]] to i32752; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[CONV]], 2753; CHECK-NEXT: [[CONV1:%.*]] = trunc i32 [[ADD]] to i8754; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDVARS_IV]]755; CHECK-NEXT: store i8 [[CONV1]], ptr [[ARRAYIDX3]], align 1756; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1757; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32758; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[LEN]]759; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]760;761entry:762 %cmp8 = icmp sgt i32 %len, 0763 br i1 %cmp8, label %for.body, label %for.cond.cleanup764 765for.cond.cleanup: ; preds = %for.body, %entry766 ret void767 768for.body: ; preds = %entry, %for.body769 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]770 %a_phi = phi i32 [ %conv, %for.body ], [ 0, %entry ]771 %arrayidx = getelementptr inbounds i8, ptr %p, i64 %indvars.iv772 %0 = load i8, ptr %arrayidx773 %conv = zext i8 %0 to i32774 %add = add nuw nsw i32 %conv, 2775 %conv1 = trunc i32 %add to i8776 %arrayidx3 = getelementptr inbounds i8, ptr %q, i64 %indvars.iv777 store i8 %conv1, ptr %arrayidx3778 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1779 %lftr.wideiv = trunc i64 %indvars.iv.next to i32780 %exitcond = icmp eq i32 %lftr.wideiv, %len781 br i1 %exitcond, label %for.cond.cleanup, label %for.body782}783 784; When we vectorize this loop, we generate correct code785; even when %len exactly divides VF (since we extract from the second last index786; and pass this to the for.cond.cleanup block). Vectorized loop returns787; the correct value a_phi = p[len -2]788define i8 @add_phifail2(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i32 %len) #0 {789; CHECK-LABEL: define i8 @add_phifail2790; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) {791; CHECK-NEXT: entry:792; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[LEN]], -1793; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64794; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1795; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 16796; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]797; CHECK: vector.ph:798; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 16799; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]800; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]801; CHECK: vector.body:802; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]803; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]804; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1805; CHECK-NEXT: [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>806; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw <16 x i32> [[TMP4]], splat (i32 2)807; CHECK-NEXT: [[TMP6:%.*]] = trunc <16 x i32> [[TMP5]] to <16 x i8>808; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDEX]]809; CHECK-NEXT: store <16 x i8> [[TMP6]], ptr [[TMP7]], align 1810; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16811; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]812; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]813; CHECK: middle.block:814; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i32> [[TMP4]], i32 15815; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <16 x i32> [[TMP4]], i32 14816; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]817; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]818; CHECK: scalar.ph:819; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]820; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]821; CHECK-NEXT: br label [[FOR_BODY:%.*]]822; CHECK: for.cond.cleanup:823; CHECK-NEXT: [[A_PHI_LCSSA:%.*]] = phi i32 [ [[A_PHI:%.*]], [[FOR_BODY]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], [[MIDDLE_BLOCK]] ]824; CHECK-NEXT: [[RET:%.*]] = trunc i32 [[A_PHI_LCSSA]] to i8825; CHECK-NEXT: ret i8 [[RET]]826; CHECK: for.body:827; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]828; CHECK-NEXT: [[A_PHI]] = phi i32 [ [[CONV:%.*]], [[FOR_BODY]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ]829; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDVARS_IV]]830; CHECK-NEXT: [[TMP9:%.*]] = load i8, ptr [[ARRAYIDX]], align 1831; CHECK-NEXT: [[CONV]] = zext i8 [[TMP9]] to i32832; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[CONV]], 2833; CHECK-NEXT: [[CONV1:%.*]] = trunc i32 [[ADD]] to i8834; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[Q]], i64 [[INDVARS_IV]]835; CHECK-NEXT: store i8 [[CONV1]], ptr [[ARRAYIDX3]], align 1836; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1837; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32838; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[LEN]]839; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]840;841entry:842 br label %for.body843 844for.cond.cleanup: ; preds = %for.body, %entry845 %ret = trunc i32 %a_phi to i8846 ret i8 %ret847 848for.body: ; preds = %entry, %for.body849 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]850 %a_phi = phi i32 [ %conv, %for.body ], [ 0, %entry ]851 %arrayidx = getelementptr inbounds i8, ptr %p, i64 %indvars.iv852 %0 = load i8, ptr %arrayidx853 %conv = zext i8 %0 to i32854 %add = add nuw nsw i32 %conv, 2855 %conv1 = trunc i32 %add to i8856 %arrayidx3 = getelementptr inbounds i8, ptr %q, i64 %indvars.iv857 store i8 %conv1, ptr %arrayidx3858 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1859 %lftr.wideiv = trunc i64 %indvars.iv.next to i32860 %exitcond = icmp eq i32 %lftr.wideiv, %len861 br i1 %exitcond, label %for.cond.cleanup, label %for.body862}863