215 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 62; RUN: opt -p loop-vectorize -mtriple riscv64 -mattr=+v -S %s | FileCheck -check-prefix=CHECK %s3; RUN: opt -p loop-vectorize -mtriple riscv64 -mattr=+v -S %s -prefer-predicate-over-epilogue=scalar-epilogue | FileCheck -check-prefix=EPILOGUE %s4 5define void @load_store_interleave_group(ptr noalias %data) {6; CHECK-LABEL: define void @load_store_interleave_group(7; CHECK-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0:[0-9]+]] {8; CHECK-NEXT: [[ENTRY:.*:]]9; CHECK-NEXT: br label %[[VECTOR_PH:.*]]10; CHECK: [[VECTOR_PH]]:11; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]12; CHECK: [[VECTOR_BODY]]:13; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]14; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]15; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)16; CHECK-NEXT: [[TMP1:%.*]] = shl nsw i64 [[EVL_BASED_IV]], 117; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP1]]18; CHECK-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP0]], 219; CHECK-NEXT: [[WIDE_VP_LOAD:%.*]] = call <vscale x 4 x i64> @llvm.vp.load.nxv4i64.p0(ptr align 8 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])20; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VP_LOAD]])21; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 022; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 123; CHECK-NEXT: [[INTERLEAVE_EVL1:%.*]] = mul nuw nsw i32 [[TMP0]], 224; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]])25; CHECK-NEXT: call void @llvm.vp.store.nxv4i64.p0(<vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr align 8 [[TMP2]], <vscale x 4 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL1]])26; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP0]] to i6427; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP5]], [[EVL_BASED_IV]]28; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]29; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 030; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]31; CHECK: [[MIDDLE_BLOCK]]:32; CHECK-NEXT: br label %[[EXIT:.*]]33; CHECK: [[EXIT]]:34; CHECK-NEXT: ret void35;36; EPILOGUE-LABEL: define void @load_store_interleave_group(37; EPILOGUE-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0:[0-9]+]] {38; EPILOGUE-NEXT: [[ENTRY:.*]]:39; EPILOGUE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()40; EPILOGUE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 141; EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]42; EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]43; EPILOGUE: [[VECTOR_PH]]:44; EPILOGUE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()45; EPILOGUE-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 246; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]]47; EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]48; EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]49; EPILOGUE: [[VECTOR_BODY]]:50; EPILOGUE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]51; EPILOGUE-NEXT: [[TMP4:%.*]] = shl nsw i64 [[INDEX]], 152; EPILOGUE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP4]]53; EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP5]], align 854; EPILOGUE-NEXT: store <vscale x 2 x i64> [[WIDE_LOAD]], ptr [[TMP5]], align 855; EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]56; EPILOGUE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]57; EPILOGUE-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]58; EPILOGUE: [[MIDDLE_BLOCK]]:59; EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 100, [[N_VEC]]60; EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]61; EPILOGUE: [[SCALAR_PH]]:62; EPILOGUE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]63; EPILOGUE-NEXT: br label %[[LOOP:.*]]64; EPILOGUE: [[LOOP]]:65; EPILOGUE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]66; EPILOGUE-NEXT: [[MUL_2:%.*]] = shl nsw i64 [[IV]], 167; EPILOGUE-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[MUL_2]]68; EPILOGUE-NEXT: [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 869; EPILOGUE-NEXT: store i64 [[L_0]], ptr [[DATA_0]], align 870; EPILOGUE-NEXT: [[ADD_1:%.*]] = or disjoint i64 [[MUL_2]], 171; EPILOGUE-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[ADD_1]]72; EPILOGUE-NEXT: [[L_1:%.*]] = load i64, ptr [[DATA_1]], align 873; EPILOGUE-NEXT: store i64 [[L_1]], ptr [[DATA_1]], align 874; EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 175; EPILOGUE-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 10076; EPILOGUE-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]77; EPILOGUE: [[EXIT]]:78; EPILOGUE-NEXT: ret void79;80entry:81 br label %loop82 83loop:84 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]85 %mul.2 = shl nsw i64 %iv, 186 %data.0 = getelementptr inbounds i64, ptr %data, i64 %mul.287 %l.0 = load i64, ptr %data.0, align 888 store i64 %l.0, ptr %data.0, align 889 %add.1 = or disjoint i64 %mul.2, 190 %data.1 = getelementptr inbounds i64, ptr %data, i64 %add.191 %l.1 = load i64, ptr %data.1, align 892 store i64 %l.1, ptr %data.1, align 893 %iv.next = add nuw nsw i64 %iv, 194 %ec = icmp eq i64 %iv.next, 10095 br i1 %ec, label %exit, label %loop96 97exit:98 ret void99}100 101 102define void @load_store_interleave_group_i32(ptr noalias %data) {103; CHECK-LABEL: define void @load_store_interleave_group_i32(104; CHECK-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0]] {105; CHECK-NEXT: [[ENTRY:.*:]]106; CHECK-NEXT: br label %[[VECTOR_PH:.*]]107; CHECK: [[VECTOR_PH]]:108; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]109; CHECK: [[VECTOR_BODY]]:110; CHECK-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]111; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]112; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)113; CHECK-NEXT: [[TMP1:%.*]] = shl nsw i64 [[EVL_BASED_IV]], 2114; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[TMP1]]115; CHECK-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP0]], 4116; CHECK-NEXT: [[WIDE_VP_LOAD:%.*]] = call <vscale x 16 x i32> @llvm.vp.load.nxv16i32.p0(ptr align 8 [[TMP2]], <vscale x 16 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])117; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VP_LOAD]])118; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0119; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1120; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2121; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 3122; CHECK-NEXT: [[INTERLEAVE_EVL1:%.*]] = mul nuw nsw i32 [[TMP0]], 4123; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 16 x i32> @llvm.vector.interleave4.nxv16i32(<vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]])124; CHECK-NEXT: call void @llvm.vp.store.nxv16i32.p0(<vscale x 16 x i32> [[INTERLEAVED_VEC]], ptr align 8 [[TMP2]], <vscale x 16 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL1]])125; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[TMP0]] to i64126; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP5]], [[EVL_BASED_IV]]127; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]]128; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[AVL_NEXT]], 0129; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]130; CHECK: [[MIDDLE_BLOCK]]:131; CHECK-NEXT: br label %[[EXIT:.*]]132; CHECK: [[EXIT]]:133; CHECK-NEXT: ret void134;135; EPILOGUE-LABEL: define void @load_store_interleave_group_i32(136; EPILOGUE-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0]] {137; EPILOGUE-NEXT: [[ENTRY:.*]]:138; EPILOGUE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()139; EPILOGUE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2140; EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP1]]141; EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]142; EPILOGUE: [[VECTOR_PH]]:143; EPILOGUE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()144; EPILOGUE-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4145; EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]]146; EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]147; EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]]148; EPILOGUE: [[VECTOR_BODY]]:149; EPILOGUE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]150; EPILOGUE-NEXT: [[TMP4:%.*]] = shl nsw i64 [[INDEX]], 2151; EPILOGUE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[TMP4]]152; EPILOGUE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 8153; EPILOGUE-NEXT: store <vscale x 4 x i32> [[WIDE_LOAD]], ptr [[TMP5]], align 8154; EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]155; EPILOGUE-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]156; EPILOGUE-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]157; EPILOGUE: [[MIDDLE_BLOCK]]:158; EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i64 100, [[N_VEC]]159; EPILOGUE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]160; EPILOGUE: [[SCALAR_PH]]:161; EPILOGUE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]162; EPILOGUE-NEXT: br label %[[LOOP:.*]]163; EPILOGUE: [[LOOP]]:164; EPILOGUE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]165; EPILOGUE-NEXT: [[MUL_2:%.*]] = shl nsw i64 [[IV]], 2166; EPILOGUE-NEXT: [[DATA_0:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[MUL_2]]167; EPILOGUE-NEXT: [[L_0:%.*]] = load i32, ptr [[DATA_0]], align 8168; EPILOGUE-NEXT: store i32 [[L_0]], ptr [[DATA_0]], align 8169; EPILOGUE-NEXT: [[ADD_1:%.*]] = or disjoint i64 [[MUL_2]], 1170; EPILOGUE-NEXT: [[DATA_1:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[ADD_1]]171; EPILOGUE-NEXT: [[L_1:%.*]] = load i32, ptr [[DATA_1]], align 8172; EPILOGUE-NEXT: store i32 [[L_1]], ptr [[DATA_1]], align 8173; EPILOGUE-NEXT: [[ADD_2:%.*]] = add i64 [[MUL_2]], 2174; EPILOGUE-NEXT: [[DATA_2:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[ADD_2]]175; EPILOGUE-NEXT: [[L_2:%.*]] = load i32, ptr [[DATA_2]], align 8176; EPILOGUE-NEXT: store i32 [[L_2]], ptr [[DATA_2]], align 8177; EPILOGUE-NEXT: [[ADD_3:%.*]] = add i64 [[MUL_2]], 3178; EPILOGUE-NEXT: [[DATA_3:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i64 [[ADD_3]]179; EPILOGUE-NEXT: [[L_3:%.*]] = load i32, ptr [[DATA_3]], align 8180; EPILOGUE-NEXT: store i32 [[L_3]], ptr [[DATA_3]], align 8181; EPILOGUE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1182; EPILOGUE-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100183; EPILOGUE-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]184; EPILOGUE: [[EXIT]]:185; EPILOGUE-NEXT: ret void186;187entry:188 br label %loop189 190loop:191 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]192 %mul.4 = shl nsw i64 %iv, 2193 %data.0 = getelementptr inbounds i32, ptr %data, i64 %mul.4194 %l.0 = load i32, ptr %data.0, align 8195 store i32 %l.0, ptr %data.0, align 8196 %add.1 = or disjoint i64 %mul.4, 1197 %data.1 = getelementptr inbounds i32, ptr %data, i64 %add.1198 %l.1 = load i32, ptr %data.1, align 8199 store i32 %l.1, ptr %data.1, align 8200 %add.2 = add i64 %mul.4, 2201 %data.2 = getelementptr inbounds i32, ptr %data, i64 %add.2202 %l.2 = load i32, ptr %data.2, align 8203 store i32 %l.2, ptr %data.2, align 8204 %add.3 = add i64 %mul.4, 3205 %data.3 = getelementptr inbounds i32, ptr %data, i64 %add.3206 %l.3 = load i32, ptr %data.3, align 8207 store i32 %l.3, ptr %data.3, align 8208 %iv.next = add nuw nsw i64 %iv, 1209 %ec = icmp eq i64 %iv.next, 100210 br i1 %ec, label %exit, label %loop211 212exit:213 ret void214}215