808 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=riscv64 -mattr=+v -S %s | FileCheck --check-prefixes=CHECK,NON-POW2 %s3; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=riscv64 -mattr=+v -S %s | FileCheck --check-prefixes=CHECK,POW2-ONLY %s4 5define void @v3_load_i32_mul_by_constant_store(ptr %src, ptr %dst) {6; NON-POW2-LABEL: @v3_load_i32_mul_by_constant_store(7; NON-POW2-NEXT: entry:8; NON-POW2-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 09; NON-POW2-NEXT: [[TMP0:%.*]] = load <3 x i32>, ptr [[GEP_SRC_0]], align 410; NON-POW2-NEXT: [[TMP1:%.*]] = mul nsw <3 x i32> [[TMP0]], splat (i32 10)11; NON-POW2-NEXT: store <3 x i32> [[TMP1]], ptr [[DST:%.*]], align 412; NON-POW2-NEXT: ret void13;14; POW2-ONLY-LABEL: @v3_load_i32_mul_by_constant_store(15; POW2-ONLY-NEXT: entry:16; POW2-ONLY-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 017; POW2-ONLY-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 218; POW2-ONLY-NEXT: [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC_2]], align 419; POW2-ONLY-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_2]], 1020; POW2-ONLY-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_0]], align 421; POW2-ONLY-NEXT: [[TMP1:%.*]] = mul nsw <2 x i32> [[TMP0]], splat (i32 10)22; POW2-ONLY-NEXT: store <2 x i32> [[TMP1]], ptr [[DST:%.*]], align 423; POW2-ONLY-NEXT: [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 224; POW2-ONLY-NEXT: store i32 [[MUL_2]], ptr [[DST_2]], align 425; POW2-ONLY-NEXT: ret void26;27entry:28 %gep.src.0 = getelementptr inbounds i32, ptr %src, i32 029 %l.src.0 = load i32, ptr %gep.src.0, align 430 %mul.0 = mul nsw i32 %l.src.0, 1031 32 %gep.src.1 = getelementptr inbounds i32, ptr %src, i32 133 %l.src.1 = load i32, ptr %gep.src.1, align 434 %mul.1 = mul nsw i32 %l.src.1, 1035 36 %gep.src.2 = getelementptr inbounds i32, ptr %src, i32 237 %l.src.2 = load i32, ptr %gep.src.2, align 438 %mul.2 = mul nsw i32 %l.src.2, 1039 40 store i32 %mul.0, ptr %dst41 42 %dst.1 = getelementptr i32, ptr %dst, i32 143 store i32 %mul.1, ptr %dst.144 45 %dst.2 = getelementptr i32, ptr %dst, i32 246 store i32 %mul.2, ptr %dst.247 48 ret void49}50 51; Should no be vectorized with a undef/poison element as padding, as52; division by undef/poison may cause UB. Must use VL predication or53; masking instead, where RISCV wins.54define void @v3_load_i32_udiv_by_constant_store(ptr %src, ptr %dst) {55; NON-POW2-LABEL: @v3_load_i32_udiv_by_constant_store(56; NON-POW2-NEXT: entry:57; NON-POW2-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 058; NON-POW2-NEXT: [[TMP0:%.*]] = load <3 x i32>, ptr [[GEP_SRC_0]], align 459; NON-POW2-NEXT: [[TMP1:%.*]] = udiv <3 x i32> splat (i32 10), [[TMP0]]60; NON-POW2-NEXT: store <3 x i32> [[TMP1]], ptr [[DST:%.*]], align 461; NON-POW2-NEXT: ret void62;63; POW2-ONLY-LABEL: @v3_load_i32_udiv_by_constant_store(64; POW2-ONLY-NEXT: entry:65; POW2-ONLY-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 066; POW2-ONLY-NEXT: [[L_SRC_0:%.*]] = load i32, ptr [[GEP_SRC_0]], align 467; POW2-ONLY-NEXT: [[MUL_0:%.*]] = udiv i32 10, [[L_SRC_0]]68; POW2-ONLY-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 169; POW2-ONLY-NEXT: [[L_SRC_1:%.*]] = load i32, ptr [[GEP_SRC_1]], align 470; POW2-ONLY-NEXT: [[MUL_1:%.*]] = udiv i32 10, [[L_SRC_1]]71; POW2-ONLY-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 272; POW2-ONLY-NEXT: [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC_2]], align 473; POW2-ONLY-NEXT: [[MUL_2:%.*]] = udiv i32 10, [[L_SRC_2]]74; POW2-ONLY-NEXT: store i32 [[MUL_0]], ptr [[DST:%.*]], align 475; POW2-ONLY-NEXT: [[DST_1:%.*]] = getelementptr i32, ptr [[DST]], i32 176; POW2-ONLY-NEXT: store i32 [[MUL_1]], ptr [[DST_1]], align 477; POW2-ONLY-NEXT: [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 278; POW2-ONLY-NEXT: store i32 [[MUL_2]], ptr [[DST_2]], align 479; POW2-ONLY-NEXT: ret void80;81entry:82 %gep.src.0 = getelementptr inbounds i32, ptr %src, i32 083 %l.src.0 = load i32, ptr %gep.src.0, align 484 %mul.0 = udiv i32 10, %l.src.085 86 %gep.src.1 = getelementptr inbounds i32, ptr %src, i32 187 %l.src.1 = load i32, ptr %gep.src.1, align 488 %mul.1 = udiv i32 10, %l.src.189 90 %gep.src.2 = getelementptr inbounds i32, ptr %src, i32 291 %l.src.2 = load i32, ptr %gep.src.2, align 492 %mul.2 = udiv i32 10, %l.src.293 94 store i32 %mul.0, ptr %dst95 96 %dst.1 = getelementptr i32, ptr %dst, i32 197 store i32 %mul.1, ptr %dst.198 99 %dst.2 = getelementptr i32, ptr %dst, i32 2100 store i32 %mul.2, ptr %dst.2101 102 ret void103}104 105 106 107define void @v3_load_i32_mul_store(ptr %src.1, ptr %src.2, ptr %dst) {108; NON-POW2-LABEL: @v3_load_i32_mul_store(109; NON-POW2-NEXT: entry:110; NON-POW2-NEXT: [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0111; NON-POW2-NEXT: [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0112; NON-POW2-NEXT: [[TMP0:%.*]] = load <3 x i32>, ptr [[GEP_SRC_1_0]], align 4113; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr [[GEP_SRC_2_0]], align 4114; NON-POW2-NEXT: [[TMP2:%.*]] = mul nsw <3 x i32> [[TMP0]], [[TMP1]]115; NON-POW2-NEXT: store <3 x i32> [[TMP2]], ptr [[DST:%.*]], align 4116; NON-POW2-NEXT: ret void117;118; POW2-ONLY-LABEL: @v3_load_i32_mul_store(119; POW2-ONLY-NEXT: entry:120; POW2-ONLY-NEXT: [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0121; POW2-ONLY-NEXT: [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0122; POW2-ONLY-NEXT: [[GEP_SRC_1_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_1]], i32 2123; POW2-ONLY-NEXT: [[L_SRC_1_2:%.*]] = load i32, ptr [[GEP_SRC_1_2]], align 4124; POW2-ONLY-NEXT: [[GEP_SRC_2_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_2]], i32 2125; POW2-ONLY-NEXT: [[L_SRC_2_2:%.*]] = load i32, ptr [[GEP_SRC_2_2]], align 4126; POW2-ONLY-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_1_2]], [[L_SRC_2_2]]127; POW2-ONLY-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_1_0]], align 4128; POW2-ONLY-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[GEP_SRC_2_0]], align 4129; POW2-ONLY-NEXT: [[TMP2:%.*]] = mul nsw <2 x i32> [[TMP0]], [[TMP1]]130; POW2-ONLY-NEXT: store <2 x i32> [[TMP2]], ptr [[DST:%.*]], align 4131; POW2-ONLY-NEXT: [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 2132; POW2-ONLY-NEXT: store i32 [[MUL_2]], ptr [[DST_2]], align 4133; POW2-ONLY-NEXT: ret void134;135entry:136 %gep.src.1.0 = getelementptr inbounds i32, ptr %src.1, i32 0137 %l.src.1.0 = load i32, ptr %gep.src.1.0, align 4138 %gep.src.2.0 = getelementptr inbounds i32, ptr %src.2, i32 0139 %l.src.2.0 = load i32, ptr %gep.src.2.0, align 4140 %mul.0 = mul nsw i32 %l.src.1.0, %l.src.2.0141 142 %gep.src.1.1 = getelementptr inbounds i32, ptr %src.1, i32 1143 %l.src.1.1 = load i32, ptr %gep.src.1.1, align 4144 %gep.src.2.1 = getelementptr inbounds i32, ptr %src.2, i32 1145 %l.src.2.1 = load i32, ptr %gep.src.2.1, align 4146 %mul.1 = mul nsw i32 %l.src.1.1, %l.src.2.1147 148 %gep.src.1.2 = getelementptr inbounds i32, ptr %src.1, i32 2149 %l.src.1.2 = load i32, ptr %gep.src.1.2, align 4150 %gep.src.2.2 = getelementptr inbounds i32, ptr %src.2, i32 2151 %l.src.2.2 = load i32, ptr %gep.src.2.2, align 4152 %mul.2 = mul nsw i32 %l.src.1.2, %l.src.2.2153 154 store i32 %mul.0, ptr %dst155 156 %dst.1 = getelementptr i32, ptr %dst, i32 1157 store i32 %mul.1, ptr %dst.1158 159 %dst.2 = getelementptr i32, ptr %dst, i32 2160 store i32 %mul.2, ptr %dst.2161 162 ret void163}164 165define void @v3_load_i32_mul_add_const_store(ptr %src.1, ptr %src.2, ptr %dst) {166; NON-POW2-LABEL: @v3_load_i32_mul_add_const_store(167; NON-POW2-NEXT: entry:168; NON-POW2-NEXT: [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0169; NON-POW2-NEXT: [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0170; NON-POW2-NEXT: [[TMP0:%.*]] = load <3 x i32>, ptr [[GEP_SRC_1_0]], align 4171; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr [[GEP_SRC_2_0]], align 4172; NON-POW2-NEXT: [[TMP2:%.*]] = mul nsw <3 x i32> [[TMP0]], [[TMP1]]173; NON-POW2-NEXT: [[TMP3:%.*]] = add <3 x i32> [[TMP2]], splat (i32 9)174; NON-POW2-NEXT: store <3 x i32> [[TMP3]], ptr [[DST:%.*]], align 4175; NON-POW2-NEXT: ret void176;177; POW2-ONLY-LABEL: @v3_load_i32_mul_add_const_store(178; POW2-ONLY-NEXT: entry:179; POW2-ONLY-NEXT: [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0180; POW2-ONLY-NEXT: [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0181; POW2-ONLY-NEXT: [[GEP_SRC_1_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_1]], i32 2182; POW2-ONLY-NEXT: [[L_SRC_1_2:%.*]] = load i32, ptr [[GEP_SRC_1_2]], align 4183; POW2-ONLY-NEXT: [[GEP_SRC_2_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_2]], i32 2184; POW2-ONLY-NEXT: [[L_SRC_2_2:%.*]] = load i32, ptr [[GEP_SRC_2_2]], align 4185; POW2-ONLY-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_1_2]], [[L_SRC_2_2]]186; POW2-ONLY-NEXT: [[ADD_2:%.*]] = add i32 [[MUL_2]], 9187; POW2-ONLY-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_1_0]], align 4188; POW2-ONLY-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[GEP_SRC_2_0]], align 4189; POW2-ONLY-NEXT: [[TMP2:%.*]] = mul nsw <2 x i32> [[TMP0]], [[TMP1]]190; POW2-ONLY-NEXT: [[TMP3:%.*]] = add <2 x i32> [[TMP2]], splat (i32 9)191; POW2-ONLY-NEXT: store <2 x i32> [[TMP3]], ptr [[DST:%.*]], align 4192; POW2-ONLY-NEXT: [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 2193; POW2-ONLY-NEXT: store i32 [[ADD_2]], ptr [[DST_2]], align 4194; POW2-ONLY-NEXT: ret void195;196entry:197 %gep.src.1.0 = getelementptr inbounds i32, ptr %src.1, i32 0198 %l.src.1.0 = load i32, ptr %gep.src.1.0, align 4199 %gep.src.2.0 = getelementptr inbounds i32, ptr %src.2, i32 0200 %l.src.2.0 = load i32, ptr %gep.src.2.0, align 4201 %mul.0 = mul nsw i32 %l.src.1.0, %l.src.2.0202 %add.0 = add i32 %mul.0, 9203 204 %gep.src.1.1 = getelementptr inbounds i32, ptr %src.1, i32 1205 %l.src.1.1 = load i32, ptr %gep.src.1.1, align 4206 %gep.src.2.1 = getelementptr inbounds i32, ptr %src.2, i32 1207 %l.src.2.1 = load i32, ptr %gep.src.2.1, align 4208 %mul.1 = mul nsw i32 %l.src.1.1, %l.src.2.1209 %add.1 = add i32 %mul.1, 9210 211 %gep.src.1.2 = getelementptr inbounds i32, ptr %src.1, i32 2212 %l.src.1.2 = load i32, ptr %gep.src.1.2, align 4213 %gep.src.2.2 = getelementptr inbounds i32, ptr %src.2, i32 2214 %l.src.2.2 = load i32, ptr %gep.src.2.2, align 4215 %mul.2 = mul nsw i32 %l.src.1.2, %l.src.2.2216 %add.2 = add i32 %mul.2, 9217 218 store i32 %add.0, ptr %dst219 220 %dst.1 = getelementptr i32, ptr %dst, i32 1221 store i32 %add.1, ptr %dst.1222 223 %dst.2 = getelementptr i32, ptr %dst, i32 2224 store i32 %add.2, ptr %dst.2225 226 ret void227}228 229define void @v3_load_f32_fadd_fadd_by_constant_store(ptr %src, ptr %dst) {230; NON-POW2-LABEL: @v3_load_f32_fadd_fadd_by_constant_store(231; NON-POW2-NEXT: entry:232; NON-POW2-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0233; NON-POW2-NEXT: [[TMP0:%.*]] = load <3 x float>, ptr [[GEP_SRC_0]], align 4234; NON-POW2-NEXT: [[TMP1:%.*]] = fadd <3 x float> [[TMP0]], splat (float 1.000000e+01)235; NON-POW2-NEXT: store <3 x float> [[TMP1]], ptr [[DST:%.*]], align 4236; NON-POW2-NEXT: ret void237;238; POW2-ONLY-LABEL: @v3_load_f32_fadd_fadd_by_constant_store(239; POW2-ONLY-NEXT: entry:240; POW2-ONLY-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0241; POW2-ONLY-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 2242; POW2-ONLY-NEXT: [[L_SRC_2:%.*]] = load float, ptr [[GEP_SRC_2]], align 4243; POW2-ONLY-NEXT: [[FADD_2:%.*]] = fadd float [[L_SRC_2]], 1.000000e+01244; POW2-ONLY-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[GEP_SRC_0]], align 4245; POW2-ONLY-NEXT: [[TMP1:%.*]] = fadd <2 x float> [[TMP0]], splat (float 1.000000e+01)246; POW2-ONLY-NEXT: store <2 x float> [[TMP1]], ptr [[DST:%.*]], align 4247; POW2-ONLY-NEXT: [[DST_2:%.*]] = getelementptr float, ptr [[DST]], i32 2248; POW2-ONLY-NEXT: store float [[FADD_2]], ptr [[DST_2]], align 4249; POW2-ONLY-NEXT: ret void250;251entry:252 %gep.src.0 = getelementptr inbounds float, ptr %src, i32 0253 %l.src.0 = load float , ptr %gep.src.0, align 4254 %fadd.0 = fadd float %l.src.0, 10.0255 256 %gep.src.1 = getelementptr inbounds float , ptr %src, i32 1257 %l.src.1 = load float, ptr %gep.src.1, align 4258 %fadd.1 = fadd float %l.src.1, 10.0259 260 %gep.src.2 = getelementptr inbounds float, ptr %src, i32 2261 %l.src.2 = load float, ptr %gep.src.2, align 4262 %fadd.2 = fadd float %l.src.2, 10.0263 264 store float %fadd.0, ptr %dst265 266 %dst.1 = getelementptr float, ptr %dst, i32 1267 store float %fadd.1, ptr %dst.1268 269 %dst.2 = getelementptr float, ptr %dst, i32 2270 store float %fadd.2, ptr %dst.2271 272 ret void273}274 275define void @phi_store3(ptr %dst) {276; NON-POW2-LABEL: @phi_store3(277; NON-POW2-NEXT: entry:278; NON-POW2-NEXT: br label [[EXIT:%.*]]279; NON-POW2: invoke.cont8.loopexit:280; NON-POW2-NEXT: br label [[EXIT]]281; NON-POW2: exit:282; NON-POW2-NEXT: [[TMP0:%.*]] = phi <3 x i32> [ <i32 1, i32 2, i32 3>, [[ENTRY:%.*]] ], [ poison, [[INVOKE_CONT8_LOOPEXIT:%.*]] ]283; NON-POW2-NEXT: store <3 x i32> [[TMP0]], ptr [[DST:%.*]], align 4284; NON-POW2-NEXT: ret void285;286; POW2-ONLY-LABEL: @phi_store3(287; POW2-ONLY-NEXT: entry:288; POW2-ONLY-NEXT: br label [[EXIT:%.*]]289; POW2-ONLY: invoke.cont8.loopexit:290; POW2-ONLY-NEXT: br label [[EXIT]]291; POW2-ONLY: exit:292; POW2-ONLY-NEXT: [[P_2:%.*]] = phi i32 [ 3, [[ENTRY:%.*]] ], [ 0, [[INVOKE_CONT8_LOOPEXIT:%.*]] ]293; POW2-ONLY-NEXT: [[TMP0:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[ENTRY]] ], [ poison, [[INVOKE_CONT8_LOOPEXIT]] ]294; POW2-ONLY-NEXT: [[DST_2:%.*]] = getelementptr i32, ptr [[DST:%.*]], i32 2295; POW2-ONLY-NEXT: store <2 x i32> [[TMP0]], ptr [[DST]], align 4296; POW2-ONLY-NEXT: store i32 [[P_2]], ptr [[DST_2]], align 4297; POW2-ONLY-NEXT: ret void298;299entry:300 br label %exit301 302invoke.cont8.loopexit: ; No predecessors!303 br label %exit304 305exit:306 %p.0 = phi i32 [ 1, %entry ], [ 0, %invoke.cont8.loopexit ]307 %p.1 = phi i32 [ 2, %entry ], [ 0, %invoke.cont8.loopexit ]308 %p.2 = phi i32 [ 3, %entry ], [ 0, %invoke.cont8.loopexit ]309 310 %dst.1 = getelementptr i32, ptr %dst, i32 1311 %dst.2 = getelementptr i32, ptr %dst, i32 2312 313 store i32 %p.0, ptr %dst, align 4314 store i32 %p.1, ptr %dst.1, align 4315 store i32 %p.2, ptr %dst.2, align 4316 ret void317}318 319define void @store_try_reorder(ptr %dst) {320; NON-POW2-LABEL: @store_try_reorder(321; NON-POW2-NEXT: entry:322; NON-POW2-NEXT: store <3 x i32> zeroinitializer, ptr [[DST:%.*]], align 4323; NON-POW2-NEXT: ret void324;325; POW2-ONLY-LABEL: @store_try_reorder(326; POW2-ONLY-NEXT: entry:327; POW2-ONLY-NEXT: store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887:%.*]], align 4328; POW2-ONLY-NEXT: [[ADD216:%.*]] = sub i32 0, 0329; POW2-ONLY-NEXT: [[ARRAYIDX_I1891:%.*]] = getelementptr i32, ptr [[ARRAYIDX_I1887]], i64 2330; POW2-ONLY-NEXT: store i32 [[ADD216]], ptr [[ARRAYIDX_I1891]], align 4331; POW2-ONLY-NEXT: ret void332;333entry:334 %add = add i32 0, 0335 store i32 %add, ptr %dst, align 4336 %add207 = sub i32 0, 0337 %arrayidx.i1887 = getelementptr i32, ptr %dst, i64 1338 store i32 %add207, ptr %arrayidx.i1887, align 4339 %add216 = sub i32 0, 0340 %arrayidx.i1891 = getelementptr i32, ptr %dst, i64 2341 store i32 %add216, ptr %arrayidx.i1891, align 4342 ret void343}344 345define void @vec3_fpext_cost(ptr %Colour, float %0) {346; NON-POW2-LABEL: @vec3_fpext_cost(347; NON-POW2-NEXT: entry:348; NON-POW2-NEXT: [[TMP1:%.*]] = insertelement <3 x float> poison, float [[TMP0:%.*]], i32 0349; NON-POW2-NEXT: [[TMP2:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <3 x i32> zeroinitializer350; NON-POW2-NEXT: [[TMP3:%.*]] = fpext <3 x float> [[TMP2]] to <3 x double>351; NON-POW2-NEXT: [[TMP4:%.*]] = call <3 x double> @llvm.fmuladd.v3f64(<3 x double> [[TMP3]], <3 x double> zeroinitializer, <3 x double> zeroinitializer)352; NON-POW2-NEXT: [[TMP5:%.*]] = fptrunc <3 x double> [[TMP4]] to <3 x float>353; NON-POW2-NEXT: store <3 x float> [[TMP5]], ptr [[COLOUR:%.*]], align 4354; NON-POW2-NEXT: ret void355;356; POW2-ONLY-LABEL: @vec3_fpext_cost(357; POW2-ONLY-NEXT: entry:358; POW2-ONLY-NEXT: [[ARRAYIDX80:%.*]] = getelementptr float, ptr [[COLOUR:%.*]], i64 2359; POW2-ONLY-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[TMP0:%.*]], i32 0360; POW2-ONLY-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> zeroinitializer361; POW2-ONLY-NEXT: [[TMP3:%.*]] = fpext <2 x float> [[TMP2]] to <2 x double>362; POW2-ONLY-NEXT: [[TMP4:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP3]], <2 x double> zeroinitializer, <2 x double> zeroinitializer)363; POW2-ONLY-NEXT: [[TMP5:%.*]] = fptrunc <2 x double> [[TMP4]] to <2 x float>364; POW2-ONLY-NEXT: store <2 x float> [[TMP5]], ptr [[COLOUR]], align 4365; POW2-ONLY-NEXT: [[CONV78:%.*]] = fpext float [[TMP0]] to double366; POW2-ONLY-NEXT: [[TMP6:%.*]] = call double @llvm.fmuladd.f64(double [[CONV78]], double 0.000000e+00, double 0.000000e+00)367; POW2-ONLY-NEXT: [[CONV82:%.*]] = fptrunc double [[TMP6]] to float368; POW2-ONLY-NEXT: store float [[CONV82]], ptr [[ARRAYIDX80]], align 4369; POW2-ONLY-NEXT: ret void370;371entry:372 %arrayidx72 = getelementptr float, ptr %Colour, i64 1373 %arrayidx80 = getelementptr float, ptr %Colour, i64 2374 %conv62 = fpext float %0 to double375 %1 = call double @llvm.fmuladd.f64(double %conv62, double 0.000000e+00, double 0.000000e+00)376 %conv66 = fptrunc double %1 to float377 store float %conv66, ptr %Colour, align 4378 %conv70 = fpext float %0 to double379 %2 = call double @llvm.fmuladd.f64(double %conv70, double 0.000000e+00, double 0.000000e+00)380 %conv74 = fptrunc double %2 to float381 store float %conv74, ptr %arrayidx72, align 4382 %conv78 = fpext float %0 to double383 %3 = call double @llvm.fmuladd.f64(double %conv78, double 0.000000e+00, double 0.000000e+00)384 %conv82 = fptrunc double %3 to float385 store float %conv82, ptr %arrayidx80, align 4386 ret void387}388 389define void @fpext_scatter(ptr %dst, double %conv) {390; CHECK-LABEL: @fpext_scatter(391; CHECK-NEXT: entry:392; CHECK-NEXT: [[CONV25:%.*]] = fptrunc double [[CONV:%.*]] to float393; CHECK-NEXT: [[LENGTHS:%.*]] = getelementptr float, ptr [[DST:%.*]], i64 0394; CHECK-NEXT: store float [[CONV25]], ptr [[LENGTHS]], align 4395; CHECK-NEXT: [[ARRAYIDX32:%.*]] = getelementptr float, ptr [[DST]], i64 1396; CHECK-NEXT: store float [[CONV25]], ptr [[ARRAYIDX32]], align 4397; CHECK-NEXT: [[ARRAYIDX37:%.*]] = getelementptr float, ptr [[DST]], i64 2398; CHECK-NEXT: store float [[CONV25]], ptr [[ARRAYIDX37]], align 4399; CHECK-NEXT: ret void400;401entry:402 %conv25 = fptrunc double %conv to float403 %Lengths = getelementptr float, ptr %dst, i64 0404 store float %conv25, ptr %Lengths, align 4405 %arrayidx32 = getelementptr float, ptr %dst, i64 1406 store float %conv25, ptr %arrayidx32, align 4407 %arrayidx37 = getelementptr float, ptr %dst, i64 2408 store float %conv25, ptr %arrayidx37, align 4409 ret void410}411 412define i32 @reduce_add(ptr %src) {413; CHECK-LABEL: @reduce_add(414; CHECK-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 0415; CHECK-NEXT: [[L_SRC_0:%.*]] = load i32, ptr [[GEP_SRC_0]], align 4416; CHECK-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 1417; CHECK-NEXT: [[L_SRC_1:%.*]] = load i32, ptr [[GEP_SRC_1]], align 4418; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 2419; CHECK-NEXT: [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC_2]], align 4420; CHECK-NEXT: [[ADD_0:%.*]] = add i32 [[L_SRC_0]], [[L_SRC_1]]421; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[ADD_0]], [[L_SRC_2]]422; CHECK-NEXT: ret i32 [[ADD_1]]423;424 %gep.src.0 = getelementptr inbounds i32, ptr %src, i32 0425 %l.src.0 = load i32, ptr %gep.src.0, align 4426 %gep.src.1 = getelementptr inbounds i32, ptr %src, i32 1427 %l.src.1 = load i32, ptr %gep.src.1, align 4428 %gep.src.2 = getelementptr inbounds i32, ptr %src, i32 2429 %l.src.2 = load i32, ptr %gep.src.2, align 4430 431 %add.0 = add i32 %l.src.0, %l.src.1432 %add.1 = add i32 %add.0, %l.src.2433 ret i32 %add.1434}435 436define float @reduce_fadd(ptr %src) {437; NON-POW2-LABEL: @reduce_fadd(438; NON-POW2-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0439; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x float>, ptr [[GEP_SRC_0]], align 4440; NON-POW2-NEXT: [[TMP2:%.*]] = call fast float @llvm.vector.reduce.fadd.v3f32(float 0.000000e+00, <3 x float> [[TMP1]])441; NON-POW2-NEXT: ret float [[TMP2]]442;443; POW2-ONLY-LABEL: @reduce_fadd(444; POW2-ONLY-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0445; POW2-ONLY-NEXT: [[L_SRC_0:%.*]] = load float, ptr [[GEP_SRC_0]], align 4446; POW2-ONLY-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 1447; POW2-ONLY-NEXT: [[L_SRC_1:%.*]] = load float, ptr [[GEP_SRC_1]], align 4448; POW2-ONLY-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 2449; POW2-ONLY-NEXT: [[L_SRC_2:%.*]] = load float, ptr [[GEP_SRC_2]], align 4450; POW2-ONLY-NEXT: [[ADD_0:%.*]] = fadd fast float [[L_SRC_0]], [[L_SRC_1]]451; POW2-ONLY-NEXT: [[ADD_1:%.*]] = fadd fast float [[ADD_0]], [[L_SRC_2]]452; POW2-ONLY-NEXT: ret float [[ADD_1]]453;454 %gep.src.0 = getelementptr inbounds float, ptr %src, i32 0455 %l.src.0 = load float, ptr %gep.src.0, align 4456 %gep.src.1 = getelementptr inbounds float, ptr %src, i32 1457 %l.src.1 = load float, ptr %gep.src.1, align 4458 %gep.src.2 = getelementptr inbounds float, ptr %src, i32 2459 %l.src.2 = load float, ptr %gep.src.2, align 4460 461 %add.0 = fadd fast float %l.src.0, %l.src.1462 %add.1 = fadd fast float %add.0, %l.src.2463 ret float %add.1464}465 466define i32 @reduce_add_after_mul(ptr %src) {467; NON-POW2-LABEL: @reduce_add_after_mul(468; NON-POW2-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 0469; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr [[GEP_SRC_0]], align 4470; NON-POW2-NEXT: [[TMP2:%.*]] = mul nsw <3 x i32> [[TMP1]], splat (i32 10)471; NON-POW2-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v3i32(<3 x i32> [[TMP2]])472; NON-POW2-NEXT: ret i32 [[TMP3]]473;474; POW2-ONLY-LABEL: @reduce_add_after_mul(475; POW2-ONLY-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 0476; POW2-ONLY-NEXT: [[L_SRC_0:%.*]] = load i32, ptr [[GEP_SRC_0]], align 4477; POW2-ONLY-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 1478; POW2-ONLY-NEXT: [[L_SRC_1:%.*]] = load i32, ptr [[GEP_SRC_1]], align 4479; POW2-ONLY-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 2480; POW2-ONLY-NEXT: [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC_2]], align 4481; POW2-ONLY-NEXT: [[MUL_0:%.*]] = mul nsw i32 [[L_SRC_0]], 10482; POW2-ONLY-NEXT: [[MUL_1:%.*]] = mul nsw i32 [[L_SRC_1]], 10483; POW2-ONLY-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_2]], 10484; POW2-ONLY-NEXT: [[ADD_0:%.*]] = add i32 [[MUL_0]], [[MUL_1]]485; POW2-ONLY-NEXT: [[ADD_1:%.*]] = add i32 [[ADD_0]], [[MUL_2]]486; POW2-ONLY-NEXT: ret i32 [[ADD_1]]487;488 %gep.src.0 = getelementptr inbounds i32, ptr %src, i32 0489 %l.src.0 = load i32, ptr %gep.src.0, align 4490 %gep.src.1 = getelementptr inbounds i32, ptr %src, i32 1491 %l.src.1 = load i32, ptr %gep.src.1, align 4492 %gep.src.2 = getelementptr inbounds i32, ptr %src, i32 2493 %l.src.2 = load i32, ptr %gep.src.2, align 4494 495 %mul.0 = mul nsw i32 %l.src.0, 10496 %mul.1 = mul nsw i32 %l.src.1, 10497 %mul.2 = mul nsw i32 %l.src.2, 10498 499 %add.0 = add i32 %mul.0, %mul.1500 %add.1 = add i32 %add.0, %mul.2501 ret i32 %add.1502}503 504define i32 @dot_product_i32(ptr %a, ptr %b) {505; NON-POW2-LABEL: @dot_product_i32(506; NON-POW2-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 0507; NON-POW2-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i32 0508; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr [[GEP_A_0]], align 4509; NON-POW2-NEXT: [[TMP2:%.*]] = load <3 x i32>, ptr [[GEP_B_0]], align 4510; NON-POW2-NEXT: [[TMP3:%.*]] = mul nsw <3 x i32> [[TMP1]], [[TMP2]]511; NON-POW2-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v3i32(<3 x i32> [[TMP3]])512; NON-POW2-NEXT: ret i32 [[TMP4]]513;514; POW2-ONLY-LABEL: @dot_product_i32(515; POW2-ONLY-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 0516; POW2-ONLY-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 2517; POW2-ONLY-NEXT: [[L_A_2:%.*]] = load i32, ptr [[GEP_A_2]], align 4518; POW2-ONLY-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i32 0519; POW2-ONLY-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 2520; POW2-ONLY-NEXT: [[L_B_2:%.*]] = load i32, ptr [[GEP_B_2]], align 4521; POW2-ONLY-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[GEP_A_0]], align 4522; POW2-ONLY-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[GEP_B_0]], align 4523; POW2-ONLY-NEXT: [[TMP3:%.*]] = mul nsw <2 x i32> [[TMP1]], [[TMP2]]524; POW2-ONLY-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_A_2]], [[L_B_2]]525; POW2-ONLY-NEXT: [[ADD_0:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP3]])526; POW2-ONLY-NEXT: [[ADD_1:%.*]] = add i32 [[ADD_0]], [[MUL_2]]527; POW2-ONLY-NEXT: ret i32 [[ADD_1]]528;529 %gep.a.0 = getelementptr inbounds i32, ptr %a, i32 0530 %l.a.0 = load i32, ptr %gep.a.0, align 4531 %gep.a.1 = getelementptr inbounds i32, ptr %a, i32 1532 %l.a.1 = load i32, ptr %gep.a.1, align 4533 %gep.a.2 = getelementptr inbounds i32, ptr %a, i32 2534 %l.a.2 = load i32, ptr %gep.a.2, align 4535 536 %gep.b.0 = getelementptr inbounds i32, ptr %b, i32 0537 %l.b.0 = load i32, ptr %gep.b.0, align 4538 %gep.b.1 = getelementptr inbounds i32, ptr %b, i32 1539 %l.b.1 = load i32, ptr %gep.b.1, align 4540 %gep.b.2 = getelementptr inbounds i32, ptr %b, i32 2541 %l.b.2 = load i32, ptr %gep.b.2, align 4542 543 %mul.0 = mul nsw i32 %l.a.0, %l.b.0544 %mul.1 = mul nsw i32 %l.a.1, %l.b.1545 %mul.2 = mul nsw i32 %l.a.2, %l.b.2546 547 %add.0 = add i32 %mul.0, %mul.1548 %add.1 = add i32 %add.0, %mul.2549 ret i32 %add.1550}551 552; Same as above, except the reduction order has been perturbed. This553; is checking for our ability to reorder.554define i32 @dot_product_i32_reorder(ptr %a, ptr %b) {555; NON-POW2-LABEL: @dot_product_i32_reorder(556; NON-POW2-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 0557; NON-POW2-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i32 0558; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr [[GEP_A_0]], align 4559; NON-POW2-NEXT: [[TMP2:%.*]] = load <3 x i32>, ptr [[GEP_B_0]], align 4560; NON-POW2-NEXT: [[TMP3:%.*]] = mul nsw <3 x i32> [[TMP1]], [[TMP2]]561; NON-POW2-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v3i32(<3 x i32> [[TMP3]])562; NON-POW2-NEXT: ret i32 [[TMP4]]563;564; POW2-ONLY-LABEL: @dot_product_i32_reorder(565; POW2-ONLY-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 0566; POW2-ONLY-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 2567; POW2-ONLY-NEXT: [[L_A_2:%.*]] = load i32, ptr [[GEP_A_2]], align 4568; POW2-ONLY-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i32 0569; POW2-ONLY-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 2570; POW2-ONLY-NEXT: [[L_B_2:%.*]] = load i32, ptr [[GEP_B_2]], align 4571; POW2-ONLY-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[GEP_A_0]], align 4572; POW2-ONLY-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[GEP_B_0]], align 4573; POW2-ONLY-NEXT: [[TMP3:%.*]] = mul nsw <2 x i32> [[TMP1]], [[TMP2]]574; POW2-ONLY-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_A_2]], [[L_B_2]]575; POW2-ONLY-NEXT: [[ADD_0:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP3]])576; POW2-ONLY-NEXT: [[ADD_1:%.*]] = add i32 [[ADD_0]], [[MUL_2]]577; POW2-ONLY-NEXT: ret i32 [[ADD_1]]578;579 %gep.a.0 = getelementptr inbounds i32, ptr %a, i32 0580 %l.a.0 = load i32, ptr %gep.a.0, align 4581 %gep.a.1 = getelementptr inbounds i32, ptr %a, i32 1582 %l.a.1 = load i32, ptr %gep.a.1, align 4583 %gep.a.2 = getelementptr inbounds i32, ptr %a, i32 2584 %l.a.2 = load i32, ptr %gep.a.2, align 4585 586 %gep.b.0 = getelementptr inbounds i32, ptr %b, i32 0587 %l.b.0 = load i32, ptr %gep.b.0, align 4588 %gep.b.1 = getelementptr inbounds i32, ptr %b, i32 1589 %l.b.1 = load i32, ptr %gep.b.1, align 4590 %gep.b.2 = getelementptr inbounds i32, ptr %b, i32 2591 %l.b.2 = load i32, ptr %gep.b.2, align 4592 593 %mul.0 = mul nsw i32 %l.a.0, %l.b.0594 %mul.1 = mul nsw i32 %l.a.1, %l.b.1595 %mul.2 = mul nsw i32 %l.a.2, %l.b.2596 597 %add.0 = add i32 %mul.1, %mul.0598 %add.1 = add i32 %add.0, %mul.2599 ret i32 %add.1600}601 602define float @dot_product_fp32(ptr %a, ptr %b) {603; NON-POW2-LABEL: @dot_product_fp32(604; NON-POW2-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i32 0605; NON-POW2-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i32 0606; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x float>, ptr [[GEP_A_0]], align 4607; NON-POW2-NEXT: [[TMP2:%.*]] = load <3 x float>, ptr [[GEP_B_0]], align 4608; NON-POW2-NEXT: [[TMP3:%.*]] = fmul fast <3 x float> [[TMP1]], [[TMP2]]609; NON-POW2-NEXT: [[TMP4:%.*]] = call fast float @llvm.vector.reduce.fadd.v3f32(float 0.000000e+00, <3 x float> [[TMP3]])610; NON-POW2-NEXT: ret float [[TMP4]]611;612; POW2-ONLY-LABEL: @dot_product_fp32(613; POW2-ONLY-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i32 0614; POW2-ONLY-NEXT: [[L_A_0:%.*]] = load float, ptr [[GEP_A_0]], align 4615; POW2-ONLY-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i32 1616; POW2-ONLY-NEXT: [[L_A_1:%.*]] = load float, ptr [[GEP_A_1]], align 4617; POW2-ONLY-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds float, ptr [[A]], i32 2618; POW2-ONLY-NEXT: [[L_A_2:%.*]] = load float, ptr [[GEP_A_2]], align 4619; POW2-ONLY-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i32 0620; POW2-ONLY-NEXT: [[L_B_0:%.*]] = load float, ptr [[GEP_B_0]], align 4621; POW2-ONLY-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds float, ptr [[B]], i32 1622; POW2-ONLY-NEXT: [[L_B_1:%.*]] = load float, ptr [[GEP_B_1]], align 4623; POW2-ONLY-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i32 2624; POW2-ONLY-NEXT: [[L_B_2:%.*]] = load float, ptr [[GEP_B_2]], align 4625; POW2-ONLY-NEXT: [[MUL_0:%.*]] = fmul fast float [[L_A_0]], [[L_B_0]]626; POW2-ONLY-NEXT: [[MUL_1:%.*]] = fmul fast float [[L_A_1]], [[L_B_1]]627; POW2-ONLY-NEXT: [[MUL_2:%.*]] = fmul fast float [[L_A_2]], [[L_B_2]]628; POW2-ONLY-NEXT: [[ADD_0:%.*]] = fadd fast float [[MUL_0]], [[MUL_1]]629; POW2-ONLY-NEXT: [[ADD_1:%.*]] = fadd fast float [[ADD_0]], [[MUL_2]]630; POW2-ONLY-NEXT: ret float [[ADD_1]]631;632 %gep.a.0 = getelementptr inbounds float, ptr %a, i32 0633 %l.a.0 = load float, ptr %gep.a.0, align 4634 %gep.a.1 = getelementptr inbounds float, ptr %a, i32 1635 %l.a.1 = load float, ptr %gep.a.1, align 4636 %gep.a.2 = getelementptr inbounds float, ptr %a, i32 2637 %l.a.2 = load float, ptr %gep.a.2, align 4638 639 %gep.b.0 = getelementptr inbounds float, ptr %b, i32 0640 %l.b.0 = load float, ptr %gep.b.0, align 4641 %gep.b.1 = getelementptr inbounds float, ptr %b, i32 1642 %l.b.1 = load float, ptr %gep.b.1, align 4643 %gep.b.2 = getelementptr inbounds float, ptr %b, i32 2644 %l.b.2 = load float, ptr %gep.b.2, align 4645 646 %mul.0 = fmul fast float %l.a.0, %l.b.0647 %mul.1 = fmul fast float %l.a.1, %l.b.1648 %mul.2 = fmul fast float %l.a.2, %l.b.2649 650 %add.0 = fadd fast float %mul.0, %mul.1651 %add.1 = fadd fast float %add.0, %mul.2652 ret float %add.1653}654 655; Same as above, except the reduction order has been perturbed. This656; is checking for our ability to reorder.657define float @dot_product_fp32_reorder(ptr %a, ptr %b) {658; NON-POW2-LABEL: @dot_product_fp32_reorder(659; NON-POW2-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i32 0660; NON-POW2-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i32 0661; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x float>, ptr [[GEP_A_0]], align 4662; NON-POW2-NEXT: [[TMP2:%.*]] = load <3 x float>, ptr [[GEP_B_0]], align 4663; NON-POW2-NEXT: [[TMP3:%.*]] = fmul fast <3 x float> [[TMP1]], [[TMP2]]664; NON-POW2-NEXT: [[TMP4:%.*]] = call fast float @llvm.vector.reduce.fadd.v3f32(float 0.000000e+00, <3 x float> [[TMP3]])665; NON-POW2-NEXT: ret float [[TMP4]]666;667; POW2-ONLY-LABEL: @dot_product_fp32_reorder(668; POW2-ONLY-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i32 0669; POW2-ONLY-NEXT: [[L_A_0:%.*]] = load float, ptr [[GEP_A_0]], align 4670; POW2-ONLY-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i32 1671; POW2-ONLY-NEXT: [[L_A_1:%.*]] = load float, ptr [[GEP_A_1]], align 4672; POW2-ONLY-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds float, ptr [[A]], i32 2673; POW2-ONLY-NEXT: [[L_A_2:%.*]] = load float, ptr [[GEP_A_2]], align 4674; POW2-ONLY-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i32 0675; POW2-ONLY-NEXT: [[L_B_0:%.*]] = load float, ptr [[GEP_B_0]], align 4676; POW2-ONLY-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds float, ptr [[B]], i32 1677; POW2-ONLY-NEXT: [[L_B_1:%.*]] = load float, ptr [[GEP_B_1]], align 4678; POW2-ONLY-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i32 2679; POW2-ONLY-NEXT: [[L_B_2:%.*]] = load float, ptr [[GEP_B_2]], align 4680; POW2-ONLY-NEXT: [[MUL_0:%.*]] = fmul fast float [[L_A_0]], [[L_B_0]]681; POW2-ONLY-NEXT: [[MUL_1:%.*]] = fmul fast float [[L_A_1]], [[L_B_1]]682; POW2-ONLY-NEXT: [[MUL_2:%.*]] = fmul fast float [[L_A_2]], [[L_B_2]]683; POW2-ONLY-NEXT: [[ADD_0:%.*]] = fadd fast float [[MUL_1]], [[MUL_0]]684; POW2-ONLY-NEXT: [[ADD_1:%.*]] = fadd fast float [[ADD_0]], [[MUL_2]]685; POW2-ONLY-NEXT: ret float [[ADD_1]]686;687 %gep.a.0 = getelementptr inbounds float, ptr %a, i32 0688 %l.a.0 = load float, ptr %gep.a.0, align 4689 %gep.a.1 = getelementptr inbounds float, ptr %a, i32 1690 %l.a.1 = load float, ptr %gep.a.1, align 4691 %gep.a.2 = getelementptr inbounds float, ptr %a, i32 2692 %l.a.2 = load float, ptr %gep.a.2, align 4693 694 %gep.b.0 = getelementptr inbounds float, ptr %b, i32 0695 %l.b.0 = load float, ptr %gep.b.0, align 4696 %gep.b.1 = getelementptr inbounds float, ptr %b, i32 1697 %l.b.1 = load float, ptr %gep.b.1, align 4698 %gep.b.2 = getelementptr inbounds float, ptr %b, i32 2699 %l.b.2 = load float, ptr %gep.b.2, align 4700 701 %mul.0 = fmul fast float %l.a.0, %l.b.0702 %mul.1 = fmul fast float %l.a.1, %l.b.1703 %mul.2 = fmul fast float %l.a.2, %l.b.2704 705 %add.0 = fadd fast float %mul.1, %mul.0706 %add.1 = fadd fast float %add.0, %mul.2707 ret float %add.1708}709 710 711define double @dot_product_fp64(ptr %a, ptr %b) {712; CHECK-LABEL: @dot_product_fp64(713; CHECK-NEXT: [[GEP_A_0:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i32 0714; CHECK-NEXT: [[L_A_0:%.*]] = load double, ptr [[GEP_A_0]], align 4715; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds double, ptr [[A]], i32 1716; CHECK-NEXT: [[L_A_1:%.*]] = load double, ptr [[GEP_A_1]], align 4717; CHECK-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds double, ptr [[A]], i32 2718; CHECK-NEXT: [[L_A_2:%.*]] = load double, ptr [[GEP_A_2]], align 4719; CHECK-NEXT: [[GEP_B_0:%.*]] = getelementptr inbounds double, ptr [[B:%.*]], i32 0720; CHECK-NEXT: [[L_B_0:%.*]] = load double, ptr [[GEP_B_0]], align 4721; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds double, ptr [[B]], i32 1722; CHECK-NEXT: [[L_B_1:%.*]] = load double, ptr [[GEP_B_1]], align 4723; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds double, ptr [[B]], i32 2724; CHECK-NEXT: [[L_B_2:%.*]] = load double, ptr [[GEP_B_2]], align 4725; CHECK-NEXT: [[MUL_0:%.*]] = fmul fast double [[L_A_0]], [[L_B_0]]726; CHECK-NEXT: [[MUL_1:%.*]] = fmul fast double [[L_A_1]], [[L_B_1]]727; CHECK-NEXT: [[MUL_2:%.*]] = fmul fast double [[L_A_2]], [[L_B_2]]728; CHECK-NEXT: [[ADD_0:%.*]] = fadd fast double [[MUL_0]], [[MUL_1]]729; CHECK-NEXT: [[ADD_1:%.*]] = fadd fast double [[ADD_0]], [[MUL_2]]730; CHECK-NEXT: ret double [[ADD_1]]731;732 %gep.a.0 = getelementptr inbounds double, ptr %a, i32 0733 %l.a.0 = load double, ptr %gep.a.0, align 4734 %gep.a.1 = getelementptr inbounds double, ptr %a, i32 1735 %l.a.1 = load double, ptr %gep.a.1, align 4736 %gep.a.2 = getelementptr inbounds double, ptr %a, i32 2737 %l.a.2 = load double, ptr %gep.a.2, align 4738 739 %gep.b.0 = getelementptr inbounds double, ptr %b, i32 0740 %l.b.0 = load double, ptr %gep.b.0, align 4741 %gep.b.1 = getelementptr inbounds double, ptr %b, i32 1742 %l.b.1 = load double, ptr %gep.b.1, align 4743 %gep.b.2 = getelementptr inbounds double, ptr %b, i32 2744 %l.b.2 = load double, ptr %gep.b.2, align 4745 746 %mul.0 = fmul fast double %l.a.0, %l.b.0747 %mul.1 = fmul fast double %l.a.1, %l.b.1748 %mul.2 = fmul fast double %l.a.2, %l.b.2749 750 %add.0 = fadd fast double %mul.0, %mul.1751 %add.1 = fadd fast double %add.0, %mul.2752 ret double %add.1753}754 755;; Covers a case where SLP would previous crash due to a756;; missing bailout in TryToFindDuplicates for the case757;; where a VL=3 list was vectorized directly (without758;; a root instruction such as a store or reduce).759define double @no_root_reshuffle(ptr %ptr) {760; CHECK-LABEL: @no_root_reshuffle(761; CHECK-NEXT: entry:762; CHECK-NEXT: [[TMP0:%.*]] = load double, ptr [[PTR:%.*]], align 8763; CHECK-NEXT: [[MUL:%.*]] = fmul fast double [[TMP0]], [[TMP0]]764; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 8765; CHECK-NEXT: [[TMP1:%.*]] = load double, ptr [[ARRAYIDX2]], align 8766; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 16767; CHECK-NEXT: [[TMP2:%.*]] = load double, ptr [[ARRAYIDX3]], align 8768; CHECK-NEXT: [[TMP3:%.*]] = fmul fast double [[TMP2]], [[TMP2]]769; CHECK-NEXT: [[MUL6:%.*]] = fmul fast double [[TMP3]], [[TMP1]]770; CHECK-NEXT: [[ADD:%.*]] = fadd fast double [[MUL6]], [[MUL]]771; CHECK-NEXT: ret double [[ADD]]772;773entry:774 %0 = load double, ptr %ptr, align 8775 %mul = fmul fast double %0, %0776 %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 8777 %1 = load double, ptr %arrayidx2, align 8778 %arrayidx3 = getelementptr inbounds i8, ptr %ptr, i64 16779 %2 = load double, ptr %arrayidx3, align 8780 %3 = fmul fast double %2, %2781 %mul6 = fmul fast double %3, %1782 %add = fadd fast double %mul6, %mul783 ret double %add784}785 786define float @reduce_fadd_after_fmul_of_buildvec(float %a, float %b, float %c) {787; CHECK-LABEL: @reduce_fadd_after_fmul_of_buildvec(788; CHECK-NEXT: [[MUL_0:%.*]] = fmul fast float [[A:%.*]], 1.000000e+01789; CHECK-NEXT: [[MUL_1:%.*]] = fmul fast float [[B:%.*]], 1.000000e+01790; CHECK-NEXT: [[MUL_2:%.*]] = fmul fast float [[C:%.*]], 1.000000e+01791; CHECK-NEXT: [[ADD_0:%.*]] = fadd fast float [[MUL_0]], [[MUL_1]]792; CHECK-NEXT: [[ADD_1:%.*]] = fadd fast float [[ADD_0]], [[MUL_2]]793; CHECK-NEXT: ret float [[ADD_1]]794;795 %mul.0 = fmul fast float %a, 10.0796 %mul.1 = fmul fast float %b, 10.0797 %mul.2 = fmul fast float %c, 10.0798 799 %add.0 = fadd fast float %mul.0, %mul.1800 %add.1 = fadd fast float %add.0, %mul.2801 ret float %add.1802}803 804 805declare float @llvm.fmuladd.f32(float, float, float)806 807declare double @llvm.fmuladd.f64(double, double, double)808