303 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -passes=slp-vectorizer -mtriple=arm64-apple-ios -S %s | FileCheck %s3 4; Test case where not vectorizing is more profitable because multiple5; fmul/{fadd,fsub} pairs can be lowered to fma instructions.6define void @slp_not_profitable_with_fast_fmf(ptr %A, ptr %B) {7; CHECK-LABEL: @slp_not_profitable_with_fast_fmf(8; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 19; CHECK-NEXT: [[A_0:%.*]] = load float, ptr [[A:%.*]], align 410; CHECK-NEXT: [[B_0:%.*]] = load float, ptr [[B]], align 411; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 212; CHECK-NEXT: [[B_2:%.*]] = load float, ptr [[GEP_B_2]], align 413; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[GEP_B_1]], align 414; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[B_0]], i32 015; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer16; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <2 x float> [[TMP3]], [[TMP1]]17; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <2 x i32> <i32 1, i32 0>18; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[A_0]], i32 019; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <2 x i32> zeroinitializer20; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <2 x float> [[TMP1]], [[TMP7]]21; CHECK-NEXT: [[TMP9:%.*]] = fsub fast <2 x float> [[TMP8]], [[TMP5]]22; CHECK-NEXT: [[TMP10:%.*]] = fadd fast <2 x float> [[TMP8]], [[TMP5]]23; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>24; CHECK-NEXT: store <2 x float> [[TMP11]], ptr [[A]], align 425; CHECK-NEXT: store float [[B_2]], ptr [[B]], align 426; CHECK-NEXT: ret void27;28 %gep.B.1 = getelementptr inbounds float, ptr %B, i64 129 %A.0 = load float, ptr %A, align 430 %B.1 = load float, ptr %gep.B.1, align 431 %mul.0 = fmul fast float %B.1, %A.032 %B.0 = load float, ptr %B, align 433 %gep.B.2 = getelementptr inbounds float, ptr %B, i64 234 %B.2 = load float, ptr %gep.B.2, align 435 %mul.1 = fmul fast float %B.2, %B.036 %sub = fsub fast float %mul.0, %mul.137 %mul.2 = fmul fast float %B.0, %B.138 %mul.3 = fmul fast float %B.2, %A.039 %add = fadd fast float %mul.3, %mul.240 store float %sub, ptr %A, align 441 %gep.A.1 = getelementptr inbounds float, ptr %A, i64 142 store float %add, ptr %gep.A.1, align 443 store float %B.2, ptr %B, align 444 ret void45}46 47define void @slp_not_profitable_with_reassoc_fmf(ptr %A, ptr %B) {48; CHECK-LABEL: @slp_not_profitable_with_reassoc_fmf(49; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 150; CHECK-NEXT: [[A_0:%.*]] = load float, ptr [[A:%.*]], align 451; CHECK-NEXT: [[B_0:%.*]] = load float, ptr [[B]], align 452; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 253; CHECK-NEXT: [[B_2:%.*]] = load float, ptr [[GEP_B_2]], align 454; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[GEP_B_1]], align 455; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[B_0]], i32 056; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer57; CHECK-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP3]], [[TMP1]]58; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <2 x i32> <i32 1, i32 0>59; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[A_0]], i32 060; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <2 x i32> zeroinitializer61; CHECK-NEXT: [[TMP8:%.*]] = fmul reassoc <2 x float> [[TMP1]], [[TMP7]]62; CHECK-NEXT: [[TMP9:%.*]] = fsub reassoc <2 x float> [[TMP8]], [[TMP5]]63; CHECK-NEXT: [[TMP10:%.*]] = fadd reassoc <2 x float> [[TMP8]], [[TMP5]]64; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>65; CHECK-NEXT: store <2 x float> [[TMP11]], ptr [[A]], align 466; CHECK-NEXT: store float [[B_2]], ptr [[B]], align 467; CHECK-NEXT: ret void68;69 %gep.B.1 = getelementptr inbounds float, ptr %B, i64 170 %A.0 = load float, ptr %A, align 471 %B.1 = load float, ptr %gep.B.1, align 472 %mul.0 = fmul reassoc float %B.1, %A.073 %B.0 = load float, ptr %B, align 474 %gep.B.2 = getelementptr inbounds float, ptr %B, i64 275 %B.2 = load float, ptr %gep.B.2, align 476 %mul.1 = fmul float %B.2, %B.077 %sub = fsub reassoc float %mul.0, %mul.178 %mul.2 = fmul float %B.0, %B.179 %mul.3 = fmul reassoc float %B.2, %A.080 %add = fadd reassoc float %mul.3, %mul.281 store float %sub, ptr %A, align 482 %gep.A.1 = getelementptr inbounds float, ptr %A, i64 183 store float %add, ptr %gep.A.1, align 484 store float %B.2, ptr %B, align 485 ret void86}87 88; FMA cannot be used due to missing fast-math flags, so SLP should kick in.89define void @slp_profitable_missing_fmf_on_fadd_fsub(ptr %A, ptr %B) {90; CHECK-LABEL: @slp_profitable_missing_fmf_on_fadd_fsub(91; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 192; CHECK-NEXT: [[A_0:%.*]] = load float, ptr [[A:%.*]], align 493; CHECK-NEXT: [[B_0:%.*]] = load float, ptr [[B]], align 494; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 295; CHECK-NEXT: [[B_2:%.*]] = load float, ptr [[GEP_B_2]], align 496; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[GEP_B_1]], align 497; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[B_0]], i32 098; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer99; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <2 x float> [[TMP3]], [[TMP1]]100; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <2 x i32> <i32 1, i32 0>101; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[A_0]], i32 0102; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <2 x i32> zeroinitializer103; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <2 x float> [[TMP1]], [[TMP7]]104; CHECK-NEXT: [[TMP9:%.*]] = fsub <2 x float> [[TMP8]], [[TMP5]]105; CHECK-NEXT: [[TMP10:%.*]] = fadd <2 x float> [[TMP8]], [[TMP5]]106; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>107; CHECK-NEXT: store <2 x float> [[TMP11]], ptr [[A]], align 4108; CHECK-NEXT: store float [[B_2]], ptr [[B]], align 4109; CHECK-NEXT: ret void110;111 %gep.B.1 = getelementptr inbounds float, ptr %B, i64 1112 %A.0 = load float, ptr %A, align 4113 %B.1 = load float, ptr %gep.B.1, align 4114 %mul.0 = fmul fast float %B.1, %A.0115 %B.0 = load float, ptr %B, align 4116 %gep.B.2 = getelementptr inbounds float, ptr %B, i64 2117 %B.2 = load float, ptr %gep.B.2, align 4118 %mul.1 = fmul fast float %B.2, %B.0119 %sub = fsub float %mul.0, %mul.1120 %mul.2 = fmul fast float %B.0, %B.1121 %mul.3 = fmul fast float %B.2, %A.0122 %add = fadd float %mul.3, %mul.2123 store float %sub, ptr %A, align 4124 %gep.A.1 = getelementptr inbounds float, ptr %A, i64 1125 store float %add, ptr %gep.A.1, align 4126 store float %B.2, ptr %B, align 4127 ret void128}129 130; FMA cannot be used due to missing fast-math flags, so SLP should kick in.131define void @slp_profitable_missing_fmf_on_fmul_fadd_fsub(ptr %A, ptr %B) {132; CHECK-LABEL: @slp_profitable_missing_fmf_on_fmul_fadd_fsub(133; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 1134; CHECK-NEXT: [[A_0:%.*]] = load float, ptr [[A:%.*]], align 4135; CHECK-NEXT: [[B_0:%.*]] = load float, ptr [[B]], align 4136; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2137; CHECK-NEXT: [[B_2:%.*]] = load float, ptr [[GEP_B_2]], align 4138; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[GEP_B_1]], align 4139; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[B_0]], i32 0140; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer141; CHECK-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP3]], [[TMP1]]142; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <2 x i32> <i32 1, i32 0>143; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[A_0]], i32 0144; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <2 x i32> zeroinitializer145; CHECK-NEXT: [[TMP8:%.*]] = fmul <2 x float> [[TMP1]], [[TMP7]]146; CHECK-NEXT: [[TMP9:%.*]] = fsub <2 x float> [[TMP8]], [[TMP5]]147; CHECK-NEXT: [[TMP10:%.*]] = fadd <2 x float> [[TMP8]], [[TMP5]]148; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>149; CHECK-NEXT: store <2 x float> [[TMP11]], ptr [[A]], align 4150; CHECK-NEXT: store float [[B_2]], ptr [[B]], align 4151; CHECK-NEXT: ret void152;153 %gep.B.1 = getelementptr inbounds float, ptr %B, i64 1154 %A.0 = load float, ptr %A, align 4155 %B.1 = load float, ptr %gep.B.1, align 4156 %mul.0 = fmul float %B.1, %A.0157 %B.0 = load float, ptr %B, align 4158 %gep.B.2 = getelementptr inbounds float, ptr %B, i64 2159 %B.2 = load float, ptr %gep.B.2, align 4160 %mul.1 = fmul float %B.2, %B.0161 %sub = fsub float %mul.0, %mul.1162 %mul.2 = fmul float %B.0, %B.1163 %mul.3 = fmul float %B.2, %A.0164 %add = fadd float %mul.3, %mul.2165 store float %sub, ptr %A, align 4166 %gep.A.1 = getelementptr inbounds float, ptr %A, i64 1167 store float %add, ptr %gep.A.1, align 4168 store float %B.2, ptr %B, align 4169 ret void170}171 172; FMA cannot be used due to missing fast-math flags, so SLP should kick in.173define void @slp_profitable_missing_fmf_nnans_only(ptr %A, ptr %B) {174; CHECK-LABEL: @slp_profitable_missing_fmf_nnans_only(175; CHECK-NEXT: [[GEP_B_1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 1176; CHECK-NEXT: [[A_0:%.*]] = load float, ptr [[A:%.*]], align 4177; CHECK-NEXT: [[B_0:%.*]] = load float, ptr [[B]], align 4178; CHECK-NEXT: [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2179; CHECK-NEXT: [[B_2:%.*]] = load float, ptr [[GEP_B_2]], align 4180; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[GEP_B_1]], align 4181; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[B_0]], i32 0182; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer183; CHECK-NEXT: [[TMP4:%.*]] = fmul nnan <2 x float> [[TMP3]], [[TMP1]]184; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <2 x i32> <i32 1, i32 0>185; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[A_0]], i32 0186; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <2 x i32> zeroinitializer187; CHECK-NEXT: [[TMP8:%.*]] = fmul nnan <2 x float> [[TMP1]], [[TMP7]]188; CHECK-NEXT: [[TMP9:%.*]] = fsub nnan <2 x float> [[TMP8]], [[TMP5]]189; CHECK-NEXT: [[TMP10:%.*]] = fadd nnan <2 x float> [[TMP8]], [[TMP5]]190; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>191; CHECK-NEXT: store <2 x float> [[TMP11]], ptr [[A]], align 4192; CHECK-NEXT: store float [[B_2]], ptr [[B]], align 4193; CHECK-NEXT: ret void194;195 %gep.B.1 = getelementptr inbounds float, ptr %B, i64 1196 %A.0 = load float, ptr %A, align 4197 %B.1 = load float, ptr %gep.B.1, align 4198 %mul.0 = fmul nnan float %B.1, %A.0199 %B.0 = load float, ptr %B, align 4200 %gep.B.2 = getelementptr inbounds float, ptr %B, i64 2201 %B.2 = load float, ptr %gep.B.2, align 4202 %mul.1 = fmul nnan float %B.2, %B.0203 %sub = fsub nnan float %mul.0, %mul.1204 %mul.2 = fmul nnan float %B.0, %B.1205 %mul.3 = fmul nnan float %B.2, %A.0206 %add = fadd nnan float %mul.3, %mul.2207 store float %sub, ptr %A, align 4208 %gep.A.1 = getelementptr inbounds float, ptr %A, i64 1209 store float %add, ptr %gep.A.1, align 4210 store float %B.2, ptr %B, align 4211 ret void212}213 214; Test case where not vectorizing is more profitable because multiple215; fmul/{fadd,fsub} pairs can be lowered to fma instructions.216define float @slp_not_profitable_in_loop(float %x, ptr %A) {217; CHECK-LABEL: @slp_not_profitable_in_loop(218; CHECK-NEXT: entry:219; CHECK-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 2220; CHECK-NEXT: [[L_1:%.*]] = load float, ptr [[GEP_A_2]], align 4221; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A]], align 4222; CHECK-NEXT: [[L_3:%.*]] = load float, ptr [[A]], align 4223; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> <float poison, float 3.000000e+00>, float [[X:%.*]], i32 0224; CHECK-NEXT: br label [[LOOP:%.*]]225; CHECK: loop:226; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]227; CHECK-NEXT: [[RED:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[RED_NEXT:%.*]], [[LOOP]] ]228; CHECK-NEXT: [[TMP2:%.*]] = fmul fast <2 x float> [[TMP1]], [[TMP0]]229; CHECK-NEXT: [[MUL12:%.*]] = fmul fast float 3.000000e+00, [[L_1]]230; CHECK-NEXT: [[MUL16:%.*]] = fmul fast float 3.000000e+00, [[L_3]]231; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 1232; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[MUL12]], [[TMP3]]233; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 0234; CHECK-NEXT: [[ADD13:%.*]] = fadd fast float [[ADD]], [[TMP4]]235; CHECK-NEXT: [[RED_NEXT]] = fadd fast float [[ADD13]], [[MUL16]]236; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1237; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV]], 10238; CHECK-NEXT: br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]]239; CHECK: exit:240; CHECK-NEXT: ret float [[RED_NEXT]]241;242entry:243 %gep.A.1 = getelementptr inbounds float, ptr %A, i64 1244 %l.0 = load float, ptr %gep.A.1, align 4245 %gep.A.2 = getelementptr inbounds float, ptr %A, i64 2246 %l.1 = load float, ptr %gep.A.2, align 4247 %l.2 = load float, ptr %A, align 4248 %l.3 = load float, ptr %A, align 4249 br label %loop250 251loop:252 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]253 %red = phi float [ 0.000000e+00, %entry ], [ %red.next, %loop ]254 %mul11 = fmul fast float 3.000000e+00, %l.0255 %mul12 = fmul fast float 3.000000e+00, %l.1256 %mul14 = fmul fast float %x, %l.2257 %mul16 = fmul fast float 3.000000e+00, %l.3258 %add = fadd fast float %mul12, %mul11259 %add13 = fadd fast float %add, %mul14260 %red.next = fadd fast float %add13, %mul16261 %iv.next = add nuw nsw i64 %iv, 1262 %cmp = icmp eq i64 %iv, 10263 br i1 %cmp, label %exit, label %loop264 265exit:266 ret float %red.next267}268 269define void @slp_profitable(ptr %A, ptr %B, float %0) {270; CHECK-LABEL: @slp_profitable(271; CHECK-NEXT: entry:272; CHECK-NEXT: [[SUB_I1096:%.*]] = fsub fast float 1.000000e+00, [[TMP0:%.*]]273; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[A:%.*]], align 4274; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0275; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer276; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <2 x float> [[TMP1]], [[TMP3]]277; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <2 x i32> <i32 1, i32 0>278; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[SUB_I1096]], i32 0279; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <2 x i32> zeroinitializer280; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <2 x float> [[TMP1]], [[TMP7]]281; CHECK-NEXT: [[TMP9:%.*]] = fadd fast <2 x float> [[TMP5]], [[TMP8]]282; CHECK-NEXT: [[TMP10:%.*]] = fsub fast <2 x float> [[TMP5]], [[TMP8]]283; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>284; CHECK-NEXT: store <2 x float> [[TMP11]], ptr [[B:%.*]], align 4285; CHECK-NEXT: ret void286;287entry:288 %gep.A.1 = getelementptr inbounds float, ptr %A, i64 1289 %sub.i1096 = fsub fast float 1.000000e+00, %0290 %1 = load float, ptr %A, align 4291 %mul.i1100 = fmul fast float %1, %sub.i1096292 %2 = load float, ptr %gep.A.1, align 4293 %mul7.i1101 = fmul fast float %2, %0294 %add.i1102 = fadd fast float %mul7.i1101, %mul.i1100295 %mul14.i = fmul fast float %1, %0296 %3 = fmul fast float %2, %sub.i1096297 %add15.i = fsub fast float %mul14.i, %3298 store float %add.i1102, ptr %B, align 4299 %gep.B.1 = getelementptr inbounds float, ptr %B, i64 1300 store float %add15.i, ptr %gep.B.1, align 4301 ret void302}303