448 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE23; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE44; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX5; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX26 7;8; dot4(ptr x, ptr y) - ((xptr y[0])+(xptr y[1])+(xptr y[2])+(xptr y[3]))9;10 11define double @dot4f64(ptr dereferenceable(32) %ptrx, ptr dereferenceable(32) %ptry) {12; CHECK-LABEL: @dot4f64(13; CHECK-NEXT: [[PTRX2:%.*]] = getelementptr inbounds double, ptr [[PTRX:%.*]], i64 214; CHECK-NEXT: [[PTRY2:%.*]] = getelementptr inbounds double, ptr [[PTRY:%.*]], i64 215; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[PTRX]], align 416; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PTRY]], align 417; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP1]], [[TMP2]]18; CHECK-NEXT: [[TMP4:%.*]] = load <2 x double>, ptr [[PTRX2]], align 419; CHECK-NEXT: [[TMP5:%.*]] = load <2 x double>, ptr [[PTRY2]], align 420; CHECK-NEXT: [[TMP6:%.*]] = fmul <2 x double> [[TMP4]], [[TMP5]]21; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP3]], i32 022; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[TMP3]], i32 123; CHECK-NEXT: [[DOT01:%.*]] = fadd double [[TMP7]], [[TMP8]]24; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP6]], i32 025; CHECK-NEXT: [[DOT012:%.*]] = fadd double [[DOT01]], [[TMP9]]26; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP6]], i32 127; CHECK-NEXT: [[DOT0123:%.*]] = fadd double [[DOT012]], [[TMP10]]28; CHECK-NEXT: ret double [[DOT0123]]29;30 %ptrx1 = getelementptr inbounds double, ptr %ptrx, i64 131 %ptry1 = getelementptr inbounds double, ptr %ptry, i64 132 %ptrx2 = getelementptr inbounds double, ptr %ptrx, i64 233 %ptry2 = getelementptr inbounds double, ptr %ptry, i64 234 %ptrx3 = getelementptr inbounds double, ptr %ptrx, i64 335 %ptry3 = getelementptr inbounds double, ptr %ptry, i64 336 %x0 = load double, ptr %ptrx, align 437 %y0 = load double, ptr %ptry, align 438 %x1 = load double, ptr %ptrx1, align 439 %y1 = load double, ptr %ptry1, align 440 %x2 = load double, ptr %ptrx2, align 441 %y2 = load double, ptr %ptry2, align 442 %x3 = load double, ptr %ptrx3, align 443 %y3 = load double, ptr %ptry3, align 444 %mul0 = fmul double %x0, %y045 %mul1 = fmul double %x1, %y146 %mul2 = fmul double %x2, %y247 %mul3 = fmul double %x3, %y348 %dot01 = fadd double %mul0, %mul149 %dot012 = fadd double %dot01, %mul250 %dot0123 = fadd double %dot012, %mul351 ret double %dot012352}53 54define float @dot4f32(ptr dereferenceable(16) %ptrx, ptr dereferenceable(16) %ptry) {55; CHECK-LABEL: @dot4f32(56; CHECK-NEXT: [[PTRX2:%.*]] = getelementptr inbounds float, ptr [[PTRX:%.*]], i64 257; CHECK-NEXT: [[PTRY2:%.*]] = getelementptr inbounds float, ptr [[PTRY:%.*]], i64 258; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[PTRX]], align 459; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[PTRY]], align 460; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP1]], [[TMP2]]61; CHECK-NEXT: [[TMP4:%.*]] = load <2 x float>, ptr [[PTRX2]], align 462; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[PTRY2]], align 463; CHECK-NEXT: [[TMP6:%.*]] = fmul <2 x float> [[TMP4]], [[TMP5]]64; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x float> [[TMP3]], i32 065; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x float> [[TMP3]], i32 166; CHECK-NEXT: [[DOT01:%.*]] = fadd float [[TMP7]], [[TMP8]]67; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x float> [[TMP6]], i32 068; CHECK-NEXT: [[DOT012:%.*]] = fadd float [[DOT01]], [[TMP9]]69; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x float> [[TMP6]], i32 170; CHECK-NEXT: [[DOT0123:%.*]] = fadd float [[DOT012]], [[TMP10]]71; CHECK-NEXT: ret float [[DOT0123]]72;73 %ptrx1 = getelementptr inbounds float, ptr %ptrx, i64 174 %ptry1 = getelementptr inbounds float, ptr %ptry, i64 175 %ptrx2 = getelementptr inbounds float, ptr %ptrx, i64 276 %ptry2 = getelementptr inbounds float, ptr %ptry, i64 277 %ptrx3 = getelementptr inbounds float, ptr %ptrx, i64 378 %ptry3 = getelementptr inbounds float, ptr %ptry, i64 379 %x0 = load float, ptr %ptrx, align 480 %y0 = load float, ptr %ptry, align 481 %x1 = load float, ptr %ptrx1, align 482 %y1 = load float, ptr %ptry1, align 483 %x2 = load float, ptr %ptrx2, align 484 %y2 = load float, ptr %ptry2, align 485 %x3 = load float, ptr %ptrx3, align 486 %y3 = load float, ptr %ptry3, align 487 %mul0 = fmul float %x0, %y088 %mul1 = fmul float %x1, %y189 %mul2 = fmul float %x2, %y290 %mul3 = fmul float %x3, %y391 %dot01 = fadd float %mul0, %mul192 %dot012 = fadd float %dot01, %mul293 %dot0123 = fadd float %dot012, %mul394 ret float %dot012395}96 97define double @dot4f64_fast(ptr dereferenceable(32) %ptrx, ptr dereferenceable(32) %ptry) {98; CHECK-LABEL: @dot4f64_fast(99; CHECK-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[PTRX:%.*]], align 4100; CHECK-NEXT: [[TMP2:%.*]] = load <4 x double>, ptr [[PTRY:%.*]], align 4101; CHECK-NEXT: [[TMP3:%.*]] = fmul <4 x double> [[TMP1]], [[TMP2]]102; CHECK-NEXT: [[TMP4:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP3]])103; CHECK-NEXT: ret double [[TMP4]]104;105 %ptrx1 = getelementptr inbounds double, ptr %ptrx, i64 1106 %ptry1 = getelementptr inbounds double, ptr %ptry, i64 1107 %ptrx2 = getelementptr inbounds double, ptr %ptrx, i64 2108 %ptry2 = getelementptr inbounds double, ptr %ptry, i64 2109 %ptrx3 = getelementptr inbounds double, ptr %ptrx, i64 3110 %ptry3 = getelementptr inbounds double, ptr %ptry, i64 3111 %x0 = load double, ptr %ptrx, align 4112 %y0 = load double, ptr %ptry, align 4113 %x1 = load double, ptr %ptrx1, align 4114 %y1 = load double, ptr %ptry1, align 4115 %x2 = load double, ptr %ptrx2, align 4116 %y2 = load double, ptr %ptry2, align 4117 %x3 = load double, ptr %ptrx3, align 4118 %y3 = load double, ptr %ptry3, align 4119 %mul0 = fmul double %x0, %y0120 %mul1 = fmul double %x1, %y1121 %mul2 = fmul double %x2, %y2122 %mul3 = fmul double %x3, %y3123 %dot01 = fadd fast double %mul0, %mul1124 %dot012 = fadd fast double %dot01, %mul2125 %dot0123 = fadd fast double %dot012, %mul3126 ret double %dot0123127}128 129define float @dot4f32_fast(ptr dereferenceable(16) %ptrx, ptr dereferenceable(16) %ptry) {130; CHECK-LABEL: @dot4f32_fast(131; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[PTRX:%.*]], align 4132; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[PTRY:%.*]], align 4133; CHECK-NEXT: [[TMP3:%.*]] = fmul <4 x float> [[TMP1]], [[TMP2]]134; CHECK-NEXT: [[TMP4:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP3]])135; CHECK-NEXT: ret float [[TMP4]]136;137 %ptrx1 = getelementptr inbounds float, ptr %ptrx, i64 1138 %ptry1 = getelementptr inbounds float, ptr %ptry, i64 1139 %ptrx2 = getelementptr inbounds float, ptr %ptrx, i64 2140 %ptry2 = getelementptr inbounds float, ptr %ptry, i64 2141 %ptrx3 = getelementptr inbounds float, ptr %ptrx, i64 3142 %ptry3 = getelementptr inbounds float, ptr %ptry, i64 3143 %x0 = load float, ptr %ptrx, align 4144 %y0 = load float, ptr %ptry, align 4145 %x1 = load float, ptr %ptrx1, align 4146 %y1 = load float, ptr %ptry1, align 4147 %x2 = load float, ptr %ptrx2, align 4148 %y2 = load float, ptr %ptry2, align 4149 %x3 = load float, ptr %ptrx3, align 4150 %y3 = load float, ptr %ptry3, align 4151 %mul0 = fmul float %x0, %y0152 %mul1 = fmul float %x1, %y1153 %mul2 = fmul float %x2, %y2154 %mul3 = fmul float %x3, %y3155 %dot01 = fadd fast float %mul0, %mul1156 %dot012 = fadd fast float %dot01, %mul2157 %dot0123 = fadd fast float %dot012, %mul3158 ret float %dot0123159}160 161;162; dot3(ptr x, ptr y) - ((xptr y[0])+(xptr y[1])+(xptr y[2]))163;164 165define double @dot3f64(ptr dereferenceable(32) %ptrx, ptr dereferenceable(32) %ptry) {166; CHECK-LABEL: @dot3f64(167; CHECK-NEXT: [[PTRX1:%.*]] = getelementptr inbounds double, ptr [[PTRX:%.*]], i64 1168; CHECK-NEXT: [[PTRY1:%.*]] = getelementptr inbounds double, ptr [[PTRY:%.*]], i64 1169; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[PTRX]], align 4170; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[PTRY]], align 4171; CHECK-NEXT: [[MUL0:%.*]] = fmul double [[X0]], [[Y0]]172; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[PTRX1]], align 4173; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PTRY1]], align 4174; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP1]], [[TMP2]]175; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0176; CHECK-NEXT: [[DOT01:%.*]] = fadd double [[MUL0]], [[TMP4]]177; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i32 1178; CHECK-NEXT: [[DOT012:%.*]] = fadd double [[DOT01]], [[TMP5]]179; CHECK-NEXT: ret double [[DOT012]]180;181 %ptrx1 = getelementptr inbounds double, ptr %ptrx, i64 1182 %ptry1 = getelementptr inbounds double, ptr %ptry, i64 1183 %ptrx2 = getelementptr inbounds double, ptr %ptrx, i64 2184 %ptry2 = getelementptr inbounds double, ptr %ptry, i64 2185 %x0 = load double, ptr %ptrx, align 4186 %y0 = load double, ptr %ptry, align 4187 %x1 = load double, ptr %ptrx1, align 4188 %y1 = load double, ptr %ptry1, align 4189 %x2 = load double, ptr %ptrx2, align 4190 %y2 = load double, ptr %ptry2, align 4191 %mul0 = fmul double %x0, %y0192 %mul1 = fmul double %x1, %y1193 %mul2 = fmul double %x2, %y2194 %dot01 = fadd double %mul0, %mul1195 %dot012 = fadd double %dot01, %mul2196 ret double %dot012197}198 199define float @dot3f32(ptr dereferenceable(16) %ptrx, ptr dereferenceable(16) %ptry) {200; CHECK-LABEL: @dot3f32(201; CHECK-NEXT: [[PTRX1:%.*]] = getelementptr inbounds float, ptr [[PTRX:%.*]], i64 1202; CHECK-NEXT: [[PTRY1:%.*]] = getelementptr inbounds float, ptr [[PTRY:%.*]], i64 1203; CHECK-NEXT: [[X0:%.*]] = load float, ptr [[PTRX]], align 4204; CHECK-NEXT: [[Y0:%.*]] = load float, ptr [[PTRY]], align 4205; CHECK-NEXT: [[MUL0:%.*]] = fmul float [[X0]], [[Y0]]206; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[PTRX1]], align 4207; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[PTRY1]], align 4208; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP1]], [[TMP2]]209; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i32 0210; CHECK-NEXT: [[DOT01:%.*]] = fadd float [[MUL0]], [[TMP4]]211; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP3]], i32 1212; CHECK-NEXT: [[DOT012:%.*]] = fadd float [[DOT01]], [[TMP5]]213; CHECK-NEXT: ret float [[DOT012]]214;215 %ptrx1 = getelementptr inbounds float, ptr %ptrx, i64 1216 %ptry1 = getelementptr inbounds float, ptr %ptry, i64 1217 %ptrx2 = getelementptr inbounds float, ptr %ptrx, i64 2218 %ptry2 = getelementptr inbounds float, ptr %ptry, i64 2219 %x0 = load float, ptr %ptrx, align 4220 %y0 = load float, ptr %ptry, align 4221 %x1 = load float, ptr %ptrx1, align 4222 %y1 = load float, ptr %ptry1, align 4223 %x2 = load float, ptr %ptrx2, align 4224 %y2 = load float, ptr %ptry2, align 4225 %mul0 = fmul float %x0, %y0226 %mul1 = fmul float %x1, %y1227 %mul2 = fmul float %x2, %y2228 %dot01 = fadd float %mul0, %mul1229 %dot012 = fadd float %dot01, %mul2230 ret float %dot012231}232 233define double @dot3f64_fast(ptr dereferenceable(32) %ptrx, ptr dereferenceable(32) %ptry) {234; CHECK-LABEL: @dot3f64_fast(235; CHECK-NEXT: [[PTRX1:%.*]] = getelementptr inbounds double, ptr [[PTRX:%.*]], i64 1236; CHECK-NEXT: [[PTRY1:%.*]] = getelementptr inbounds double, ptr [[PTRY:%.*]], i64 1237; CHECK-NEXT: [[X0:%.*]] = load double, ptr [[PTRX]], align 4238; CHECK-NEXT: [[Y0:%.*]] = load double, ptr [[PTRY]], align 4239; CHECK-NEXT: [[MUL0:%.*]] = fmul double [[X0]], [[Y0]]240; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[PTRX1]], align 4241; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PTRY1]], align 4242; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP1]], [[TMP2]]243; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0244; CHECK-NEXT: [[DOT01:%.*]] = fadd fast double [[MUL0]], [[TMP4]]245; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i32 1246; CHECK-NEXT: [[DOT012:%.*]] = fadd fast double [[DOT01]], [[TMP5]]247; CHECK-NEXT: ret double [[DOT012]]248;249 %ptrx1 = getelementptr inbounds double, ptr %ptrx, i64 1250 %ptry1 = getelementptr inbounds double, ptr %ptry, i64 1251 %ptrx2 = getelementptr inbounds double, ptr %ptrx, i64 2252 %ptry2 = getelementptr inbounds double, ptr %ptry, i64 2253 %x0 = load double, ptr %ptrx, align 4254 %y0 = load double, ptr %ptry, align 4255 %x1 = load double, ptr %ptrx1, align 4256 %y1 = load double, ptr %ptry1, align 4257 %x2 = load double, ptr %ptrx2, align 4258 %y2 = load double, ptr %ptry2, align 4259 %mul0 = fmul double %x0, %y0260 %mul1 = fmul double %x1, %y1261 %mul2 = fmul double %x2, %y2262 %dot01 = fadd fast double %mul0, %mul1263 %dot012 = fadd fast double %dot01, %mul2264 ret double %dot012265}266 267define float @dot3f32_fast(ptr dereferenceable(16) %ptrx, ptr dereferenceable(16) %ptry) {268; CHECK-LABEL: @dot3f32_fast(269; CHECK-NEXT: [[PTRX1:%.*]] = getelementptr inbounds float, ptr [[PTRX:%.*]], i64 1270; CHECK-NEXT: [[PTRY1:%.*]] = getelementptr inbounds float, ptr [[PTRY:%.*]], i64 1271; CHECK-NEXT: [[X0:%.*]] = load float, ptr [[PTRX]], align 4272; CHECK-NEXT: [[Y0:%.*]] = load float, ptr [[PTRY]], align 4273; CHECK-NEXT: [[MUL0:%.*]] = fmul float [[X0]], [[Y0]]274; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[PTRX1]], align 4275; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[PTRY1]], align 4276; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP1]], [[TMP2]]277; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i32 0278; CHECK-NEXT: [[DOT01:%.*]] = fadd fast float [[MUL0]], [[TMP4]]279; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP3]], i32 1280; CHECK-NEXT: [[DOT012:%.*]] = fadd fast float [[DOT01]], [[TMP5]]281; CHECK-NEXT: ret float [[DOT012]]282;283 %ptrx1 = getelementptr inbounds float, ptr %ptrx, i64 1284 %ptry1 = getelementptr inbounds float, ptr %ptry, i64 1285 %ptrx2 = getelementptr inbounds float, ptr %ptrx, i64 2286 %ptry2 = getelementptr inbounds float, ptr %ptry, i64 2287 %x0 = load float, ptr %ptrx, align 4288 %y0 = load float, ptr %ptry, align 4289 %x1 = load float, ptr %ptrx1, align 4290 %y1 = load float, ptr %ptry1, align 4291 %x2 = load float, ptr %ptrx2, align 4292 %y2 = load float, ptr %ptry2, align 4293 %mul0 = fmul float %x0, %y0294 %mul1 = fmul float %x1, %y1295 %mul2 = fmul float %x2, %y2296 %dot01 = fadd fast float %mul0, %mul1297 %dot012 = fadd fast float %dot01, %mul2298 ret float %dot012299}300 301;302; dot2(ptr x, ptr y) - ((xptr y[0])+(xptr y[1]))303;304 305define double @dot2f64(ptr dereferenceable(16) %ptrx, ptr dereferenceable(16) %ptry) {306; CHECK-LABEL: @dot2f64(307; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[PTRX:%.*]], align 4308; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PTRY:%.*]], align 4309; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP1]], [[TMP2]]310; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0311; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i32 1312; CHECK-NEXT: [[DOT01:%.*]] = fadd double [[TMP4]], [[TMP5]]313; CHECK-NEXT: ret double [[DOT01]]314;315 %ptrx1 = getelementptr inbounds double, ptr %ptrx, i64 1316 %ptry1 = getelementptr inbounds double, ptr %ptry, i64 1317 %x0 = load double, ptr %ptrx, align 4318 %y0 = load double, ptr %ptry, align 4319 %x1 = load double, ptr %ptrx1, align 4320 %y1 = load double, ptr %ptry1, align 4321 %mul0 = fmul double %x0, %y0322 %mul1 = fmul double %x1, %y1323 %dot01 = fadd double %mul0, %mul1324 ret double %dot01325}326 327define float @dot2f32(ptr dereferenceable(16) %ptrx, ptr dereferenceable(16) %ptry) {328; CHECK-LABEL: @dot2f32(329; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[PTRX:%.*]], align 4330; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[PTRY:%.*]], align 4331; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP1]], [[TMP2]]332; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i32 0333; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP3]], i32 1334; CHECK-NEXT: [[DOT01:%.*]] = fadd float [[TMP4]], [[TMP5]]335; CHECK-NEXT: ret float [[DOT01]]336;337 %ptrx1 = getelementptr inbounds float, ptr %ptrx, i64 1338 %ptry1 = getelementptr inbounds float, ptr %ptry, i64 1339 %x0 = load float, ptr %ptrx, align 4340 %y0 = load float, ptr %ptry, align 4341 %x1 = load float, ptr %ptrx1, align 4342 %y1 = load float, ptr %ptry1, align 4343 %mul0 = fmul float %x0, %y0344 %mul1 = fmul float %x1, %y1345 %dot01 = fadd float %mul0, %mul1346 ret float %dot01347}348 349define double @dot2f64_fast(ptr dereferenceable(16) %ptrx, ptr dereferenceable(16) %ptry) {350; SSE2-LABEL: @dot2f64_fast(351; SSE2-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[PTRX:%.*]], align 4352; SSE2-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PTRY:%.*]], align 4353; SSE2-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP1]], [[TMP2]]354; SSE2-NEXT: [[TMP4:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP3]])355; SSE2-NEXT: ret double [[TMP4]]356;357; SSE4-LABEL: @dot2f64_fast(358; SSE4-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[PTRX:%.*]], align 4359; SSE4-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PTRY:%.*]], align 4360; SSE4-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP1]], [[TMP2]]361; SSE4-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0362; SSE4-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i32 1363; SSE4-NEXT: [[DOT01:%.*]] = fadd fast double [[TMP4]], [[TMP5]]364; SSE4-NEXT: ret double [[DOT01]]365;366; AVX-LABEL: @dot2f64_fast(367; AVX-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[PTRX:%.*]], align 4368; AVX-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PTRY:%.*]], align 4369; AVX-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP1]], [[TMP2]]370; AVX-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0371; AVX-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i32 1372; AVX-NEXT: [[DOT01:%.*]] = fadd fast double [[TMP4]], [[TMP5]]373; AVX-NEXT: ret double [[DOT01]]374;375; AVX2-LABEL: @dot2f64_fast(376; AVX2-NEXT: [[PTRX1:%.*]] = getelementptr inbounds double, ptr [[PTRX:%.*]], i64 1377; AVX2-NEXT: [[PTRY1:%.*]] = getelementptr inbounds double, ptr [[PTRY:%.*]], i64 1378; AVX2-NEXT: [[X0:%.*]] = load double, ptr [[PTRX]], align 4379; AVX2-NEXT: [[Y0:%.*]] = load double, ptr [[PTRY]], align 4380; AVX2-NEXT: [[X1:%.*]] = load double, ptr [[PTRX1]], align 4381; AVX2-NEXT: [[Y1:%.*]] = load double, ptr [[PTRY1]], align 4382; AVX2-NEXT: [[MUL0:%.*]] = fmul double [[X0]], [[Y0]]383; AVX2-NEXT: [[MUL1:%.*]] = fmul double [[X1]], [[Y1]]384; AVX2-NEXT: [[DOT01:%.*]] = fadd fast double [[MUL0]], [[MUL1]]385; AVX2-NEXT: ret double [[DOT01]]386;387 %ptrx1 = getelementptr inbounds double, ptr %ptrx, i64 1388 %ptry1 = getelementptr inbounds double, ptr %ptry, i64 1389 %x0 = load double, ptr %ptrx, align 4390 %y0 = load double, ptr %ptry, align 4391 %x1 = load double, ptr %ptrx1, align 4392 %y1 = load double, ptr %ptry1, align 4393 %mul0 = fmul double %x0, %y0394 %mul1 = fmul double %x1, %y1395 %dot01 = fadd fast double %mul0, %mul1396 ret double %dot01397}398 399define float @dot2f32_fast(ptr dereferenceable(16) %ptrx, ptr dereferenceable(16) %ptry) {400; SSE2-LABEL: @dot2f32_fast(401; SSE2-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[PTRX:%.*]], align 4402; SSE2-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[PTRY:%.*]], align 4403; SSE2-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP1]], [[TMP2]]404; SSE2-NEXT: [[TMP4:%.*]] = call fast float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> [[TMP3]])405; SSE2-NEXT: ret float [[TMP4]]406;407; SSE4-LABEL: @dot2f32_fast(408; SSE4-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[PTRX:%.*]], align 4409; SSE4-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[PTRY:%.*]], align 4410; SSE4-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP1]], [[TMP2]]411; SSE4-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i32 0412; SSE4-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP3]], i32 1413; SSE4-NEXT: [[DOT01:%.*]] = fadd fast float [[TMP4]], [[TMP5]]414; SSE4-NEXT: ret float [[DOT01]]415;416; AVX-LABEL: @dot2f32_fast(417; AVX-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[PTRX:%.*]], align 4418; AVX-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[PTRY:%.*]], align 4419; AVX-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP1]], [[TMP2]]420; AVX-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i32 0421; AVX-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP3]], i32 1422; AVX-NEXT: [[DOT01:%.*]] = fadd fast float [[TMP4]], [[TMP5]]423; AVX-NEXT: ret float [[DOT01]]424;425; AVX2-LABEL: @dot2f32_fast(426; AVX2-NEXT: [[PTRX1:%.*]] = getelementptr inbounds float, ptr [[PTRX:%.*]], i64 1427; AVX2-NEXT: [[PTRY1:%.*]] = getelementptr inbounds float, ptr [[PTRY:%.*]], i64 1428; AVX2-NEXT: [[X0:%.*]] = load float, ptr [[PTRX]], align 4429; AVX2-NEXT: [[Y0:%.*]] = load float, ptr [[PTRY]], align 4430; AVX2-NEXT: [[X1:%.*]] = load float, ptr [[PTRX1]], align 4431; AVX2-NEXT: [[Y1:%.*]] = load float, ptr [[PTRY1]], align 4432; AVX2-NEXT: [[MUL0:%.*]] = fmul float [[X0]], [[Y0]]433; AVX2-NEXT: [[MUL1:%.*]] = fmul float [[X1]], [[Y1]]434; AVX2-NEXT: [[DOT01:%.*]] = fadd fast float [[MUL0]], [[MUL1]]435; AVX2-NEXT: ret float [[DOT01]]436;437 %ptrx1 = getelementptr inbounds float, ptr %ptrx, i64 1438 %ptry1 = getelementptr inbounds float, ptr %ptry, i64 1439 %x0 = load float, ptr %ptrx, align 4440 %y0 = load float, ptr %ptry, align 4441 %x1 = load float, ptr %ptrx1, align 4442 %y1 = load float, ptr %ptry1, align 4443 %mul0 = fmul float %x0, %y0444 %mul1 = fmul float %x1, %y1445 %dot01 = fadd fast float %mul0, %mul1446 ret float %dot01447}448