1137 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSSE3-SLOW3; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,fast-hops | FileCheck %s --check-prefixes=SSSE3-FAST4; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX-SLOW,AVX1-SLOW5; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops | FileCheck %s --check-prefixes=AVX-FAST,AVX1-FAST6; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX-SLOW,AVX2-SLOW7; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2,fast-hops | FileCheck %s --check-prefixes=AVX-FAST,AVX2-FAST8 9; Vectorized Pairwise Sum Reductions10; e.g.11; inline STYPE sum(VTYPE x) {12; return (x[0] + x[1]) + (x[2] + x[3]);13; }14;15; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) {16; return (VTYPE) { sum( A0 ), sum( A1 ), sum( A2 ), sum( A3 ) };17; }18 19define <4 x float> @pair_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {20; SSSE3-SLOW-LABEL: pair_sum_v4f32_v4f32:21; SSSE3-SLOW: # %bb.0:22; SSSE3-SLOW-NEXT: haddps %xmm1, %xmm023; SSSE3-SLOW-NEXT: haddps %xmm2, %xmm324; SSSE3-SLOW-NEXT: haddps %xmm3, %xmm025; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1,3,2]26; SSSE3-SLOW-NEXT: retq27;28; SSSE3-FAST-LABEL: pair_sum_v4f32_v4f32:29; SSSE3-FAST: # %bb.0:30; SSSE3-FAST-NEXT: haddps %xmm1, %xmm031; SSSE3-FAST-NEXT: haddps %xmm3, %xmm232; SSSE3-FAST-NEXT: haddps %xmm2, %xmm033; SSSE3-FAST-NEXT: retq34;35; AVX-SLOW-LABEL: pair_sum_v4f32_v4f32:36; AVX-SLOW: # %bb.0:37; AVX-SLOW-NEXT: vhaddps %xmm1, %xmm0, %xmm038; AVX-SLOW-NEXT: vhaddps %xmm2, %xmm2, %xmm139; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm2 = xmm0[0,2],xmm1[0,1]40; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[1,3],xmm1[1,1]41; AVX-SLOW-NEXT: vhaddps %xmm3, %xmm3, %xmm142; AVX-SLOW-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]43; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[0]44; AVX-SLOW-NEXT: vaddps %xmm0, %xmm1, %xmm045; AVX-SLOW-NEXT: retq46;47; AVX-FAST-LABEL: pair_sum_v4f32_v4f32:48; AVX-FAST: # %bb.0:49; AVX-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm050; AVX-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm151; AVX-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm052; AVX-FAST-NEXT: retq53 %5 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2>54 %6 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3>55 %7 = fadd <2 x float> %5, %656 %8 = shufflevector <2 x float> %7, <2 x float> poison, <2 x i32> <i32 1, i32 undef>57 %9 = fadd <2 x float> %7, %858 %10 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 0, i32 2>59 %11 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 1, i32 3>60 %12 = fadd <2 x float> %10, %1161 %13 = shufflevector <2 x float> %12, <2 x float> poison, <2 x i32> <i32 1, i32 undef>62 %14 = fadd <2 x float> %12, %1363 %15 = shufflevector <2 x float> %9, <2 x float> %14, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>64 %16 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 0, i32 2>65 %17 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 1, i32 3>66 %18 = fadd <2 x float> %16, %1767 %19 = shufflevector <2 x float> %18, <2 x float> poison, <2 x i32> <i32 1, i32 undef>68 %20 = fadd <2 x float> %18, %1969 %21 = shufflevector <2 x float> %20, <2 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>70 %22 = shufflevector <4 x float> %15, <4 x float> %21, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>71 %23 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 0, i32 2>72 %24 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 1, i32 3>73 %25 = fadd <2 x float> %23, %2474 %26 = shufflevector <2 x float> %25, <2 x float> poison, <2 x i32> <i32 1, i32 undef>75 %27 = fadd <2 x float> %25, %2676 %28 = shufflevector <2 x float> %27, <2 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>77 %29 = shufflevector <4 x float> %22, <4 x float> %28, <4 x i32> <i32 0, i32 1, i32 2, i32 4>78 ret <4 x float> %2979}80 81define <4 x i32> @pair_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) {82; SSSE3-SLOW-LABEL: pair_sum_v4i32_v4i32:83; SSSE3-SLOW: # %bb.0:84; SSSE3-SLOW-NEXT: phaddd %xmm1, %xmm085; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,3,1,3]86; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]87; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm088; SSSE3-SLOW-NEXT: phaddd %xmm2, %xmm389; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]90; SSSE3-SLOW-NEXT: paddd %xmm3, %xmm191; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,0]92; SSSE3-SLOW-NEXT: retq93;94; SSSE3-FAST-LABEL: pair_sum_v4i32_v4i32:95; SSSE3-FAST: # %bb.0:96; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm097; SSSE3-FAST-NEXT: phaddd %xmm3, %xmm298; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm099; SSSE3-FAST-NEXT: retq100;101; AVX1-SLOW-LABEL: pair_sum_v4i32_v4i32:102; AVX1-SLOW: # %bb.0:103; AVX1-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm0104; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,3,1,3]105; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]106; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0107; AVX1-SLOW-NEXT: vphaddd %xmm2, %xmm2, %xmm1108; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]109; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1110; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]111; AVX1-SLOW-NEXT: vphaddd %xmm3, %xmm3, %xmm1112; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,0,0,0]113; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm2, %xmm1114; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]115; AVX1-SLOW-NEXT: retq116;117; AVX-FAST-LABEL: pair_sum_v4i32_v4i32:118; AVX-FAST: # %bb.0:119; AVX-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0120; AVX-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm1121; AVX-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0122; AVX-FAST-NEXT: retq123;124; AVX2-SLOW-LABEL: pair_sum_v4i32_v4i32:125; AVX2-SLOW: # %bb.0:126; AVX2-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm0127; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,3,1,3]128; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]129; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0130; AVX2-SLOW-NEXT: vphaddd %xmm2, %xmm2, %xmm1131; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]132; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1133; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]134; AVX2-SLOW-NEXT: vphaddd %xmm3, %xmm3, %xmm1135; AVX2-SLOW-NEXT: vpbroadcastd %xmm1, %xmm2136; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm2, %xmm1137; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]138; AVX2-SLOW-NEXT: retq139 %5 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2>140 %6 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3>141 %7 = add <2 x i32> %5, %6142 %8 = shufflevector <2 x i32> %7, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>143 %9 = add <2 x i32> %7, %8144 %10 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 0, i32 2>145 %11 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 1, i32 3>146 %12 = add <2 x i32> %10, %11147 %13 = shufflevector <2 x i32> %12, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>148 %14 = add <2 x i32> %12, %13149 %15 = shufflevector <2 x i32> %9, <2 x i32> %14, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>150 %16 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 0, i32 2>151 %17 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 1, i32 3>152 %18 = add <2 x i32> %16, %17153 %19 = shufflevector <2 x i32> %18, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>154 %20 = add <2 x i32> %18, %19155 %21 = shufflevector <2 x i32> %20, <2 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>156 %22 = shufflevector <4 x i32> %15, <4 x i32> %21, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>157 %23 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 0, i32 2>158 %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 1, i32 3>159 %25 = add <2 x i32> %23, %24160 %26 = shufflevector <2 x i32> %25, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>161 %27 = add <2 x i32> %25, %26162 %28 = shufflevector <2 x i32> %27, <2 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>163 %29 = shufflevector <4 x i32> %22, <4 x i32> %28, <4 x i32> <i32 0, i32 1, i32 2, i32 4>164 ret <4 x i32> %29165}166 167define <8 x float> @pair_sum_v8f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3, <4 x float> %4, <4 x float> %5, <4 x float> %6, <4 x float> %7) {168; SSSE3-SLOW-LABEL: pair_sum_v8f32_v4f32:169; SSSE3-SLOW: # %bb.0:170; SSSE3-SLOW-NEXT: haddps %xmm1, %xmm0171; SSSE3-SLOW-NEXT: movaps %xmm0, %xmm1172; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,3],xmm0[1,3]173; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2,1,3]174; SSSE3-SLOW-NEXT: addps %xmm1, %xmm0175; SSSE3-SLOW-NEXT: haddps %xmm3, %xmm2176; SSSE3-SLOW-NEXT: haddps %xmm4, %xmm5177; SSSE3-SLOW-NEXT: haddps %xmm5, %xmm2178; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1,3,2]179; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]180; SSSE3-SLOW-NEXT: haddps %xmm7, %xmm6181; SSSE3-SLOW-NEXT: haddps %xmm6, %xmm6182; SSSE3-SLOW-NEXT: movhlps {{.*#+}} xmm6 = xmm2[1],xmm6[1]183; SSSE3-SLOW-NEXT: movaps %xmm6, %xmm1184; SSSE3-SLOW-NEXT: retq185;186; SSSE3-FAST-LABEL: pair_sum_v8f32_v4f32:187; SSSE3-FAST: # %bb.0:188; SSSE3-FAST-NEXT: haddps %xmm1, %xmm0189; SSSE3-FAST-NEXT: haddps %xmm0, %xmm0190; SSSE3-FAST-NEXT: haddps %xmm3, %xmm2191; SSSE3-FAST-NEXT: haddps %xmm5, %xmm4192; SSSE3-FAST-NEXT: haddps %xmm4, %xmm2193; SSSE3-FAST-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]194; SSSE3-FAST-NEXT: haddps %xmm7, %xmm6195; SSSE3-FAST-NEXT: haddps %xmm6, %xmm4196; SSSE3-FAST-NEXT: movaps %xmm4, %xmm1197; SSSE3-FAST-NEXT: retq198;199; AVX1-SLOW-LABEL: pair_sum_v8f32_v4f32:200; AVX1-SLOW: # %bb.0:201; AVX1-SLOW-NEXT: vhaddps %xmm1, %xmm0, %xmm0202; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm0[1,3,1,3]203; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]204; AVX1-SLOW-NEXT: vaddps %xmm1, %xmm0, %xmm0205; AVX1-SLOW-NEXT: vhaddps %xmm4, %xmm4, %xmm1206; AVX1-SLOW-NEXT: vhaddps %xmm5, %xmm5, %xmm4207; AVX1-SLOW-NEXT: vhaddps %xmm3, %xmm2, %xmm2208; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,1]209; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]210; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]211; AVX1-SLOW-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3]212; AVX1-SLOW-NEXT: vaddps %xmm1, %xmm3, %xmm1213; AVX1-SLOW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]214; AVX1-SLOW-NEXT: vshufpd {{.*#+}} xmm1 = xmm1[1,0]215; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1216; AVX1-SLOW-NEXT: vhaddps %xmm7, %xmm6, %xmm2217; AVX1-SLOW-NEXT: vhaddps %xmm2, %xmm2, %xmm2218; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0219; AVX1-SLOW-NEXT: vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]220; AVX1-SLOW-NEXT: retq221;222; AVX1-FAST-LABEL: pair_sum_v8f32_v4f32:223; AVX1-FAST: # %bb.0:224; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0225; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0226; AVX1-FAST-NEXT: vhaddps %xmm4, %xmm4, %xmm1227; AVX1-FAST-NEXT: vhaddps %xmm5, %xmm5, %xmm4228; AVX1-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm2229; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,1]230; AVX1-FAST-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]231; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]232; AVX1-FAST-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3]233; AVX1-FAST-NEXT: vaddps %xmm1, %xmm3, %xmm1234; AVX1-FAST-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]235; AVX1-FAST-NEXT: vshufpd {{.*#+}} xmm1 = xmm1[1,0]236; AVX1-FAST-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1237; AVX1-FAST-NEXT: vhaddps %xmm7, %xmm6, %xmm2238; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm2, %xmm2239; AVX1-FAST-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0240; AVX1-FAST-NEXT: vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]241; AVX1-FAST-NEXT: retq242;243; AVX2-SLOW-LABEL: pair_sum_v8f32_v4f32:244; AVX2-SLOW: # %bb.0:245; AVX2-SLOW-NEXT: vhaddps %xmm1, %xmm0, %xmm0246; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm0[1,3,1,3]247; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2,1,3]248; AVX2-SLOW-NEXT: vaddps %xmm1, %xmm0, %xmm0249; AVX2-SLOW-NEXT: vhaddps %xmm4, %xmm4, %xmm1250; AVX2-SLOW-NEXT: vhaddps %xmm5, %xmm5, %xmm4251; AVX2-SLOW-NEXT: vhaddps %xmm3, %xmm2, %xmm2252; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,1]253; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]254; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]255; AVX2-SLOW-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3]256; AVX2-SLOW-NEXT: vaddps %xmm1, %xmm3, %xmm1257; AVX2-SLOW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]258; AVX2-SLOW-NEXT: vshufpd {{.*#+}} xmm1 = xmm1[1,0]259; AVX2-SLOW-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0260; AVX2-SLOW-NEXT: vhaddps %xmm7, %xmm6, %xmm1261; AVX2-SLOW-NEXT: vhaddps %xmm1, %xmm1, %xmm1262; AVX2-SLOW-NEXT: vbroadcastsd %xmm1, %ymm1263; AVX2-SLOW-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]264; AVX2-SLOW-NEXT: retq265;266; AVX2-FAST-LABEL: pair_sum_v8f32_v4f32:267; AVX2-FAST: # %bb.0:268; AVX2-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0269; AVX2-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0270; AVX2-FAST-NEXT: vhaddps %xmm4, %xmm4, %xmm1271; AVX2-FAST-NEXT: vhaddps %xmm5, %xmm5, %xmm4272; AVX2-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm2273; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,1]274; AVX2-FAST-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]275; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]276; AVX2-FAST-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3]277; AVX2-FAST-NEXT: vaddps %xmm1, %xmm3, %xmm1278; AVX2-FAST-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]279; AVX2-FAST-NEXT: vshufpd {{.*#+}} xmm1 = xmm1[1,0]280; AVX2-FAST-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0281; AVX2-FAST-NEXT: vhaddps %xmm7, %xmm6, %xmm1282; AVX2-FAST-NEXT: vhaddps %xmm0, %xmm1, %xmm1283; AVX2-FAST-NEXT: vbroadcastsd %xmm1, %ymm1284; AVX2-FAST-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]285; AVX2-FAST-NEXT: retq286 %9 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2>287 %10 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3>288 %11 = fadd <2 x float> %9, %10289 %12 = shufflevector <2 x float> %11, <2 x float> poison, <2 x i32> <i32 1, i32 undef>290 %13 = fadd <2 x float> %11, %12291 %14 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 0, i32 2>292 %15 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 1, i32 3>293 %16 = fadd <2 x float> %14, %15294 %17 = shufflevector <2 x float> %16, <2 x float> poison, <2 x i32> <i32 1, i32 undef>295 %18 = fadd <2 x float> %16, %17296 %19 = shufflevector <2 x float> %13, <2 x float> %18, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>297 %20 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 0, i32 2>298 %21 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 1, i32 3>299 %22 = fadd <2 x float> %20, %21300 %23 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 0, i32 2>301 %24 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 1, i32 3>302 %25 = fadd <2 x float> %23, %24303 %26 = shufflevector <4 x float> %4, <4 x float> poison, <2 x i32> <i32 0, i32 2>304 %27 = shufflevector <4 x float> %4, <4 x float> poison, <2 x i32> <i32 1, i32 3>305 %28 = fadd <2 x float> %26, %27306 %29 = shufflevector <2 x float> %28, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>307 %30 = shufflevector <4 x float> %5, <4 x float> poison, <2 x i32> <i32 0, i32 2>308 %31 = shufflevector <4 x float> %5, <4 x float> poison, <2 x i32> <i32 1, i32 3>309 %32 = fadd <2 x float> %30, %31310 %33 = shufflevector <2 x float> %32, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>311 %34 = shufflevector <2 x float> %22, <2 x float> %25, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>312 %35 = shufflevector <4 x float> %34, <4 x float> %29, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>313 %36 = shufflevector <4 x float> %35, <4 x float> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 4>314 %37 = shufflevector <2 x float> %22, <2 x float> %25, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>315 %38 = shufflevector <4 x float> %37, <4 x float> %29, <4 x i32> <i32 0, i32 1, i32 5, i32 undef>316 %39 = shufflevector <4 x float> %38, <4 x float> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 5>317 %40 = fadd <4 x float> %36, %39318 %41 = shufflevector <4 x float> %40, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>319 %42 = shufflevector <8 x float> %19, <8 x float> %41, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef>320 %43 = shufflevector <4 x float> %6, <4 x float> poison, <2 x i32> <i32 0, i32 2>321 %44 = shufflevector <4 x float> %6, <4 x float> poison, <2 x i32> <i32 1, i32 3>322 %45 = fadd <2 x float> %43, %44323 %46 = shufflevector <4 x float> %7, <4 x float> poison, <2 x i32> <i32 0, i32 2>324 %47 = shufflevector <4 x float> %7, <4 x float> poison, <2 x i32> <i32 1, i32 3>325 %48 = fadd <2 x float> %46, %47326 %49 = shufflevector <2 x float> %45, <2 x float> %48, <2 x i32> <i32 0, i32 2>327 %50 = shufflevector <2 x float> %45, <2 x float> %48, <2 x i32> <i32 1, i32 3>328 %51 = fadd <2 x float> %49, %50329 %52 = shufflevector <2 x float> %51, <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>330 %53 = shufflevector <8 x float> %42, <8 x float> %52, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>331 ret <8 x float> %53332}333 334define <8 x i32> @pair_sum_v8i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3, <4 x i32> %4, <4 x i32> %5, <4 x i32> %6, <4 x i32> %7) {335; SSSE3-SLOW-LABEL: pair_sum_v8i32_v4i32:336; SSSE3-SLOW: # %bb.0:337; SSSE3-SLOW-NEXT: phaddd %xmm1, %xmm0338; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,3,1,3]339; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]340; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm0341; SSSE3-SLOW-NEXT: phaddd %xmm3, %xmm2342; SSSE3-SLOW-NEXT: phaddd %xmm4, %xmm5343; SSSE3-SLOW-NEXT: phaddd %xmm5, %xmm2344; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,1,3,2]345; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]346; SSSE3-SLOW-NEXT: phaddd %xmm7, %xmm6347; SSSE3-SLOW-NEXT: phaddd %xmm6, %xmm6348; SSSE3-SLOW-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1],xmm6[1]349; SSSE3-SLOW-NEXT: retq350;351; SSSE3-FAST-LABEL: pair_sum_v8i32_v4i32:352; SSSE3-FAST: # %bb.0:353; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm0354; SSSE3-FAST-NEXT: phaddd %xmm0, %xmm0355; SSSE3-FAST-NEXT: phaddd %xmm3, %xmm2356; SSSE3-FAST-NEXT: phaddd %xmm5, %xmm4357; SSSE3-FAST-NEXT: phaddd %xmm4, %xmm2358; SSSE3-FAST-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]359; SSSE3-FAST-NEXT: phaddd %xmm6, %xmm6360; SSSE3-FAST-NEXT: phaddd %xmm7, %xmm7361; SSSE3-FAST-NEXT: phaddd %xmm7, %xmm6362; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,3],xmm6[0,2]363; SSSE3-FAST-NEXT: movaps %xmm2, %xmm1364; SSSE3-FAST-NEXT: retq365;366; AVX1-SLOW-LABEL: pair_sum_v8i32_v4i32:367; AVX1-SLOW: # %bb.0:368; AVX1-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm0369; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,3,1,3]370; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]371; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0372; AVX1-SLOW-NEXT: vphaddd %xmm4, %xmm4, %xmm1373; AVX1-SLOW-NEXT: vphaddd %xmm5, %xmm5, %xmm4374; AVX1-SLOW-NEXT: vphaddd %xmm3, %xmm2, %xmm2375; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]376; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]377; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[0,0,0,0]378; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5],xmm5[6,7]379; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]380; AVX1-SLOW-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3]381; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm3, %xmm1382; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]383; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]384; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1385; AVX1-SLOW-NEXT: vphaddd %xmm7, %xmm6, %xmm2386; AVX1-SLOW-NEXT: vphaddd %xmm2, %xmm2, %xmm2387; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0388; AVX1-SLOW-NEXT: vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]389; AVX1-SLOW-NEXT: retq390;391; AVX1-FAST-LABEL: pair_sum_v8i32_v4i32:392; AVX1-FAST: # %bb.0:393; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0394; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0395; AVX1-FAST-NEXT: vphaddd %xmm4, %xmm4, %xmm1396; AVX1-FAST-NEXT: vphaddd %xmm5, %xmm5, %xmm4397; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm2398; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]399; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm1[4,5,6,7]400; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[0,0,0,0]401; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5],xmm5[6,7]402; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]403; AVX1-FAST-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3]404; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm3, %xmm1405; AVX1-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]406; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]407; AVX1-FAST-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1408; AVX1-FAST-NEXT: vphaddd %xmm7, %xmm6, %xmm2409; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm2, %xmm2410; AVX1-FAST-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0411; AVX1-FAST-NEXT: vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]412; AVX1-FAST-NEXT: retq413;414; AVX2-SLOW-LABEL: pair_sum_v8i32_v4i32:415; AVX2-SLOW: # %bb.0:416; AVX2-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm0417; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,3,1,3]418; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]419; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0420; AVX2-SLOW-NEXT: vphaddd %xmm4, %xmm4, %xmm1421; AVX2-SLOW-NEXT: vphaddd %xmm5, %xmm5, %xmm4422; AVX2-SLOW-NEXT: vphaddd %xmm3, %xmm2, %xmm2423; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]424; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm1[2,3]425; AVX2-SLOW-NEXT: vpbroadcastd %xmm4, %xmm5426; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1,2],xmm5[3]427; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]428; AVX2-SLOW-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3]429; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm3, %xmm1430; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]431; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]432; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0433; AVX2-SLOW-NEXT: vphaddd %xmm7, %xmm6, %xmm1434; AVX2-SLOW-NEXT: vphaddd %xmm1, %xmm1, %xmm1435; AVX2-SLOW-NEXT: vpbroadcastq %xmm1, %ymm1436; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]437; AVX2-SLOW-NEXT: retq438;439; AVX2-FAST-LABEL: pair_sum_v8i32_v4i32:440; AVX2-FAST: # %bb.0:441; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0442; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0443; AVX2-FAST-NEXT: vphaddd %xmm4, %xmm4, %xmm1444; AVX2-FAST-NEXT: vphaddd %xmm5, %xmm5, %xmm4445; AVX2-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm2446; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]447; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm1[2,3]448; AVX2-FAST-NEXT: vpbroadcastd %xmm4, %xmm5449; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm3 = xmm3[0,1,2],xmm5[3]450; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]451; AVX2-FAST-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3]452; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm3, %xmm1453; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]454; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]455; AVX2-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0456; AVX2-FAST-NEXT: vphaddd %xmm7, %xmm6, %xmm1457; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm1, %xmm1458; AVX2-FAST-NEXT: vpbroadcastq %xmm1, %ymm1459; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]460; AVX2-FAST-NEXT: retq461 %9 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2>462 %10 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3>463 %11 = add <2 x i32> %9, %10464 %12 = shufflevector <2 x i32> %11, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>465 %13 = add <2 x i32> %11, %12466 %14 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 0, i32 2>467 %15 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 1, i32 3>468 %16 = add <2 x i32> %14, %15469 %17 = shufflevector <2 x i32> %16, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>470 %18 = add <2 x i32> %16, %17471 %19 = shufflevector <2 x i32> %13, <2 x i32> %18, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>472 %20 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 0, i32 2>473 %21 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 1, i32 3>474 %22 = add <2 x i32> %20, %21475 %23 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 0, i32 2>476 %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 1, i32 3>477 %25 = add <2 x i32> %23, %24478 %26 = shufflevector <4 x i32> %4, <4 x i32> poison, <2 x i32> <i32 0, i32 2>479 %27 = shufflevector <4 x i32> %4, <4 x i32> poison, <2 x i32> <i32 1, i32 3>480 %28 = add <2 x i32> %26, %27481 %29 = shufflevector <2 x i32> %28, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>482 %30 = shufflevector <4 x i32> %5, <4 x i32> poison, <2 x i32> <i32 0, i32 2>483 %31 = shufflevector <4 x i32> %5, <4 x i32> poison, <2 x i32> <i32 1, i32 3>484 %32 = add <2 x i32> %30, %31485 %33 = shufflevector <2 x i32> %32, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>486 %34 = shufflevector <2 x i32> %22, <2 x i32> %25, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>487 %35 = shufflevector <4 x i32> %34, <4 x i32> %29, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>488 %36 = shufflevector <4 x i32> %35, <4 x i32> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 4>489 %37 = shufflevector <2 x i32> %22, <2 x i32> %25, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>490 %38 = shufflevector <4 x i32> %37, <4 x i32> %29, <4 x i32> <i32 0, i32 1, i32 5, i32 undef>491 %39 = shufflevector <4 x i32> %38, <4 x i32> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 5>492 %40 = add <4 x i32> %36, %39493 %41 = shufflevector <4 x i32> %40, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>494 %42 = shufflevector <8 x i32> %19, <8 x i32> %41, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef>495 %43 = shufflevector <4 x i32> %6, <4 x i32> poison, <2 x i32> <i32 0, i32 2>496 %44 = shufflevector <4 x i32> %6, <4 x i32> poison, <2 x i32> <i32 1, i32 3>497 %45 = add <2 x i32> %43, %44498 %46 = shufflevector <4 x i32> %7, <4 x i32> poison, <2 x i32> <i32 0, i32 2>499 %47 = shufflevector <4 x i32> %7, <4 x i32> poison, <2 x i32> <i32 1, i32 3>500 %48 = add <2 x i32> %46, %47501 %49 = shufflevector <2 x i32> %45, <2 x i32> %48, <2 x i32> <i32 0, i32 2>502 %50 = shufflevector <2 x i32> %45, <2 x i32> %48, <2 x i32> <i32 1, i32 3>503 %51 = add <2 x i32> %49, %50504 %52 = shufflevector <2 x i32> %51, <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>505 %53 = shufflevector <8 x i32> %42, <8 x i32> %52, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>506 ret <8 x i32> %53507}508 509; Vectorized Sequential Sum Reductions510; e.g.511; inline STYPE sum(VTYPE x) {512; return ((x[0] + x[1]) + x[2]) + x[3];513; }514;515; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) {516; return (VTYPE) { sum( A0 ), sum( A1 ), sum( A2 ), sum( A3 ) };517; }518 519define <4 x float> @sequential_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {520; SSSE3-SLOW-LABEL: sequential_sum_v4f32_v4f32:521; SSSE3-SLOW: # %bb.0:522; SSSE3-SLOW-NEXT: movaps %xmm0, %xmm5523; SSSE3-SLOW-NEXT: haddps %xmm1, %xmm5524; SSSE3-SLOW-NEXT: movaps %xmm0, %xmm4525; SSSE3-SLOW-NEXT: unpckhps {{.*#+}} xmm4 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]526; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3],xmm0[3,3]527; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm0 = xmm2[1,1,3,3]528; SSSE3-SLOW-NEXT: addps %xmm2, %xmm0529; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,2],xmm0[0,1]530; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,3]531; SSSE3-SLOW-NEXT: addps %xmm5, %xmm4532; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,3]533; SSSE3-SLOW-NEXT: addps %xmm1, %xmm4534; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm0 = xmm3[1,1,3,3]535; SSSE3-SLOW-NEXT: addps %xmm3, %xmm0536; SSSE3-SLOW-NEXT: movaps %xmm3, %xmm1537; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]538; SSSE3-SLOW-NEXT: addps %xmm0, %xmm1539; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]540; SSSE3-SLOW-NEXT: addps %xmm1, %xmm3541; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3]542; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm3[2,0]543; SSSE3-SLOW-NEXT: movaps %xmm4, %xmm0544; SSSE3-SLOW-NEXT: retq545;546; SSSE3-FAST-LABEL: sequential_sum_v4f32_v4f32:547; SSSE3-FAST: # %bb.0:548; SSSE3-FAST-NEXT: movaps %xmm0, %xmm4549; SSSE3-FAST-NEXT: movaps %xmm0, %xmm5550; SSSE3-FAST-NEXT: haddps %xmm1, %xmm5551; SSSE3-FAST-NEXT: unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]552; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3],xmm4[3,3]553; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]554; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm2[3,3]555; SSSE3-FAST-NEXT: haddps %xmm2, %xmm2556; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,2],xmm2[0,1]557; SSSE3-FAST-NEXT: addps %xmm5, %xmm0558; SSSE3-FAST-NEXT: addps %xmm1, %xmm0559; SSSE3-FAST-NEXT: movaps %xmm3, %xmm1560; SSSE3-FAST-NEXT: haddps %xmm3, %xmm1561; SSSE3-FAST-NEXT: movaps %xmm3, %xmm2562; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1]563; SSSE3-FAST-NEXT: addps %xmm1, %xmm2564; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]565; SSSE3-FAST-NEXT: addps %xmm2, %xmm3566; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm0[2,3]567; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]568; SSSE3-FAST-NEXT: retq569;570; AVX-SLOW-LABEL: sequential_sum_v4f32_v4f32:571; AVX-SLOW: # %bb.0:572; AVX-SLOW-NEXT: vhaddps %xmm1, %xmm0, %xmm4573; AVX-SLOW-NEXT: vunpckhps {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]574; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm1 = xmm1[1,0]575; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero576; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]577; AVX-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1578; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm4[0,2],xmm1[0,1]579; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[3,3]580; AVX-SLOW-NEXT: vmovsd {{.*#+}} xmm2 = xmm5[0],xmm2[1]581; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm3[1,1,3,3]582; AVX-SLOW-NEXT: vaddps %xmm3, %xmm4, %xmm4583; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0]584; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[2]585; AVX-SLOW-NEXT: vaddps %xmm1, %xmm2, %xmm1586; AVX-SLOW-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3]587; AVX-SLOW-NEXT: vaddps %xmm1, %xmm0, %xmm0588; AVX-SLOW-NEXT: retq589;590; AVX-FAST-LABEL: sequential_sum_v4f32_v4f32:591; AVX-FAST: # %bb.0:592; AVX-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm4593; AVX-FAST-NEXT: vunpckhps {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]594; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm1 = xmm1[1,0]595; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero596; AVX-FAST-NEXT: vhaddps %xmm2, %xmm2, %xmm1597; AVX-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm4[0,2],xmm1[0,1]598; AVX-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[3,3]599; AVX-FAST-NEXT: vmovsd {{.*#+}} xmm2 = xmm5[0],xmm2[1]600; AVX-FAST-NEXT: vhaddps %xmm3, %xmm3, %xmm4601; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0]602; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[2]603; AVX-FAST-NEXT: vaddps %xmm1, %xmm2, %xmm1604; AVX-FAST-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3]605; AVX-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0606; AVX-FAST-NEXT: retq607 %5 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 0, i32 4>608 %6 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 1, i32 5>609 %7 = fadd <2 x float> %5, %6610 %8 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 2, i32 6>611 %9 = fadd <2 x float> %8, %7612 %10 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 3, i32 7>613 %11 = fadd <2 x float> %10, %9614 %12 = shufflevector <2 x float> %11, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>615 %13 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>616 %14 = fadd <4 x float> %13, %2617 %15 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>618 %16 = fadd <4 x float> %15, %14619 %17 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>620 %18 = fadd <4 x float> %17, %16621 %19 = shufflevector <4 x float> %12, <4 x float> %18, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>622 %20 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>623 %21 = fadd <4 x float> %20, %3624 %22 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>625 %23 = fadd <4 x float> %22, %21626 %24 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>627 %25 = fadd <4 x float> %24, %23628 %26 = shufflevector <4 x float> %19, <4 x float> %25, <4 x i32> <i32 0, i32 1, i32 2, i32 4>629 ret <4 x float> %26630}631 632define <4 x i32> @sequential_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) {633; SSSE3-SLOW-LABEL: sequential_sum_v4i32_v4i32:634; SSSE3-SLOW: # %bb.0:635; SSSE3-SLOW-NEXT: movdqa %xmm0, %xmm4636; SSSE3-SLOW-NEXT: phaddd %xmm1, %xmm4637; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]638; SSSE3-SLOW-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]639; SSSE3-SLOW-NEXT: paddd %xmm0, %xmm4640; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]641; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm5 = xmm2[0,1,0,1]642; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm5643; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm5644; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]645; SSSE3-SLOW-NEXT: paddd %xmm3, %xmm1646; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]647; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm6648; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm5[2,3]649; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm6[2,0]650; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3],xmm2[3,3]651; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,3],xmm3[2,0]652; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm0653; SSSE3-SLOW-NEXT: retq654;655; SSSE3-FAST-LABEL: sequential_sum_v4i32_v4i32:656; SSSE3-FAST: # %bb.0:657; SSSE3-FAST-NEXT: movdqa %xmm0, %xmm4658; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm4659; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]660; SSSE3-FAST-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]661; SSSE3-FAST-NEXT: paddd %xmm0, %xmm4662; SSSE3-FAST-NEXT: movdqa %xmm2, %xmm1663; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm1664; SSSE3-FAST-NEXT: paddd %xmm2, %xmm1665; SSSE3-FAST-NEXT: movdqa %xmm3, %xmm5666; SSSE3-FAST-NEXT: phaddd %xmm3, %xmm5667; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]668; SSSE3-FAST-NEXT: paddd %xmm5, %xmm6669; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm1[2,3]670; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm6[2,0]671; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3],xmm2[3,3]672; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,3],xmm3[2,0]673; SSSE3-FAST-NEXT: paddd %xmm4, %xmm0674; SSSE3-FAST-NEXT: retq675;676; AVX1-SLOW-LABEL: sequential_sum_v4i32_v4i32:677; AVX1-SLOW: # %bb.0:678; AVX1-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm4679; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]680; AVX1-SLOW-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]681; AVX1-SLOW-NEXT: vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]682; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]683; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]684; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1685; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm4[0],xmm1[0]686; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[3,3,3,3]687; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]688; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0,1,2,3],xmm2[4,5,6,7]689; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm2, %xmm1690; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0691; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]692; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[0,0,0,0]693; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1694; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]695; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2696; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm2, %xmm1697; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]698; AVX1-SLOW-NEXT: retq699;700; AVX1-FAST-LABEL: sequential_sum_v4i32_v4i32:701; AVX1-FAST: # %bb.0:702; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm4703; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]704; AVX1-FAST-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]705; AVX1-FAST-NEXT: vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]706; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]707; AVX1-FAST-NEXT: vphaddd %xmm2, %xmm2, %xmm1708; AVX1-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm4[0],xmm1[0]709; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[3,3,3,3]710; AVX1-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]711; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm5[0,1,2,3],xmm2[4,5,6,7]712; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1713; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0714; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm3, %xmm1715; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]716; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]717; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2718; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1719; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]720; AVX1-FAST-NEXT: retq721;722; AVX2-SLOW-LABEL: sequential_sum_v4i32_v4i32:723; AVX2-SLOW: # %bb.0:724; AVX2-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm4725; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]726; AVX2-SLOW-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]727; AVX2-SLOW-NEXT: vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]728; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]729; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]730; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1731; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm4[0],xmm1[0]732; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[3,3,3,3]733; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]734; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]735; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm2, %xmm1736; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0737; AVX2-SLOW-NEXT: vpbroadcastq %xmm3, %xmm1738; AVX2-SLOW-NEXT: vpbroadcastd %xmm3, %xmm2739; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1740; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]741; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2742; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm2, %xmm1743; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]744; AVX2-SLOW-NEXT: retq745;746; AVX2-FAST-LABEL: sequential_sum_v4i32_v4i32:747; AVX2-FAST: # %bb.0:748; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm4749; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]750; AVX2-FAST-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]751; AVX2-FAST-NEXT: vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1]752; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]753; AVX2-FAST-NEXT: vphaddd %xmm2, %xmm2, %xmm1754; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm4[0],xmm1[0]755; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[3,3,3,3]756; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]757; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]758; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1759; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0760; AVX2-FAST-NEXT: vphaddd %xmm3, %xmm3, %xmm1761; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,2,2,2]762; AVX2-FAST-NEXT: vpbroadcastd %xmm1, %xmm1763; AVX2-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2764; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1765; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]766; AVX2-FAST-NEXT: retq767 %5 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 0, i32 4>768 %6 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 1, i32 5>769 %7 = add <2 x i32> %5, %6770 %8 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 2, i32 6>771 %9 = add <2 x i32> %8, %7772 %10 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 3, i32 7>773 %11 = add <2 x i32> %10, %9774 %12 = shufflevector <2 x i32> %11, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>775 %13 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>776 %14 = add <4 x i32> %13, %2777 %15 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>778 %16 = add <4 x i32> %15, %14779 %17 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>780 %18 = add <4 x i32> %17, %16781 %19 = shufflevector <4 x i32> %12, <4 x i32> %18, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>782 %20 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>783 %21 = add <4 x i32> %20, %3784 %22 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>785 %23 = add <4 x i32> %22, %21786 %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>787 %25 = add <4 x i32> %24, %23788 %26 = shufflevector <4 x i32> %19, <4 x i32> %25, <4 x i32> <i32 0, i32 1, i32 2, i32 4>789 ret <4 x i32> %26790}791 792; Vectorized Reductions793; e.g.794; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) {795; return (VTYPE) { reduce( A0 ), reduce( A1 ), reduce( A2 ), reduce( A3 ) };796; }797 798define <4 x float> @reduction_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {799; SSSE3-SLOW-LABEL: reduction_sum_v4f32_v4f32:800; SSSE3-SLOW: # %bb.0:801; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]802; SSSE3-SLOW-NEXT: addss %xmm0, %xmm4803; SSSE3-SLOW-NEXT: movaps %xmm0, %xmm5804; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1]805; SSSE3-SLOW-NEXT: addss %xmm4, %xmm5806; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]807; SSSE3-SLOW-NEXT: addss %xmm5, %xmm0808; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]809; SSSE3-SLOW-NEXT: addss %xmm1, %xmm4810; SSSE3-SLOW-NEXT: movaps %xmm1, %xmm5811; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1]812; SSSE3-SLOW-NEXT: addss %xmm4, %xmm5813; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]814; SSSE3-SLOW-NEXT: addss %xmm5, %xmm1815; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]816; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]817; SSSE3-SLOW-NEXT: addss %xmm2, %xmm1818; SSSE3-SLOW-NEXT: movaps %xmm2, %xmm4819; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1]820; SSSE3-SLOW-NEXT: addss %xmm1, %xmm4821; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]822; SSSE3-SLOW-NEXT: addss %xmm4, %xmm2823; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]824; SSSE3-SLOW-NEXT: addss %xmm3, %xmm1825; SSSE3-SLOW-NEXT: movaps %xmm3, %xmm4826; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]827; SSSE3-SLOW-NEXT: addss %xmm1, %xmm4828; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]829; SSSE3-SLOW-NEXT: addss %xmm4, %xmm3830; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]831; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]832; SSSE3-SLOW-NEXT: retq833;834; SSSE3-FAST-LABEL: reduction_sum_v4f32_v4f32:835; SSSE3-FAST: # %bb.0:836; SSSE3-FAST-NEXT: movaps %xmm0, %xmm4837; SSSE3-FAST-NEXT: haddps %xmm0, %xmm4838; SSSE3-FAST-NEXT: movaps %xmm0, %xmm5839; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1]840; SSSE3-FAST-NEXT: addss %xmm4, %xmm5841; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]842; SSSE3-FAST-NEXT: addss %xmm5, %xmm0843; SSSE3-FAST-NEXT: movaps %xmm1, %xmm4844; SSSE3-FAST-NEXT: haddps %xmm1, %xmm4845; SSSE3-FAST-NEXT: movaps %xmm1, %xmm5846; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1]847; SSSE3-FAST-NEXT: addss %xmm4, %xmm5848; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]849; SSSE3-FAST-NEXT: addss %xmm5, %xmm1850; SSSE3-FAST-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]851; SSSE3-FAST-NEXT: movaps %xmm2, %xmm1852; SSSE3-FAST-NEXT: haddps %xmm2, %xmm1853; SSSE3-FAST-NEXT: movaps %xmm2, %xmm4854; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1]855; SSSE3-FAST-NEXT: addss %xmm1, %xmm4856; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]857; SSSE3-FAST-NEXT: addss %xmm4, %xmm2858; SSSE3-FAST-NEXT: movaps %xmm3, %xmm1859; SSSE3-FAST-NEXT: haddps %xmm3, %xmm1860; SSSE3-FAST-NEXT: movaps %xmm3, %xmm4861; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]862; SSSE3-FAST-NEXT: addss %xmm1, %xmm4863; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]864; SSSE3-FAST-NEXT: addss %xmm4, %xmm3865; SSSE3-FAST-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]866; SSSE3-FAST-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]867; SSSE3-FAST-NEXT: retq868;869; AVX-SLOW-LABEL: reduction_sum_v4f32_v4f32:870; AVX-SLOW: # %bb.0:871; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]872; AVX-SLOW-NEXT: vaddss %xmm4, %xmm0, %xmm4873; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm5 = xmm0[1,0]874; AVX-SLOW-NEXT: vaddss %xmm5, %xmm4, %xmm4875; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]876; AVX-SLOW-NEXT: vaddss %xmm0, %xmm4, %xmm0877; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]878; AVX-SLOW-NEXT: vaddss %xmm4, %xmm1, %xmm4879; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm5 = xmm1[1,0]880; AVX-SLOW-NEXT: vaddss %xmm5, %xmm4, %xmm4881; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]882; AVX-SLOW-NEXT: vaddss %xmm1, %xmm4, %xmm1883; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]884; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]885; AVX-SLOW-NEXT: vaddss %xmm1, %xmm2, %xmm1886; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm4 = xmm2[1,0]887; AVX-SLOW-NEXT: vaddss %xmm4, %xmm1, %xmm1888; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]889; AVX-SLOW-NEXT: vaddss %xmm2, %xmm1, %xmm1890; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]891; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]892; AVX-SLOW-NEXT: vaddss %xmm1, %xmm3, %xmm1893; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm2 = xmm3[1,0]894; AVX-SLOW-NEXT: vaddss %xmm2, %xmm1, %xmm1895; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm2 = xmm3[3,3,3,3]896; AVX-SLOW-NEXT: vaddss %xmm2, %xmm1, %xmm1897; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]898; AVX-SLOW-NEXT: retq899;900; AVX-FAST-LABEL: reduction_sum_v4f32_v4f32:901; AVX-FAST: # %bb.0:902; AVX-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm4903; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm5 = xmm0[1,0]904; AVX-FAST-NEXT: vaddss %xmm5, %xmm4, %xmm4905; AVX-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]906; AVX-FAST-NEXT: vaddss %xmm0, %xmm4, %xmm0907; AVX-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm4908; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm5 = xmm1[1,0]909; AVX-FAST-NEXT: vaddss %xmm5, %xmm4, %xmm4910; AVX-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]911; AVX-FAST-NEXT: vaddss %xmm1, %xmm4, %xmm1912; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]913; AVX-FAST-NEXT: vhaddps %xmm2, %xmm2, %xmm1914; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm4 = xmm2[1,0]915; AVX-FAST-NEXT: vaddss %xmm4, %xmm1, %xmm1916; AVX-FAST-NEXT: vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]917; AVX-FAST-NEXT: vaddss %xmm2, %xmm1, %xmm1918; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]919; AVX-FAST-NEXT: vhaddps %xmm3, %xmm3, %xmm1920; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm2 = xmm3[1,0]921; AVX-FAST-NEXT: vaddss %xmm2, %xmm1, %xmm1922; AVX-FAST-NEXT: vshufps {{.*#+}} xmm2 = xmm3[3,3,3,3]923; AVX-FAST-NEXT: vaddss %xmm2, %xmm1, %xmm1924; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]925; AVX-FAST-NEXT: retq926 %5 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %0)927 %6 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %1)928 %7 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %2)929 %8 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %3)930 %9 = insertelement <4 x float> undef, float %5, i32 0931 %10 = insertelement <4 x float> %9, float %6, i32 1932 %11 = insertelement <4 x float> %10, float %7, i32 2933 %12 = insertelement <4 x float> %11, float %8, i32 3934 ret <4 x float> %12935}936declare float @llvm.vector.reduce.fadd.f32.v4f32(float, <4 x float>)937 938define <4 x float> @reduction_sum_v4f32_v4f32_reassoc(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {939; SSSE3-SLOW-LABEL: reduction_sum_v4f32_v4f32_reassoc:940; SSSE3-SLOW: # %bb.0:941; SSSE3-SLOW-NEXT: movaps %xmm0, %xmm4942; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]943; SSSE3-SLOW-NEXT: addps %xmm4, %xmm0944; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]945; SSSE3-SLOW-NEXT: movaps %xmm1, %xmm5946; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1]947; SSSE3-SLOW-NEXT: addps %xmm1, %xmm5948; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm1 = xmm5[1,1,3,3]949; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]950; SSSE3-SLOW-NEXT: movaps %xmm2, %xmm1951; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]952; SSSE3-SLOW-NEXT: addps %xmm2, %xmm1953; SSSE3-SLOW-NEXT: movaps %xmm3, %xmm2954; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1]955; SSSE3-SLOW-NEXT: addps %xmm3, %xmm2956; SSSE3-SLOW-NEXT: movaps %xmm2, %xmm3957; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm3 = xmm3[0],xmm1[0]958; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]959; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]960; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]961; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,0]962; SSSE3-SLOW-NEXT: addps %xmm4, %xmm0963; SSSE3-SLOW-NEXT: retq964;965; SSSE3-FAST-LABEL: reduction_sum_v4f32_v4f32_reassoc:966; SSSE3-FAST: # %bb.0:967; SSSE3-FAST-NEXT: movaps %xmm0, %xmm4968; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]969; SSSE3-FAST-NEXT: addps %xmm4, %xmm0970; SSSE3-FAST-NEXT: movaps %xmm1, %xmm4971; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm1[1]972; SSSE3-FAST-NEXT: addps %xmm1, %xmm4973; SSSE3-FAST-NEXT: haddps %xmm4, %xmm0974; SSSE3-FAST-NEXT: movaps %xmm2, %xmm1975; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]976; SSSE3-FAST-NEXT: addps %xmm2, %xmm1977; SSSE3-FAST-NEXT: movaps %xmm3, %xmm2978; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1]979; SSSE3-FAST-NEXT: addps %xmm3, %xmm2980; SSSE3-FAST-NEXT: haddps %xmm2, %xmm1981; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]982; SSSE3-FAST-NEXT: retq983;984; AVX-SLOW-LABEL: reduction_sum_v4f32_v4f32_reassoc:985; AVX-SLOW: # %bb.0:986; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm4 = xmm0[1,0]987; AVX-SLOW-NEXT: vaddps %xmm4, %xmm0, %xmm0988; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm4 = xmm1[1,0]989; AVX-SLOW-NEXT: vaddps %xmm4, %xmm1, %xmm1990; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm4 = xmm2[1,0]991; AVX-SLOW-NEXT: vaddps %xmm4, %xmm2, %xmm2992; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm4 = xmm3[1,0]993; AVX-SLOW-NEXT: vaddps %xmm4, %xmm3, %xmm3994; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm4 = xmm3[1,1],xmm2[1,1]995; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm5 = xmm0[1],xmm1[1],zero,zero996; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,0]997; AVX-SLOW-NEXT: vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0]998; AVX-SLOW-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]999; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]1000; AVX-SLOW-NEXT: vaddps %xmm4, %xmm0, %xmm01001; AVX-SLOW-NEXT: retq1002;1003; AVX-FAST-LABEL: reduction_sum_v4f32_v4f32_reassoc:1004; AVX-FAST: # %bb.0:1005; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm4 = xmm0[1,0]1006; AVX-FAST-NEXT: vaddps %xmm4, %xmm0, %xmm01007; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm4 = xmm1[1,0]1008; AVX-FAST-NEXT: vaddps %xmm4, %xmm1, %xmm11009; AVX-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm01010; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm1 = xmm2[1,0]1011; AVX-FAST-NEXT: vaddps %xmm1, %xmm2, %xmm11012; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm2 = xmm3[1,0]1013; AVX-FAST-NEXT: vaddps %xmm2, %xmm3, %xmm21014; AVX-FAST-NEXT: vhaddps %xmm2, %xmm1, %xmm11015; AVX-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]1016; AVX-FAST-NEXT: retq1017 %5 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %0)1018 %6 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %1)1019 %7 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %2)1020 %8 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %3)1021 %9 = insertelement <4 x float> undef, float %5, i32 01022 %10 = insertelement <4 x float> %9, float %6, i32 11023 %11 = insertelement <4 x float> %10, float %7, i32 21024 %12 = insertelement <4 x float> %11, float %8, i32 31025 ret <4 x float> %121026}1027 1028define <4 x i32> @reduction_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) {1029; SSSE3-SLOW-LABEL: reduction_sum_v4i32_v4i32:1030; SSSE3-SLOW: # %bb.0:1031; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]1032; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm01033; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]1034; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]1035; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm51036; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm5[1,1,1,1]1037; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]1038; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]1039; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm11040; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]1041; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]1042; SSSE3-SLOW-NEXT: paddd %xmm3, %xmm61043; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm3 = xmm6[1,1,1,1]1044; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]1045; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]1046; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]1047; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]1048; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]1049; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm01050; SSSE3-SLOW-NEXT: retq1051;1052; SSSE3-FAST-LABEL: reduction_sum_v4i32_v4i32:1053; SSSE3-FAST: # %bb.0:1054; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]1055; SSSE3-FAST-NEXT: paddd %xmm4, %xmm01056; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]1057; SSSE3-FAST-NEXT: paddd %xmm1, %xmm41058; SSSE3-FAST-NEXT: phaddd %xmm4, %xmm01059; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]1060; SSSE3-FAST-NEXT: paddd %xmm2, %xmm11061; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]1062; SSSE3-FAST-NEXT: paddd %xmm3, %xmm21063; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm11064; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]1065; SSSE3-FAST-NEXT: retq1066;1067; AVX1-SLOW-LABEL: reduction_sum_v4i32_v4i32:1068; AVX1-SLOW: # %bb.0:1069; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]1070; AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm01071; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]1072; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]1073; AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm1, %xmm11074; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]1075; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]1076; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]1077; AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm2, %xmm21078; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]1079; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]1080; AVX1-SLOW-NEXT: vpaddd %xmm6, %xmm3, %xmm31081; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]1082; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]1083; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]1084; AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm2, %xmm21085; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]1086; AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm01087; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]1088; AVX1-SLOW-NEXT: retq1089;1090; AVX-FAST-LABEL: reduction_sum_v4i32_v4i32:1091; AVX-FAST: # %bb.0:1092; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]1093; AVX-FAST-NEXT: vpaddd %xmm4, %xmm0, %xmm01094; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]1095; AVX-FAST-NEXT: vpaddd %xmm4, %xmm1, %xmm11096; AVX-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm01097; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]1098; AVX-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm11099; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]1100; AVX-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm21101; AVX-FAST-NEXT: vphaddd %xmm2, %xmm1, %xmm11102; AVX-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]1103; AVX-FAST-NEXT: retq1104;1105; AVX2-SLOW-LABEL: reduction_sum_v4i32_v4i32:1106; AVX2-SLOW: # %bb.0:1107; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]1108; AVX2-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm01109; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]1110; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]1111; AVX2-SLOW-NEXT: vpaddd %xmm5, %xmm1, %xmm11112; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]1113; AVX2-SLOW-NEXT: vpaddd %xmm5, %xmm2, %xmm21114; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]1115; AVX2-SLOW-NEXT: vpaddd %xmm5, %xmm3, %xmm31116; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]1117; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0],xmm1[1],xmm4[2,3]1118; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3]1119; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]1120; AVX2-SLOW-NEXT: vpbroadcastd %xmm3, %xmm11121; AVX2-SLOW-NEXT: vpbroadcastd %xmm2, %xmm21122; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]1123; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]1124; AVX2-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm01125; AVX2-SLOW-NEXT: retq1126 %5 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %0)1127 %6 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %1)1128 %7 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %2)1129 %8 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %3)1130 %9 = insertelement <4 x i32> undef, i32 %5, i32 01131 %10 = insertelement <4 x i32> %9, i32 %6, i32 11132 %11 = insertelement <4 x i32> %10, i32 %7, i32 21133 %12 = insertelement <4 x i32> %11, i32 %8, i32 31134 ret <4 x i32> %121135}1136declare i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32>)1137