346 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx512fp16,avx512vl -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -machine-combiner-verify-pattern-order=true < %s | FileCheck %s3 4; Incremental updates of the instruction depths should be enough for this test5; case.6; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=avx512fp16,avx512vl -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math -machine-combiner-inc-threshold=0 < %s | FileCheck %s7 8; Verify that the first two adds are independent regardless of how the inputs are9; commuted. The destination registers are used as source registers for the third add.10 11define half @reassociate_adds1(half %x0, half %x1, half %x2, half %x3) {12; CHECK-LABEL: reassociate_adds1:13; CHECK: # %bb.0:14; CHECK-NEXT: vaddsh %xmm1, %xmm0, %xmm015; CHECK-NEXT: vaddsh %xmm3, %xmm2, %xmm116; CHECK-NEXT: vaddsh %xmm1, %xmm0, %xmm017; CHECK-NEXT: retq18 %t0 = fadd reassoc nsz half %x0, %x119 %t1 = fadd reassoc nsz half %t0, %x220 %t2 = fadd reassoc nsz half %t1, %x321 ret half %t222}23 24define half @reassociate_adds2(half %x0, half %x1, half %x2, half %x3) {25; CHECK-LABEL: reassociate_adds2:26; CHECK: # %bb.0:27; CHECK-NEXT: vaddsh %xmm1, %xmm0, %xmm028; CHECK-NEXT: vaddsh %xmm3, %xmm2, %xmm129; CHECK-NEXT: vaddsh %xmm0, %xmm1, %xmm030; CHECK-NEXT: retq31 %t0 = fadd reassoc nsz half %x0, %x132 %t1 = fadd reassoc nsz half %x2, %t033 %t2 = fadd reassoc nsz half %t1, %x334 ret half %t235}36 37define half @reassociate_adds3(half %x0, half %x1, half %x2, half %x3) {38; CHECK-LABEL: reassociate_adds3:39; CHECK: # %bb.0:40; CHECK-NEXT: vaddsh %xmm1, %xmm0, %xmm041; CHECK-NEXT: vaddsh %xmm2, %xmm3, %xmm142; CHECK-NEXT: vaddsh %xmm0, %xmm1, %xmm043; CHECK-NEXT: retq44 %t0 = fadd reassoc nsz half %x0, %x145 %t1 = fadd reassoc nsz half %t0, %x246 %t2 = fadd reassoc nsz half %x3, %t147 ret half %t248}49 50define half @reassociate_adds4(half %x0, half %x1, half %x2, half %x3) {51; CHECK-LABEL: reassociate_adds4:52; CHECK: # %bb.0:53; CHECK-NEXT: vaddsh %xmm1, %xmm0, %xmm054; CHECK-NEXT: vaddsh %xmm2, %xmm3, %xmm155; CHECK-NEXT: vaddsh %xmm0, %xmm1, %xmm056; CHECK-NEXT: retq57 %t0 = fadd reassoc nsz half %x0, %x158 %t1 = fadd reassoc nsz half %x2, %t059 %t2 = fadd reassoc nsz half %x3, %t160 ret half %t261}62 63; Verify that we reassociate some of these ops. The optimal balanced tree of adds is not64; produced because that would cost more compile time.65 66define half @reassociate_adds5(half %x0, half %x1, half %x2, half %x3, half %x4, half %x5, half %x6, half %x7) {67; CHECK-LABEL: reassociate_adds5:68; CHECK: # %bb.0:69; CHECK-NEXT: vaddsh %xmm1, %xmm0, %xmm070; CHECK-NEXT: vaddsh %xmm3, %xmm2, %xmm171; CHECK-NEXT: vaddsh %xmm1, %xmm0, %xmm072; CHECK-NEXT: vaddsh %xmm5, %xmm4, %xmm173; CHECK-NEXT: vaddsh %xmm6, %xmm1, %xmm174; CHECK-NEXT: vaddsh %xmm1, %xmm0, %xmm075; CHECK-NEXT: vaddsh %xmm7, %xmm0, %xmm076; CHECK-NEXT: retq77 %t0 = fadd reassoc nsz half %x0, %x178 %t1 = fadd reassoc nsz half %t0, %x279 %t2 = fadd reassoc nsz half %t1, %x380 %t3 = fadd reassoc nsz half %t2, %x481 %t4 = fadd reassoc nsz half %t3, %x582 %t5 = fadd reassoc nsz half %t4, %x683 %t6 = fadd reassoc nsz half %t5, %x784 ret half %t685}86 87; Verify that we only need two associative operations to reassociate the operands.88; Also, we should reassociate such that the result of the high latency division89; is used by the final 'add' rather than reassociating the %x3 operand with the90; division. The latter reassociation would not improve anything.91 92define half @reassociate_adds6(half %x0, half %x1, half %x2, half %x3) {93; CHECK-LABEL: reassociate_adds6:94; CHECK: # %bb.0:95; CHECK-NEXT: vdivsh %xmm1, %xmm0, %xmm096; CHECK-NEXT: vaddsh %xmm2, %xmm3, %xmm197; CHECK-NEXT: vaddsh %xmm0, %xmm1, %xmm098; CHECK-NEXT: retq99 %t0 = fdiv reassoc nsz half %x0, %x1100 %t1 = fadd reassoc nsz half %x2, %t0101 %t2 = fadd reassoc nsz half %x3, %t1102 ret half %t2103}104 105; Verify that SSE and AVX scalar single-precision multiplies are reassociated.106 107define half @reassociate_muls1(half %x0, half %x1, half %x2, half %x3) {108; CHECK-LABEL: reassociate_muls1:109; CHECK: # %bb.0:110; CHECK-NEXT: vdivsh %xmm1, %xmm0, %xmm0111; CHECK-NEXT: vmulsh %xmm2, %xmm3, %xmm1112; CHECK-NEXT: vmulsh %xmm0, %xmm1, %xmm0113; CHECK-NEXT: retq114 %t0 = fdiv reassoc nsz half %x0, %x1115 %t1 = fmul reassoc nsz half %x2, %t0116 %t2 = fmul reassoc nsz half %x3, %t1117 ret half %t2118}119 120; Verify that SSE and AVX 128-bit vector half-precision adds are reassociated.121 122define <8 x half> @reassociate_adds_v8f16(<8 x half> %x0, <8 x half> %x1, <8 x half> %x2, <8 x half> %x3) {123; CHECK-LABEL: reassociate_adds_v8f16:124; CHECK: # %bb.0:125; CHECK-NEXT: vdivph %xmm1, %xmm0, %xmm0126; CHECK-NEXT: vaddph %xmm2, %xmm3, %xmm1127; CHECK-NEXT: vaddph %xmm0, %xmm1, %xmm0128; CHECK-NEXT: retq129 %t0 = fdiv reassoc nsz <8 x half> %x0, %x1130 %t1 = fadd reassoc nsz <8 x half> %x2, %t0131 %t2 = fadd reassoc nsz <8 x half> %x3, %t1132 ret <8 x half> %t2133}134 135; Verify that SSE and AVX 128-bit vector half-precision multiplies are reassociated.136 137define <8 x half> @reassociate_muls_v8f16(<8 x half> %x0, <8 x half> %x1, <8 x half> %x2, <8 x half> %x3) {138; CHECK-LABEL: reassociate_muls_v8f16:139; CHECK: # %bb.0:140; CHECK-NEXT: vaddph %xmm1, %xmm0, %xmm0141; CHECK-NEXT: vmulph %xmm2, %xmm3, %xmm1142; CHECK-NEXT: vmulph %xmm0, %xmm1, %xmm0143; CHECK-NEXT: retq144 %t0 = fadd reassoc nsz <8 x half> %x0, %x1145 %t1 = fmul reassoc nsz <8 x half> %x2, %t0146 %t2 = fmul reassoc nsz <8 x half> %x3, %t1147 ret <8 x half> %t2148}149 150; Verify that AVX 256-bit vector half-precision adds are reassociated.151 152define <16 x half> @reassociate_adds_v16f16(<16 x half> %x0, <16 x half> %x1, <16 x half> %x2, <16 x half> %x3) {153; CHECK-LABEL: reassociate_adds_v16f16:154; CHECK: # %bb.0:155; CHECK-NEXT: vdivph %ymm1, %ymm0, %ymm0156; CHECK-NEXT: vaddph %ymm2, %ymm3, %ymm1157; CHECK-NEXT: vaddph %ymm0, %ymm1, %ymm0158; CHECK-NEXT: retq159 %t0 = fdiv reassoc nsz <16 x half> %x0, %x1160 %t1 = fadd reassoc nsz <16 x half> %x2, %t0161 %t2 = fadd reassoc nsz <16 x half> %x3, %t1162 ret <16 x half> %t2163}164 165; Verify that AVX 256-bit vector half-precision multiplies are reassociated.166 167define <16 x half> @reassociate_muls_v16f16(<16 x half> %x0, <16 x half> %x1, <16 x half> %x2, <16 x half> %x3) {168; CHECK-LABEL: reassociate_muls_v16f16:169; CHECK: # %bb.0:170; CHECK-NEXT: vaddph %ymm1, %ymm0, %ymm0171; CHECK-NEXT: vmulph %ymm2, %ymm3, %ymm1172; CHECK-NEXT: vmulph %ymm0, %ymm1, %ymm0173; CHECK-NEXT: retq174 %t0 = fadd reassoc nsz <16 x half> %x0, %x1175 %t1 = fmul reassoc nsz <16 x half> %x2, %t0176 %t2 = fmul reassoc nsz <16 x half> %x3, %t1177 ret <16 x half> %t2178}179 180; Verify that AVX512 512-bit vector half-precision adds are reassociated.181 182define <32 x half> @reassociate_adds_v32f16(<32 x half> %x0, <32 x half> %x1, <32 x half> %x2, <32 x half> %x3) {183; CHECK-LABEL: reassociate_adds_v32f16:184; CHECK: # %bb.0:185; CHECK-NEXT: vdivph %zmm1, %zmm0, %zmm0186; CHECK-NEXT: vaddph %zmm2, %zmm3, %zmm1187; CHECK-NEXT: vaddph %zmm0, %zmm1, %zmm0188; CHECK-NEXT: retq189 %t0 = fdiv reassoc nsz <32 x half> %x0, %x1190 %t1 = fadd reassoc nsz <32 x half> %x2, %t0191 %t2 = fadd reassoc nsz <32 x half> %x3, %t1192 ret <32 x half> %t2193}194 195; Verify that AVX512 512-bit vector half-precision multiplies are reassociated.196 197define <32 x half> @reassociate_muls_v32f16(<32 x half> %x0, <32 x half> %x1, <32 x half> %x2, <32 x half> %x3) {198; CHECK-LABEL: reassociate_muls_v32f16:199; CHECK: # %bb.0:200; CHECK-NEXT: vaddph %zmm1, %zmm0, %zmm0201; CHECK-NEXT: vmulph %zmm2, %zmm3, %zmm1202; CHECK-NEXT: vmulph %zmm0, %zmm1, %zmm0203; CHECK-NEXT: retq204 %t0 = fadd reassoc nsz <32 x half> %x0, %x1205 %t1 = fmul reassoc nsz <32 x half> %x2, %t0206 %t2 = fmul reassoc nsz <32 x half> %x3, %t1207 ret <32 x half> %t2208}209 210; Verify that SSE and AVX scalar half-precision minimum ops are reassociated.211 212define half @reassociate_mins_half(half %x0, half %x1, half %x2, half %x3) {213; CHECK-LABEL: reassociate_mins_half:214; CHECK: # %bb.0:215; CHECK-NEXT: vdivsh %xmm1, %xmm0, %xmm0216; CHECK-NEXT: vminsh %xmm2, %xmm3, %xmm1217; CHECK-NEXT: vminsh %xmm0, %xmm1, %xmm0218; CHECK-NEXT: retq219 %t0 = fdiv half %x0, %x1220 %cmp1 = fcmp olt half %x2, %t0221 %sel1 = select i1 %cmp1, half %x2, half %t0222 %cmp2 = fcmp olt half %x3, %sel1223 %sel2 = select i1 %cmp2, half %x3, half %sel1224 ret half %sel2225}226 227; Verify that SSE and AVX scalar half-precision maximum ops are reassociated.228 229define half @reassociate_maxs_half(half %x0, half %x1, half %x2, half %x3) {230; CHECK-LABEL: reassociate_maxs_half:231; CHECK: # %bb.0:232; CHECK-NEXT: vdivsh %xmm1, %xmm0, %xmm0233; CHECK-NEXT: vmaxsh %xmm2, %xmm3, %xmm1234; CHECK-NEXT: vmaxsh %xmm0, %xmm1, %xmm0235; CHECK-NEXT: retq236 %t0 = fdiv half %x0, %x1237 %cmp1 = fcmp ogt half %x2, %t0238 %sel1 = select i1 %cmp1, half %x2, half %t0239 %cmp2 = fcmp ogt half %x3, %sel1240 %sel2 = select i1 %cmp2, half %x3, half %sel1241 ret half %sel2242}243 244; Verify that SSE and AVX 128-bit vector half-precision minimum ops are reassociated.245 246define <8 x half> @reassociate_mins_v8f16(<8 x half> %x0, <8 x half> %x1, <8 x half> %x2, <8 x half> %x3) {247; CHECK-LABEL: reassociate_mins_v8f16:248; CHECK: # %bb.0:249; CHECK-NEXT: vaddph %xmm1, %xmm0, %xmm0250; CHECK-NEXT: vminph %xmm2, %xmm3, %xmm1251; CHECK-NEXT: vminph %xmm0, %xmm1, %xmm0252; CHECK-NEXT: retq253 %t0 = fadd <8 x half> %x0, %x1254 %cmp1 = fcmp olt <8 x half> %x2, %t0255 %sel1 = select <8 x i1> %cmp1, <8 x half> %x2, <8 x half> %t0256 %cmp2 = fcmp olt <8 x half> %x3, %sel1257 %sel2 = select <8 x i1> %cmp2, <8 x half> %x3, <8 x half> %sel1258 ret <8 x half> %sel2259}260 261; Verify that SSE and AVX 128-bit vector half-precision maximum ops are reassociated.262 263define <8 x half> @reassociate_maxs_v8f16(<8 x half> %x0, <8 x half> %x1, <8 x half> %x2, <8 x half> %x3) {264; CHECK-LABEL: reassociate_maxs_v8f16:265; CHECK: # %bb.0:266; CHECK-NEXT: vaddph %xmm1, %xmm0, %xmm0267; CHECK-NEXT: vmaxph %xmm2, %xmm3, %xmm1268; CHECK-NEXT: vmaxph %xmm0, %xmm1, %xmm0269; CHECK-NEXT: retq270 %t0 = fadd <8 x half> %x0, %x1271 %cmp1 = fcmp ogt <8 x half> %x2, %t0272 %sel1 = select <8 x i1> %cmp1, <8 x half> %x2, <8 x half> %t0273 %cmp2 = fcmp ogt <8 x half> %x3, %sel1274 %sel2 = select <8 x i1> %cmp2, <8 x half> %x3, <8 x half> %sel1275 ret <8 x half> %sel2276}277 278; Verify that AVX 256-bit vector half-precision minimum ops are reassociated.279 280define <16 x half> @reassociate_mins_v16f16(<16 x half> %x0, <16 x half> %x1, <16 x half> %x2, <16 x half> %x3) {281; CHECK-LABEL: reassociate_mins_v16f16:282; CHECK: # %bb.0:283; CHECK-NEXT: vaddph %ymm1, %ymm0, %ymm0284; CHECK-NEXT: vminph %ymm2, %ymm3, %ymm1285; CHECK-NEXT: vminph %ymm0, %ymm1, %ymm0286; CHECK-NEXT: retq287 %t0 = fadd <16 x half> %x0, %x1288 %cmp1 = fcmp olt <16 x half> %x2, %t0289 %sel1 = select <16 x i1> %cmp1, <16 x half> %x2, <16 x half> %t0290 %cmp2 = fcmp olt <16 x half> %x3, %sel1291 %sel2 = select <16 x i1> %cmp2, <16 x half> %x3, <16 x half> %sel1292 ret <16 x half> %sel2293}294 295; Verify that AVX 256-bit vector half-precision maximum ops are reassociated.296 297define <16 x half> @reassociate_maxs_v16f16(<16 x half> %x0, <16 x half> %x1, <16 x half> %x2, <16 x half> %x3) {298; CHECK-LABEL: reassociate_maxs_v16f16:299; CHECK: # %bb.0:300; CHECK-NEXT: vaddph %ymm1, %ymm0, %ymm0301; CHECK-NEXT: vmaxph %ymm2, %ymm3, %ymm1302; CHECK-NEXT: vmaxph %ymm0, %ymm1, %ymm0303; CHECK-NEXT: retq304 %t0 = fadd <16 x half> %x0, %x1305 %cmp1 = fcmp ogt <16 x half> %x2, %t0306 %sel1 = select <16 x i1> %cmp1, <16 x half> %x2, <16 x half> %t0307 %cmp2 = fcmp ogt <16 x half> %x3, %sel1308 %sel2 = select <16 x i1> %cmp2, <16 x half> %x3, <16 x half> %sel1309 ret <16 x half> %sel2310}311 312; Verify that AVX512 512-bit vector half-precision minimum ops are reassociated.313 314define <32 x half> @reassociate_mins_v32f16(<32 x half> %x0, <32 x half> %x1, <32 x half> %x2, <32 x half> %x3) {315; CHECK-LABEL: reassociate_mins_v32f16:316; CHECK: # %bb.0:317; CHECK-NEXT: vaddph %zmm1, %zmm0, %zmm0318; CHECK-NEXT: vminph %zmm2, %zmm3, %zmm1319; CHECK-NEXT: vminph %zmm0, %zmm1, %zmm0320; CHECK-NEXT: retq321 %t0 = fadd <32 x half> %x0, %x1322 %cmp1 = fcmp olt <32 x half> %x2, %t0323 %sel1 = select <32 x i1> %cmp1, <32 x half> %x2, <32 x half> %t0324 %cmp2 = fcmp olt <32 x half> %x3, %sel1325 %sel2 = select <32 x i1> %cmp2, <32 x half> %x3, <32 x half> %sel1326 ret <32 x half> %sel2327}328 329; Verify that AVX512 512-bit vector half-precision maximum ops are reassociated.330 331define <32 x half> @reassociate_maxs_v16f32(<32 x half> %x0, <32 x half> %x1, <32 x half> %x2, <32 x half> %x3) {332; CHECK-LABEL: reassociate_maxs_v16f32:333; CHECK: # %bb.0:334; CHECK-NEXT: vaddph %zmm1, %zmm0, %zmm0335; CHECK-NEXT: vmaxph %zmm2, %zmm3, %zmm1336; CHECK-NEXT: vmaxph %zmm0, %zmm1, %zmm0337; CHECK-NEXT: retq338 %t0 = fadd <32 x half> %x0, %x1339 %cmp1 = fcmp ogt <32 x half> %x2, %t0340 %sel1 = select <32 x i1> %cmp1, <32 x half> %x2, <32 x half> %t0341 %cmp2 = fcmp ogt <32 x half> %x3, %sel1342 %sel2 = select <32 x i1> %cmp2, <32 x half> %x3, <32 x half> %sel1343 ret <32 x half> %sel2344}345 346