1027 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64--linux-gnu -mcpu=x86-64 -mattr=+sse2 | FileCheck %s --check-prefix=CHECK --check-prefix=SSE3; RUN: llc < %s -mtriple=x86_64--linux-gnu -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX14; RUN: llc < %s -mtriple=x86_64--linux-gnu -mcpu=x86-64 -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX5125 6declare double @__sqrt_finite(double)7declare float @__sqrtf_finite(float)8declare x86_fp80 @__sqrtl_finite(x86_fp80)9declare float @llvm.sqrt.f32(float)10declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)11declare <8 x float> @llvm.sqrt.v8f32(<8 x float>)12declare <16 x float> @llvm.sqrt.v16f32(<16 x float>)13declare double @llvm.sqrt.f64(double)14declare <2 x double> @llvm.sqrt.v2f64(<2 x double>)15 16declare float @llvm.fabs.f32(float)17declare <4 x float> @llvm.fabs.v4f32(<4 x float>)18declare double @llvm.fabs.f64(double)19 20define double @finite_f64_no_estimate(double %d) #0 {21; SSE-LABEL: finite_f64_no_estimate:22; SSE: # %bb.0:23; SSE-NEXT: sqrtsd %xmm0, %xmm024; SSE-NEXT: retq25;26; AVX-LABEL: finite_f64_no_estimate:27; AVX: # %bb.0:28; AVX-NEXT: vsqrtsd %xmm0, %xmm0, %xmm029; AVX-NEXT: retq30 %call = tail call double @__sqrt_finite(double %d) #231 ret double %call32}33 34; No estimates for doubles.35 36define double @finite_f64_estimate(double %d) #1 {37; SSE-LABEL: finite_f64_estimate:38; SSE: # %bb.0:39; SSE-NEXT: sqrtsd %xmm0, %xmm040; SSE-NEXT: retq41;42; AVX-LABEL: finite_f64_estimate:43; AVX: # %bb.0:44; AVX-NEXT: vsqrtsd %xmm0, %xmm0, %xmm045; AVX-NEXT: retq46 %call = tail call double @__sqrt_finite(double %d) #247 ret double %call48}49 50define float @finite_f32_no_estimate(float %f) #0 {51; SSE-LABEL: finite_f32_no_estimate:52; SSE: # %bb.0:53; SSE-NEXT: sqrtss %xmm0, %xmm054; SSE-NEXT: retq55;56; AVX-LABEL: finite_f32_no_estimate:57; AVX: # %bb.0:58; AVX-NEXT: vsqrtss %xmm0, %xmm0, %xmm059; AVX-NEXT: retq60 %call = tail call float @__sqrtf_finite(float %f) #261 ret float %call62}63 64define float @finite_f32_estimate_ieee(float %f) #1 {65; SSE-LABEL: finite_f32_estimate_ieee:66; SSE: # %bb.0:67; SSE-NEXT: sqrtss %xmm0, %xmm068; SSE-NEXT: retq69;70; AVX-LABEL: finite_f32_estimate_ieee:71; AVX: # %bb.0:72; AVX-NEXT: vsqrtss %xmm0, %xmm0, %xmm073; AVX-NEXT: retq74 %call = tail call float @__sqrtf_finite(float %f) #275 ret float %call76}77 78define float @finite_f32_estimate_ieee_ninf(float %f) #1 {79; SSE-LABEL: finite_f32_estimate_ieee_ninf:80; SSE: # %bb.0:81; SSE-NEXT: sqrtss %xmm0, %xmm082; SSE-NEXT: retq83;84; AVX-LABEL: finite_f32_estimate_ieee_ninf:85; AVX: # %bb.0:86; AVX-NEXT: vsqrtss %xmm0, %xmm0, %xmm087; AVX-NEXT: retq88 %call = tail call ninf afn float @__sqrtf_finite(float %f) #289 ret float %call90}91 92define float @finite_f32_estimate_daz(float %f) #4 {93; SSE-LABEL: finite_f32_estimate_daz:94; SSE: # %bb.0:95; SSE-NEXT: sqrtss %xmm0, %xmm096; SSE-NEXT: retq97;98; AVX-LABEL: finite_f32_estimate_daz:99; AVX: # %bb.0:100; AVX-NEXT: vsqrtss %xmm0, %xmm0, %xmm0101; AVX-NEXT: retq102 %call = tail call float @__sqrtf_finite(float %f) #2103 ret float %call104}105 106define float @finite_f32_estimate_daz_ninf(float %f) #4 {107; SSE-LABEL: finite_f32_estimate_daz_ninf:108; SSE: # %bb.0:109; SSE-NEXT: sqrtss %xmm0, %xmm0110; SSE-NEXT: retq111;112; AVX-LABEL: finite_f32_estimate_daz_ninf:113; AVX: # %bb.0:114; AVX-NEXT: vsqrtss %xmm0, %xmm0, %xmm0115; AVX-NEXT: retq116 %call = tail call ninf afn float @__sqrtf_finite(float %f) #2117 ret float %call118}119 120define x86_fp80 @finite_f80_no_estimate(x86_fp80 %ld) #0 {121; CHECK-LABEL: finite_f80_no_estimate:122; CHECK: # %bb.0:123; CHECK-NEXT: fldt {{[0-9]+}}(%rsp)124; CHECK-NEXT: fsqrt125; CHECK-NEXT: retq126 %call = tail call x86_fp80 @__sqrtl_finite(x86_fp80 %ld) #2127 ret x86_fp80 %call128}129 130; Don't die on the impossible.131 132define x86_fp80 @finite_f80_estimate_but_no(x86_fp80 %ld) #1 {133; CHECK-LABEL: finite_f80_estimate_but_no:134; CHECK: # %bb.0:135; CHECK-NEXT: fldt {{[0-9]+}}(%rsp)136; CHECK-NEXT: fsqrt137; CHECK-NEXT: retq138 %call = tail call x86_fp80 @__sqrtl_finite(x86_fp80 %ld) #2139 ret x86_fp80 %call140}141 142; PR34994 - https://bugs.llvm.org/show_bug.cgi?id=34994143 144define float @sqrtf_check_denorms(float %x) #3 {145; SSE-LABEL: sqrtf_check_denorms:146; SSE: # %bb.0:147; SSE-NEXT: sqrtss %xmm0, %xmm0148; SSE-NEXT: retq149;150; AVX-LABEL: sqrtf_check_denorms:151; AVX: # %bb.0:152; AVX-NEXT: vsqrtss %xmm0, %xmm0, %xmm0153; AVX-NEXT: retq154 %call = tail call float @__sqrtf_finite(float %x) #2155 ret float %call156}157 158define float @sqrtf_check_denorms_ninf(float %x) #3 {159; SSE-LABEL: sqrtf_check_denorms_ninf:160; SSE: # %bb.0:161; SSE-NEXT: sqrtss %xmm0, %xmm0162; SSE-NEXT: retq163;164; AVX-LABEL: sqrtf_check_denorms_ninf:165; AVX: # %bb.0:166; AVX-NEXT: vsqrtss %xmm0, %xmm0, %xmm0167; AVX-NEXT: retq168 %call = tail call ninf afn float @__sqrtf_finite(float %x) #2169 ret float %call170}171 172define <4 x float> @sqrt_v4f32_check_denorms(<4 x float> %x) #3 {173; SSE-LABEL: sqrt_v4f32_check_denorms:174; SSE: # %bb.0:175; SSE-NEXT: sqrtps %xmm0, %xmm0176; SSE-NEXT: retq177;178; AVX-LABEL: sqrt_v4f32_check_denorms:179; AVX: # %bb.0:180; AVX-NEXT: vsqrtps %xmm0, %xmm0181; AVX-NEXT: retq182 %call = tail call <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) #2183 ret <4 x float> %call184}185 186define <4 x float> @sqrt_v4f32_check_denorms_ieee_ninf(<4 x float> %x) #7 {187; SSE-LABEL: sqrt_v4f32_check_denorms_ieee_ninf:188; SSE: # %bb.0:189; SSE-NEXT: rsqrtps %xmm0, %xmm1190; SSE-NEXT: movaps %xmm0, %xmm2191; SSE-NEXT: mulps %xmm1, %xmm2192; SSE-NEXT: movaps {{.*#+}} xmm3 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]193; SSE-NEXT: mulps %xmm2, %xmm3194; SSE-NEXT: mulps %xmm1, %xmm2195; SSE-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2196; SSE-NEXT: mulps %xmm3, %xmm2197; SSE-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0198; SSE-NEXT: movaps {{.*#+}} xmm1 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38]199; SSE-NEXT: cmpleps %xmm0, %xmm1200; SSE-NEXT: andps %xmm2, %xmm1201; SSE-NEXT: movaps %xmm1, %xmm0202; SSE-NEXT: retq203;204; AVX1-LABEL: sqrt_v4f32_check_denorms_ieee_ninf:205; AVX1: # %bb.0:206; AVX1-NEXT: vrsqrtps %xmm0, %xmm1207; AVX1-NEXT: vmulps %xmm1, %xmm0, %xmm2208; AVX1-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3209; AVX1-NEXT: vmulps %xmm1, %xmm2, %xmm1210; AVX1-NEXT: vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1211; AVX1-NEXT: vmulps %xmm1, %xmm3, %xmm1212; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0213; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38]214; AVX1-NEXT: vcmpleps %xmm0, %xmm2, %xmm0215; AVX1-NEXT: vandps %xmm1, %xmm0, %xmm0216; AVX1-NEXT: retq217;218; AVX512-LABEL: sqrt_v4f32_check_denorms_ieee_ninf:219; AVX512: # %bb.0:220; AVX512-NEXT: vrsqrtps %xmm0, %xmm1221; AVX512-NEXT: vmulps %xmm1, %xmm0, %xmm2222; AVX512-NEXT: vbroadcastss {{.*#+}} xmm3 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]223; AVX512-NEXT: vfmadd231ps {{.*#+}} xmm3 = (xmm2 * xmm1) + xmm3224; AVX512-NEXT: vbroadcastss {{.*#+}} xmm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]225; AVX512-NEXT: vmulps %xmm1, %xmm2, %xmm1226; AVX512-NEXT: vmulps %xmm3, %xmm1, %xmm1227; AVX512-NEXT: vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]228; AVX512-NEXT: vandps %xmm2, %xmm0, %xmm0229; AVX512-NEXT: vbroadcastss {{.*#+}} xmm2 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38]230; AVX512-NEXT: vcmpleps %xmm0, %xmm2, %xmm0231; AVX512-NEXT: vandps %xmm1, %xmm0, %xmm0232; AVX512-NEXT: retq233 %call = tail call fast ninf afn <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) #2234 ret <4 x float> %call235}236 237define <4 x float> @sqrt_v4f32_check_denorms_dynamic_ninf(<4 x float> %x) #8 {238; SSE-LABEL: sqrt_v4f32_check_denorms_dynamic_ninf:239; SSE: # %bb.0:240; SSE-NEXT: rsqrtps %xmm0, %xmm1241; SSE-NEXT: movaps %xmm0, %xmm2242; SSE-NEXT: mulps %xmm1, %xmm2243; SSE-NEXT: movaps {{.*#+}} xmm3 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]244; SSE-NEXT: mulps %xmm2, %xmm3245; SSE-NEXT: mulps %xmm1, %xmm2246; SSE-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2247; SSE-NEXT: mulps %xmm3, %xmm2248; SSE-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0249; SSE-NEXT: movaps {{.*#+}} xmm1 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38]250; SSE-NEXT: cmpleps %xmm0, %xmm1251; SSE-NEXT: andps %xmm2, %xmm1252; SSE-NEXT: movaps %xmm1, %xmm0253; SSE-NEXT: retq254;255; AVX1-LABEL: sqrt_v4f32_check_denorms_dynamic_ninf:256; AVX1: # %bb.0:257; AVX1-NEXT: vrsqrtps %xmm0, %xmm1258; AVX1-NEXT: vmulps %xmm1, %xmm0, %xmm2259; AVX1-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3260; AVX1-NEXT: vmulps %xmm1, %xmm2, %xmm1261; AVX1-NEXT: vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1262; AVX1-NEXT: vmulps %xmm1, %xmm3, %xmm1263; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0264; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38]265; AVX1-NEXT: vcmpleps %xmm0, %xmm2, %xmm0266; AVX1-NEXT: vandps %xmm1, %xmm0, %xmm0267; AVX1-NEXT: retq268;269; AVX512-LABEL: sqrt_v4f32_check_denorms_dynamic_ninf:270; AVX512: # %bb.0:271; AVX512-NEXT: vrsqrtps %xmm0, %xmm1272; AVX512-NEXT: vmulps %xmm1, %xmm0, %xmm2273; AVX512-NEXT: vbroadcastss {{.*#+}} xmm3 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]274; AVX512-NEXT: vfmadd231ps {{.*#+}} xmm3 = (xmm2 * xmm1) + xmm3275; AVX512-NEXT: vbroadcastss {{.*#+}} xmm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]276; AVX512-NEXT: vmulps %xmm1, %xmm2, %xmm1277; AVX512-NEXT: vmulps %xmm3, %xmm1, %xmm1278; AVX512-NEXT: vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]279; AVX512-NEXT: vandps %xmm2, %xmm0, %xmm0280; AVX512-NEXT: vbroadcastss {{.*#+}} xmm2 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38]281; AVX512-NEXT: vcmpleps %xmm0, %xmm2, %xmm0282; AVX512-NEXT: vandps %xmm1, %xmm0, %xmm0283; AVX512-NEXT: retq284 %call = tail call fast ninf afn <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) #2285 ret <4 x float> %call286}287 288define float @f32_no_estimate(float %x) #0 {289; SSE-LABEL: f32_no_estimate:290; SSE: # %bb.0:291; SSE-NEXT: sqrtss %xmm0, %xmm1292; SSE-NEXT: movss {{.*#+}} xmm0 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]293; SSE-NEXT: divss %xmm1, %xmm0294; SSE-NEXT: retq295;296; AVX-LABEL: f32_no_estimate:297; AVX: # %bb.0:298; AVX-NEXT: vsqrtss %xmm0, %xmm0, %xmm0299; AVX-NEXT: vmovss {{.*#+}} xmm1 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]300; AVX-NEXT: vdivss %xmm0, %xmm1, %xmm0301; AVX-NEXT: retq302 %sqrt = tail call float @llvm.sqrt.f32(float %x)303 %div = fdiv fast float 1.0, %sqrt304 ret float %div305}306 307define float @f32_estimate(float %x) #1 {308; SSE-LABEL: f32_estimate:309; SSE: # %bb.0:310; SSE-NEXT: rsqrtss %xmm0, %xmm1311; SSE-NEXT: mulss %xmm1, %xmm0312; SSE-NEXT: mulss %xmm1, %xmm0313; SSE-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0314; SSE-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1315; SSE-NEXT: mulss %xmm1, %xmm0316; SSE-NEXT: retq317;318; AVX1-LABEL: f32_estimate:319; AVX1: # %bb.0:320; AVX1-NEXT: vrsqrtss %xmm0, %xmm0, %xmm1321; AVX1-NEXT: vmulss %xmm1, %xmm0, %xmm0322; AVX1-NEXT: vmulss %xmm1, %xmm0, %xmm0323; AVX1-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0324; AVX1-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1325; AVX1-NEXT: vmulss %xmm0, %xmm1, %xmm0326; AVX1-NEXT: retq327;328; AVX512-LABEL: f32_estimate:329; AVX512: # %bb.0:330; AVX512-NEXT: vrsqrtss %xmm0, %xmm0, %xmm1331; AVX512-NEXT: vmulss %xmm1, %xmm0, %xmm0332; AVX512-NEXT: vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem333; AVX512-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1334; AVX512-NEXT: vmulss %xmm0, %xmm1, %xmm0335; AVX512-NEXT: retq336 %sqrt = tail call float @llvm.sqrt.f32(float %x)337 %div = fdiv fast float 1.0, %sqrt338 ret float %div339}340 341define float @f32_estimate2(float %x) #5 {342; SSE-LABEL: f32_estimate2:343; SSE: # %bb.0:344; SSE-NEXT: sqrtss %xmm0, %xmm0345; SSE-NEXT: retq346;347; AVX-LABEL: f32_estimate2:348; AVX: # %bb.0:349; AVX-NEXT: vsqrtss %xmm0, %xmm0, %xmm0350; AVX-NEXT: retq351 %sqrt = tail call fast float @llvm.sqrt.f32(float %x)352 ret float %sqrt353}354 355define <4 x float> @v4f32_no_estimate(<4 x float> %x) #0 {356; SSE-LABEL: v4f32_no_estimate:357; SSE: # %bb.0:358; SSE-NEXT: sqrtps %xmm0, %xmm1359; SSE-NEXT: movaps {{.*#+}} xmm0 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]360; SSE-NEXT: divps %xmm1, %xmm0361; SSE-NEXT: retq362;363; AVX-LABEL: v4f32_no_estimate:364; AVX: # %bb.0:365; AVX-NEXT: vsqrtps %xmm0, %xmm0366; AVX-NEXT: vbroadcastss {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]367; AVX-NEXT: vdivps %xmm0, %xmm1, %xmm0368; AVX-NEXT: retq369 %sqrt = tail call <4 x float> @llvm.sqrt.v4f32(<4 x float> %x)370 %div = fdiv fast <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, %sqrt371 ret <4 x float> %div372}373 374define <4 x float> @v4f32_estimate(<4 x float> %x) #1 {375; SSE-LABEL: v4f32_estimate:376; SSE: # %bb.0:377; SSE-NEXT: rsqrtps %xmm0, %xmm1378; SSE-NEXT: mulps %xmm1, %xmm0379; SSE-NEXT: mulps %xmm1, %xmm0380; SSE-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0381; SSE-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1382; SSE-NEXT: mulps %xmm1, %xmm0383; SSE-NEXT: retq384;385; AVX1-LABEL: v4f32_estimate:386; AVX1: # %bb.0:387; AVX1-NEXT: vrsqrtps %xmm0, %xmm1388; AVX1-NEXT: vmulps %xmm1, %xmm0, %xmm0389; AVX1-NEXT: vmulps %xmm1, %xmm0, %xmm0390; AVX1-NEXT: vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0391; AVX1-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1392; AVX1-NEXT: vmulps %xmm0, %xmm1, %xmm0393; AVX1-NEXT: retq394;395; AVX512-LABEL: v4f32_estimate:396; AVX512: # %bb.0:397; AVX512-NEXT: vrsqrtps %xmm0, %xmm1398; AVX512-NEXT: vmulps %xmm1, %xmm0, %xmm0399; AVX512-NEXT: vbroadcastss {{.*#+}} xmm2 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]400; AVX512-NEXT: vfmadd231ps {{.*#+}} xmm2 = (xmm1 * xmm0) + xmm2401; AVX512-NEXT: vbroadcastss {{.*#+}} xmm0 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]402; AVX512-NEXT: vmulps %xmm0, %xmm1, %xmm0403; AVX512-NEXT: vmulps %xmm2, %xmm0, %xmm0404; AVX512-NEXT: retq405 %sqrt = tail call <4 x float> @llvm.sqrt.v4f32(<4 x float> %x)406 %div = fdiv fast <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, %sqrt407 ret <4 x float> %div408}409 410define <4 x float> @v4f32_estimate2(<4 x float> %x) #5 {411; SSE-LABEL: v4f32_estimate2:412; SSE: # %bb.0:413; SSE-NEXT: movaps {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]414; SSE-NEXT: andps %xmm0, %xmm1415; SSE-NEXT: movaps {{.*#+}} xmm2 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38]416; SSE-NEXT: cmpleps %xmm1, %xmm2417; SSE-NEXT: rsqrtps %xmm0, %xmm1418; SSE-NEXT: mulps %xmm1, %xmm0419; SSE-NEXT: andps %xmm2, %xmm0420; SSE-NEXT: retq421;422; AVX1-LABEL: v4f32_estimate2:423; AVX1: # %bb.0:424; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1425; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38]426; AVX1-NEXT: vcmpleps %xmm1, %xmm2, %xmm1427; AVX1-NEXT: vrsqrtps %xmm0, %xmm2428; AVX1-NEXT: vmulps %xmm2, %xmm0, %xmm0429; AVX1-NEXT: vandps %xmm0, %xmm1, %xmm0430; AVX1-NEXT: retq431;432; AVX512-LABEL: v4f32_estimate2:433; AVX512: # %bb.0:434; AVX512-NEXT: vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]435; AVX512-NEXT: vandps %xmm1, %xmm0, %xmm1436; AVX512-NEXT: vbroadcastss {{.*#+}} xmm2 = [1.17549435E-38,1.17549435E-38,1.17549435E-38,1.17549435E-38]437; AVX512-NEXT: vcmpleps %xmm1, %xmm2, %xmm1438; AVX512-NEXT: vrsqrtps %xmm0, %xmm2439; AVX512-NEXT: vmulps %xmm2, %xmm0, %xmm0440; AVX512-NEXT: vandps %xmm0, %xmm1, %xmm0441; AVX512-NEXT: retq442 %sqrt = tail call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %x)443 ret <4 x float> %sqrt444}445 446define <8 x float> @v8f32_no_estimate(<8 x float> %x) #0 {447; SSE-LABEL: v8f32_no_estimate:448; SSE: # %bb.0:449; SSE-NEXT: sqrtps %xmm1, %xmm2450; SSE-NEXT: sqrtps %xmm0, %xmm3451; SSE-NEXT: movaps {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]452; SSE-NEXT: movaps %xmm1, %xmm0453; SSE-NEXT: divps %xmm3, %xmm0454; SSE-NEXT: divps %xmm2, %xmm1455; SSE-NEXT: retq456;457; AVX1-LABEL: v8f32_no_estimate:458; AVX1: # %bb.0:459; AVX1-NEXT: vsqrtps %ymm0, %ymm0460; AVX1-NEXT: vmovaps {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]461; AVX1-NEXT: vdivps %ymm0, %ymm1, %ymm0462; AVX1-NEXT: retq463;464; AVX512-LABEL: v8f32_no_estimate:465; AVX512: # %bb.0:466; AVX512-NEXT: vsqrtps %ymm0, %ymm0467; AVX512-NEXT: vbroadcastss {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]468; AVX512-NEXT: vdivps %ymm0, %ymm1, %ymm0469; AVX512-NEXT: retq470 %sqrt = tail call <8 x float> @llvm.sqrt.v8f32(<8 x float> %x)471 %div = fdiv fast <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %sqrt472 ret <8 x float> %div473}474 475define <8 x float> @v8f32_estimate(<8 x float> %x) #1 {476; SSE-LABEL: v8f32_estimate:477; SSE: # %bb.0:478; SSE-NEXT: rsqrtps %xmm0, %xmm2479; SSE-NEXT: movaps {{.*#+}} xmm3 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]480; SSE-NEXT: mulps %xmm2, %xmm0481; SSE-NEXT: mulps %xmm2, %xmm0482; SSE-NEXT: mulps %xmm3, %xmm2483; SSE-NEXT: movaps {{.*#+}} xmm4 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]484; SSE-NEXT: addps %xmm4, %xmm0485; SSE-NEXT: mulps %xmm2, %xmm0486; SSE-NEXT: rsqrtps %xmm1, %xmm2487; SSE-NEXT: mulps %xmm2, %xmm3488; SSE-NEXT: mulps %xmm2, %xmm1489; SSE-NEXT: mulps %xmm2, %xmm1490; SSE-NEXT: addps %xmm4, %xmm1491; SSE-NEXT: mulps %xmm3, %xmm1492; SSE-NEXT: retq493;494; AVX1-LABEL: v8f32_estimate:495; AVX1: # %bb.0:496; AVX1-NEXT: vrsqrtps %ymm0, %ymm1497; AVX1-NEXT: vmulps %ymm1, %ymm0, %ymm0498; AVX1-NEXT: vmulps %ymm1, %ymm0, %ymm0499; AVX1-NEXT: vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0500; AVX1-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1501; AVX1-NEXT: vmulps %ymm0, %ymm1, %ymm0502; AVX1-NEXT: retq503;504; AVX512-LABEL: v8f32_estimate:505; AVX512: # %bb.0:506; AVX512-NEXT: vrsqrtps %ymm0, %ymm1507; AVX512-NEXT: vmulps %ymm1, %ymm0, %ymm0508; AVX512-NEXT: vbroadcastss {{.*#+}} ymm2 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]509; AVX512-NEXT: vbroadcastss {{.*#+}} ymm3 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]510; AVX512-NEXT: vfmadd231ps {{.*#+}} ymm2 = (ymm1 * ymm0) + ymm2511; AVX512-NEXT: vmulps %ymm3, %ymm1, %ymm0512; AVX512-NEXT: vmulps %ymm2, %ymm0, %ymm0513; AVX512-NEXT: retq514 %sqrt = tail call <8 x float> @llvm.sqrt.v8f32(<8 x float> %x)515 %div = fdiv fast <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %sqrt516 ret <8 x float> %div517}518 519define <16 x float> @v16f32_no_estimate(<16 x float> %x) #0 {520; SSE-LABEL: v16f32_no_estimate:521; SSE: # %bb.0:522; SSE-NEXT: sqrtps %xmm3, %xmm4523; SSE-NEXT: sqrtps %xmm2, %xmm5524; SSE-NEXT: sqrtps %xmm1, %xmm2525; SSE-NEXT: sqrtps %xmm0, %xmm1526; SSE-NEXT: movaps {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]527; SSE-NEXT: movaps %xmm3, %xmm0528; SSE-NEXT: divps %xmm1, %xmm0529; SSE-NEXT: movaps %xmm3, %xmm1530; SSE-NEXT: divps %xmm2, %xmm1531; SSE-NEXT: movaps %xmm3, %xmm2532; SSE-NEXT: divps %xmm5, %xmm2533; SSE-NEXT: divps %xmm4, %xmm3534; SSE-NEXT: retq535;536; AVX1-LABEL: v16f32_no_estimate:537; AVX1: # %bb.0:538; AVX1-NEXT: vsqrtps %ymm1, %ymm1539; AVX1-NEXT: vsqrtps %ymm0, %ymm0540; AVX1-NEXT: vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]541; AVX1-NEXT: vdivps %ymm0, %ymm2, %ymm0542; AVX1-NEXT: vdivps %ymm1, %ymm2, %ymm1543; AVX1-NEXT: retq544;545; AVX512-LABEL: v16f32_no_estimate:546; AVX512: # %bb.0:547; AVX512-NEXT: vsqrtps %zmm0, %zmm0548; AVX512-NEXT: vbroadcastss {{.*#+}} zmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]549; AVX512-NEXT: vdivps %zmm0, %zmm1, %zmm0550; AVX512-NEXT: retq551 %sqrt = tail call <16 x float> @llvm.sqrt.v16f32(<16 x float> %x)552 %div = fdiv fast <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %sqrt553 ret <16 x float> %div554}555 556define <16 x float> @v16f32_estimate(<16 x float> %x) #1 {557; SSE-LABEL: v16f32_estimate:558; SSE: # %bb.0:559; SSE-NEXT: rsqrtps %xmm0, %xmm6560; SSE-NEXT: movaps {{.*#+}} xmm4 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]561; SSE-NEXT: mulps %xmm6, %xmm0562; SSE-NEXT: mulps %xmm6, %xmm0563; SSE-NEXT: mulps %xmm4, %xmm6564; SSE-NEXT: movaps {{.*#+}} xmm5 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]565; SSE-NEXT: addps %xmm5, %xmm0566; SSE-NEXT: mulps %xmm6, %xmm0567; SSE-NEXT: rsqrtps %xmm1, %xmm6568; SSE-NEXT: mulps %xmm6, %xmm1569; SSE-NEXT: mulps %xmm6, %xmm1570; SSE-NEXT: mulps %xmm4, %xmm6571; SSE-NEXT: addps %xmm5, %xmm1572; SSE-NEXT: mulps %xmm6, %xmm1573; SSE-NEXT: rsqrtps %xmm2, %xmm6574; SSE-NEXT: mulps %xmm6, %xmm2575; SSE-NEXT: mulps %xmm6, %xmm2576; SSE-NEXT: mulps %xmm4, %xmm6577; SSE-NEXT: addps %xmm5, %xmm2578; SSE-NEXT: mulps %xmm6, %xmm2579; SSE-NEXT: rsqrtps %xmm3, %xmm6580; SSE-NEXT: mulps %xmm6, %xmm4581; SSE-NEXT: mulps %xmm6, %xmm3582; SSE-NEXT: mulps %xmm6, %xmm3583; SSE-NEXT: addps %xmm5, %xmm3584; SSE-NEXT: mulps %xmm4, %xmm3585; SSE-NEXT: retq586;587; AVX1-LABEL: v16f32_estimate:588; AVX1: # %bb.0:589; AVX1-NEXT: vrsqrtps %ymm0, %ymm2590; AVX1-NEXT: vmovaps {{.*#+}} ymm3 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]591; AVX1-NEXT: vmulps %ymm3, %ymm2, %ymm4592; AVX1-NEXT: vmulps %ymm2, %ymm0, %ymm0593; AVX1-NEXT: vmulps %ymm2, %ymm0, %ymm0594; AVX1-NEXT: vmovaps {{.*#+}} ymm2 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]595; AVX1-NEXT: vaddps %ymm2, %ymm0, %ymm0596; AVX1-NEXT: vrsqrtps %ymm1, %ymm5597; AVX1-NEXT: vmulps %ymm0, %ymm4, %ymm0598; AVX1-NEXT: vmulps %ymm3, %ymm5, %ymm3599; AVX1-NEXT: vmulps %ymm5, %ymm1, %ymm1600; AVX1-NEXT: vmulps %ymm5, %ymm1, %ymm1601; AVX1-NEXT: vaddps %ymm2, %ymm1, %ymm1602; AVX1-NEXT: vmulps %ymm1, %ymm3, %ymm1603; AVX1-NEXT: retq604;605; AVX512-LABEL: v16f32_estimate:606; AVX512: # %bb.0:607; AVX512-NEXT: vrsqrt14ps %zmm0, %zmm1608; AVX512-NEXT: vmulps %zmm1, %zmm0, %zmm0609; AVX512-NEXT: vfmadd213ps {{.*#+}} zmm0 = (zmm1 * zmm0) + mem610; AVX512-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1611; AVX512-NEXT: vmulps %zmm0, %zmm1, %zmm0612; AVX512-NEXT: retq613 %sqrt = tail call <16 x float> @llvm.sqrt.v16f32(<16 x float> %x)614 %div = fdiv fast <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %sqrt615 ret <16 x float> %div616}617 618; x / (fabs(y) * sqrt(z)) --> x * rsqrt(y*y*z)619 620define float @div_sqrt_fabs_f32(float %x, float %y, float %z) {621; SSE-LABEL: div_sqrt_fabs_f32:622; SSE: # %bb.0:623; SSE-NEXT: mulss %xmm1, %xmm1624; SSE-NEXT: mulss %xmm2, %xmm1625; SSE-NEXT: xorps %xmm2, %xmm2626; SSE-NEXT: rsqrtss %xmm1, %xmm2627; SSE-NEXT: mulss %xmm2, %xmm1628; SSE-NEXT: mulss %xmm2, %xmm1629; SSE-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1630; SSE-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2631; SSE-NEXT: mulss %xmm2, %xmm0632; SSE-NEXT: mulss %xmm1, %xmm0633; SSE-NEXT: retq634;635; AVX1-LABEL: div_sqrt_fabs_f32:636; AVX1: # %bb.0:637; AVX1-NEXT: vmulss %xmm1, %xmm1, %xmm1638; AVX1-NEXT: vmulss %xmm2, %xmm1, %xmm1639; AVX1-NEXT: vrsqrtss %xmm1, %xmm1, %xmm2640; AVX1-NEXT: vmulss %xmm2, %xmm1, %xmm1641; AVX1-NEXT: vmulss %xmm2, %xmm1, %xmm1642; AVX1-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1643; AVX1-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2644; AVX1-NEXT: vmulss %xmm2, %xmm0, %xmm0645; AVX1-NEXT: vmulss %xmm1, %xmm0, %xmm0646; AVX1-NEXT: retq647;648; AVX512-LABEL: div_sqrt_fabs_f32:649; AVX512: # %bb.0:650; AVX512-NEXT: vmulss %xmm1, %xmm1, %xmm1651; AVX512-NEXT: vmulss %xmm2, %xmm1, %xmm1652; AVX512-NEXT: vrsqrtss %xmm1, %xmm1, %xmm2653; AVX512-NEXT: vmulss %xmm2, %xmm1, %xmm1654; AVX512-NEXT: vfmadd213ss {{.*#+}} xmm1 = (xmm2 * xmm1) + mem655; AVX512-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2656; AVX512-NEXT: vmulss %xmm2, %xmm0, %xmm0657; AVX512-NEXT: vmulss %xmm1, %xmm0, %xmm0658; AVX512-NEXT: retq659 %s = call fast float @llvm.sqrt.f32(float %z)660 %a = call fast float @llvm.fabs.f32(float %y)661 %m = fmul fast float %s, %a662 %d = fdiv fast float %x, %m663 ret float %d664}665 666; x / (fabs(y) * sqrt(z)) --> x * rsqrt(y*y*z)667 668define <4 x float> @div_sqrt_fabs_v4f32(<4 x float> %x, <4 x float> %y, <4 x float> %z) {669; SSE-LABEL: div_sqrt_fabs_v4f32:670; SSE: # %bb.0:671; SSE-NEXT: mulps %xmm1, %xmm1672; SSE-NEXT: mulps %xmm2, %xmm1673; SSE-NEXT: rsqrtps %xmm1, %xmm2674; SSE-NEXT: mulps %xmm2, %xmm1675; SSE-NEXT: mulps %xmm2, %xmm1676; SSE-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1677; SSE-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2678; SSE-NEXT: mulps %xmm1, %xmm2679; SSE-NEXT: mulps %xmm2, %xmm0680; SSE-NEXT: retq681;682; AVX1-LABEL: div_sqrt_fabs_v4f32:683; AVX1: # %bb.0:684; AVX1-NEXT: vmulps %xmm1, %xmm1, %xmm1685; AVX1-NEXT: vmulps %xmm2, %xmm1, %xmm1686; AVX1-NEXT: vrsqrtps %xmm1, %xmm2687; AVX1-NEXT: vmulps %xmm2, %xmm1, %xmm1688; AVX1-NEXT: vmulps %xmm2, %xmm1, %xmm1689; AVX1-NEXT: vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1690; AVX1-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2691; AVX1-NEXT: vmulps %xmm1, %xmm2, %xmm1692; AVX1-NEXT: vmulps %xmm1, %xmm0, %xmm0693; AVX1-NEXT: retq694;695; AVX512-LABEL: div_sqrt_fabs_v4f32:696; AVX512: # %bb.0:697; AVX512-NEXT: vmulps %xmm1, %xmm1, %xmm1698; AVX512-NEXT: vmulps %xmm2, %xmm1, %xmm1699; AVX512-NEXT: vrsqrtps %xmm1, %xmm2700; AVX512-NEXT: vmulps %xmm2, %xmm1, %xmm1701; AVX512-NEXT: vbroadcastss {{.*#+}} xmm3 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]702; AVX512-NEXT: vfmadd231ps {{.*#+}} xmm3 = (xmm2 * xmm1) + xmm3703; AVX512-NEXT: vbroadcastss {{.*#+}} xmm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]704; AVX512-NEXT: vmulps %xmm1, %xmm2, %xmm1705; AVX512-NEXT: vmulps %xmm3, %xmm1, %xmm1706; AVX512-NEXT: vmulps %xmm1, %xmm0, %xmm0707; AVX512-NEXT: retq708 %s = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %z)709 %a = call <4 x float> @llvm.fabs.v4f32(<4 x float> %y)710 %m = fmul contract reassoc <4 x float> %a, %s711 %d = fdiv contract reassoc arcp <4 x float> %x, %m712 ret <4 x float> %d713}714 715; This has 'arcp' but does not have 'reassoc' FMF.716; We allow converting the sqrt to an estimate, but717; do not pull the divisor into the estimate.718; x / (fabs(y) * sqrt(z)) --> x * rsqrt(z) / fabs(y)719 720define <4 x float> @div_sqrt_fabs_v4f32_fmf(<4 x float> %x, <4 x float> %y, <4 x float> %z) {721; SSE-LABEL: div_sqrt_fabs_v4f32_fmf:722; SSE: # %bb.0:723; SSE-NEXT: rsqrtps %xmm2, %xmm3724; SSE-NEXT: mulps %xmm3, %xmm2725; SSE-NEXT: mulps %xmm3, %xmm2726; SSE-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2727; SSE-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3728; SSE-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1729; SSE-NEXT: mulps %xmm2, %xmm3730; SSE-NEXT: divps %xmm1, %xmm3731; SSE-NEXT: mulps %xmm3, %xmm0732; SSE-NEXT: retq733;734; AVX1-LABEL: div_sqrt_fabs_v4f32_fmf:735; AVX1: # %bb.0:736; AVX1-NEXT: vrsqrtps %xmm2, %xmm3737; AVX1-NEXT: vmulps %xmm3, %xmm2, %xmm2738; AVX1-NEXT: vmulps %xmm3, %xmm2, %xmm2739; AVX1-NEXT: vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2740; AVX1-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3741; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1742; AVX1-NEXT: vmulps %xmm2, %xmm3, %xmm2743; AVX1-NEXT: vdivps %xmm1, %xmm2, %xmm1744; AVX1-NEXT: vmulps %xmm1, %xmm0, %xmm0745; AVX1-NEXT: retq746;747; AVX512-LABEL: div_sqrt_fabs_v4f32_fmf:748; AVX512: # %bb.0:749; AVX512-NEXT: vrsqrtps %xmm2, %xmm3750; AVX512-NEXT: vbroadcastss {{.*#+}} xmm4 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]751; AVX512-NEXT: vmulps %xmm4, %xmm3, %xmm4752; AVX512-NEXT: vmulps %xmm3, %xmm2, %xmm2753; AVX512-NEXT: vmulps %xmm3, %xmm2, %xmm2754; AVX512-NEXT: vbroadcastss {{.*#+}} xmm3 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]755; AVX512-NEXT: vaddps %xmm3, %xmm2, %xmm2756; AVX512-NEXT: vmulps %xmm2, %xmm4, %xmm2757; AVX512-NEXT: vbroadcastss {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN]758; AVX512-NEXT: vandps %xmm3, %xmm1, %xmm1759; AVX512-NEXT: vdivps %xmm1, %xmm2, %xmm1760; AVX512-NEXT: vmulps %xmm1, %xmm0, %xmm0761; AVX512-NEXT: retq762 %s = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %z)763 %a = call <4 x float> @llvm.fabs.v4f32(<4 x float> %y)764 %m = fmul <4 x float> %a, %s765 %d = fdiv arcp <4 x float> %x, %m766 ret <4 x float> %d767}768 769; No estimates for f64, so do not convert fabs into an fmul.770 771define double @div_sqrt_fabs_f64(double %x, double %y, double %z) {772; SSE-LABEL: div_sqrt_fabs_f64:773; SSE: # %bb.0:774; SSE-NEXT: andpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1775; SSE-NEXT: sqrtsd %xmm2, %xmm2776; SSE-NEXT: mulsd %xmm2, %xmm1777; SSE-NEXT: divsd %xmm1, %xmm0778; SSE-NEXT: retq779;780; AVX-LABEL: div_sqrt_fabs_f64:781; AVX: # %bb.0:782; AVX-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1783; AVX-NEXT: vsqrtsd %xmm2, %xmm2, %xmm2784; AVX-NEXT: vmulsd %xmm1, %xmm2, %xmm1785; AVX-NEXT: vdivsd %xmm1, %xmm0, %xmm0786; AVX-NEXT: retq787 %s = call fast double @llvm.sqrt.f64(double %z)788 %a = call fast double @llvm.fabs.f64(double %y)789 %m = fmul fast double %s, %a790 %d = fdiv fast double %x, %m791 ret double %d792}793 794; This is a special case for the general pattern above -795; if the sqrt operand is the same as the other mul op,796; then fabs may be omitted.797; x / (y * sqrt(y)) --> x * rsqrt(y*y*y)798 799define float @div_sqrt_f32(float %x, float %y) {800; SSE-LABEL: div_sqrt_f32:801; SSE: # %bb.0:802; SSE-NEXT: movaps %xmm1, %xmm2803; SSE-NEXT: mulss %xmm1, %xmm2804; SSE-NEXT: mulss %xmm1, %xmm2805; SSE-NEXT: xorps %xmm1, %xmm1806; SSE-NEXT: rsqrtss %xmm2, %xmm1807; SSE-NEXT: mulss %xmm1, %xmm2808; SSE-NEXT: mulss %xmm1, %xmm2809; SSE-NEXT: addss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2810; SSE-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1811; SSE-NEXT: mulss %xmm1, %xmm0812; SSE-NEXT: mulss %xmm2, %xmm0813; SSE-NEXT: retq814;815; AVX1-LABEL: div_sqrt_f32:816; AVX1: # %bb.0:817; AVX1-NEXT: vmulss %xmm1, %xmm1, %xmm2818; AVX1-NEXT: vmulss %xmm1, %xmm2, %xmm1819; AVX1-NEXT: vrsqrtss %xmm1, %xmm1, %xmm2820; AVX1-NEXT: vmulss %xmm2, %xmm1, %xmm1821; AVX1-NEXT: vmulss %xmm2, %xmm1, %xmm1822; AVX1-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1823; AVX1-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2824; AVX1-NEXT: vmulss %xmm2, %xmm0, %xmm0825; AVX1-NEXT: vmulss %xmm1, %xmm0, %xmm0826; AVX1-NEXT: retq827;828; AVX512-LABEL: div_sqrt_f32:829; AVX512: # %bb.0:830; AVX512-NEXT: vmulss %xmm1, %xmm1, %xmm2831; AVX512-NEXT: vmulss %xmm1, %xmm2, %xmm1832; AVX512-NEXT: vrsqrtss %xmm1, %xmm1, %xmm2833; AVX512-NEXT: vmulss %xmm2, %xmm1, %xmm1834; AVX512-NEXT: vfmadd213ss {{.*#+}} xmm1 = (xmm2 * xmm1) + mem835; AVX512-NEXT: vmulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2836; AVX512-NEXT: vmulss %xmm2, %xmm0, %xmm0837; AVX512-NEXT: vmulss %xmm1, %xmm0, %xmm0838; AVX512-NEXT: retq839 %s = call fast float @llvm.sqrt.f32(float %y)840 %m = fmul fast float %s, %y841 %d = fdiv fast float %x, %m842 ret float %d843}844 845; This is a special case for the general pattern above -846; if the sqrt operand is the same as the other mul op,847; then fabs may be omitted.848; x / (y * sqrt(y)) --> x * rsqrt(y*y*y)849 850define <4 x float> @div_sqrt_v4f32(<4 x float> %x, <4 x float> %y) {851; SSE-LABEL: div_sqrt_v4f32:852; SSE: # %bb.0:853; SSE-NEXT: movaps %xmm1, %xmm2854; SSE-NEXT: mulps %xmm1, %xmm2855; SSE-NEXT: mulps %xmm1, %xmm2856; SSE-NEXT: rsqrtps %xmm2, %xmm1857; SSE-NEXT: mulps %xmm1, %xmm2858; SSE-NEXT: mulps %xmm1, %xmm2859; SSE-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2860; SSE-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1861; SSE-NEXT: mulps %xmm2, %xmm1862; SSE-NEXT: mulps %xmm1, %xmm0863; SSE-NEXT: retq864;865; AVX1-LABEL: div_sqrt_v4f32:866; AVX1: # %bb.0:867; AVX1-NEXT: vmulps %xmm1, %xmm1, %xmm2868; AVX1-NEXT: vmulps %xmm1, %xmm2, %xmm1869; AVX1-NEXT: vrsqrtps %xmm1, %xmm2870; AVX1-NEXT: vmulps %xmm2, %xmm1, %xmm1871; AVX1-NEXT: vmulps %xmm2, %xmm1, %xmm1872; AVX1-NEXT: vaddps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1873; AVX1-NEXT: vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2874; AVX1-NEXT: vmulps %xmm1, %xmm2, %xmm1875; AVX1-NEXT: vmulps %xmm1, %xmm0, %xmm0876; AVX1-NEXT: retq877;878; AVX512-LABEL: div_sqrt_v4f32:879; AVX512: # %bb.0:880; AVX512-NEXT: vmulps %xmm1, %xmm1, %xmm2881; AVX512-NEXT: vmulps %xmm1, %xmm2, %xmm1882; AVX512-NEXT: vrsqrtps %xmm1, %xmm2883; AVX512-NEXT: vmulps %xmm2, %xmm1, %xmm1884; AVX512-NEXT: vbroadcastss {{.*#+}} xmm3 = [-3.0E+0,-3.0E+0,-3.0E+0,-3.0E+0]885; AVX512-NEXT: vfmadd231ps {{.*#+}} xmm3 = (xmm2 * xmm1) + xmm3886; AVX512-NEXT: vbroadcastss {{.*#+}} xmm1 = [-5.0E-1,-5.0E-1,-5.0E-1,-5.0E-1]887; AVX512-NEXT: vmulps %xmm1, %xmm2, %xmm1888; AVX512-NEXT: vmulps %xmm3, %xmm1, %xmm1889; AVX512-NEXT: vmulps %xmm1, %xmm0, %xmm0890; AVX512-NEXT: retq891 %s = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %y)892 %m = fmul contract reassoc <4 x float> %y, %s893 %d = fdiv contract reassoc arcp <4 x float> %x, %m894 ret <4 x float> %d895}896 897define double @sqrt_fdiv_common_operand(double %x) nounwind {898; SSE-LABEL: sqrt_fdiv_common_operand:899; SSE: # %bb.0:900; SSE-NEXT: sqrtsd %xmm0, %xmm0901; SSE-NEXT: retq902;903; AVX-LABEL: sqrt_fdiv_common_operand:904; AVX: # %bb.0:905; AVX-NEXT: vsqrtsd %xmm0, %xmm0, %xmm0906; AVX-NEXT: retq907 %sqrt = call fast double @llvm.sqrt.f64(double %x)908 %r = fdiv fast double %x, %sqrt909 ret double %r910}911 912define <2 x double> @sqrt_fdiv_common_operand_vec(<2 x double> %x) nounwind {913; SSE-LABEL: sqrt_fdiv_common_operand_vec:914; SSE: # %bb.0:915; SSE-NEXT: sqrtpd %xmm0, %xmm0916; SSE-NEXT: retq917;918; AVX-LABEL: sqrt_fdiv_common_operand_vec:919; AVX: # %bb.0:920; AVX-NEXT: vsqrtpd %xmm0, %xmm0921; AVX-NEXT: retq922 %sqrt = call <2 x double> @llvm.sqrt.v2f64(<2 x double> %x)923 %r = fdiv arcp nsz reassoc <2 x double> %x, %sqrt924 ret <2 x double> %r925}926 927define double @sqrt_fdiv_common_operand_extra_use(double %x, ptr %p) nounwind {928; SSE-LABEL: sqrt_fdiv_common_operand_extra_use:929; SSE: # %bb.0:930; SSE-NEXT: sqrtsd %xmm0, %xmm0931; SSE-NEXT: movsd %xmm0, (%rdi)932; SSE-NEXT: retq933;934; AVX-LABEL: sqrt_fdiv_common_operand_extra_use:935; AVX: # %bb.0:936; AVX-NEXT: vsqrtsd %xmm0, %xmm0, %xmm0937; AVX-NEXT: vmovsd %xmm0, (%rdi)938; AVX-NEXT: retq939 %sqrt = call fast double @llvm.sqrt.f64(double %x)940 store double %sqrt, ptr %p941 %r = fdiv fast double %x, %sqrt942 ret double %r943}944 945define double @sqrt_simplify_before_recip(double %x, ptr %p) nounwind {946; SSE-LABEL: sqrt_simplify_before_recip:947; SSE: # %bb.0:948; SSE-NEXT: sqrtsd %xmm0, %xmm0949; SSE-NEXT: movsd {{.*#+}} xmm1 = [1.0E+0,0.0E+0]950; SSE-NEXT: divsd %xmm0, %xmm1951; SSE-NEXT: movsd %xmm1, (%rdi)952; SSE-NEXT: retq953;954; AVX-LABEL: sqrt_simplify_before_recip:955; AVX: # %bb.0:956; AVX-NEXT: vsqrtsd %xmm0, %xmm0, %xmm0957; AVX-NEXT: vmovsd {{.*#+}} xmm1 = [1.0E+0,0.0E+0]958; AVX-NEXT: vdivsd %xmm0, %xmm1, %xmm1959; AVX-NEXT: vmovsd %xmm1, (%rdi)960; AVX-NEXT: retq961 %sqrt = tail call fast double @llvm.sqrt.f64(double %x)962 %rsqrt = fdiv fast double 1.0, %sqrt963 %sqrt_fast = fdiv fast double %x, %sqrt964 store double %rsqrt, ptr %p, align 8965 ret double %sqrt_fast966}967 968define <2 x double> @sqrt_simplify_before_recip_vec(<2 x double> %x, ptr %p) nounwind {969; SSE-LABEL: sqrt_simplify_before_recip_vec:970; SSE: # %bb.0:971; SSE-NEXT: sqrtpd %xmm0, %xmm0972; SSE-NEXT: movapd {{.*#+}} xmm1 = [1.0E+0,1.0E+0]973; SSE-NEXT: divpd %xmm0, %xmm1974; SSE-NEXT: movupd %xmm1, (%rdi)975; SSE-NEXT: retq976;977; AVX-LABEL: sqrt_simplify_before_recip_vec:978; AVX: # %bb.0:979; AVX-NEXT: vsqrtpd %xmm0, %xmm0980; AVX-NEXT: vmovddup {{.*#+}} xmm1 = [1.0E+0,1.0E+0]981; AVX-NEXT: # xmm1 = mem[0,0]982; AVX-NEXT: vdivpd %xmm0, %xmm1, %xmm1983; AVX-NEXT: vmovupd %xmm1, (%rdi)984; AVX-NEXT: retq985 %sqrt = tail call fast <2 x double> @llvm.sqrt.v2f64(<2 x double> %x)986 %rsqrt = fdiv fast <2 x double> <double 1.0, double 1.0>, %sqrt987 %sqrt_fast = fdiv fast <2 x double> %x, %sqrt988 store <2 x double> %rsqrt, ptr %p, align 8989 ret <2 x double> %sqrt_fast990}991 992define double @sqrt_simplify_before_recip_order(double %x, ptr %p) nounwind {993; SSE-LABEL: sqrt_simplify_before_recip_order:994; SSE: # %bb.0:995; SSE-NEXT: sqrtsd %xmm0, %xmm0996; SSE-NEXT: movsd {{.*#+}} xmm1 = [4.2E+1,0.0E+0]997; SSE-NEXT: divsd %xmm0, %xmm1998; SSE-NEXT: movsd %xmm1, (%rdi)999; SSE-NEXT: retq1000;1001; AVX-LABEL: sqrt_simplify_before_recip_order:1002; AVX: # %bb.0:1003; AVX-NEXT: vsqrtsd %xmm0, %xmm0, %xmm01004; AVX-NEXT: vmovsd {{.*#+}} xmm1 = [4.2E+1,0.0E+0]1005; AVX-NEXT: vdivsd %xmm0, %xmm1, %xmm11006; AVX-NEXT: vmovsd %xmm1, (%rdi)1007; AVX-NEXT: retq1008 %sqrt = tail call fast double @llvm.sqrt.f64(double %x)1009 %sqrt_fast = fdiv fast double %x, %sqrt1010 %rsqrt = fdiv fast double 42.0, %sqrt1011 store double %rsqrt, ptr %p, align 81012 ret double %sqrt_fast1013}1014 1015attributes #0 = { "reciprocal-estimates"="!sqrtf,!vec-sqrtf,!divf,!vec-divf" }1016attributes #1 = { "reciprocal-estimates"="sqrt,vec-sqrt" }1017attributes #2 = { nounwind readnone }1018attributes #3 = { "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="preserve-sign,ieee" }1019attributes #4 = { "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="ieee,preserve-sign" }1020attributes #5 = { "reciprocal-estimates"="all:0" }1021attributes #6 = { "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="preserve-sign,dynamic" }1022 1023; Attributes without1024; TODO: Merge with previous attributes when this attribute can be deleted.1025attributes #7 = { "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="preserve-sign,ieee" } ; #31026attributes #8 = { "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="preserve-sign,dynamic" } ; #61027