512 lines · plain
1; Check that nvvm intrinsics get simplified to target-generic intrinsics where2; possible.3;4; We run this test twice; once with ftz on, and again with ftz off. Behold the5; hackery:6 7; RUN: cat %s > %t.ftz8; RUN: echo 'attributes #0 = { "denormal-fp-math-f32" = "preserve-sign" }' >> %t.ftz9; RUN: opt < %t.ftz -passes=instcombine -mtriple=nvptx64-nvidia-cuda -S | FileCheck %s --check-prefix=CHECK --check-prefix=FTZ10 11; RUN: cat %s > %t.noftz12; RUN: echo 'attributes #0 = { "denormal-fp-math-f32" = "ieee" }' >> %t.noftz13; RUN: opt < %t.noftz -passes=instcombine -mtriple=nvptx64-nvidia-cuda -S | FileCheck %s --check-prefix=CHECK --check-prefix=NOFTZ14 15; We handle nvvm intrinsics with ftz variants as follows:16; - If the module is in ftz mode, the ftz variant is transformed into the17; regular llvm intrinsic, and the non-ftz variant is left alone.18; - If the module is not in ftz mode, it's the reverse: Only the non-ftz19; variant is transformed, and the ftz variant is left alone.20 21; Check NVVM intrinsics that map directly to LLVM target-generic intrinsics.22 23; CHECK-LABEL: @ceil_double24define double @ceil_double(double %a) #0 {25; CHECK: call double @llvm.ceil.f6426 %ret = call double @llvm.nvvm.ceil.d(double %a)27 ret double %ret28}29; CHECK-LABEL: @ceil_float30define float @ceil_float(float %a) #0 {31; NOFTZ: call float @llvm.ceil.f3232; FTZ: call float @llvm.nvvm.ceil.f33 %ret = call float @llvm.nvvm.ceil.f(float %a)34 ret float %ret35}36; CHECK-LABEL: @ceil_float_ftz37define float @ceil_float_ftz(float %a) #0 {38; NOFTZ: call float @llvm.nvvm.ceil.ftz.f39; FTZ: call float @llvm.ceil.f3240 %ret = call float @llvm.nvvm.ceil.ftz.f(float %a)41 ret float %ret42}43 44; CHECK-LABEL: @fabs_double45define double @fabs_double(double %a) #0 {46; CHECK: call double @llvm.nvvm.fabs.f6447 %ret = call double @llvm.nvvm.fabs.d(double %a)48 ret double %ret49}50; CHECK-LABEL: @fabs_float51define float @fabs_float(float %a) #0 {52; CHECK: call float @llvm.nvvm.fabs.f53 %ret = call float @llvm.nvvm.fabs.f(float %a)54 ret float %ret55}56; CHECK-LABEL: @fabs_float_ftz57define float @fabs_float_ftz(float %a) #0 {58; CHECK: call float @llvm.nvvm.fabs.ftz.f59 %ret = call float @llvm.nvvm.fabs.ftz.f(float %a)60 ret float %ret61}62 63; CHECK-LABEL: @floor_double64define double @floor_double(double %a) #0 {65; CHECK: call double @llvm.floor.f6466 %ret = call double @llvm.nvvm.floor.d(double %a)67 ret double %ret68}69; CHECK-LABEL: @floor_float70define float @floor_float(float %a) #0 {71; NOFTZ: call float @llvm.floor.f3272; FTZ: call float @llvm.nvvm.floor.f73 %ret = call float @llvm.nvvm.floor.f(float %a)74 ret float %ret75}76; CHECK-LABEL: @floor_float_ftz77define float @floor_float_ftz(float %a) #0 {78; NOFTZ: call float @llvm.nvvm.floor.ftz.f79; FTZ: call float @llvm.floor.f3280 %ret = call float @llvm.nvvm.floor.ftz.f(float %a)81 ret float %ret82}83 84; CHECK-LABEL: @fma_double85define double @fma_double(double %a, double %b, double %c) #0 {86; CHECK: call double @llvm.fma.f6487 %ret = call double @llvm.nvvm.fma.rn.d(double %a, double %b, double %c)88 ret double %ret89}90; CHECK-LABEL: @fma_float91define float @fma_float(float %a, float %b, float %c) #0 {92; NOFTZ: call float @llvm.fma.f3293; FTZ: call float @llvm.nvvm.fma.rn.f94 %ret = call float @llvm.nvvm.fma.rn.f(float %a, float %b, float %c)95 ret float %ret96}97; CHECK-LABEL: @fma_float_ftz98define float @fma_float_ftz(float %a, float %b, float %c) #0 {99; NOFTZ: call float @llvm.nvvm.fma.rn.ftz.f100; FTZ: call float @llvm.fma.f32101 %ret = call float @llvm.nvvm.fma.rn.ftz.f(float %a, float %b, float %c)102 ret float %ret103}104 105; CHECK-LABEL: @fmax_double106define double @fmax_double(double %a, double %b) #0 {107; CHECK: call double @llvm.maxnum.f64108 %ret = call double @llvm.nvvm.fmax.d(double %a, double %b)109 ret double %ret110}111; CHECK-LABEL: @fmax_float112define float @fmax_float(float %a, float %b) #0 {113; NOFTZ: call float @llvm.maxnum.f32114; FTZ: call float @llvm.nvvm.fmax.f115 %ret = call float @llvm.nvvm.fmax.f(float %a, float %b)116 ret float %ret117}118; CHECK-LABEL: @fmax_float_ftz119define float @fmax_float_ftz(float %a, float %b) #0 {120; NOFTZ: call float @llvm.nvvm.fmax.ftz.f121; FTZ: call float @llvm.maxnum.f32122 %ret = call float @llvm.nvvm.fmax.ftz.f(float %a, float %b)123 ret float %ret124}125 126; CHECK-LABEL: @fmin_double127define double @fmin_double(double %a, double %b) #0 {128; CHECK: call double @llvm.minnum.f64129 %ret = call double @llvm.nvvm.fmin.d(double %a, double %b)130 ret double %ret131}132; CHECK-LABEL: @fmin_float133define float @fmin_float(float %a, float %b) #0 {134; NOFTZ: call float @llvm.minnum.f32135; FTZ: call float @llvm.nvvm.fmin.f136 %ret = call float @llvm.nvvm.fmin.f(float %a, float %b)137 ret float %ret138}139; CHECK-LABEL: @fmin_float_ftz140define float @fmin_float_ftz(float %a, float %b) #0 {141; NOFTZ: call float @llvm.nvvm.fmin.ftz.f142; FTZ: call float @llvm.minnum.f32143 %ret = call float @llvm.nvvm.fmin.ftz.f(float %a, float %b)144 ret float %ret145}146 147; CHECK-LABEL: @round_double148define double @round_double(double %a) #0 {149; CHECK: call double @llvm.nvvm.round.d150 %ret = call double @llvm.nvvm.round.d(double %a)151 ret double %ret152}153; CHECK-LABEL: @round_float154define float @round_float(float %a) #0 {155; CHECK: call float @llvm.nvvm.round.f156 %ret = call float @llvm.nvvm.round.f(float %a)157 ret float %ret158}159; CHECK-LABEL: @round_float_ftz160define float @round_float_ftz(float %a) #0 {161; CHECK: call float @llvm.nvvm.round.ftz.f162 %ret = call float @llvm.nvvm.round.ftz.f(float %a)163 ret float %ret164}165 166; CHECK-LABEL: @trunc_double167define double @trunc_double(double %a) #0 {168; CHECK: call double @llvm.trunc.f64169 %ret = call double @llvm.nvvm.trunc.d(double %a)170 ret double %ret171}172; CHECK-LABEL: @trunc_float173define float @trunc_float(float %a) #0 {174; NOFTZ: call float @llvm.trunc.f32175; FTZ: call float @llvm.nvvm.trunc.f176 %ret = call float @llvm.nvvm.trunc.f(float %a)177 ret float %ret178}179; CHECK-LABEL: @trunc_float_ftz180define float @trunc_float_ftz(float %a) #0 {181; NOFTZ: call float @llvm.nvvm.trunc.ftz.f182; FTZ: call float @llvm.trunc.f32183 %ret = call float @llvm.nvvm.trunc.ftz.f(float %a)184 ret float %ret185}186 187; Check NVVM intrinsics that correspond to LLVM cast operations.188; fp -> integer casts should not be converted, as the semantics189; for NaN/Inf/Overflow inputs are different.190; Only integer -> fp casts should be converted.191 192; CHECK-LABEL: @test_d2i193define i32 @test_d2i(double %a) #0 {194; CHECK: call i32 @llvm.nvvm.d2i.rz(double %a)195; CHECK-NOT: fptosi double %a to i32196 %ret = call i32 @llvm.nvvm.d2i.rz(double %a)197 ret i32 %ret198}199; CHECK-LABEL: @test_f2i200define i32 @test_f2i(float %a) #0 {201; CHECK: call i32 @llvm.nvvm.f2i.rz(float %a)202; CHECK-NOT: fptosi float %a to i32203 %ret = call i32 @llvm.nvvm.f2i.rz(float %a)204 ret i32 %ret205}206; CHECK-LABEL: @test_d2ll207define i64 @test_d2ll(double %a) #0 {208; CHECK: call i64 @llvm.nvvm.d2ll.rz(double %a)209; CHECK-NOT: fptosi double %a to i64210 %ret = call i64 @llvm.nvvm.d2ll.rz(double %a)211 ret i64 %ret212}213; CHECK-LABEL: @test_f2ll214define i64 @test_f2ll(float %a) #0 {215; CHECK: call i64 @llvm.nvvm.f2ll.rz(float %a)216; CHECK-NOT: fptosi float %a to i64217 %ret = call i64 @llvm.nvvm.f2ll.rz(float %a)218 ret i64 %ret219}220; CHECK-LABEL: @test_d2ui221define i32 @test_d2ui(double %a) #0 {222; CHECK: call i32 @llvm.nvvm.d2ui.rz(double %a)223; CHECK-NOT: fptoui double %a to i32224 %ret = call i32 @llvm.nvvm.d2ui.rz(double %a)225 ret i32 %ret226}227; CHECK-LABEL: @test_f2ui228define i32 @test_f2ui(float %a) #0 {229; CHECK: call i32 @llvm.nvvm.f2ui.rz(float %a)230; CHECK-NOT: fptoui float %a to i32231 %ret = call i32 @llvm.nvvm.f2ui.rz(float %a)232 ret i32 %ret233}234; CHECK-LABEL: @test_d2ull235define i64 @test_d2ull(double %a) #0 {236; CHECK: call i64 @llvm.nvvm.d2ull.rz(double %a)237; CHECK-NOT: fptoui double %a to i64238 %ret = call i64 @llvm.nvvm.d2ull.rz(double %a)239 ret i64 %ret240}241; CHECK-LABEL: @test_f2ull242define i64 @test_f2ull(float %a) #0 {243; CHECK: call i64 @llvm.nvvm.f2ull.rz(float %a)244; CHECK-NOT: fptoui float %a to i64245 %ret = call i64 @llvm.nvvm.f2ull.rz(float %a)246 ret i64 %ret247}248 249; CHECK-LABEL: @test_i2d250define double @test_i2d(i32 %a) #0 {251; CHECK: sitofp i32 %a to double252 %ret = call double @llvm.nvvm.i2d.rn(i32 %a)253 ret double %ret254}255; CHECK-LABEL: @test_i2f256define float @test_i2f(i32 %a) #0 {257; CHECK: sitofp i32 %a to float258 %ret = call float @llvm.nvvm.i2f.rn(i32 %a)259 ret float %ret260}261; CHECK-LABEL: @test_ll2d262define double @test_ll2d(i64 %a) #0 {263; CHECK: sitofp i64 %a to double264 %ret = call double @llvm.nvvm.ll2d.rn(i64 %a)265 ret double %ret266}267; CHECK-LABEL: @test_ll2f268define float @test_ll2f(i64 %a) #0 {269; CHECK: sitofp i64 %a to float270 %ret = call float @llvm.nvvm.ll2f.rn(i64 %a)271 ret float %ret272}273; CHECK-LABEL: @test_ui2d274define double @test_ui2d(i32 %a) #0 {275; CHECK: uitofp i32 %a to double276 %ret = call double @llvm.nvvm.ui2d.rn(i32 %a)277 ret double %ret278}279; CHECK-LABEL: @test_ui2f280define float @test_ui2f(i32 %a) #0 {281; CHECK: uitofp i32 %a to float282 %ret = call float @llvm.nvvm.ui2f.rn(i32 %a)283 ret float %ret284}285; CHECK-LABEL: @test_ull2d286define double @test_ull2d(i64 %a) #0 {287; CHECK: uitofp i64 %a to double288 %ret = call double @llvm.nvvm.ull2d.rn(i64 %a)289 ret double %ret290}291; CHECK-LABEL: @test_ull2f292define float @test_ull2f(i64 %a) #0 {293; CHECK: uitofp i64 %a to float294 %ret = call float @llvm.nvvm.ull2f.rn(i64 %a)295 ret float %ret296}297 298; Check NVVM intrinsics that map to LLVM binary operations.299 300; CHECK-LABEL: @test_add_rn_d301define double @test_add_rn_d(double %a, double %b) #0 {302; CHECK: call double @llvm.nvvm.add.rn.d303 %ret = call double @llvm.nvvm.add.rn.d(double %a, double %b)304 ret double %ret305}306; CHECK-LABEL: @test_add_rn_f307define float @test_add_rn_f(float %a, float %b) #0 {308; CHECK: call float @llvm.nvvm.add.rn.f309 %ret = call float @llvm.nvvm.add.rn.f(float %a, float %b)310 ret float %ret311}312; CHECK-LABEL: @test_add_rn_f_ftz313define float @test_add_rn_f_ftz(float %a, float %b) #0 {314; CHECK: call float @llvm.nvvm.add.rn.ftz.f(float %a, float %b)315 %ret = call float @llvm.nvvm.add.rn.ftz.f(float %a, float %b)316 ret float %ret317}318 319; CHECK-LABEL: @test_mul_rn_d320define double @test_mul_rn_d(double %a, double %b) #0 {321; CHECK: call double @llvm.nvvm.mul.rn.d322 %ret = call double @llvm.nvvm.mul.rn.d(double %a, double %b)323 ret double %ret324}325; CHECK-LABEL: @test_mul_rn_f326define float @test_mul_rn_f(float %a, float %b) #0 {327; CHECK: call float @llvm.nvvm.mul.rn.f328 %ret = call float @llvm.nvvm.mul.rn.f(float %a, float %b)329 ret float %ret330}331; CHECK-LABEL: @test_mul_rn_f_ftz332define float @test_mul_rn_f_ftz(float %a, float %b) #0 {333; CHECK: call float @llvm.nvvm.mul.rn.ftz.f(float %a, float %b)334 %ret = call float @llvm.nvvm.mul.rn.ftz.f(float %a, float %b)335 ret float %ret336}337 338; CHECK-LABEL: @test_div_rn_d339define double @test_div_rn_d(double %a, double %b) #0 {340; CHECK: fdiv341 %ret = call double @llvm.nvvm.div.rn.d(double %a, double %b)342 ret double %ret343}344; CHECK-LABEL: @test_div_rn_f345define float @test_div_rn_f(float %a, float %b) #0 {346; CHECK: call float @llvm.nvvm.div.rn.f347 %ret = call float @llvm.nvvm.div.rn.f(float %a, float %b)348 ret float %ret349}350; CHECK-LABEL: @test_div_rn_f_ftz351define float @test_div_rn_f_ftz(float %a, float %b) #0 {352; CHECK: call float @llvm.nvvm.div.rn.ftz.f(float %a, float %b)353 %ret = call float @llvm.nvvm.div.rn.ftz.f(float %a, float %b)354 ret float %ret355}356 357; Check NVVM intrinsics that require us to emit custom IR.358 359; CHECK-LABEL: @test_rcp_rn_f360define float @test_rcp_rn_f(float %a) #0 {361; CHECK: call float @llvm.nvvm.rcp.rn.f362 %ret = call float @llvm.nvvm.rcp.rn.f(float %a)363 ret float %ret364}365; CHECK-LABEL: @test_rcp_rn_f_ftz366define float @test_rcp_rn_f_ftz(float %a) #0 {367; CHECK: call float @llvm.nvvm.rcp.rn.ftz.f(float %a)368 %ret = call float @llvm.nvvm.rcp.rn.ftz.f(float %a)369 ret float %ret370}371 372; CHECK-LABEL: @test_sqrt_rn_d373define double @test_sqrt_rn_d(double %a) #0 {374; CHECK: call double @llvm.sqrt.f64(double %a)375 %ret = call double @llvm.nvvm.sqrt.rn.d(double %a)376 ret double %ret377}378; nvvm.sqrt.f is a special case: It goes to a llvm.sqrt.f379; CHECK-LABEL: @test_sqrt_f380define float @test_sqrt_f(float %a) #0 {381; CHECK: call float @llvm.sqrt.f32(float %a)382 %ret = call float @llvm.nvvm.sqrt.f(float %a)383 ret float %ret384}385; CHECK-LABEL: @test_sqrt_rn_f386define float @test_sqrt_rn_f(float %a) #0 {387; CHECK: call float @llvm.nvvm.sqrt.rn.f388 %ret = call float @llvm.nvvm.sqrt.rn.f(float %a)389 ret float %ret390}391; CHECK-LABEL: @test_sqrt_rn_f_ftz392define float @test_sqrt_rn_f_ftz(float %a) #0 {393; CHECK: call float @llvm.nvvm.sqrt.rn.ftz.f(float %a)394 %ret = call float @llvm.nvvm.sqrt.rn.ftz.f(float %a)395 ret float %ret396}397 398; CHECK-LABEL: @test_fshl_clamp_1399define i32 @test_fshl_clamp_1(i32 %a, i32 %b) {400; CHECK: call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 3)401 %call = call i32 @llvm.nvvm.fshl.clamp.i32(i32 %a, i32 %b, i32 3)402 ret i32 %call403}404 405; CHECK-LABEL: @test_fshl_clamp_2406define i32 @test_fshl_clamp_2(i32 %a, i32 %b) {407; CHECK: ret i32 %b408 %call = call i32 @llvm.nvvm.fshl.clamp.i32(i32 %a, i32 %b, i32 300)409 ret i32 %call410}411 412; CHECK-LABEL: @test_fshl_clamp_3413define i32 @test_fshl_clamp_3(i32 %a, i32 %b, i32 %c) {414; CHECK: call i32 @llvm.nvvm.fshl.clamp.i32(i32 %a, i32 %b, i32 %c)415 %call = call i32 @llvm.nvvm.fshl.clamp.i32(i32 %a, i32 %b, i32 %c)416 ret i32 %call417}418 419; CHECK-LABEL: @test_fshr_clamp_1420define i32 @test_fshr_clamp_1(i32 %a, i32 %b) {421; CHECK: call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 29)422 %call = call i32 @llvm.nvvm.fshr.clamp.i32(i32 %a, i32 %b, i32 3)423 ret i32 %call424}425 426; CHECK-LABEL: @test_fshr_clamp_2427define i32 @test_fshr_clamp_2(i32 %a, i32 %b) {428; CHECK: ret i32 %a429 %call = call i32 @llvm.nvvm.fshr.clamp.i32(i32 %a, i32 %b, i32 300)430 ret i32 %call431}432 433; CHECK-LABEL: @test_fshr_clamp_3434define i32 @test_fshr_clamp_3(i32 %a, i32 %b, i32 %c) {435; CHECK: call i32 @llvm.nvvm.fshr.clamp.i32(i32 %a, i32 %b, i32 %c)436 %call = call i32 @llvm.nvvm.fshr.clamp.i32(i32 %a, i32 %b, i32 %c)437 ret i32 %call438}439 440declare double @llvm.nvvm.add.rn.d(double, double)441declare float @llvm.nvvm.add.rn.f(float, float)442declare float @llvm.nvvm.add.rn.ftz.f(float, float)443declare double @llvm.nvvm.ceil.d(double)444declare float @llvm.nvvm.ceil.f(float)445declare float @llvm.nvvm.ceil.ftz.f(float)446declare float @llvm.nvvm.d2f.rm(double)447declare float @llvm.nvvm.d2f.rm.ftz(double)448declare float @llvm.nvvm.d2f.rp(double)449declare float @llvm.nvvm.d2f.rp.ftz(double)450declare float @llvm.nvvm.d2f.rz(double)451declare float @llvm.nvvm.d2f.rz.ftz(double)452declare i32 @llvm.nvvm.d2i.rz(double)453declare i64 @llvm.nvvm.d2ll.rz(double)454declare i32 @llvm.nvvm.d2ui.rz(double)455declare i64 @llvm.nvvm.d2ull.rz(double)456declare double @llvm.nvvm.div.rn.d(double, double)457declare float @llvm.nvvm.div.rn.f(float, float)458declare float @llvm.nvvm.div.rn.ftz.f(float, float)459declare i16 @llvm.nvvm.f2h.rz(float)460declare i16 @llvm.nvvm.f2h.rz.ftz(float)461declare i32 @llvm.nvvm.f2i.rz(float)462declare i32 @llvm.nvvm.f2i.rz.ftz(float)463declare i64 @llvm.nvvm.f2ll.rz(float)464declare i64 @llvm.nvvm.f2ll.rz.ftz(float)465declare i32 @llvm.nvvm.f2ui.rz(float)466declare i32 @llvm.nvvm.f2ui.rz.ftz(float)467declare i64 @llvm.nvvm.f2ull.rz(float)468declare i64 @llvm.nvvm.f2ull.rz.ftz(float)469declare double @llvm.nvvm.fabs.d(double)470declare float @llvm.nvvm.fabs.f(float)471declare float @llvm.nvvm.fabs.ftz.f(float)472declare double @llvm.nvvm.floor.d(double)473declare float @llvm.nvvm.floor.f(float)474declare float @llvm.nvvm.floor.ftz.f(float)475declare double @llvm.nvvm.fma.rn.d(double, double, double)476declare float @llvm.nvvm.fma.rn.f(float, float, float)477declare float @llvm.nvvm.fma.rn.ftz.f(float, float, float)478declare double @llvm.nvvm.fmax.d(double, double)479declare float @llvm.nvvm.fmax.f(float, float)480declare float @llvm.nvvm.fmax.ftz.f(float, float)481declare double @llvm.nvvm.fmin.d(double, double)482declare float @llvm.nvvm.fmin.f(float, float)483declare float @llvm.nvvm.fmin.ftz.f(float, float)484declare double @llvm.nvvm.i2d.rn(i32)485declare float @llvm.nvvm.i2f.rn(i32)486declare double @llvm.nvvm.ll2d.rn(i64)487declare float @llvm.nvvm.ll2f.rn(i64)488declare double @llvm.nvvm.lohi.i2d(i32, i32)489declare double @llvm.nvvm.mul.rn.d(double, double)490declare float @llvm.nvvm.mul.rn.f(float, float)491declare float @llvm.nvvm.mul.rn.ftz.f(float, float)492declare double @llvm.nvvm.rcp.rm.d(double)493declare double @llvm.nvvm.rcp.rn.d(double)494declare float @llvm.nvvm.rcp.rn.f(float)495declare float @llvm.nvvm.rcp.rn.ftz.f(float)496declare double @llvm.nvvm.round.d(double)497declare float @llvm.nvvm.round.f(float)498declare float @llvm.nvvm.round.ftz.f(float)499declare float @llvm.nvvm.sqrt.f(float)500declare double @llvm.nvvm.sqrt.rn.d(double)501declare float @llvm.nvvm.sqrt.rn.f(float)502declare float @llvm.nvvm.sqrt.rn.ftz.f(float)503declare double @llvm.nvvm.trunc.d(double)504declare float @llvm.nvvm.trunc.f(float)505declare float @llvm.nvvm.trunc.ftz.f(float)506declare double @llvm.nvvm.ui2d.rn(i32)507declare float @llvm.nvvm.ui2f.rn(i32)508declare double @llvm.nvvm.ull2d.rn(i64)509declare float @llvm.nvvm.ull2f.rn(i64)510declare i32 @llvm.nvvm.fshr.clamp.i32(i32, i32, i32)511declare i32 @llvm.nvvm.fshl.clamp.i32(i32, i32, i32)512