244 lines · plain
1//=- AMDGPUCombine.td - Define AMDGPU Combine Rules ----------*- tablegen -*-=//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9include "llvm/Target/GlobalISel/Combine.td"10 11// TODO: This really belongs after legalization after scalarization.12 13def fmin_fmax_legacy_matchdata : GIDefMatchData<"FMinFMaxLegacyInfo">;14 15let Predicates = [HasFminFmaxLegacy] in16def fcmp_select_to_fmin_fmax_legacy : GICombineRule<17 (defs root:$select, fmin_fmax_legacy_matchdata:$matchinfo),18 (match (G_FCMP $cond, $pred, $lhs, $rhs):$fcmp,19 (G_SELECT f32:$dst, $cond, $true, $false):$select,20 [{ return matchFMinFMaxLegacy(*${select}, *${fcmp}, ${matchinfo}); }]),21 (apply [{ applySelectFCmpToFMinFMaxLegacy(*${select}, ${matchinfo}); }])>;22 23 24def uchar_to_float : GICombineRule<25 (defs root:$itofp),26 (match (wip_match_opcode G_UITOFP, G_SITOFP):$itofp,27 [{ return matchUCharToFloat(*${itofp}); }]),28 (apply [{ applyUCharToFloat(*${itofp}); }])>;29 30 31def rcp_sqrt_to_rsq : GICombineRule<32 (defs root:$rcp, build_fn_matchinfo:$matchinfo),33 (match (wip_match_opcode G_INTRINSIC, G_FSQRT):$rcp,34 [{ return matchRcpSqrtToRsq(*${rcp}, ${matchinfo}); }]),35 (apply [{ Helper.applyBuildFn(*${rcp}, ${matchinfo}); }])>;36 37def fdiv_by_sqrt_to_rsq_f16 : GICombineRule<38 (defs root:$root),39 (match (G_FSQRT f16:$sqrt, $x, (MIFlags FmContract)),40 (G_FDIV f16:$dst, $y, $sqrt, (MIFlags FmContract)):$root,41 [{ return matchFDivSqrtToRsqF16(*${root}); }]),42 (apply [{ applyFDivSqrtToRsqF16(*${root}, ${x}.getReg()); }])>;43 44def cvt_f32_ubyteN_matchdata : GIDefMatchData<"CvtF32UByteMatchInfo">;45 46def cvt_f32_ubyteN : GICombineRule<47 (defs root:$cvt_f32_ubyteN, cvt_f32_ubyteN_matchdata:$matchinfo),48 (match (wip_match_opcode G_AMDGPU_CVT_F32_UBYTE0,49 G_AMDGPU_CVT_F32_UBYTE1,50 G_AMDGPU_CVT_F32_UBYTE2,51 G_AMDGPU_CVT_F32_UBYTE3):$cvt_f32_ubyteN,52 [{ return matchCvtF32UByteN(*${cvt_f32_ubyteN}, ${matchinfo}); }]),53 (apply [{ applyCvtF32UByteN(*${cvt_f32_ubyteN}, ${matchinfo}); }])>;54 55def clamp_i64_to_i16_matchdata : GIDefMatchData<"ClampI64ToI16MatchInfo">;56 57def clamp_i64_to_i16 : GICombineRule<58 (defs root:$clamp_i64_to_i16, clamp_i64_to_i16_matchdata:$matchinfo),59 (match (wip_match_opcode G_TRUNC):$clamp_i64_to_i16,60 [{ return matchClampI64ToI16(*${clamp_i64_to_i16}, MRI, MF, ${matchinfo}); }]),61 (apply [{ applyClampI64ToI16(*${clamp_i64_to_i16}, ${matchinfo}); }])>;62 63def med3_matchdata : GIDefMatchData<"Med3MatchInfo">;64 65def int_minmax_to_med3 : GICombineRule<66 (defs root:$min_or_max, med3_matchdata:$matchinfo),67 (match (wip_match_opcode G_SMAX,68 G_SMIN,69 G_UMAX,70 G_UMIN):$min_or_max,71 [{ return matchIntMinMaxToMed3(*${min_or_max}, ${matchinfo}); }]),72 (apply [{ applyMed3(*${min_or_max}, ${matchinfo}); }])>;73 74let Predicates = [Predicate<"Subtarget->d16PreservesUnusedBits()">] in75def d16_load : GICombineRule<76 (defs root:$bitcast),77 (combine (G_BITCAST $dst, $src):$bitcast,78 [{ return combineD16Load(*${bitcast} ); }])>;79 80def fp_minmax_to_med3 : GICombineRule<81 (defs root:$min_or_max, med3_matchdata:$matchinfo),82 (match (wip_match_opcode G_FMAXNUM,83 G_FMINNUM,84 G_FMAXNUM_IEEE,85 G_FMINNUM_IEEE):$min_or_max,86 [{ return matchFPMinMaxToMed3(*${min_or_max}, ${matchinfo}); }]),87 (apply [{ applyMed3(*${min_or_max}, ${matchinfo}); }])>;88 89def fp_minmax_to_clamp : GICombineRule<90 (defs root:$min_or_max, register_matchinfo:$matchinfo),91 (match (wip_match_opcode G_FMAXNUM,92 G_FMINNUM,93 G_FMAXNUM_IEEE,94 G_FMINNUM_IEEE):$min_or_max,95 [{ return matchFPMinMaxToClamp(*${min_or_max}, ${matchinfo}); }]),96 (apply [{ applyClamp(*${min_or_max}, ${matchinfo}); }])>;97 98def fmed3_intrinsic_to_clamp : GICombineRule<99 (defs root:$fmed3, register_matchinfo:$matchinfo),100 (match (wip_match_opcode G_AMDGPU_FMED3):$fmed3,101 [{ return matchFPMed3ToClamp(*${fmed3}, ${matchinfo}); }]),102 (apply [{ applyClamp(*${fmed3}, ${matchinfo}); }])>;103 104def remove_fcanonicalize : GICombineRule<105 (defs root:$fcanonicalize, register_matchinfo:$matchinfo),106 (match (wip_match_opcode G_FCANONICALIZE):$fcanonicalize,107 [{ return matchRemoveFcanonicalize(*${fcanonicalize}, ${matchinfo}); }]),108 (apply [{ Helper.replaceSingleDefInstWithReg(*${fcanonicalize}, ${matchinfo}); }])>;109 110def foldable_fneg_matchdata : GIDefMatchData<"MachineInstr *">;111 112def foldable_fneg : GICombineRule<113 (defs root:$ffn, foldable_fneg_matchdata:$matchinfo),114 (match (wip_match_opcode G_FNEG):$ffn,115 [{ return Helper.matchFoldableFneg(*${ffn}, ${matchinfo}); }]),116 (apply [{ Helper.applyFoldableFneg(*${ffn}, ${matchinfo}); }])>;117 118// Detects s_mul_u64 instructions whose higher bits are zero/sign extended.119def smulu64 : GICombineRule<120 (defs root:$smul, unsigned_matchinfo:$matchinfo),121 (match (wip_match_opcode G_MUL):$smul,122 [{ return matchCombine_s_mul_u64(*${smul}, ${matchinfo}); }]),123 (apply [{ Helper.replaceOpcodeWith(*${smul}, ${matchinfo}); }])>;124 125def sign_exension_in_reg_matchdata : GIDefMatchData<"std::pair<MachineInstr *, unsigned>">;126 127def sign_extension_in_reg : GICombineRule<128 (defs root:$sign_inreg, sign_exension_in_reg_matchdata:$matchinfo),129 (match (wip_match_opcode G_SEXT_INREG):$sign_inreg,130 [{ return matchCombineSignExtendInReg(*${sign_inreg}, ${matchinfo}); }]),131 (apply [{ applyCombineSignExtendInReg(*${sign_inreg}, ${matchinfo}); }])>;132 133// Do the following combines :134// fmul x, select(y, A, B) -> fldexp (x, select i32 (y, a, b))135// fmul x, select(y, -A, -B) -> fldexp ((fneg x), select i32 (y, a, b))136def combine_fmul_with_select_to_fldexp : GICombineRule<137 (defs root:$root, build_fn_matchinfo:$matchinfo),138 (match (G_FMUL $dst, $x, $select):$root,139 (G_SELECT $select, $y, $A, $B):$sel,140 [{ return Helper.matchCombineFmulWithSelectToFldexp(*${root}, *${sel}, ${matchinfo}); }]),141 (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;142 143// (shift x, (zext amt)) -> (shift x, (and (anyext amt), mask)144//145// The pattern is longer, but is better for matching during ISel.146class canonicalize_zext_shift_amt<Instruction opc> : GICombineRule<147 (defs root:$dst),148 (match (G_ZEXT $amt, $amtsrc):$zext,149 (opc $dst, $src, $amt):$shift),150 (apply [{ applyCanonicalizeZextShiftAmt(*${shift}, *${zext}); }])>;151 152def canonicalize_zext_lshr : canonicalize_zext_shift_amt<G_LSHR>;153def canonicalize_zext_ashr : canonicalize_zext_shift_amt<G_ASHR>;154def canonicalize_zext_shl : canonicalize_zext_shift_amt<G_SHL>;155 156def zext_of_shift_amount_combines : GICombineGroup<[157 canonicalize_zext_lshr, canonicalize_zext_ashr, canonicalize_zext_shl158]>;159 160// (and/or i64:x, i64:y) -> i64:(merge (and/or lo_32(x), lo_32(y)), (and/or hi_32(x), hi_32(y)))161// when either x or y is all ones in low or high parts162class combine_binop_s64_with_s32_mask<Instruction opcode> : GICombineRule<163 (defs root:$dst),164 (match (opcode $dst, i64:$x, i64:$y):$dst,165 [{ return Helper.matchConstantIs32BitMask(${x}.getReg()) ||166 Helper.matchConstantIs32BitMask(${y}.getReg()); }]),167 (apply (G_UNMERGE_VALUES i32:$x_lo, i32:$x_hi, $x),168 (G_UNMERGE_VALUES i32:$y_lo, i32:$y_hi, $y),169 (opcode i32:$lo, $x_lo, $y_lo),170 (opcode i32:$hi, $x_hi, $y_hi),171 (G_MERGE_VALUES $dst, $lo, $hi))>;172 173def combine_or_s64_with_s32_mask : combine_binop_s64_with_s32_mask<G_OR>;174def combine_and_s64_with_s32_mask : combine_binop_s64_with_s32_mask<G_AND>;175def binop_s64_with_s32_mask_combines : GICombineGroup<[176 combine_or_s64_with_s32_mask, combine_and_s64_with_s32_mask177]>;178 179// (or i64:x, (zext i32:y)) -> i64:(merge (or lo_32(x), i32:y), hi_32(x))180// (or (zext i32:y), i64:x) -> i64:(merge (or lo_32(x), i32:y), hi_32(x))181def or_s64_zext_s32_frag : GICombinePatFrag<(outs root:$dst), (ins $src_s64, $src_s32),182 [(pattern (G_OR $dst, i64:$src_s64, i64:$zext_val), (G_ZEXT i64:$zext_val, i32:$src_s32)),183 (pattern (G_OR $dst, i64:$zext_val, i64:$src_s64), (G_ZEXT i64:$zext_val, i32:$src_s32))]>;184 185def combine_or_s64_s32 : GICombineRule<186 (defs root:$dst),187 (match (or_s64_zext_s32_frag $dst, i64:$x, i32:$y):$dst),188 (apply (G_UNMERGE_VALUES $x_lo, $x_hi, $x),189 (G_OR $or, $x_lo, $y),190 (G_MERGE_VALUES $dst, $or, $x_hi))>;191 192let Predicates = [Has16BitInsts, NotHasMed3_16] in {193// For gfx8, expand f16-fmed3-as-f32 into a min/max f16 sequence. This194// saves one instruction compared to the promotion.195//196// FIXME: Should have ComplexPattern like in/out matchers197//198// FIXME: We should be able to match either G_AMDGPU_FMED3 or199// G_INTRINSIC @llvm.amdgcn.fmed3. Currently the legalizer will200// replace the intrinsic with G_AMDGPU_FMED3 since we can't write a201// pattern to match it.202def expand_promoted_fmed3 : GICombineRule<203 (defs root:$fptrunc_dst),204 (match (G_FPTRUNC $fptrunc_dst, $fmed3_dst):$fptrunc,205 (G_AMDGPU_FMED3 $fmed3_dst, $src0, $src1, $src2),206 [{ return Helper.matchExpandPromotedF16FMed3(*${fptrunc}, ${src0}.getReg(), ${src1}.getReg(), ${src2}.getReg()); }]),207 (apply [{ Helper.applyExpandPromotedF16FMed3(*${fptrunc}, ${src0}.getReg(), ${src1}.getReg(), ${src2}.getReg()); }])208>;209 210} // End Predicates = [NotHasMed3_16]211 212// Combines which should only apply on SI/CI213def gfx6gfx7_combines : GICombineGroup<[fcmp_select_to_fmin_fmax_legacy]>;214 215// Combines which should only apply on VI216def gfx8_combines : GICombineGroup<[expand_promoted_fmed3]>;217 218def AMDGPUPreLegalizerCombiner: GICombiner<219 "AMDGPUPreLegalizerCombinerImpl",220 [all_combines, combine_fmul_with_select_to_fldexp, clamp_i64_to_i16,221 foldable_fneg, combine_shuffle_vector_to_build_vector,222 binop_s64_with_s32_mask_combines, combine_or_s64_s32]> {223 let CombineAllMethodName = "tryCombineAllImpl";224}225 226def AMDGPUPostLegalizerCombiner: GICombiner<227 "AMDGPUPostLegalizerCombinerImpl",228 [all_combines, gfx6gfx7_combines, gfx8_combines, combine_fmul_with_select_to_fldexp,229 uchar_to_float, cvt_f32_ubyteN, remove_fcanonicalize, foldable_fneg,230 rcp_sqrt_to_rsq, fdiv_by_sqrt_to_rsq_f16, sign_extension_in_reg, smulu64,231 binop_s64_with_s32_mask_combines, combine_or_s64_s32]> {232 let CombineAllMethodName = "tryCombineAllImpl";233}234 235def AMDGPURegBankCombiner : GICombiner<236 "AMDGPURegBankCombinerImpl",237 [unmerge_merge, unmerge_cst, unmerge_undef,238 zext_trunc_fold, int_minmax_to_med3, ptr_add_immed_chain,239 fp_minmax_to_clamp, fp_minmax_to_med3, fmed3_intrinsic_to_clamp,240 identity_combines, redundant_and, constant_fold_cast_op,241 cast_of_cast_combines, sext_trunc, zext_of_shift_amount_combines,242 d16_load]> {243}244