373 lines · plain
1//=- AArch64Combine.td - Define AArch64 Combine Rules ---------*-tablegen -*-=//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9//10//===----------------------------------------------------------------------===//11 12include "llvm/Target/GlobalISel/Combine.td"13 14def fconstant_to_constant : GICombineRule<15 (defs root:$root),16 (match (G_FCONSTANT $dst, $src):$root,17 [{ return matchFConstantToConstant(*${root}, MRI); }]),18 (apply [{ applyFConstantToConstant(*${root}); }])>;19 20def icmp_redundant_trunc_matchdata : GIDefMatchData<"Register">;21def icmp_redundant_trunc : GICombineRule<22 (defs root:$root, icmp_redundant_trunc_matchdata:$matchinfo),23 (match (G_ICMP $dst, $tst, $src1, $src2):$root,24 [{ return matchICmpRedundantTrunc(*${root}, MRI, Helper.getValueTracking(), ${matchinfo}); }]),25 (apply [{ applyICmpRedundantTrunc(*${root}, MRI, B, Observer, ${matchinfo}); }])>;26 27// AArch64-specific offset folding for G_GLOBAL_VALUE.28def fold_global_offset_matchdata : GIDefMatchData<"std::pair<uint64_t, uint64_t>">;29def fold_global_offset : GICombineRule<30 (defs root:$root, fold_global_offset_matchdata:$matchinfo),31 (match (wip_match_opcode G_GLOBAL_VALUE):$root,32 [{ return matchFoldGlobalOffset(*${root}, MRI, ${matchinfo}); }]),33 (apply [{ applyFoldGlobalOffset(*${root}, MRI, B, Observer, ${matchinfo});}])34>;35 36// Boolean: 0 = G_ZEXT, 1 = G_SEXT37def ext_addv_to_udot_addv_matchinfo : GIDefMatchData<"std::tuple<Register, Register, bool>">;38let Predicates = [HasDotProd] in {39def ext_addv_to_udot_addv : GICombineRule<40 (defs root:$root, ext_addv_to_udot_addv_matchinfo:$matchinfo),41 (match (wip_match_opcode G_VECREDUCE_ADD):$root,42 [{ return matchExtAddvToDotAddv(*${root}, MRI, STI, ${matchinfo}); }]),43 (apply [{ applyExtAddvToDotAddv(*${root}, MRI, B, Observer, STI, ${matchinfo}); }])44>;45}46 47def ext_uaddv_to_uaddlv_matchinfo : GIDefMatchData<"std::pair<Register, bool>">;48def ext_uaddv_to_uaddlv : GICombineRule<49 (defs root:$root, ext_uaddv_to_uaddlv_matchinfo:$matchinfo),50 (match (wip_match_opcode G_VECREDUCE_ADD):$root,51 [{ return matchExtUaddvToUaddlv(*${root}, MRI, ${matchinfo}); }]),52 (apply [{ applyExtUaddvToUaddlv(*${root}, MRI, B, Observer, ${matchinfo}); }])53>;54 55class push_opcode_through_ext<Instruction opcode, Instruction extOpcode> : GICombineRule <56 (defs root:$root),57 (match (extOpcode $ext1, $src1):$ExtMI,58 (extOpcode $ext2, $src2),59 (opcode $dst, $ext1, $ext2):$root,60 [{ return matchPushAddSubExt(*${root}, MRI, ${dst}.getReg(), ${src1}.getReg(), ${src2}.getReg()); }]),61 (apply [{ applyPushAddSubExt(*${root}, MRI, B, ${ExtMI}->getOpcode() == TargetOpcode::G_SEXT, ${dst}.getReg(), ${src1}.getReg(), ${src2}.getReg()); }])>;62 63def push_sub_through_zext : push_opcode_through_ext<G_SUB, G_ZEXT>;64def push_add_through_zext : push_opcode_through_ext<G_ADD, G_ZEXT>;65def push_mul_through_zext : push_opcode_through_ext<G_MUL, G_ZEXT>;66def push_sub_through_sext : push_opcode_through_ext<G_SUB, G_SEXT>;67def push_add_through_sext : push_opcode_through_ext<G_ADD, G_SEXT>;68def push_mul_through_sext : push_opcode_through_ext<G_MUL, G_SEXT>;69 70def AArch64PreLegalizerCombiner: GICombiner<71 "AArch64PreLegalizerCombinerImpl", [all_combines,72 icmp_redundant_trunc,73 fold_global_offset,74 ext_addv_to_udot_addv,75 ext_uaddv_to_uaddlv,76 push_sub_through_zext,77 push_add_through_zext,78 push_mul_through_zext,79 push_sub_through_sext,80 push_add_through_sext,81 push_mul_through_sext]> {82 let CombineAllMethodName = "tryCombineAllImpl";83}84 85def AArch64O0PreLegalizerCombiner: GICombiner<86 "AArch64O0PreLegalizerCombinerImpl", [optnone_combines]> {87 let CombineAllMethodName = "tryCombineAllImpl";88}89 90// Matchdata for combines which replace a G_SHUFFLE_VECTOR with a91// target-specific opcode.92def shuffle_matchdata : GIDefMatchData<"ShuffleVectorPseudo">;93 94def rev : GICombineRule<95 (defs root:$root, shuffle_matchdata:$matchinfo),96 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,97 [{ return matchREV(*${root}, MRI, ${matchinfo}); }]),98 (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }])99>;100 101def zip : GICombineRule<102 (defs root:$root, shuffle_matchdata:$matchinfo),103 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,104 [{ return matchZip(*${root}, MRI, ${matchinfo}); }]),105 (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }])106>;107 108def uzp : GICombineRule<109 (defs root:$root, shuffle_matchdata:$matchinfo),110 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,111 [{ return matchUZP(*${root}, MRI, ${matchinfo}); }]),112 (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }])113>;114 115def dup: GICombineRule <116 (defs root:$root, shuffle_matchdata:$matchinfo),117 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,118 [{ return matchDup(*${root}, MRI, ${matchinfo}); }]),119 (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }])120>;121 122def trn : GICombineRule<123 (defs root:$root, shuffle_matchdata:$matchinfo),124 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,125 [{ return matchTRN(*${root}, MRI, ${matchinfo}); }]),126 (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }])127>;128 129def ext: GICombineRule <130 (defs root:$root, shuffle_matchdata:$matchinfo),131 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,132 [{ return matchEXT(*${root}, MRI, ${matchinfo}); }]),133 (apply [{ applyEXT(*${root}, ${matchinfo}); }])134>;135 136def fullrev: GICombineRule <137 (defs root:$root, shuffle_matchdata:$matchinfo),138 (match (G_IMPLICIT_DEF $src2),139 (G_SHUFFLE_VECTOR $src, $src1, $src2, $mask):$root,140 [{ return ShuffleVectorInst::isReverseMask(${mask}.getShuffleMask(),141 ${mask}.getShuffleMask().size()); }]),142 (apply [{ applyFullRev(*${root}, MRI); }])143>;144 145def insertelt_nonconst: GICombineRule <146 (defs root:$root, shuffle_matchdata:$matchinfo),147 (match (wip_match_opcode G_INSERT_VECTOR_ELT):$root,148 [{ return matchNonConstInsert(*${root}, MRI); }]),149 (apply [{ applyNonConstInsert(*${root}, MRI, B); }])150>;151 152def shuf_to_ins_matchdata : GIDefMatchData<"std::tuple<Register, int, Register, int>">;153def shuf_to_ins: GICombineRule <154 (defs root:$root, shuf_to_ins_matchdata:$matchinfo),155 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,156 [{ return matchINS(*${root}, MRI, ${matchinfo}); }]),157 (apply [{ applyINS(*${root}, MRI, B, ${matchinfo}); }])158>;159 160def vashr_vlshr_imm_matchdata : GIDefMatchData<"int64_t">;161def vashr_vlshr_imm : GICombineRule<162 (defs root:$root, vashr_vlshr_imm_matchdata:$matchinfo),163 (match (wip_match_opcode G_ASHR, G_LSHR):$root,164 [{ return matchVAshrLshrImm(*${root}, MRI, ${matchinfo}); }]),165 (apply [{ applyVAshrLshrImm(*${root}, MRI, ${matchinfo}); }])166>;167 168def form_duplane_matchdata :169 GIDefMatchData<"std::pair<unsigned, int>">;170def form_duplane : GICombineRule <171 (defs root:$root, form_duplane_matchdata:$matchinfo),172 (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,173 [{ return matchDupLane(*${root}, MRI, ${matchinfo}); }]),174 (apply [{ applyDupLane(*${root}, MRI, B, ${matchinfo}); }])175>;176 177// Clean up G_UNMERGE(G_DUPLANE16) -> G_DUPLANE16178class unmerge_duplane<Instruction Op> : GICombineRule <179 (defs root:$root),180 (match (Op $a, $src, $c),181 (G_UNMERGE_VALUES $d1, $d2, $a):$root,182 [{ return MRI.getType(${d1}.getReg()).getSizeInBits() == 64; }]),183 (apply (GIReplaceReg $d2, $d1), (Op $d1, $src, $c))184>;185def unmerge_duplane8 : unmerge_duplane<G_DUPLANE8>;186def unmerge_duplane16 : unmerge_duplane<G_DUPLANE16>;187def unmerge_duplane32 : unmerge_duplane<G_DUPLANE32>;188// G_DUPLANE64 is not included as the result in scalar.189def unmerge_duplanes : GICombineGroup<[unmerge_duplane8, unmerge_duplane16,190 unmerge_duplane32]>;191 192def shuffle_vector_lowering : GICombineGroup<[dup, form_duplane, rev, ext, zip,193 uzp, trn, fullrev, shuf_to_ins]>;194 195// Turn G_UNMERGE_VALUES -> G_EXTRACT_VECTOR_ELT's196def vector_unmerge_lowering : GICombineRule <197 (defs root:$root),198 (match (wip_match_opcode G_UNMERGE_VALUES):$root,199 [{ return matchScalarizeVectorUnmerge(*${root}, MRI); }]),200 (apply [{ applyScalarizeVectorUnmerge(*${root}, MRI, B); }])201>;202 203def adjust_icmp_imm_matchdata :204 GIDefMatchData<"std::pair<uint64_t, CmpInst::Predicate>">;205def adjust_icmp_imm : GICombineRule <206 (defs root:$root, adjust_icmp_imm_matchdata:$matchinfo),207 (match (G_ICMP $dst, $tst, $src1, $src2):$root,208 [{ return matchAdjustICmpImmAndPred(*${root}, MRI, ${matchinfo}); }]),209 (apply [{ applyAdjustICmpImmAndPred(*${root}, ${matchinfo}, B, Observer); }])210>;211 212def swap_icmp_operands : GICombineRule <213 (defs root:$root),214 (match (G_ICMP $dst, $tst, $src1, $src2):$root,215 [{ return trySwapICmpOperands(*${root}, MRI); }]),216 (apply [{ applySwapICmpOperands(*${root}, Observer); }])217>;218 219def icmp_lowering : GICombineGroup<[adjust_icmp_imm, swap_icmp_operands]>;220 221def extractvecelt_pairwise_add_matchdata : GIDefMatchData<"std::tuple<unsigned, LLT, Register>">;222def extractvecelt_pairwise_add : GICombineRule<223 (defs root:$root, extractvecelt_pairwise_add_matchdata:$matchinfo),224 (match (wip_match_opcode G_EXTRACT_VECTOR_ELT):$root,225 [{ return matchExtractVecEltPairwiseAdd(*${root}, MRI, ${matchinfo}); }]),226 (apply [{ applyExtractVecEltPairwiseAdd(*${root}, MRI, B, ${matchinfo}); }])227>;228 229def mul_const_matchdata : GIDefMatchData<"std::function<void(MachineIRBuilder&, Register)>">;230def mul_const : GICombineRule<231 (defs root:$root, mul_const_matchdata:$matchinfo),232 (match (wip_match_opcode G_MUL):$root,233 [{ return matchAArch64MulConstCombine(*${root}, MRI, ${matchinfo}); }]),234 (apply [{ applyAArch64MulConstCombine(*${root}, MRI, B, ${matchinfo}); }])235>;236 237def mull_matchdata : GIDefMatchData<"std::tuple<bool, Register, Register>">;238def extmultomull : GICombineRule<239 (defs root:$root, mull_matchdata:$matchinfo),240 (match (G_MUL $dst, $src1, $src2):$root,241 [{ return matchExtMulToMULL(*${root}, MRI, VT, ${matchinfo}); }]),242 (apply [{ applyExtMulToMULL(*${root}, MRI, B, Observer, ${matchinfo}); }])243>;244 245def lower_mulv2s64 : GICombineRule<246 (defs root:$root),247 (match (G_MUL $dst, $src1, $src2):$root,248 [{ return matchMulv2s64(*${root}, MRI); }]),249 (apply [{ applyMulv2s64(*${root}, MRI, B, Observer); }])250>;251 252def build_vector_to_dup : GICombineRule<253 (defs root:$root),254 (match (wip_match_opcode G_BUILD_VECTOR):$root,255 [{ return matchBuildVectorToDup(*${root}, MRI); }]),256 (apply [{ applyBuildVectorToDup(*${root}, MRI, B); }])257>;258 259def build_vector_to_vector_insert : GICombineRule<260 (defs root:$root),261 (match (G_BUILD_VECTOR $dst, GIVariadic<>:$unused):$root,262 [{ return matchLowerBuildToInsertVecElt(*${root}, MRI); }]),263 (apply [{ applyLowerBuildToInsertVecElt(*${root}, MRI, B); }])264>;265 266def build_vector_lowering : GICombineGroup<[build_vector_to_dup,267 build_vector_to_vector_insert]>;268 269def lower_vector_fcmp : GICombineRule<270 (defs root:$root),271 (match (G_FCMP $dst, $tst, $src1, $src2):$root,272 [{ return matchLowerVectorFCMP(*${root}, MRI, B); }]),273 (apply [{ applyLowerVectorFCMP(*${root}, MRI, B); }])>;274 275def form_truncstore_matchdata : GIDefMatchData<"Register">;276def form_truncstore : GICombineRule<277 (defs root:$root, form_truncstore_matchdata:$matchinfo),278 (match (G_STORE $src, $addr):$root,279 [{ return matchFormTruncstore(*${root}, MRI, ${matchinfo}); }]),280 (apply [{ applyFormTruncstore(*${root}, MRI, B, Observer, ${matchinfo}); }])281>;282 283def fold_merge_to_zext : GICombineRule<284 (defs root:$d),285 (match (wip_match_opcode G_MERGE_VALUES):$d,286 [{ return matchFoldMergeToZext(*${d}, MRI); }]),287 (apply [{ applyFoldMergeToZext(*${d}, MRI, B, Observer); }])288>;289 290def mutate_anyext_to_zext : GICombineRule<291 (defs root:$d),292 (match (wip_match_opcode G_ANYEXT):$d,293 [{ return matchMutateAnyExtToZExt(*${d}, MRI); }]),294 (apply [{ applyMutateAnyExtToZExt(*${d}, MRI, B, Observer); }])295>;296 297def split_store_zero_128 : GICombineRule<298 (defs root:$d),299 (match (G_STORE $src, $addr):$d,300 [{ return matchSplitStoreZero128(*${d}, MRI); }]),301 (apply [{ applySplitStoreZero128(*${d}, MRI, B, Observer); }])302>;303 304def vector_sext_inreg_to_shift : GICombineRule<305 (defs root:$d),306 (match (wip_match_opcode G_SEXT_INREG):$d,307 [{ return matchVectorSextInReg(*${d}, MRI); }]),308 (apply [{ applyVectorSextInReg(*${d}, MRI, B, Observer); }])309>;310 311def unmerge_ext_to_unmerge_matchdata : GIDefMatchData<"Register">;312def unmerge_ext_to_unmerge : GICombineRule<313 (defs root:$d, unmerge_ext_to_unmerge_matchdata:$matchinfo),314 (match (wip_match_opcode G_UNMERGE_VALUES):$d,315 [{ return matchUnmergeExtToUnmerge(*${d}, MRI, ${matchinfo}); }]),316 (apply [{ applyUnmergeExtToUnmerge(*${d}, MRI, B, Observer, ${matchinfo}); }])317>;318 319def regtriple_matchdata : GIDefMatchData<"std::tuple<Register, Register, Register>">;320def or_to_bsp: GICombineRule <321 (defs root:$root, regtriple_matchdata:$matchinfo),322 (match (G_OR $dst, $src1, $src2):$root,323 [{ return matchOrToBSP(*${root}, MRI, ${matchinfo}); }]),324 (apply [{ applyOrToBSP(*${root}, MRI, B, ${matchinfo}); }])325>;326 327// Combines Mul(And(Srl(X, 15), 0x10001), 0xffff) into CMLTz328def combine_mul_cmlt : GICombineRule<329 (defs root:$root, register_matchinfo:$matchinfo),330 (match (wip_match_opcode G_MUL):$root,331 [{ return matchCombineMulCMLT(*${root}, MRI, ${matchinfo}); }]),332 (apply [{ applyCombineMulCMLT(*${root}, MRI, B, ${matchinfo}); }])333>;334 335// Post-legalization combines which should happen at all optimization levels.336// (E.g. ones that facilitate matching for the selector) For example, matching337// pseudos.338def AArch64PostLegalizerLowering339 : GICombiner<"AArch64PostLegalizerLoweringImpl",340 [shuffle_vector_lowering, vashr_vlshr_imm,341 icmp_lowering, build_vector_lowering,342 lower_vector_fcmp, form_truncstore, fconstant_to_constant,343 vector_sext_inreg_to_shift,344 unmerge_ext_to_unmerge, lower_mulv2s64,345 vector_unmerge_lowering, insertelt_nonconst,346 unmerge_duplanes]> {347}348 349// Post-legalization combines which are primarily optimizations.350def AArch64PostLegalizerCombiner351 : GICombiner<"AArch64PostLegalizerCombinerImpl",352 [copy_prop, cast_of_cast_combines, constant_fold_fp_ops, 353 buildvector_of_truncate, integer_of_truncate,354 mutate_anyext_to_zext, combines_for_extload, 355 combine_indexed_load_store, sext_trunc_sextload,356 hoist_logic_op_with_same_opcode_hands,357 redundant_and, xor_of_and_with_same_reg,358 extractvecelt_pairwise_add, redundant_or,359 mul_const, redundant_sext_inreg,360 form_bitfield_extract, rotate_out_of_range,361 icmp_to_true_false_known_bits, overflow_combines,362 select_combines, fold_merge_to_zext, merge_combines,363 constant_fold_binops, identity_combines,364 ptr_add_immed_chain, overlapping_and,365 split_store_zero_128, undef_combines,366 select_to_minmax, or_to_bsp, combine_concat_vector,367 commute_constant_to_rhs, extract_vec_elt_combines,368 push_freeze_to_prevent_poison_from_propagating,369 combine_mul_cmlt, combine_use_vector_truncate,370 extmultomull, truncsat_combines, lshr_of_trunc_of_lshr,371 funnel_shift_from_or_shift_constants_are_legal]> {372}373