1316 lines · cpp
1//=== AArch64PostLegalizerLowering.cpp --------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// Post-legalization lowering for instructions.11///12/// This is used to offload pattern matching from the selector.13///14/// For example, this combiner will notice that a G_SHUFFLE_VECTOR is actually15/// a G_ZIP, G_UZP, etc.16///17/// General optimization combines should be handled by either the18/// AArch64PostLegalizerCombiner or the AArch64PreLegalizerCombiner.19///20//===----------------------------------------------------------------------===//21 22#include "AArch64ExpandImm.h"23#include "AArch64GlobalISelUtils.h"24#include "AArch64PerfectShuffle.h"25#include "AArch64Subtarget.h"26#include "GISel/AArch64LegalizerInfo.h"27#include "MCTargetDesc/AArch64MCTargetDesc.h"28#include "Utils/AArch64BaseInfo.h"29#include "llvm/CodeGen/GlobalISel/Combiner.h"30#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"31#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"32#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"33#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"34#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"35#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"36#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"37#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"38#include "llvm/CodeGen/GlobalISel/Utils.h"39#include "llvm/CodeGen/MachineFrameInfo.h"40#include "llvm/CodeGen/MachineFunctionPass.h"41#include "llvm/CodeGen/MachineInstrBuilder.h"42#include "llvm/CodeGen/MachineRegisterInfo.h"43#include "llvm/CodeGen/TargetOpcodes.h"44#include "llvm/CodeGen/TargetPassConfig.h"45#include "llvm/IR/InstrTypes.h"46#include "llvm/Support/ErrorHandling.h"47#include <optional>48 49#define GET_GICOMBINER_DEPS50#include "AArch64GenPostLegalizeGILowering.inc"51#undef GET_GICOMBINER_DEPS52 53#define DEBUG_TYPE "aarch64-postlegalizer-lowering"54 55using namespace llvm;56using namespace MIPatternMatch;57using namespace AArch64GISelUtils;58 59namespace {60 61#define GET_GICOMBINER_TYPES62#include "AArch64GenPostLegalizeGILowering.inc"63#undef GET_GICOMBINER_TYPES64 65/// Represents a pseudo instruction which replaces a G_SHUFFLE_VECTOR.66///67/// Used for matching target-supported shuffles before codegen.68struct ShuffleVectorPseudo {69 unsigned Opc; ///< Opcode for the instruction. (E.g. G_ZIP1)70 Register Dst; ///< Destination register.71 SmallVector<SrcOp, 2> SrcOps; ///< Source registers.72 ShuffleVectorPseudo(unsigned Opc, Register Dst,73 std::initializer_list<SrcOp> SrcOps)74 : Opc(Opc), Dst(Dst), SrcOps(SrcOps){};75 ShuffleVectorPseudo() = default;76};77 78/// Return true if a G_FCONSTANT instruction is known to be better-represented79/// as a G_CONSTANT.80bool matchFConstantToConstant(MachineInstr &MI, MachineRegisterInfo &MRI) {81 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT);82 Register DstReg = MI.getOperand(0).getReg();83 const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();84 if (DstSize != 16 && DstSize != 32 && DstSize != 64)85 return false;86 87 // When we're storing a value, it doesn't matter what register bank it's on.88 // Since not all floating point constants can be materialized using a fmov,89 // it makes more sense to just use a GPR.90 return all_of(MRI.use_nodbg_instructions(DstReg),91 [](const MachineInstr &Use) { return Use.mayStore(); });92}93 94/// Change a G_FCONSTANT into a G_CONSTANT.95void applyFConstantToConstant(MachineInstr &MI) {96 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT);97 MachineIRBuilder MIB(MI);98 const APFloat &ImmValAPF = MI.getOperand(1).getFPImm()->getValueAPF();99 MIB.buildConstant(MI.getOperand(0).getReg(), ImmValAPF.bitcastToAPInt());100 MI.eraseFromParent();101}102 103/// Check if a G_EXT instruction can handle a shuffle mask \p M when the vector104/// sources of the shuffle are different.105std::optional<std::pair<bool, uint64_t>> getExtMask(ArrayRef<int> M,106 unsigned NumElts) {107 // Look for the first non-undef element.108 auto FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; });109 if (FirstRealElt == M.end())110 return std::nullopt;111 112 // Use APInt to handle overflow when calculating expected element.113 unsigned MaskBits = APInt(32, NumElts * 2).logBase2();114 APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1, false, true);115 116 // The following shuffle indices must be the successive elements after the117 // first real element.118 if (any_of(119 make_range(std::next(FirstRealElt), M.end()),120 [&ExpectedElt](int Elt) { return Elt != ExpectedElt++ && Elt >= 0; }))121 return std::nullopt;122 123 // The index of an EXT is the first element if it is not UNDEF.124 // Watch out for the beginning UNDEFs. The EXT index should be the expected125 // value of the first element. E.g.126 // <-1, -1, 3, ...> is treated as <1, 2, 3, ...>.127 // <-1, -1, 0, 1, ...> is treated as <2*NumElts-2, 2*NumElts-1, 0, 1, ...>.128 // ExpectedElt is the last mask index plus 1.129 uint64_t Imm = ExpectedElt.getZExtValue();130 bool ReverseExt = false;131 132 // There are two difference cases requiring to reverse input vectors.133 // For example, for vector <4 x i32> we have the following cases,134 // Case 1: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, -1, 0>)135 // Case 2: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, 7, 0>)136 // For both cases, we finally use mask <5, 6, 7, 0>, which requires137 // to reverse two input vectors.138 if (Imm < NumElts)139 ReverseExt = true;140 else141 Imm -= NumElts;142 return std::make_pair(ReverseExt, Imm);143}144 145/// Helper function for matchINS.146///147/// \returns a value when \p M is an ins mask for \p NumInputElements.148///149/// First element of the returned pair is true when the produced150/// G_INSERT_VECTOR_ELT destination should be the LHS of the G_SHUFFLE_VECTOR.151///152/// Second element is the destination lane for the G_INSERT_VECTOR_ELT.153std::optional<std::pair<bool, int>> isINSMask(ArrayRef<int> M,154 int NumInputElements) {155 if (M.size() != static_cast<size_t>(NumInputElements))156 return std::nullopt;157 int NumLHSMatch = 0, NumRHSMatch = 0;158 int LastLHSMismatch = -1, LastRHSMismatch = -1;159 for (int Idx = 0; Idx < NumInputElements; ++Idx) {160 if (M[Idx] == -1) {161 ++NumLHSMatch;162 ++NumRHSMatch;163 continue;164 }165 M[Idx] == Idx ? ++NumLHSMatch : LastLHSMismatch = Idx;166 M[Idx] == Idx + NumInputElements ? ++NumRHSMatch : LastRHSMismatch = Idx;167 }168 const int NumNeededToMatch = NumInputElements - 1;169 if (NumLHSMatch == NumNeededToMatch)170 return std::make_pair(true, LastLHSMismatch);171 if (NumRHSMatch == NumNeededToMatch)172 return std::make_pair(false, LastRHSMismatch);173 return std::nullopt;174}175 176/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with a177/// G_REV instruction. Returns the appropriate G_REV opcode in \p Opc.178bool matchREV(MachineInstr &MI, MachineRegisterInfo &MRI,179 ShuffleVectorPseudo &MatchInfo) {180 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);181 ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();182 Register Dst = MI.getOperand(0).getReg();183 Register Src = MI.getOperand(1).getReg();184 LLT Ty = MRI.getType(Dst);185 unsigned EltSize = Ty.getScalarSizeInBits();186 187 // Element size for a rev cannot be 64.188 if (EltSize == 64)189 return false;190 191 unsigned NumElts = Ty.getNumElements();192 193 // Try to produce a G_REV instruction194 for (unsigned LaneSize : {64U, 32U, 16U}) {195 if (isREVMask(ShuffleMask, EltSize, NumElts, LaneSize)) {196 unsigned Opcode;197 if (LaneSize == 64U)198 Opcode = AArch64::G_REV64;199 else if (LaneSize == 32U)200 Opcode = AArch64::G_REV32;201 else202 Opcode = AArch64::G_REV16;203 204 MatchInfo = ShuffleVectorPseudo(Opcode, Dst, {Src});205 return true;206 }207 }208 209 return false;210}211 212/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with213/// a G_TRN1 or G_TRN2 instruction.214bool matchTRN(MachineInstr &MI, MachineRegisterInfo &MRI,215 ShuffleVectorPseudo &MatchInfo) {216 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);217 unsigned WhichResult;218 ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();219 Register Dst = MI.getOperand(0).getReg();220 unsigned NumElts = MRI.getType(Dst).getNumElements();221 if (!isTRNMask(ShuffleMask, NumElts, WhichResult))222 return false;223 unsigned Opc = (WhichResult == 0) ? AArch64::G_TRN1 : AArch64::G_TRN2;224 Register V1 = MI.getOperand(1).getReg();225 Register V2 = MI.getOperand(2).getReg();226 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});227 return true;228}229 230/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with231/// a G_UZP1 or G_UZP2 instruction.232///233/// \param [in] MI - The shuffle vector instruction.234/// \param [out] MatchInfo - Either G_UZP1 or G_UZP2 on success.235bool matchUZP(MachineInstr &MI, MachineRegisterInfo &MRI,236 ShuffleVectorPseudo &MatchInfo) {237 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);238 unsigned WhichResult;239 ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();240 Register Dst = MI.getOperand(0).getReg();241 unsigned NumElts = MRI.getType(Dst).getNumElements();242 if (!isUZPMask(ShuffleMask, NumElts, WhichResult))243 return false;244 unsigned Opc = (WhichResult == 0) ? AArch64::G_UZP1 : AArch64::G_UZP2;245 Register V1 = MI.getOperand(1).getReg();246 Register V2 = MI.getOperand(2).getReg();247 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});248 return true;249}250 251bool matchZip(MachineInstr &MI, MachineRegisterInfo &MRI,252 ShuffleVectorPseudo &MatchInfo) {253 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);254 unsigned WhichResult;255 unsigned OperandOrder;256 ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();257 Register Dst = MI.getOperand(0).getReg();258 unsigned NumElts = MRI.getType(Dst).getNumElements();259 if (!isZIPMask(ShuffleMask, NumElts, WhichResult, OperandOrder))260 return false;261 unsigned Opc = (WhichResult == 0) ? AArch64::G_ZIP1 : AArch64::G_ZIP2;262 Register V1 = MI.getOperand(OperandOrder == 0 ? 1 : 2).getReg();263 Register V2 = MI.getOperand(OperandOrder == 0 ? 2 : 1).getReg();264 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});265 return true;266}267 268/// Helper function for matchDup.269bool matchDupFromInsertVectorElt(int Lane, MachineInstr &MI,270 MachineRegisterInfo &MRI,271 ShuffleVectorPseudo &MatchInfo) {272 if (Lane != 0)273 return false;274 275 // Try to match a vector splat operation into a dup instruction.276 // We're looking for this pattern:277 //278 // %scalar:gpr(s64) = COPY $x0279 // %undef:fpr(<2 x s64>) = G_IMPLICIT_DEF280 // %cst0:gpr(s32) = G_CONSTANT i32 0281 // %zerovec:fpr(<2 x s32>) = G_BUILD_VECTOR %cst0(s32), %cst0(s32)282 // %ins:fpr(<2 x s64>) = G_INSERT_VECTOR_ELT %undef, %scalar(s64), %cst0(s32)283 // %splat:fpr(<2 x s64>) = G_SHUFFLE_VECTOR %ins(<2 x s64>), %undef,284 // %zerovec(<2 x s32>)285 //286 // ...into:287 // %splat = G_DUP %scalar288 289 // Begin matching the insert.290 auto *InsMI = getOpcodeDef(TargetOpcode::G_INSERT_VECTOR_ELT,291 MI.getOperand(1).getReg(), MRI);292 if (!InsMI)293 return false;294 // Match the undef vector operand.295 if (!getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, InsMI->getOperand(1).getReg(),296 MRI))297 return false;298 299 // Match the index constant 0.300 if (!mi_match(InsMI->getOperand(3).getReg(), MRI, m_ZeroInt()))301 return false;302 303 MatchInfo = ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(0).getReg(),304 {InsMI->getOperand(2).getReg()});305 return true;306}307 308/// Helper function for matchDup.309bool matchDupFromBuildVector(int Lane, MachineInstr &MI,310 MachineRegisterInfo &MRI,311 ShuffleVectorPseudo &MatchInfo) {312 assert(Lane >= 0 && "Expected positive lane?");313 int NumElements = MRI.getType(MI.getOperand(1).getReg()).getNumElements();314 // Test if the LHS is a BUILD_VECTOR. If it is, then we can just reference the315 // lane's definition directly.316 auto *BuildVecMI =317 getOpcodeDef(TargetOpcode::G_BUILD_VECTOR,318 MI.getOperand(Lane < NumElements ? 1 : 2).getReg(), MRI);319 // If Lane >= NumElements then it is point to RHS, just check from RHS320 if (NumElements <= Lane)321 Lane -= NumElements;322 323 if (!BuildVecMI)324 return false;325 Register Reg = BuildVecMI->getOperand(Lane + 1).getReg();326 MatchInfo =327 ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(0).getReg(), {Reg});328 return true;329}330 331bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI,332 ShuffleVectorPseudo &MatchInfo) {333 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);334 auto MaybeLane = getSplatIndex(MI);335 if (!MaybeLane)336 return false;337 int Lane = *MaybeLane;338 // If this is undef splat, generate it via "just" vdup, if possible.339 if (Lane < 0)340 Lane = 0;341 if (matchDupFromInsertVectorElt(Lane, MI, MRI, MatchInfo))342 return true;343 if (matchDupFromBuildVector(Lane, MI, MRI, MatchInfo))344 return true;345 return false;346}347 348// Check if an EXT instruction can handle the shuffle mask when the vector349// sources of the shuffle are the same.350bool isSingletonExtMask(ArrayRef<int> M, LLT Ty) {351 unsigned NumElts = Ty.getNumElements();352 353 // Assume that the first shuffle index is not UNDEF. Fail if it is.354 if (M[0] < 0)355 return false;356 357 // If this is a VEXT shuffle, the immediate value is the index of the first358 // element. The other shuffle indices must be the successive elements after359 // the first one.360 unsigned ExpectedElt = M[0];361 for (unsigned I = 1; I < NumElts; ++I) {362 // Increment the expected index. If it wraps around, just follow it363 // back to index zero and keep going.364 ++ExpectedElt;365 if (ExpectedElt == NumElts)366 ExpectedElt = 0;367 368 if (M[I] < 0)369 continue; // Ignore UNDEF indices.370 if (ExpectedElt != static_cast<unsigned>(M[I]))371 return false;372 }373 374 return true;375}376 377bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI,378 ShuffleVectorPseudo &MatchInfo) {379 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);380 Register Dst = MI.getOperand(0).getReg();381 LLT DstTy = MRI.getType(Dst);382 Register V1 = MI.getOperand(1).getReg();383 Register V2 = MI.getOperand(2).getReg();384 auto Mask = MI.getOperand(3).getShuffleMask();385 uint64_t Imm;386 auto ExtInfo = getExtMask(Mask, DstTy.getNumElements());387 uint64_t ExtFactor = MRI.getType(V1).getScalarSizeInBits() / 8;388 389 if (!ExtInfo) {390 if (!getOpcodeDef<GImplicitDef>(V2, MRI) ||391 !isSingletonExtMask(Mask, DstTy))392 return false;393 394 Imm = Mask[0] * ExtFactor;395 MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1, V1, Imm});396 return true;397 }398 bool ReverseExt;399 std::tie(ReverseExt, Imm) = *ExtInfo;400 if (ReverseExt)401 std::swap(V1, V2);402 Imm *= ExtFactor;403 MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1, V2, Imm});404 return true;405}406 407/// Replace a G_SHUFFLE_VECTOR instruction with a pseudo.408/// \p Opc is the opcode to use. \p MI is the G_SHUFFLE_VECTOR.409void applyShuffleVectorPseudo(MachineInstr &MI,410 ShuffleVectorPseudo &MatchInfo) {411 MachineIRBuilder MIRBuilder(MI);412 MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst}, MatchInfo.SrcOps);413 MI.eraseFromParent();414}415 416/// Replace a G_SHUFFLE_VECTOR instruction with G_EXT.417/// Special-cased because the constant operand must be emitted as a G_CONSTANT418/// for the imported tablegen patterns to work.419void applyEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) {420 MachineIRBuilder MIRBuilder(MI);421 if (MatchInfo.SrcOps[2].getImm() == 0)422 MIRBuilder.buildCopy(MatchInfo.Dst, MatchInfo.SrcOps[0]);423 else {424 // Tablegen patterns expect an i32 G_CONSTANT as the final op.425 auto Cst =426 MIRBuilder.buildConstant(LLT::scalar(32), MatchInfo.SrcOps[2].getImm());427 MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst},428 {MatchInfo.SrcOps[0], MatchInfo.SrcOps[1], Cst});429 }430 MI.eraseFromParent();431}432 433void applyFullRev(MachineInstr &MI, MachineRegisterInfo &MRI) {434 Register Dst = MI.getOperand(0).getReg();435 Register Src = MI.getOperand(1).getReg();436 LLT DstTy = MRI.getType(Dst);437 assert(DstTy.getSizeInBits() == 128 &&438 "Expected 128bit vector in applyFullRev");439 MachineIRBuilder MIRBuilder(MI);440 auto Cst = MIRBuilder.buildConstant(LLT::scalar(32), 8);441 auto Rev = MIRBuilder.buildInstr(AArch64::G_REV64, {DstTy}, {Src});442 MIRBuilder.buildInstr(AArch64::G_EXT, {Dst}, {Rev, Rev, Cst});443 MI.eraseFromParent();444}445 446bool matchNonConstInsert(MachineInstr &MI, MachineRegisterInfo &MRI) {447 assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT);448 449 auto ValAndVReg =450 getIConstantVRegValWithLookThrough(MI.getOperand(3).getReg(), MRI);451 return !ValAndVReg;452}453 454void applyNonConstInsert(MachineInstr &MI, MachineRegisterInfo &MRI,455 MachineIRBuilder &Builder) {456 auto &Insert = cast<GInsertVectorElement>(MI);457 Builder.setInstrAndDebugLoc(Insert);458 459 Register Offset = Insert.getIndexReg();460 LLT VecTy = MRI.getType(Insert.getReg(0));461 LLT EltTy = MRI.getType(Insert.getElementReg());462 LLT IdxTy = MRI.getType(Insert.getIndexReg());463 464 if (VecTy.isScalableVector())465 return;466 467 // Create a stack slot and store the vector into it468 MachineFunction &MF = Builder.getMF();469 Align Alignment(470 std::min<uint64_t>(VecTy.getSizeInBytes().getKnownMinValue(), 16));471 int FrameIdx = MF.getFrameInfo().CreateStackObject(VecTy.getSizeInBytes(),472 Alignment, false);473 LLT FramePtrTy = LLT::pointer(0, 64);474 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIdx);475 auto StackTemp = Builder.buildFrameIndex(FramePtrTy, FrameIdx);476 477 Builder.buildStore(Insert.getOperand(1), StackTemp, PtrInfo, Align(8));478 479 // Get the pointer to the element, and be sure not to hit undefined behavior480 // if the index is out of bounds.481 assert(isPowerOf2_64(VecTy.getNumElements()) &&482 "Expected a power-2 vector size");483 auto Mask = Builder.buildConstant(IdxTy, VecTy.getNumElements() - 1);484 Register And = Builder.buildAnd(IdxTy, Offset, Mask).getReg(0);485 auto EltSize = Builder.buildConstant(IdxTy, EltTy.getSizeInBytes());486 Register Mul = Builder.buildMul(IdxTy, And, EltSize).getReg(0);487 Register EltPtr =488 Builder.buildPtrAdd(MRI.getType(StackTemp.getReg(0)), StackTemp, Mul)489 .getReg(0);490 491 // Write the inserted element492 Builder.buildStore(Insert.getElementReg(), EltPtr, PtrInfo, Align(1));493 // Reload the whole vector.494 Builder.buildLoad(Insert.getReg(0), StackTemp, PtrInfo, Align(8));495 Insert.eraseFromParent();496}497 498/// Match a G_SHUFFLE_VECTOR with a mask which corresponds to a499/// G_INSERT_VECTOR_ELT and G_EXTRACT_VECTOR_ELT pair.500///501/// e.g.502/// %shuf = G_SHUFFLE_VECTOR %left, %right, shufflemask(0, 0)503///504/// Can be represented as505///506/// %extract = G_EXTRACT_VECTOR_ELT %left, 0507/// %ins = G_INSERT_VECTOR_ELT %left, %extract, 1508///509bool matchINS(MachineInstr &MI, MachineRegisterInfo &MRI,510 std::tuple<Register, int, Register, int> &MatchInfo) {511 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);512 ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();513 Register Dst = MI.getOperand(0).getReg();514 int NumElts = MRI.getType(Dst).getNumElements();515 auto DstIsLeftAndDstLane = isINSMask(ShuffleMask, NumElts);516 if (!DstIsLeftAndDstLane)517 return false;518 bool DstIsLeft;519 int DstLane;520 std::tie(DstIsLeft, DstLane) = *DstIsLeftAndDstLane;521 Register Left = MI.getOperand(1).getReg();522 Register Right = MI.getOperand(2).getReg();523 Register DstVec = DstIsLeft ? Left : Right;524 Register SrcVec = Left;525 526 int SrcLane = ShuffleMask[DstLane];527 if (SrcLane >= NumElts) {528 SrcVec = Right;529 SrcLane -= NumElts;530 }531 532 MatchInfo = std::make_tuple(DstVec, DstLane, SrcVec, SrcLane);533 return true;534}535 536void applyINS(MachineInstr &MI, MachineRegisterInfo &MRI,537 MachineIRBuilder &Builder,538 std::tuple<Register, int, Register, int> &MatchInfo) {539 Builder.setInstrAndDebugLoc(MI);540 Register Dst = MI.getOperand(0).getReg();541 auto ScalarTy = MRI.getType(Dst).getElementType();542 Register DstVec, SrcVec;543 int DstLane, SrcLane;544 std::tie(DstVec, DstLane, SrcVec, SrcLane) = MatchInfo;545 auto SrcCst = Builder.buildConstant(LLT::scalar(64), SrcLane);546 auto Extract = Builder.buildExtractVectorElement(ScalarTy, SrcVec, SrcCst);547 auto DstCst = Builder.buildConstant(LLT::scalar(64), DstLane);548 Builder.buildInsertVectorElement(Dst, DstVec, Extract, DstCst);549 MI.eraseFromParent();550}551 552/// isVShiftRImm - Check if this is a valid vector for the immediate553/// operand of a vector shift right operation. The value must be in the range:554/// 1 <= Value <= ElementBits for a right shift.555bool isVShiftRImm(Register Reg, MachineRegisterInfo &MRI, LLT Ty,556 int64_t &Cnt) {557 assert(Ty.isVector() && "vector shift count is not a vector type");558 MachineInstr *MI = MRI.getVRegDef(Reg);559 auto Cst = getAArch64VectorSplatScalar(*MI, MRI);560 if (!Cst)561 return false;562 Cnt = *Cst;563 int64_t ElementBits = Ty.getScalarSizeInBits();564 return Cnt >= 1 && Cnt <= ElementBits;565}566 567/// Match a vector G_ASHR or G_LSHR with a valid immediate shift.568bool matchVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI,569 int64_t &Imm) {570 assert(MI.getOpcode() == TargetOpcode::G_ASHR ||571 MI.getOpcode() == TargetOpcode::G_LSHR);572 LLT Ty = MRI.getType(MI.getOperand(1).getReg());573 if (!Ty.isVector())574 return false;575 return isVShiftRImm(MI.getOperand(2).getReg(), MRI, Ty, Imm);576}577 578void applyVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI,579 int64_t &Imm) {580 unsigned Opc = MI.getOpcode();581 assert(Opc == TargetOpcode::G_ASHR || Opc == TargetOpcode::G_LSHR);582 unsigned NewOpc =583 Opc == TargetOpcode::G_ASHR ? AArch64::G_VASHR : AArch64::G_VLSHR;584 MachineIRBuilder MIB(MI);585 MIB.buildInstr(NewOpc, {MI.getOperand(0)}, {MI.getOperand(1)}).addImm(Imm);586 MI.eraseFromParent();587}588 589/// Determine if it is possible to modify the \p RHS and predicate \p P of a590/// G_ICMP instruction such that the right-hand side is an arithmetic immediate.591///592/// \returns A pair containing the updated immediate and predicate which may593/// be used to optimize the instruction.594///595/// \note This assumes that the comparison has been legalized.596std::optional<std::pair<uint64_t, CmpInst::Predicate>>597tryAdjustICmpImmAndPred(Register RHS, CmpInst::Predicate P,598 const MachineRegisterInfo &MRI) {599 const auto &Ty = MRI.getType(RHS);600 if (Ty.isVector())601 return std::nullopt;602 unsigned Size = Ty.getSizeInBits();603 assert((Size == 32 || Size == 64) && "Expected 32 or 64 bit compare only?");604 605 // If the RHS is not a constant, or the RHS is already a valid arithmetic606 // immediate, then there is nothing to change.607 auto ValAndVReg = getIConstantVRegValWithLookThrough(RHS, MRI);608 if (!ValAndVReg)609 return std::nullopt;610 uint64_t OriginalC = ValAndVReg->Value.getZExtValue();611 uint64_t C = OriginalC;612 if (isLegalArithImmed(C))613 return std::nullopt;614 615 // We have a non-arithmetic immediate. Check if adjusting the immediate and616 // adjusting the predicate will result in a legal arithmetic immediate.617 switch (P) {618 default:619 return std::nullopt;620 case CmpInst::ICMP_SLT:621 case CmpInst::ICMP_SGE:622 // Check for623 //624 // x slt c => x sle c - 1625 // x sge c => x sgt c - 1626 //627 // When c is not the smallest possible negative number.628 if ((Size == 64 && static_cast<int64_t>(C) == INT64_MIN) ||629 (Size == 32 && static_cast<int32_t>(C) == INT32_MIN))630 return std::nullopt;631 P = (P == CmpInst::ICMP_SLT) ? CmpInst::ICMP_SLE : CmpInst::ICMP_SGT;632 C -= 1;633 break;634 case CmpInst::ICMP_ULT:635 case CmpInst::ICMP_UGE:636 // Check for637 //638 // x ult c => x ule c - 1639 // x uge c => x ugt c - 1640 //641 // When c is not zero.642 assert(C != 0 && "C should not be zero here!");643 P = (P == CmpInst::ICMP_ULT) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT;644 C -= 1;645 break;646 case CmpInst::ICMP_SLE:647 case CmpInst::ICMP_SGT:648 // Check for649 //650 // x sle c => x slt c + 1651 // x sgt c => s sge c + 1652 //653 // When c is not the largest possible signed integer.654 if ((Size == 32 && static_cast<int32_t>(C) == INT32_MAX) ||655 (Size == 64 && static_cast<int64_t>(C) == INT64_MAX))656 return std::nullopt;657 P = (P == CmpInst::ICMP_SLE) ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGE;658 C += 1;659 break;660 case CmpInst::ICMP_ULE:661 case CmpInst::ICMP_UGT:662 // Check for663 //664 // x ule c => x ult c + 1665 // x ugt c => s uge c + 1666 //667 // When c is not the largest possible unsigned integer.668 if ((Size == 32 && static_cast<uint32_t>(C) == UINT32_MAX) ||669 (Size == 64 && C == UINT64_MAX))670 return std::nullopt;671 P = (P == CmpInst::ICMP_ULE) ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE;672 C += 1;673 break;674 }675 676 // Check if the new constant is valid, and return the updated constant and677 // predicate if it is.678 if (Size == 32)679 C = static_cast<uint32_t>(C);680 if (isLegalArithImmed(C))681 return {{C, P}};682 683 auto NumberOfInstrToLoadImm = [=](uint64_t Imm) {684 SmallVector<AArch64_IMM::ImmInsnModel> Insn;685 AArch64_IMM::expandMOVImm(Imm, 32, Insn);686 return Insn.size();687 };688 689 if (NumberOfInstrToLoadImm(OriginalC) > NumberOfInstrToLoadImm(C))690 return {{C, P}};691 692 return std::nullopt;693}694 695/// Determine whether or not it is possible to update the RHS and predicate of696/// a G_ICMP instruction such that the RHS will be selected as an arithmetic697/// immediate.698///699/// \p MI - The G_ICMP instruction700/// \p MatchInfo - The new RHS immediate and predicate on success701///702/// See tryAdjustICmpImmAndPred for valid transformations.703bool matchAdjustICmpImmAndPred(704 MachineInstr &MI, const MachineRegisterInfo &MRI,705 std::pair<uint64_t, CmpInst::Predicate> &MatchInfo) {706 assert(MI.getOpcode() == TargetOpcode::G_ICMP);707 Register RHS = MI.getOperand(3).getReg();708 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());709 if (auto MaybeNewImmAndPred = tryAdjustICmpImmAndPred(RHS, Pred, MRI)) {710 MatchInfo = *MaybeNewImmAndPred;711 return true;712 }713 return false;714}715 716void applyAdjustICmpImmAndPred(717 MachineInstr &MI, std::pair<uint64_t, CmpInst::Predicate> &MatchInfo,718 MachineIRBuilder &MIB, GISelChangeObserver &Observer) {719 MIB.setInstrAndDebugLoc(MI);720 MachineOperand &RHS = MI.getOperand(3);721 MachineRegisterInfo &MRI = *MIB.getMRI();722 auto Cst = MIB.buildConstant(MRI.cloneVirtualRegister(RHS.getReg()),723 MatchInfo.first);724 Observer.changingInstr(MI);725 RHS.setReg(Cst->getOperand(0).getReg());726 MI.getOperand(1).setPredicate(MatchInfo.second);727 Observer.changedInstr(MI);728}729 730bool matchDupLane(MachineInstr &MI, MachineRegisterInfo &MRI,731 std::pair<unsigned, int> &MatchInfo) {732 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);733 Register Src1Reg = MI.getOperand(1).getReg();734 const LLT SrcTy = MRI.getType(Src1Reg);735 const LLT DstTy = MRI.getType(MI.getOperand(0).getReg());736 737 auto LaneIdx = getSplatIndex(MI);738 if (!LaneIdx)739 return false;740 741 // The lane idx should be within the first source vector.742 if (*LaneIdx >= SrcTy.getNumElements())743 return false;744 745 if (DstTy != SrcTy)746 return false;747 748 LLT ScalarTy = SrcTy.getElementType();749 unsigned ScalarSize = ScalarTy.getSizeInBits();750 751 unsigned Opc = 0;752 switch (SrcTy.getNumElements()) {753 case 2:754 if (ScalarSize == 64)755 Opc = AArch64::G_DUPLANE64;756 else if (ScalarSize == 32)757 Opc = AArch64::G_DUPLANE32;758 break;759 case 4:760 if (ScalarSize == 32)761 Opc = AArch64::G_DUPLANE32;762 else if (ScalarSize == 16)763 Opc = AArch64::G_DUPLANE16;764 break;765 case 8:766 if (ScalarSize == 8)767 Opc = AArch64::G_DUPLANE8;768 else if (ScalarSize == 16)769 Opc = AArch64::G_DUPLANE16;770 break;771 case 16:772 if (ScalarSize == 8)773 Opc = AArch64::G_DUPLANE8;774 break;775 default:776 break;777 }778 if (!Opc)779 return false;780 781 MatchInfo.first = Opc;782 MatchInfo.second = *LaneIdx;783 return true;784}785 786void applyDupLane(MachineInstr &MI, MachineRegisterInfo &MRI,787 MachineIRBuilder &B, std::pair<unsigned, int> &MatchInfo) {788 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);789 Register Src1Reg = MI.getOperand(1).getReg();790 const LLT SrcTy = MRI.getType(Src1Reg);791 792 B.setInstrAndDebugLoc(MI);793 auto Lane = B.buildConstant(LLT::scalar(64), MatchInfo.second);794 795 Register DupSrc = MI.getOperand(1).getReg();796 // For types like <2 x s32>, we can use G_DUPLANE32, with a <4 x s32> source.797 // To do this, we can use a G_CONCAT_VECTORS to do the widening.798 if (SrcTy.getSizeInBits() == 64) {799 auto Undef = B.buildUndef(SrcTy);800 DupSrc = B.buildConcatVectors(SrcTy.multiplyElements(2),801 {Src1Reg, Undef.getReg(0)})802 .getReg(0);803 }804 B.buildInstr(MatchInfo.first, {MI.getOperand(0).getReg()}, {DupSrc, Lane});805 MI.eraseFromParent();806}807 808bool matchScalarizeVectorUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI) {809 auto &Unmerge = cast<GUnmerge>(MI);810 Register Src1Reg = Unmerge.getReg(Unmerge.getNumOperands() - 1);811 const LLT SrcTy = MRI.getType(Src1Reg);812 if (SrcTy.getSizeInBits() != 128 && SrcTy.getSizeInBits() != 64)813 return false;814 return SrcTy.isVector() && !SrcTy.isScalable() &&815 Unmerge.getNumOperands() == (unsigned)SrcTy.getNumElements() + 1;816}817 818void applyScalarizeVectorUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,819 MachineIRBuilder &B) {820 auto &Unmerge = cast<GUnmerge>(MI);821 Register Src1Reg = Unmerge.getReg(Unmerge.getNumOperands() - 1);822 const LLT SrcTy = MRI.getType(Src1Reg);823 assert((SrcTy.isVector() && !SrcTy.isScalable()) &&824 "Expected a fixed length vector");825 826 for (int I = 0; I < SrcTy.getNumElements(); ++I)827 B.buildExtractVectorElementConstant(Unmerge.getReg(I), Src1Reg, I);828 MI.eraseFromParent();829}830 831bool matchBuildVectorToDup(MachineInstr &MI, MachineRegisterInfo &MRI) {832 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);833 834 // Later, during selection, we'll try to match imported patterns using835 // immAllOnesV and immAllZerosV. These require G_BUILD_VECTOR. Don't lower836 // G_BUILD_VECTORs which could match those patterns.837 if (isBuildVectorAllZeros(MI, MRI) || isBuildVectorAllOnes(MI, MRI))838 return false;839 840 return getAArch64VectorSplat(MI, MRI).has_value();841}842 843void applyBuildVectorToDup(MachineInstr &MI, MachineRegisterInfo &MRI,844 MachineIRBuilder &B) {845 B.setInstrAndDebugLoc(MI);846 B.buildInstr(AArch64::G_DUP, {MI.getOperand(0).getReg()},847 {MI.getOperand(1).getReg()});848 MI.eraseFromParent();849}850 851/// \returns how many instructions would be saved by folding a G_ICMP's shift852/// and/or extension operations.853unsigned getCmpOperandFoldingProfit(Register CmpOp, MachineRegisterInfo &MRI) {854 // No instructions to save if there's more than one use or no uses.855 if (!MRI.hasOneNonDBGUse(CmpOp))856 return 0;857 858 // FIXME: This is duplicated with the selector. (See: selectShiftedRegister)859 auto IsSupportedExtend = [&](const MachineInstr &MI) {860 if (MI.getOpcode() == TargetOpcode::G_SEXT_INREG)861 return true;862 if (MI.getOpcode() != TargetOpcode::G_AND)863 return false;864 auto ValAndVReg =865 getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI);866 if (!ValAndVReg)867 return false;868 uint64_t Mask = ValAndVReg->Value.getZExtValue();869 return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);870 };871 872 MachineInstr *Def = getDefIgnoringCopies(CmpOp, MRI);873 if (IsSupportedExtend(*Def))874 return 1;875 876 unsigned Opc = Def->getOpcode();877 if (Opc != TargetOpcode::G_SHL && Opc != TargetOpcode::G_ASHR &&878 Opc != TargetOpcode::G_LSHR)879 return 0;880 881 auto MaybeShiftAmt =882 getIConstantVRegValWithLookThrough(Def->getOperand(2).getReg(), MRI);883 if (!MaybeShiftAmt)884 return 0;885 uint64_t ShiftAmt = MaybeShiftAmt->Value.getZExtValue();886 MachineInstr *ShiftLHS =887 getDefIgnoringCopies(Def->getOperand(1).getReg(), MRI);888 889 // Check if we can fold an extend and a shift.890 // FIXME: This is duplicated with the selector. (See:891 // selectArithExtendedRegister)892 if (IsSupportedExtend(*ShiftLHS))893 return (ShiftAmt <= 4) ? 2 : 1;894 895 LLT Ty = MRI.getType(Def->getOperand(0).getReg());896 if (Ty.isVector())897 return 0;898 unsigned ShiftSize = Ty.getSizeInBits();899 if ((ShiftSize == 32 && ShiftAmt <= 31) ||900 (ShiftSize == 64 && ShiftAmt <= 63))901 return 1;902 return 0;903}904 905/// \returns true if it would be profitable to swap the LHS and RHS of a G_ICMP906/// instruction \p MI.907bool trySwapICmpOperands(MachineInstr &MI, MachineRegisterInfo &MRI) {908 assert(MI.getOpcode() == TargetOpcode::G_ICMP);909 // Swap the operands if it would introduce a profitable folding opportunity.910 // (e.g. a shift + extend).911 //912 // For example:913 // lsl w13, w11, #1914 // cmp w13, w12915 // can be turned into:916 // cmp w12, w11, lsl #1917 918 // Don't swap if there's a constant on the RHS, because we know we can fold919 // that.920 Register RHS = MI.getOperand(3).getReg();921 auto RHSCst = getIConstantVRegValWithLookThrough(RHS, MRI);922 if (RHSCst && isLegalArithImmed(RHSCst->Value.getSExtValue()))923 return false;924 925 Register LHS = MI.getOperand(2).getReg();926 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());927 auto GetRegForProfit = [&](Register Reg) {928 MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);929 return isCMN(Def, Pred, MRI) ? Def->getOperand(2).getReg() : Reg;930 };931 932 // Don't have a constant on the RHS. If we swap the LHS and RHS of the933 // compare, would we be able to fold more instructions?934 Register TheLHS = GetRegForProfit(LHS);935 Register TheRHS = GetRegForProfit(RHS);936 937 // If the LHS is more likely to give us a folding opportunity, then swap the938 // LHS and RHS.939 return (getCmpOperandFoldingProfit(TheLHS, MRI) >940 getCmpOperandFoldingProfit(TheRHS, MRI));941}942 943void applySwapICmpOperands(MachineInstr &MI, GISelChangeObserver &Observer) {944 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());945 Register LHS = MI.getOperand(2).getReg();946 Register RHS = MI.getOperand(3).getReg();947 Observer.changedInstr(MI);948 MI.getOperand(1).setPredicate(CmpInst::getSwappedPredicate(Pred));949 MI.getOperand(2).setReg(RHS);950 MI.getOperand(3).setReg(LHS);951 Observer.changedInstr(MI);952}953 954/// \returns a function which builds a vector floating point compare instruction955/// for a condition code \p CC.956/// \param [in] NoNans - True if the target has NoNansFPMath.957std::function<Register(MachineIRBuilder &)>958getVectorFCMP(AArch64CC::CondCode CC, Register LHS, Register RHS, bool NoNans,959 MachineRegisterInfo &MRI) {960 LLT DstTy = MRI.getType(LHS);961 assert(DstTy.isVector() && "Expected vector types only?");962 assert(DstTy == MRI.getType(RHS) && "Src and Dst types must match!");963 switch (CC) {964 default:965 llvm_unreachable("Unexpected condition code!");966 case AArch64CC::NE:967 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {968 auto FCmp = MIB.buildInstr(AArch64::G_FCMEQ, {DstTy}, {LHS, RHS});969 return MIB.buildNot(DstTy, FCmp).getReg(0);970 };971 case AArch64CC::EQ:972 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {973 return MIB.buildInstr(AArch64::G_FCMEQ, {DstTy}, {LHS, RHS}).getReg(0);974 };975 case AArch64CC::GE:976 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {977 return MIB.buildInstr(AArch64::G_FCMGE, {DstTy}, {LHS, RHS}).getReg(0);978 };979 case AArch64CC::GT:980 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {981 return MIB.buildInstr(AArch64::G_FCMGT, {DstTy}, {LHS, RHS}).getReg(0);982 };983 case AArch64CC::LS:984 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {985 return MIB.buildInstr(AArch64::G_FCMGE, {DstTy}, {RHS, LHS}).getReg(0);986 };987 case AArch64CC::MI:988 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {989 return MIB.buildInstr(AArch64::G_FCMGT, {DstTy}, {RHS, LHS}).getReg(0);990 };991 }992}993 994/// Try to lower a vector G_FCMP \p MI into an AArch64-specific pseudo.995bool matchLowerVectorFCMP(MachineInstr &MI, MachineRegisterInfo &MRI,996 MachineIRBuilder &MIB) {997 assert(MI.getOpcode() == TargetOpcode::G_FCMP);998 const auto &ST = MI.getMF()->getSubtarget<AArch64Subtarget>();999 1000 Register Dst = MI.getOperand(0).getReg();1001 LLT DstTy = MRI.getType(Dst);1002 if (!DstTy.isVector() || !ST.hasNEON())1003 return false;1004 Register LHS = MI.getOperand(2).getReg();1005 unsigned EltSize = MRI.getType(LHS).getScalarSizeInBits();1006 if (EltSize == 16 && !ST.hasFullFP16())1007 return false;1008 if (EltSize != 16 && EltSize != 32 && EltSize != 64)1009 return false;1010 1011 return true;1012}1013 1014/// Try to lower a vector G_FCMP \p MI into an AArch64-specific pseudo.1015void applyLowerVectorFCMP(MachineInstr &MI, MachineRegisterInfo &MRI,1016 MachineIRBuilder &MIB) {1017 assert(MI.getOpcode() == TargetOpcode::G_FCMP);1018 const auto &ST = MI.getMF()->getSubtarget<AArch64Subtarget>();1019 1020 const auto &CmpMI = cast<GFCmp>(MI);1021 1022 Register Dst = CmpMI.getReg(0);1023 CmpInst::Predicate Pred = CmpMI.getCond();1024 Register LHS = CmpMI.getLHSReg();1025 Register RHS = CmpMI.getRHSReg();1026 1027 LLT DstTy = MRI.getType(Dst);1028 1029 bool Invert = false;1030 AArch64CC::CondCode CC, CC2 = AArch64CC::AL;1031 if ((Pred == CmpInst::Predicate::FCMP_ORD ||1032 Pred == CmpInst::Predicate::FCMP_UNO) &&1033 isBuildVectorAllZeros(*MRI.getVRegDef(RHS), MRI)) {1034 // The special case "fcmp ord %a, 0" is the canonical check that LHS isn't1035 // NaN, so equivalent to a == a and doesn't need the two comparisons an1036 // "ord" normally would.1037 // Similarly, "fcmp uno %a, 0" is the canonical check that LHS is NaN and is1038 // thus equivalent to a != a.1039 RHS = LHS;1040 CC = Pred == CmpInst::Predicate::FCMP_ORD ? AArch64CC::EQ : AArch64CC::NE;1041 } else1042 changeVectorFCMPPredToAArch64CC(Pred, CC, CC2, Invert);1043 1044 // Instead of having an apply function, just build here to simplify things.1045 MIB.setInstrAndDebugLoc(MI);1046 1047 const bool NoNans =1048 ST.getTargetLowering()->getTargetMachine().Options.NoNaNsFPMath;1049 1050 auto Cmp = getVectorFCMP(CC, LHS, RHS, NoNans, MRI);1051 Register CmpRes;1052 if (CC2 == AArch64CC::AL)1053 CmpRes = Cmp(MIB);1054 else {1055 auto Cmp2 = getVectorFCMP(CC2, LHS, RHS, NoNans, MRI);1056 auto Cmp2Dst = Cmp2(MIB);1057 auto Cmp1Dst = Cmp(MIB);1058 CmpRes = MIB.buildOr(DstTy, Cmp1Dst, Cmp2Dst).getReg(0);1059 }1060 if (Invert)1061 CmpRes = MIB.buildNot(DstTy, CmpRes).getReg(0);1062 MRI.replaceRegWith(Dst, CmpRes);1063 MI.eraseFromParent();1064}1065 1066// Matches G_BUILD_VECTOR where at least one source operand is not a constant1067bool matchLowerBuildToInsertVecElt(MachineInstr &MI, MachineRegisterInfo &MRI) {1068 auto *GBuildVec = cast<GBuildVector>(&MI);1069 1070 // Check if the values are all constants1071 for (unsigned I = 0; I < GBuildVec->getNumSources(); ++I) {1072 auto ConstVal =1073 getAnyConstantVRegValWithLookThrough(GBuildVec->getSourceReg(I), MRI);1074 1075 if (!ConstVal.has_value())1076 return true;1077 }1078 1079 return false;1080}1081 1082void applyLowerBuildToInsertVecElt(MachineInstr &MI, MachineRegisterInfo &MRI,1083 MachineIRBuilder &B) {1084 auto *GBuildVec = cast<GBuildVector>(&MI);1085 LLT DstTy = MRI.getType(GBuildVec->getReg(0));1086 Register DstReg = B.buildUndef(DstTy).getReg(0);1087 1088 for (unsigned I = 0; I < GBuildVec->getNumSources(); ++I) {1089 Register SrcReg = GBuildVec->getSourceReg(I);1090 if (mi_match(SrcReg, MRI, m_GImplicitDef()))1091 continue;1092 auto IdxReg = B.buildConstant(LLT::scalar(64), I);1093 DstReg =1094 B.buildInsertVectorElement(DstTy, DstReg, SrcReg, IdxReg).getReg(0);1095 }1096 B.buildCopy(GBuildVec->getReg(0), DstReg);1097 GBuildVec->eraseFromParent();1098}1099 1100bool matchFormTruncstore(MachineInstr &MI, MachineRegisterInfo &MRI,1101 Register &SrcReg) {1102 assert(MI.getOpcode() == TargetOpcode::G_STORE);1103 Register DstReg = MI.getOperand(0).getReg();1104 if (MRI.getType(DstReg).isVector())1105 return false;1106 // Match a store of a truncate.1107 if (!mi_match(DstReg, MRI, m_GTrunc(m_Reg(SrcReg))))1108 return false;1109 // Only form truncstores for value types of max 64b.1110 return MRI.getType(SrcReg).getSizeInBits() <= 64;1111}1112 1113void applyFormTruncstore(MachineInstr &MI, MachineRegisterInfo &MRI,1114 MachineIRBuilder &B, GISelChangeObserver &Observer,1115 Register &SrcReg) {1116 assert(MI.getOpcode() == TargetOpcode::G_STORE);1117 Observer.changingInstr(MI);1118 MI.getOperand(0).setReg(SrcReg);1119 Observer.changedInstr(MI);1120}1121 1122// Lower vector G_SEXT_INREG back to shifts for selection. We allowed them to1123// form in the first place for combine opportunities, so any remaining ones1124// at this stage need be lowered back.1125bool matchVectorSextInReg(MachineInstr &MI, MachineRegisterInfo &MRI) {1126 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);1127 Register DstReg = MI.getOperand(0).getReg();1128 LLT DstTy = MRI.getType(DstReg);1129 return DstTy.isVector();1130}1131 1132void applyVectorSextInReg(MachineInstr &MI, MachineRegisterInfo &MRI,1133 MachineIRBuilder &B, GISelChangeObserver &Observer) {1134 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);1135 B.setInstrAndDebugLoc(MI);1136 LegalizerHelper Helper(*MI.getMF(), Observer, B);1137 Helper.lower(MI, 0, /* Unused hint type */ LLT());1138}1139 1140/// Combine <N x t>, unused = unmerge(G_EXT <2*N x t> v, undef, N)1141/// => unused, <N x t> = unmerge v1142bool matchUnmergeExtToUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,1143 Register &MatchInfo) {1144 auto &Unmerge = cast<GUnmerge>(MI);1145 if (Unmerge.getNumDefs() != 2)1146 return false;1147 if (!MRI.use_nodbg_empty(Unmerge.getReg(1)))1148 return false;1149 1150 LLT DstTy = MRI.getType(Unmerge.getReg(0));1151 if (!DstTy.isVector())1152 return false;1153 1154 MachineInstr *Ext = getOpcodeDef(AArch64::G_EXT, Unmerge.getSourceReg(), MRI);1155 if (!Ext)1156 return false;1157 1158 Register ExtSrc1 = Ext->getOperand(1).getReg();1159 Register ExtSrc2 = Ext->getOperand(2).getReg();1160 auto LowestVal =1161 getIConstantVRegValWithLookThrough(Ext->getOperand(3).getReg(), MRI);1162 if (!LowestVal || LowestVal->Value.getZExtValue() != DstTy.getSizeInBytes())1163 return false;1164 1165 if (!getOpcodeDef<GImplicitDef>(ExtSrc2, MRI))1166 return false;1167 1168 MatchInfo = ExtSrc1;1169 return true;1170}1171 1172void applyUnmergeExtToUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,1173 MachineIRBuilder &B,1174 GISelChangeObserver &Observer, Register &SrcReg) {1175 Observer.changingInstr(MI);1176 // Swap dst registers.1177 Register Dst1 = MI.getOperand(0).getReg();1178 MI.getOperand(0).setReg(MI.getOperand(1).getReg());1179 MI.getOperand(1).setReg(Dst1);1180 MI.getOperand(2).setReg(SrcReg);1181 Observer.changedInstr(MI);1182}1183 1184// Match mul({z/s}ext , {z/s}ext) => {u/s}mull OR1185// Match v2s64 mul instructions, which will then be scalarised later on1186// Doing these two matches in one function to ensure that the order of matching1187// will always be the same.1188// Try lowering MUL to MULL before trying to scalarize if needed.1189bool matchMulv2s64(MachineInstr &MI, MachineRegisterInfo &MRI) {1190 // Get the instructions that defined the source operand1191 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());1192 return DstTy == LLT::fixed_vector(2, 64);1193}1194 1195void applyMulv2s64(MachineInstr &MI, MachineRegisterInfo &MRI,1196 MachineIRBuilder &B, GISelChangeObserver &Observer) {1197 assert(MI.getOpcode() == TargetOpcode::G_MUL &&1198 "Expected a G_MUL instruction");1199 1200 // Get the instructions that defined the source operand1201 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());1202 assert(DstTy == LLT::fixed_vector(2, 64) && "Expected v2s64 Mul");1203 LegalizerHelper Helper(*MI.getMF(), Observer, B);1204 Helper.fewerElementsVector(1205 MI, 0,1206 DstTy.changeElementCount(DstTy.getElementCount().divideCoefficientBy(2)));1207}1208 1209class AArch64PostLegalizerLoweringImpl : public Combiner {1210protected:1211 const CombinerHelper Helper;1212 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig;1213 const AArch64Subtarget &STI;1214 1215public:1216 AArch64PostLegalizerLoweringImpl(1217 MachineFunction &MF, CombinerInfo &CInfo, const TargetPassConfig *TPC,1218 GISelCSEInfo *CSEInfo,1219 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig,1220 const AArch64Subtarget &STI);1221 1222 static const char *getName() { return "AArch6400PreLegalizerCombiner"; }1223 1224 bool tryCombineAll(MachineInstr &I) const override;1225 1226private:1227#define GET_GICOMBINER_CLASS_MEMBERS1228#include "AArch64GenPostLegalizeGILowering.inc"1229#undef GET_GICOMBINER_CLASS_MEMBERS1230};1231 1232#define GET_GICOMBINER_IMPL1233#include "AArch64GenPostLegalizeGILowering.inc"1234#undef GET_GICOMBINER_IMPL1235 1236AArch64PostLegalizerLoweringImpl::AArch64PostLegalizerLoweringImpl(1237 MachineFunction &MF, CombinerInfo &CInfo, const TargetPassConfig *TPC,1238 GISelCSEInfo *CSEInfo,1239 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig,1240 const AArch64Subtarget &STI)1241 : Combiner(MF, CInfo, TPC, /*VT*/ nullptr, CSEInfo),1242 Helper(Observer, B, /*IsPreLegalize*/ true), RuleConfig(RuleConfig),1243 STI(STI),1244#define GET_GICOMBINER_CONSTRUCTOR_INITS1245#include "AArch64GenPostLegalizeGILowering.inc"1246#undef GET_GICOMBINER_CONSTRUCTOR_INITS1247{1248}1249 1250class AArch64PostLegalizerLowering : public MachineFunctionPass {1251public:1252 static char ID;1253 1254 AArch64PostLegalizerLowering();1255 1256 StringRef getPassName() const override {1257 return "AArch64PostLegalizerLowering";1258 }1259 1260 bool runOnMachineFunction(MachineFunction &MF) override;1261 void getAnalysisUsage(AnalysisUsage &AU) const override;1262 1263private:1264 AArch64PostLegalizerLoweringImplRuleConfig RuleConfig;1265};1266} // end anonymous namespace1267 1268void AArch64PostLegalizerLowering::getAnalysisUsage(AnalysisUsage &AU) const {1269 AU.addRequired<TargetPassConfig>();1270 AU.setPreservesCFG();1271 getSelectionDAGFallbackAnalysisUsage(AU);1272 MachineFunctionPass::getAnalysisUsage(AU);1273}1274 1275AArch64PostLegalizerLowering::AArch64PostLegalizerLowering()1276 : MachineFunctionPass(ID) {1277 if (!RuleConfig.parseCommandLineOption())1278 report_fatal_error("Invalid rule identifier");1279}1280 1281bool AArch64PostLegalizerLowering::runOnMachineFunction(MachineFunction &MF) {1282 if (MF.getProperties().hasFailedISel())1283 return false;1284 assert(MF.getProperties().hasLegalized() && "Expected a legalized function?");1285 auto *TPC = &getAnalysis<TargetPassConfig>();1286 const Function &F = MF.getFunction();1287 1288 const AArch64Subtarget &ST = MF.getSubtarget<AArch64Subtarget>();1289 CombinerInfo CInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,1290 /*LegalizerInfo*/ nullptr, /*OptEnabled=*/true,1291 F.hasOptSize(), F.hasMinSize());1292 // Disable fixed-point iteration to reduce compile-time1293 CInfo.MaxIterations = 1;1294 CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;1295 // PostLegalizerCombiner performs DCE, so a full DCE pass is unnecessary.1296 CInfo.EnableFullDCE = false;1297 AArch64PostLegalizerLoweringImpl Impl(MF, CInfo, TPC, /*CSEInfo*/ nullptr,1298 RuleConfig, ST);1299 return Impl.combineMachineInstrs();1300}1301 1302char AArch64PostLegalizerLowering::ID = 0;1303INITIALIZE_PASS_BEGIN(AArch64PostLegalizerLowering, DEBUG_TYPE,1304 "Lower AArch64 MachineInstrs after legalization", false,1305 false)1306INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)1307INITIALIZE_PASS_END(AArch64PostLegalizerLowering, DEBUG_TYPE,1308 "Lower AArch64 MachineInstrs after legalization", false,1309 false)1310 1311namespace llvm {1312FunctionPass *createAArch64PostLegalizerLowering() {1313 return new AArch64PostLegalizerLowering();1314}1315} // end namespace llvm1316