2567 lines · cpp
1 2//===- GlobalISelEmitter.cpp - Generate an instruction selector -----------===//3//4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.5// See https://llvm.org/LICENSE.txt for license information.6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception7//8//===----------------------------------------------------------------------===//9//10/// \file11/// This tablegen backend emits code for use by the GlobalISel instruction12/// selector. See include/llvm/Target/GlobalISel/Target.td.13///14/// This file analyzes the patterns recognized by the SelectionDAGISel tablegen15/// backend, filters out the ones that are unsupported, maps16/// SelectionDAG-specific constructs to their GlobalISel counterpart17/// (when applicable: MVT to LLT; SDNode to generic Instruction).18///19/// Not all patterns are supported: pass the tablegen invocation20/// "-warn-on-skipped-patterns" to emit a warning when a pattern is skipped,21/// as well as why.22///23/// The generated file defines a single method:24/// bool <Target>InstructionSelector::selectImpl(MachineInstr &I) const;25/// intended to be used in InstructionSelector::select as the first-step26/// selector for the patterns that don't require complex C++.27///28/// FIXME: We'll probably want to eventually define a base29/// "TargetGenInstructionSelector" class.30///31//===----------------------------------------------------------------------===//32 33#include "Basic/CodeGenIntrinsics.h"34#include "Common/CodeGenDAGPatterns.h"35#include "Common/CodeGenInstruction.h"36#include "Common/CodeGenRegisters.h"37#include "Common/CodeGenTarget.h"38#include "Common/GlobalISel/GlobalISelMatchTable.h"39#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h"40#include "Common/InfoByHwMode.h"41#include "llvm/ADT/Statistic.h"42#include "llvm/CodeGenTypes/LowLevelType.h"43#include "llvm/CodeGenTypes/MachineValueType.h"44#include "llvm/Support/CodeGenCoverage.h"45#include "llvm/Support/CommandLine.h"46#include "llvm/Support/Error.h"47#include "llvm/Support/ScopedPrinter.h"48#include "llvm/TableGen/Error.h"49#include "llvm/TableGen/Record.h"50#include "llvm/TableGen/TableGenBackend.h"51#include <string>52 53using namespace llvm;54using namespace llvm::gi;55 56using action_iterator = RuleMatcher::action_iterator;57 58#define DEBUG_TYPE "gisel-emitter"59 60STATISTIC(NumPatternTotal, "Total number of patterns");61STATISTIC(NumPatternImported, "Number of patterns imported from SelectionDAG");62STATISTIC(NumPatternImportsSkipped, "Number of SelectionDAG imports skipped");63STATISTIC(NumPatternsTested,64 "Number of patterns executed according to coverage information");65 66static cl::OptionCategory GlobalISelEmitterCat("Options for -gen-global-isel");67 68static cl::opt<bool> WarnOnSkippedPatterns(69 "warn-on-skipped-patterns",70 cl::desc("Explain why a pattern was skipped for inclusion "71 "in the GlobalISel selector"),72 cl::init(false), cl::cat(GlobalISelEmitterCat));73 74static cl::opt<bool> GenerateCoverage(75 "instrument-gisel-coverage",76 cl::desc("Generate coverage instrumentation for GlobalISel"),77 cl::init(false), cl::cat(GlobalISelEmitterCat));78 79static cl::opt<std::string> UseCoverageFile(80 "gisel-coverage-file", cl::init(""),81 cl::desc("Specify file to retrieve coverage information from"),82 cl::cat(GlobalISelEmitterCat));83 84static cl::opt<bool> OptimizeMatchTable(85 "optimize-match-table",86 cl::desc("Generate an optimized version of the match table"),87 cl::init(true), cl::cat(GlobalISelEmitterCat));88 89static std::string explainPredicates(const TreePatternNode &N) {90 std::string Explanation;91 StringRef Separator = "";92 for (const TreePredicateCall &Call : N.getPredicateCalls()) {93 const TreePredicateFn &P = Call.Fn;94 Explanation +=95 (Separator + P.getOrigPatFragRecord()->getRecord()->getName()).str();96 Separator = ", ";97 98 if (P.isAlwaysTrue())99 Explanation += " always-true";100 if (P.isImmediatePattern())101 Explanation += " immediate";102 103 if (P.isUnindexed())104 Explanation += " unindexed";105 106 if (P.isNonExtLoad())107 Explanation += " non-extload";108 if (P.isAnyExtLoad())109 Explanation += " extload";110 if (P.isSignExtLoad())111 Explanation += " sextload";112 if (P.isZeroExtLoad())113 Explanation += " zextload";114 115 if (P.isNonTruncStore())116 Explanation += " non-truncstore";117 if (P.isTruncStore())118 Explanation += " truncstore";119 120 if (const Record *VT = P.getMemoryVT())121 Explanation += (" MemVT=" + VT->getName()).str();122 if (const Record *VT = P.getScalarMemoryVT())123 Explanation += (" ScalarVT(MemVT)=" + VT->getName()).str();124 125 if (const ListInit *AddrSpaces = P.getAddressSpaces()) {126 raw_string_ostream OS(Explanation);127 OS << " AddressSpaces=[";128 129 StringRef AddrSpaceSeparator;130 for (const Init *Val : AddrSpaces->getElements()) {131 const IntInit *IntVal = dyn_cast<IntInit>(Val);132 if (!IntVal)133 continue;134 135 OS << AddrSpaceSeparator << IntVal->getValue();136 AddrSpaceSeparator = ", ";137 }138 139 OS << ']';140 }141 142 int64_t MinAlign = P.getMinAlignment();143 if (MinAlign > 0)144 Explanation += " MinAlign=" + utostr(MinAlign);145 146 if (P.isAtomicOrderingMonotonic())147 Explanation += " monotonic";148 if (P.isAtomicOrderingAcquire())149 Explanation += " acquire";150 if (P.isAtomicOrderingRelease())151 Explanation += " release";152 if (P.isAtomicOrderingAcquireRelease())153 Explanation += " acq_rel";154 if (P.isAtomicOrderingSequentiallyConsistent())155 Explanation += " seq_cst";156 if (P.isAtomicOrderingAcquireOrStronger())157 Explanation += " >=acquire";158 if (P.isAtomicOrderingWeakerThanAcquire())159 Explanation += " <acquire";160 if (P.isAtomicOrderingReleaseOrStronger())161 Explanation += " >=release";162 if (P.isAtomicOrderingWeakerThanRelease())163 Explanation += " <release";164 }165 return Explanation;166}167 168static std::string explainOperator(const Record *Operator) {169 if (Operator->isSubClassOf("SDNode"))170 return (" (" + Operator->getValueAsString("Opcode") + ")").str();171 172 if (Operator->isSubClassOf("Intrinsic"))173 return (" (Operator is an Intrinsic, " + Operator->getName() + ")").str();174 175 if (Operator->isSubClassOf("ComplexPattern"))176 return (" (Operator is an unmapped ComplexPattern, " + Operator->getName() +177 ")")178 .str();179 180 if (Operator->isSubClassOf("SDNodeXForm"))181 return (" (Operator is an unmapped SDNodeXForm, " + Operator->getName() +182 ")")183 .str();184 185 return (" (Operator " + Operator->getName() + " not understood)").str();186}187 188/// Helper function to let the emitter report skip reason error messages.189static Error failedImport(const Twine &Reason) {190 return make_error<StringError>(Reason, inconvertibleErrorCode());191}192 193static Error isTrivialOperatorNode(const TreePatternNode &N) {194 std::string Explanation;195 std::string Separator;196 197 bool HasUnsupportedPredicate = false;198 for (const TreePredicateCall &Call : N.getPredicateCalls()) {199 const TreePredicateFn &Predicate = Call.Fn;200 201 if (Predicate.isAlwaysTrue())202 continue;203 204 if (Predicate.isImmediatePattern())205 continue;206 207 if (Predicate.hasNoUse() || Predicate.hasOneUse())208 continue;209 210 if (Predicate.isNonExtLoad() || Predicate.isAnyExtLoad() ||211 Predicate.isSignExtLoad() || Predicate.isZeroExtLoad())212 continue;213 214 if (Predicate.isNonTruncStore() || Predicate.isTruncStore())215 continue;216 217 if (Predicate.isLoad() && Predicate.getMemoryVT())218 continue;219 220 if (Predicate.isStore() && Predicate.getMemoryVT())221 continue;222 223 if (Predicate.isLoad() || Predicate.isStore()) {224 if (Predicate.isUnindexed())225 continue;226 }227 228 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {229 const ListInit *AddrSpaces = Predicate.getAddressSpaces();230 if (AddrSpaces && !AddrSpaces->empty())231 continue;232 233 if (Predicate.getMinAlignment() > 0)234 continue;235 }236 237 if (Predicate.isAtomic() && Predicate.getMemoryVT())238 continue;239 240 if (Predicate.isAtomic() &&241 (Predicate.isAtomicOrderingMonotonic() ||242 Predicate.isAtomicOrderingAcquire() ||243 Predicate.isAtomicOrderingRelease() ||244 Predicate.isAtomicOrderingAcquireRelease() ||245 Predicate.isAtomicOrderingSequentiallyConsistent() ||246 Predicate.isAtomicOrderingAcquireOrStronger() ||247 Predicate.isAtomicOrderingWeakerThanAcquire() ||248 Predicate.isAtomicOrderingReleaseOrStronger() ||249 Predicate.isAtomicOrderingWeakerThanRelease()))250 continue;251 252 if (Predicate.hasGISelPredicateCode())253 continue;254 255 HasUnsupportedPredicate = true;256 Explanation = Separator + "Has a predicate (" + explainPredicates(N) + ")";257 Separator = ", ";258 Explanation += (Separator + "first-failing:" +259 Predicate.getOrigPatFragRecord()->getRecord()->getName())260 .str();261 break;262 }263 264 if (!HasUnsupportedPredicate)265 return Error::success();266 267 return failedImport(Explanation);268}269 270static std::string getScopedName(unsigned Scope, const std::string &Name) {271 return ("pred:" + Twine(Scope) + ":" + Name).str();272}273 274static std::string getMangledRootDefName(StringRef DefOperandName) {275 return ("DstI[" + DefOperandName + "]").str();276}277 278//===- GlobalISelEmitter class --------------------------------------------===//279 280static Expected<LLTCodeGen> getInstResultType(const TreePatternNode &Dst,281 const CodeGenTarget &Target) {282 // While we allow more than one output (both implicit and explicit defs)283 // below, we only expect one explicit def here.284 assert(Dst.getOperator()->isSubClassOf("Instruction"));285 const CodeGenInstruction &InstInfo = Target.getInstruction(Dst.getOperator());286 if (!InstInfo.Operands.NumDefs)287 return failedImport("Dst pattern child needs a def");288 289 ArrayRef<TypeSetByHwMode> ChildTypes = Dst.getExtTypes();290 if (ChildTypes.size() < 1)291 return failedImport("Dst pattern child has no result");292 293 // If there are multiple results, just take the first one (this is how294 // SelectionDAG does it).295 std::optional<LLTCodeGen> MaybeOpTy;296 if (ChildTypes.front().isMachineValueType()) {297 MaybeOpTy = MVTToLLT(ChildTypes.front().getMachineValueType().SimpleTy);298 }299 300 if (!MaybeOpTy)301 return failedImport("Dst operand has an unsupported type");302 return *MaybeOpTy;303}304 305namespace {306class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter {307public:308 explicit GlobalISelEmitter(const RecordKeeper &RK);309 310 void emitAdditionalImpl(raw_ostream &OS) override;311 312 void emitMIPredicateFns(raw_ostream &OS) override;313 void emitLeafPredicateFns(raw_ostream &OS) override;314 void emitI64ImmPredicateFns(raw_ostream &OS) override;315 void emitAPFloatImmPredicateFns(raw_ostream &OS) override;316 void emitAPIntImmPredicateFns(raw_ostream &OS) override;317 void emitTestSimplePredicate(raw_ostream &OS) override;318 void emitRunCustomAction(raw_ostream &OS) override;319 320 const CodeGenTarget &getTarget() const override { return Target; }321 StringRef getClassName() const override { return ClassName; }322 323 void run(raw_ostream &OS);324 325private:326 std::string ClassName;327 328 const RecordKeeper &RK;329 const CodeGenDAGPatterns CGP;330 const CodeGenTarget &Target;331 CodeGenRegBank &CGRegs;332 333 ArrayRef<const Record *> AllPatFrags;334 335 /// Keep track of the equivalence between SDNodes and Instruction by mapping336 /// SDNodes to the GINodeEquiv mapping. We need to map to the GINodeEquiv to337 /// check for attributes on the relation such as CheckMMOIsNonAtomic.338 /// This is defined using 'GINodeEquiv' in the target description.339 DenseMap<const Record *, const Record *> NodeEquivs;340 341 /// Keep track of the equivalence between ComplexPattern's and342 /// GIComplexOperandMatcher. Map entries are specified by subclassing343 /// GIComplexPatternEquiv.344 DenseMap<const Record *, const Record *> ComplexPatternEquivs;345 346 /// Keep track of the equivalence between SDNodeXForm's and347 /// GICustomOperandRenderer. Map entries are specified by subclassing348 /// GISDNodeXFormEquiv.349 DenseMap<const Record *, const Record *> SDNodeXFormEquivs;350 351 /// Keep track of Scores of PatternsToMatch similar to how the DAG does.352 /// This adds compatibility for RuleMatchers to use this for ordering rules.353 DenseMap<uint64_t, int> RuleMatcherScores;354 355 // Rule coverage information.356 std::optional<CodeGenCoverage> RuleCoverage;357 358 /// Variables used to help with collecting of named operands for predicates359 /// with 'let PredicateCodeUsesOperands = 1'. WaitingForNamedOperands is set360 /// to the number of named operands that predicate expects. Store locations in361 /// StoreIdxForName correspond to the order in which operand names appear in362 /// predicate's argument list.363 /// When we visit named operand and WaitingForNamedOperands is not zero, add364 /// matcher that will record operand and decrease counter.365 unsigned WaitingForNamedOperands = 0;366 StringMap<unsigned> StoreIdxForName;367 368 void gatherOpcodeValues();369 void gatherTypeIDValues();370 void gatherNodeEquivs();371 372 const Record *findNodeEquiv(const Record *N) const;373 const CodeGenInstruction *getEquivNode(const Record &Equiv,374 const TreePatternNode &N) const;375 376 Error importRulePredicates(RuleMatcher &M,377 ArrayRef<const Record *> Predicates);378 Expected<InstructionMatcher &>379 createAndImportSelDAGMatcher(RuleMatcher &Rule,380 InstructionMatcher &InsnMatcher,381 const TreePatternNode &Src, unsigned &TempOpIdx);382 Error importComplexPatternOperandMatcher(OperandMatcher &OM, const Record *R,383 unsigned &TempOpIdx) const;384 Error importChildMatcher(RuleMatcher &Rule, InstructionMatcher &InsnMatcher,385 const TreePatternNode &SrcChild,386 bool OperandIsAPointer, bool OperandIsImmArg,387 unsigned OpIdx, unsigned &TempOpIdx);388 389 Expected<BuildMIAction &>390 createAndImportInstructionRenderer(RuleMatcher &M,391 InstructionMatcher &InsnMatcher,392 const TreePatternNode &Dst) const;393 Expected<action_iterator> createAndImportSubInstructionRenderer(394 action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst,395 unsigned TempReg) const;396 Expected<action_iterator>397 createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M,398 const TreePatternNode &Dst) const;399 400 Expected<action_iterator>401 importExplicitDefRenderers(action_iterator InsertPt, RuleMatcher &M,402 BuildMIAction &DstMIBuilder,403 const TreePatternNode &Dst, bool IsRoot) const;404 405 Expected<action_iterator>406 importExplicitUseRenderers(action_iterator InsertPt, RuleMatcher &M,407 BuildMIAction &DstMIBuilder,408 const TreePatternNode &Dst) const;409 410 Error importNamedNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,411 const TreePatternNode &N) const;412 413 Error importLeafNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,414 const TreePatternNode &N,415 action_iterator InsertPt) const;416 417 Error importXFormNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,418 const TreePatternNode &N) const;419 420 Error importInstructionNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,421 const TreePatternNode &N,422 action_iterator &InsertPt) const;423 424 Error importNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,425 const TreePatternNode &N,426 action_iterator &InsertPt) const;427 428 Error importImplicitDefRenderers(BuildMIAction &DstMIBuilder,429 ArrayRef<const Record *> ImplicitDefs) const;430 431 /// Analyze pattern \p P, returning a matcher for it if possible.432 /// Otherwise, return an Error explaining why we don't support it.433 Expected<RuleMatcher> runOnPattern(const PatternToMatch &P);434 435 void declareSubtargetFeature(const Record *Predicate);436 437 unsigned declareHwModeCheck(StringRef HwModeFeatures);438 439 MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules, bool Optimize,440 bool WithCoverage);441 442 /// Infer a CodeGenRegisterClass for the type of \p SuperRegNode. The returned443 /// CodeGenRegisterClass will support the CodeGenRegisterClass of444 /// \p SubRegNode, and the subregister index defined by \p SubRegIdxNode.445 /// If no register class is found, return nullptr.446 const CodeGenRegisterClass *447 inferSuperRegisterClassForNode(const TypeSetByHwMode &Ty,448 const TreePatternNode &SuperRegNode,449 const TreePatternNode &SubRegIdxNode) const;450 const CodeGenSubRegIndex *451 inferSubRegIndexForNode(const TreePatternNode &SubRegIdxNode) const;452 453 /// Infer a CodeGenRegisterClass which suppoorts \p Ty and \p SubRegIdxNode.454 /// Return nullptr if no such class exists.455 const CodeGenRegisterClass *456 inferSuperRegisterClass(const TypeSetByHwMode &Ty,457 const TreePatternNode &SubRegIdxNode) const;458 459 /// Return the CodeGenRegisterClass associated with \p Leaf if it has one.460 const CodeGenRegisterClass *461 getRegClassFromLeaf(const TreePatternNode &Leaf) const;462 463 /// Return a CodeGenRegisterClass for \p N if one can be found. Return464 /// nullptr otherwise.465 const CodeGenRegisterClass *466 inferRegClassFromPattern(const TreePatternNode &N) const;467 468 const CodeGenRegisterClass *469 inferRegClassFromRegisterClassLike(const Record *RecClassLike) const;470 471 const CodeGenRegisterClass *472 inferRegClassFromInstructionPattern(const TreePatternNode &N,473 unsigned ResIdx) const;474 475 Error constrainOperands(action_iterator InsertPt, RuleMatcher &M,476 unsigned InsnID, const TreePatternNode &Dst) const;477 478 /// Return the size of the MemoryVT in this predicate, if possible.479 std::optional<unsigned>480 getMemSizeBitsFromPredicate(const TreePredicateFn &Predicate);481 482 // Add builtin predicates.483 Expected<InstructionMatcher &>484 addBuiltinPredicates(const Record *SrcGIEquivOrNull,485 const TreePredicateFn &Predicate,486 InstructionMatcher &InsnMatcher, bool &HasAddedMatcher);487};488} // namespace489 490static StringRef getPatFragPredicateEnumName(const Record *R) {491 return R->getName();492}493 494void GlobalISelEmitter::gatherOpcodeValues() {495 InstructionOpcodeMatcher::initOpcodeValuesMap(Target);496}497 498void GlobalISelEmitter::gatherTypeIDValues() {499 LLTOperandMatcher::initTypeIDValuesMap();500}501 502void GlobalISelEmitter::gatherNodeEquivs() {503 assert(NodeEquivs.empty());504 for (const Record *Equiv : RK.getAllDerivedDefinitions("GINodeEquiv"))505 NodeEquivs[Equiv->getValueAsDef("Node")] = Equiv;506 507 assert(ComplexPatternEquivs.empty());508 for (const Record *Equiv :509 RK.getAllDerivedDefinitions("GIComplexPatternEquiv")) {510 const Record *SelDAGEquiv = Equiv->getValueAsDef("SelDAGEquivalent");511 if (!SelDAGEquiv)512 continue;513 ComplexPatternEquivs[SelDAGEquiv] = Equiv;514 }515 516 assert(SDNodeXFormEquivs.empty());517 for (const Record *Equiv :518 RK.getAllDerivedDefinitions("GISDNodeXFormEquiv")) {519 const Record *SelDAGEquiv = Equiv->getValueAsDef("SelDAGEquivalent");520 if (!SelDAGEquiv)521 continue;522 SDNodeXFormEquivs[SelDAGEquiv] = Equiv;523 }524}525 526const Record *GlobalISelEmitter::findNodeEquiv(const Record *N) const {527 return NodeEquivs.lookup(N);528}529 530const CodeGenInstruction *531GlobalISelEmitter::getEquivNode(const Record &Equiv,532 const TreePatternNode &N) const {533 if (N.getNumChildren() >= 1) {534 // setcc operation maps to two different G_* instructions based on the type.535 if (!Equiv.isValueUnset("IfFloatingPoint") &&536 MVT(N.getChild(0).getSimpleType(0)).isFloatingPoint())537 return &Target.getInstruction(Equiv.getValueAsDef("IfFloatingPoint"));538 }539 540 if (!Equiv.isValueUnset("IfConvergent") &&541 N.getIntrinsicInfo(CGP)->isConvergent)542 return &Target.getInstruction(Equiv.getValueAsDef("IfConvergent"));543 544 for (const TreePredicateCall &Call : N.getPredicateCalls()) {545 const TreePredicateFn &Predicate = Call.Fn;546 if (!Equiv.isValueUnset("IfSignExtend") &&547 (Predicate.isLoad() || Predicate.isAtomic()) &&548 Predicate.isSignExtLoad())549 return &Target.getInstruction(Equiv.getValueAsDef("IfSignExtend"));550 if (!Equiv.isValueUnset("IfZeroExtend") &&551 (Predicate.isLoad() || Predicate.isAtomic()) &&552 Predicate.isZeroExtLoad())553 return &Target.getInstruction(Equiv.getValueAsDef("IfZeroExtend"));554 }555 556 return &Target.getInstruction(Equiv.getValueAsDef("I"));557}558 559GlobalISelEmitter::GlobalISelEmitter(const RecordKeeper &RK)560 : GlobalISelMatchTableExecutorEmitter(), RK(RK), CGP(RK),561 Target(CGP.getTargetInfo()), CGRegs(Target.getRegBank()) {562 ClassName = Target.getName().str() + "InstructionSelector";563}564 565//===- Emitter ------------------------------------------------------------===//566 567Error GlobalISelEmitter::importRulePredicates(568 RuleMatcher &M, ArrayRef<const Record *> Predicates) {569 for (const Record *Pred : Predicates) {570 if (Pred->getValueAsString("CondString").empty())571 continue;572 declareSubtargetFeature(Pred);573 M.addRequiredFeature(Pred);574 }575 576 return Error::success();577}578 579std::optional<unsigned> GlobalISelEmitter::getMemSizeBitsFromPredicate(580 const TreePredicateFn &Predicate) {581 std::optional<LLTCodeGen> MemTyOrNone =582 MVTToLLT(getValueType(Predicate.getMemoryVT()));583 584 if (!MemTyOrNone)585 return std::nullopt;586 587 // Align so unusual types like i1 don't get rounded down.588 return llvm::alignTo(589 static_cast<unsigned>(MemTyOrNone->get().getSizeInBits()), 8);590}591 592Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(593 const Record *SrcGIEquivOrNull, const TreePredicateFn &Predicate,594 InstructionMatcher &InsnMatcher, bool &HasAddedMatcher) {595 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {596 if (const ListInit *AddrSpaces = Predicate.getAddressSpaces()) {597 SmallVector<unsigned, 4> ParsedAddrSpaces;598 599 for (const Init *Val : AddrSpaces->getElements()) {600 const IntInit *IntVal = dyn_cast<IntInit>(Val);601 if (!IntVal)602 return failedImport("Address space is not an integer");603 ParsedAddrSpaces.push_back(IntVal->getValue());604 }605 606 if (!ParsedAddrSpaces.empty()) {607 InsnMatcher.addPredicate<MemoryAddressSpacePredicateMatcher>(608 0, ParsedAddrSpaces);609 return InsnMatcher;610 }611 }612 613 int64_t MinAlign = Predicate.getMinAlignment();614 if (MinAlign > 0) {615 InsnMatcher.addPredicate<MemoryAlignmentPredicateMatcher>(0, MinAlign);616 return InsnMatcher;617 }618 }619 620 // G_LOAD is used for both non-extending and any-extending loads.621 if (Predicate.isLoad() || Predicate.isAtomic()) {622 if (Predicate.isNonExtLoad()) {623 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(624 0, MemoryVsLLTSizePredicateMatcher::EqualTo, 0);625 return InsnMatcher;626 }627 if (Predicate.isAnyExtLoad()) {628 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(629 0, MemoryVsLLTSizePredicateMatcher::LessThan, 0);630 return InsnMatcher;631 }632 }633 634 if (Predicate.isStore()) {635 if (Predicate.isTruncStore()) {636 if (Predicate.getMemoryVT() != nullptr) {637 // FIXME: If MemoryVT is set, we end up with 2 checks for the MMO size.638 auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);639 if (!MemSizeInBits)640 return failedImport("MemVT could not be converted to LLT");641 642 InsnMatcher.addPredicate<MemorySizePredicateMatcher>(0, *MemSizeInBits /643 8);644 } else {645 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(646 0, MemoryVsLLTSizePredicateMatcher::LessThan, 0);647 }648 return InsnMatcher;649 }650 if (Predicate.isNonTruncStore()) {651 // We need to check the sizes match here otherwise we could incorrectly652 // match truncating stores with non-truncating ones.653 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(654 0, MemoryVsLLTSizePredicateMatcher::EqualTo, 0);655 }656 }657 658 assert(SrcGIEquivOrNull != nullptr && "Invalid SrcGIEquivOrNull value");659 // No check required. We already did it by swapping the opcode.660 if (!SrcGIEquivOrNull->isValueUnset("IfSignExtend") &&661 Predicate.isSignExtLoad())662 return InsnMatcher;663 664 // No check required. We already did it by swapping the opcode.665 if (!SrcGIEquivOrNull->isValueUnset("IfZeroExtend") &&666 Predicate.isZeroExtLoad())667 return InsnMatcher;668 669 // No check required. G_STORE by itself is a non-extending store.670 if (Predicate.isNonTruncStore())671 return InsnMatcher;672 673 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {674 if (Predicate.getMemoryVT() != nullptr) {675 auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);676 if (!MemSizeInBits)677 return failedImport("MemVT could not be converted to LLT");678 679 InsnMatcher.addPredicate<MemorySizePredicateMatcher>(0,680 *MemSizeInBits / 8);681 return InsnMatcher;682 }683 }684 685 if (Predicate.isLoad() || Predicate.isStore()) {686 // No check required. A G_LOAD/G_STORE is an unindexed load.687 if (Predicate.isUnindexed())688 return InsnMatcher;689 }690 691 if (Predicate.isAtomic()) {692 if (Predicate.isAtomicOrderingMonotonic()) {693 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("Monotonic");694 return InsnMatcher;695 }696 if (Predicate.isAtomicOrderingAcquire()) {697 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("Acquire");698 return InsnMatcher;699 }700 if (Predicate.isAtomicOrderingRelease()) {701 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("Release");702 return InsnMatcher;703 }704 if (Predicate.isAtomicOrderingAcquireRelease()) {705 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(706 "AcquireRelease");707 return InsnMatcher;708 }709 if (Predicate.isAtomicOrderingSequentiallyConsistent()) {710 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(711 "SequentiallyConsistent");712 return InsnMatcher;713 }714 }715 716 if (Predicate.isAtomicOrderingAcquireOrStronger()) {717 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(718 "Acquire", AtomicOrderingMMOPredicateMatcher::AO_OrStronger);719 return InsnMatcher;720 }721 if (Predicate.isAtomicOrderingWeakerThanAcquire()) {722 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(723 "Acquire", AtomicOrderingMMOPredicateMatcher::AO_WeakerThan);724 return InsnMatcher;725 }726 727 if (Predicate.isAtomicOrderingReleaseOrStronger()) {728 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(729 "Release", AtomicOrderingMMOPredicateMatcher::AO_OrStronger);730 return InsnMatcher;731 }732 if (Predicate.isAtomicOrderingWeakerThanRelease()) {733 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(734 "Release", AtomicOrderingMMOPredicateMatcher::AO_WeakerThan);735 return InsnMatcher;736 }737 HasAddedMatcher = false;738 return InsnMatcher;739}740 741Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(742 RuleMatcher &Rule, InstructionMatcher &InsnMatcher,743 const TreePatternNode &Src, unsigned &TempOpIdx) {744 const auto SavedFlags = Rule.setGISelFlags(Src.getGISelFlagsRecord());745 746 const Record *SrcGIEquivOrNull = nullptr;747 const CodeGenInstruction *SrcGIOrNull = nullptr;748 749 // Start with the defined operands (i.e., the results of the root operator).750 if (Src.isLeaf()) {751 const Init *SrcInit = Src.getLeafValue();752 if (isa<IntInit>(SrcInit)) {753 InsnMatcher.addPredicate<InstructionOpcodeMatcher>(754 &Target.getInstruction(RK.getDef("G_CONSTANT")));755 } else {756 return failedImport(757 "Unable to deduce gMIR opcode to handle Src (which is a leaf)");758 }759 } else {760 SrcGIEquivOrNull = findNodeEquiv(Src.getOperator());761 if (!SrcGIEquivOrNull)762 return failedImport("Pattern operator lacks an equivalent Instruction" +763 explainOperator(Src.getOperator()));764 SrcGIOrNull = getEquivNode(*SrcGIEquivOrNull, Src);765 766 // The operators look good: match the opcode767 InsnMatcher.addPredicate<InstructionOpcodeMatcher>(SrcGIOrNull);768 }769 770 // Since there are no opcodes for atomic loads and stores comparing to771 // SelectionDAG, we add CheckMMOIsNonAtomic predicate immediately after the772 // opcode predicate to make a logical combination of them.773 if (SrcGIEquivOrNull &&774 SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic"))775 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("NotAtomic");776 else if (SrcGIEquivOrNull &&777 SrcGIEquivOrNull->getValueAsBit("CheckMMOIsAtomic")) {778 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(779 "Unordered", AtomicOrderingMMOPredicateMatcher::AO_OrStronger);780 }781 782 unsigned OpIdx = 0;783 for (const TypeSetByHwMode &VTy : Src.getExtTypes()) {784 // Results don't have a name unless they are the root node. The caller will785 // set the name if appropriate.786 const bool OperandIsAPointer =787 SrcGIOrNull && SrcGIOrNull->isOutOperandAPointer(OpIdx);788 OperandMatcher &OM = InsnMatcher.addOperand(OpIdx++, "", TempOpIdx);789 if (auto Error = OM.addTypeCheckPredicate(VTy, OperandIsAPointer))790 return failedImport(toString(std::move(Error)) +791 " for result of Src pattern operator");792 }793 794 for (const TreePredicateCall &Call : Src.getPredicateCalls()) {795 const TreePredicateFn &Predicate = Call.Fn;796 bool HasAddedBuiltinMatcher = true;797 if (Predicate.isAlwaysTrue())798 continue;799 800 if (Predicate.isImmediatePattern()) {801 InsnMatcher.addPredicate<InstructionImmPredicateMatcher>(Predicate);802 continue;803 }804 805 auto InsnMatcherOrError = addBuiltinPredicates(806 SrcGIEquivOrNull, Predicate, InsnMatcher, HasAddedBuiltinMatcher);807 if (auto Error = InsnMatcherOrError.takeError())808 return std::move(Error);809 810 // FIXME: This should be part of addBuiltinPredicates(). If we add this at811 // the start of addBuiltinPredicates() without returning, then there might812 // be cases where we hit the last return before which the813 // HasAddedBuiltinMatcher will be set to false. The predicate could be814 // missed if we add it in the middle or at the end due to return statements815 // after the addPredicate<>() calls.816 if (Predicate.hasNoUse()) {817 InsnMatcher.addPredicate<NoUsePredicateMatcher>();818 HasAddedBuiltinMatcher = true;819 }820 if (Predicate.hasOneUse()) {821 InsnMatcher.addPredicate<OneUsePredicateMatcher>();822 HasAddedBuiltinMatcher = true;823 }824 825 if (Predicate.hasGISelPredicateCode()) {826 if (Predicate.usesOperands()) {827 assert(WaitingForNamedOperands == 0 &&828 "previous predicate didn't find all operands or "829 "nested predicate that uses operands");830 TreePattern *TP = Predicate.getOrigPatFragRecord();831 WaitingForNamedOperands = TP->getNumArgs();832 for (unsigned I = 0; I < WaitingForNamedOperands; ++I)833 StoreIdxForName[getScopedName(Call.Scope, TP->getArgName(I))] = I;834 }835 InsnMatcher.addPredicate<GenericInstructionPredicateMatcher>(Predicate);836 continue;837 }838 if (!HasAddedBuiltinMatcher) {839 return failedImport("Src pattern child has predicate (" +840 explainPredicates(Src) + ")");841 }842 }843 844 if (Src.isLeaf()) {845 const Init *SrcInit = Src.getLeafValue();846 if (const IntInit *SrcIntInit = dyn_cast<IntInit>(SrcInit)) {847 OperandMatcher &OM =848 InsnMatcher.addOperand(OpIdx++, Src.getName().str(), TempOpIdx);849 OM.addPredicate<LiteralIntOperandMatcher>(SrcIntInit->getValue());850 } else {851 return failedImport(852 "Unable to deduce gMIR opcode to handle Src (which is a leaf)");853 }854 } else {855 assert(SrcGIOrNull &&856 "Expected to have already found an equivalent Instruction");857 if (SrcGIOrNull->getName() == "G_CONSTANT" ||858 SrcGIOrNull->getName() == "G_FCONSTANT" ||859 SrcGIOrNull->getName() == "G_FRAME_INDEX") {860 // imm/fpimm still have operands but we don't need to do anything with it861 // here since we don't support ImmLeaf predicates yet. However, we still862 // need to note the hidden operand to get GIM_CheckNumOperands correct.863 InsnMatcher.addOperand(OpIdx++, "", TempOpIdx);864 return InsnMatcher;865 }866 867 // Special case because the operand order is changed from setcc. The868 // predicate operand needs to be swapped from the last operand to the first869 // source.870 871 unsigned NumChildren = Src.getNumChildren();872 bool IsFCmp = SrcGIOrNull->getName() == "G_FCMP";873 874 if (IsFCmp || SrcGIOrNull->getName() == "G_ICMP") {875 const TreePatternNode &SrcChild = Src.getChild(NumChildren - 1);876 if (SrcChild.isLeaf()) {877 const DefInit *DI = dyn_cast<DefInit>(SrcChild.getLeafValue());878 const Record *CCDef = DI ? DI->getDef() : nullptr;879 if (!CCDef || !CCDef->isSubClassOf("CondCode"))880 return failedImport("Unable to handle CondCode");881 882 OperandMatcher &OM = InsnMatcher.addOperand(883 OpIdx++, SrcChild.getName().str(), TempOpIdx);884 StringRef PredType = IsFCmp ? CCDef->getValueAsString("FCmpPredicate")885 : CCDef->getValueAsString("ICmpPredicate");886 887 if (!PredType.empty()) {888 OM.addPredicate<CmpPredicateOperandMatcher>(PredType.str());889 // Process the other 2 operands normally.890 --NumChildren;891 }892 }893 }894 895 // Match the used operands (i.e. the children of the operator).896 bool IsIntrinsic =897 SrcGIOrNull->getName() == "G_INTRINSIC" ||898 SrcGIOrNull->getName() == "G_INTRINSIC_W_SIDE_EFFECTS" ||899 SrcGIOrNull->getName() == "G_INTRINSIC_CONVERGENT" ||900 SrcGIOrNull->getName() == "G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS";901 const CodeGenIntrinsic *II = Src.getIntrinsicInfo(CGP);902 if (IsIntrinsic && !II)903 return failedImport("Expected IntInit containing intrinsic ID)");904 905 for (unsigned I = 0; I != NumChildren; ++I) {906 const TreePatternNode &SrcChild = Src.getChild(I);907 908 // We need to determine the meaning of a literal integer based on the909 // context. If this is a field required to be an immediate (such as an910 // immarg intrinsic argument), the required predicates are different than911 // a constant which may be materialized in a register. If we have an912 // argument that is required to be an immediate, we should not emit an LLT913 // type check, and should not be looking for a G_CONSTANT defined914 // register.915 bool OperandIsImmArg = SrcGIOrNull->isInOperandImmArg(I);916 917 // SelectionDAG allows pointers to be represented with iN since it doesn't918 // distinguish between pointers and integers but they are different types919 // in GlobalISel. Coerce integers to pointers to address space 0 if the920 // context indicates a pointer.921 //922 bool OperandIsAPointer = SrcGIOrNull->isInOperandAPointer(I);923 924 if (IsIntrinsic) {925 // For G_INTRINSIC/G_INTRINSIC_W_SIDE_EFFECTS, the operand immediately926 // following the defs is an intrinsic ID.927 if (I == 0) {928 OperandMatcher &OM = InsnMatcher.addOperand(929 OpIdx++, SrcChild.getName().str(), TempOpIdx);930 OM.addPredicate<IntrinsicIDOperandMatcher>(II);931 continue;932 }933 934 // We have to check intrinsics for llvm_anyptr_ty and immarg parameters.935 //936 // Note that we have to look at the i-1th parameter, because we don't937 // have the intrinsic ID in the intrinsic's parameter list.938 OperandIsAPointer |= II->isParamAPointer(I - 1);939 OperandIsImmArg |= II->isParamImmArg(I - 1);940 }941 942 if (auto Error =943 importChildMatcher(Rule, InsnMatcher, SrcChild, OperandIsAPointer,944 OperandIsImmArg, OpIdx++, TempOpIdx))945 return std::move(Error);946 }947 }948 949 return InsnMatcher;950}951 952Error GlobalISelEmitter::importComplexPatternOperandMatcher(953 OperandMatcher &OM, const Record *R, unsigned &TempOpIdx) const {954 const auto &ComplexPattern = ComplexPatternEquivs.find(R);955 if (ComplexPattern == ComplexPatternEquivs.end())956 return failedImport("SelectionDAG ComplexPattern (" + R->getName() +957 ") not mapped to GlobalISel");958 959 OM.addPredicate<ComplexPatternOperandMatcher>(OM, *ComplexPattern->second);960 TempOpIdx++;961 return Error::success();962}963 964// Get the name to use for a pattern operand. For an anonymous physical register965// input, this should use the register name.966static StringRef getSrcChildName(const TreePatternNode &SrcChild,967 const Record *&PhysReg) {968 StringRef SrcChildName = SrcChild.getName();969 if (SrcChildName.empty() && SrcChild.isLeaf()) {970 if (auto *ChildDefInit = dyn_cast<DefInit>(SrcChild.getLeafValue())) {971 auto *ChildRec = ChildDefInit->getDef();972 if (ChildRec->isSubClassOf("Register")) {973 SrcChildName = ChildRec->getName();974 PhysReg = ChildRec;975 }976 }977 }978 979 return SrcChildName;980}981 982Error GlobalISelEmitter::importChildMatcher(983 RuleMatcher &Rule, InstructionMatcher &InsnMatcher,984 const TreePatternNode &SrcChild, bool OperandIsAPointer,985 bool OperandIsImmArg, unsigned OpIdx, unsigned &TempOpIdx) {986 987 const Record *PhysReg = nullptr;988 std::string SrcChildName = getSrcChildName(SrcChild, PhysReg).str();989 if (!SrcChild.isLeaf() &&990 SrcChild.getOperator()->isSubClassOf("ComplexPattern")) {991 // The "name" of a non-leaf complex pattern (MY_PAT $op1, $op2) is992 // "MY_PAT:op1:op2" and the ones with same "name" represent same operand.993 std::string PatternName = SrcChild.getOperator()->getName().str();994 for (const TreePatternNode &Child : SrcChild.children()) {995 PatternName += ":";996 PatternName += Child.getName();997 }998 SrcChildName = PatternName;999 }1000 1001 OperandMatcher &OM =1002 PhysReg ? InsnMatcher.addPhysRegInput(PhysReg, OpIdx, TempOpIdx)1003 : InsnMatcher.addOperand(OpIdx, SrcChildName, TempOpIdx);1004 if (OM.isSameAsAnotherOperand())1005 return Error::success();1006 1007 ArrayRef<TypeSetByHwMode> ChildTypes = SrcChild.getExtTypes();1008 if (ChildTypes.size() != 1)1009 return failedImport("Src pattern child has multiple results");1010 1011 // Check MBB's before the type check since they are not a known type.1012 if (!SrcChild.isLeaf()) {1013 if (SrcChild.getOperator()->getName() == "bb") {1014 OM.addPredicate<MBBOperandMatcher>();1015 return Error::success();1016 }1017 if (SrcChild.getOperator()->getName() == "timm") {1018 OM.addPredicate<ImmOperandMatcher>();1019 1020 // Add predicates, if any1021 for (const TreePredicateCall &Call : SrcChild.getPredicateCalls()) {1022 const TreePredicateFn &Predicate = Call.Fn;1023 1024 // Only handle immediate patterns for now1025 if (Predicate.isImmediatePattern()) {1026 OM.addPredicate<OperandImmPredicateMatcher>(Predicate);1027 }1028 }1029 1030 return Error::success();1031 }1032 } else if (auto *ChildDefInit = dyn_cast<DefInit>(SrcChild.getLeafValue())) {1033 auto *ChildRec = ChildDefInit->getDef();1034 if (ChildRec->isSubClassOf("ValueType") && !SrcChild.hasName()) {1035 // An unnamed ValueType as in (sext_inreg GPR:$foo, i8). GISel represents1036 // this as a literal constant with the scalar size.1037 MVT VT = llvm::getValueType(ChildRec);1038 OM.addPredicate<LiteralIntOperandMatcher>(VT.getScalarSizeInBits());1039 return Error::success();1040 }1041 }1042 1043 // Immediate arguments have no meaningful type to check as they don't have1044 // registers.1045 if (!OperandIsImmArg) {1046 if (auto Error =1047 OM.addTypeCheckPredicate(ChildTypes.front(), OperandIsAPointer))1048 return failedImport(toString(std::move(Error)) + " for Src operand (" +1049 to_string(SrcChild) + ")");1050 }1051 1052 // Try look up SrcChild for a (named) predicate operand if there is any.1053 if (WaitingForNamedOperands) {1054 auto &ScopedNames = SrcChild.getNamesAsPredicateArg();1055 if (!ScopedNames.empty()) {1056 auto PA = ScopedNames.begin();1057 std::string Name = getScopedName(PA->getScope(), PA->getIdentifier());1058 OM.addPredicate<RecordNamedOperandMatcher>(StoreIdxForName[Name], Name);1059 --WaitingForNamedOperands;1060 }1061 }1062 1063 // Check for nested instructions.1064 if (!SrcChild.isLeaf()) {1065 if (SrcChild.getOperator()->isSubClassOf("ComplexPattern")) {1066 // When a ComplexPattern is used as an operator, it should do the same1067 // thing as when used as a leaf. However, the children of the operator1068 // name the sub-operands that make up the complex operand and we must1069 // prepare to reference them in the renderer too.1070 unsigned RendererID = TempOpIdx;1071 if (auto Error = importComplexPatternOperandMatcher(1072 OM, SrcChild.getOperator(), TempOpIdx))1073 return Error;1074 1075 for (unsigned I = 0, E = SrcChild.getNumChildren(); I != E; ++I) {1076 auto &SubOperand = SrcChild.getChild(I);1077 if (!SubOperand.getName().empty()) {1078 if (auto Error = Rule.defineComplexSubOperand(1079 SubOperand.getName(), SrcChild.getOperator(), RendererID, I,1080 SrcChildName))1081 return Error;1082 }1083 }1084 1085 return Error::success();1086 }1087 1088 auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(1089 InsnMatcher.getRuleMatcher(), SrcChild.getName());1090 if (!MaybeInsnOperand) {1091 // This isn't strictly true. If the user were to provide exactly the same1092 // matchers as the original operand then we could allow it. However, it's1093 // simpler to not permit the redundant specification.1094 return failedImport(1095 "Nested instruction cannot be the same as another operand");1096 }1097 1098 // Map the node to a gMIR instruction.1099 InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;1100 auto InsnMatcherOrError = createAndImportSelDAGMatcher(1101 Rule, InsnOperand.getInsnMatcher(), SrcChild, TempOpIdx);1102 if (auto Error = InsnMatcherOrError.takeError())1103 return Error;1104 1105 return Error::success();1106 }1107 1108 if (SrcChild.hasAnyPredicate()) {1109 for (const TreePredicateCall &Call : SrcChild.getPredicateCalls()) {1110 const TreePredicateFn &Predicate = Call.Fn;1111 1112 if (!Predicate.hasGISelLeafPredicateCode())1113 return failedImport("Src pattern child has unsupported predicate");1114 OM.addPredicate<OperandLeafPredicateMatcher>(Predicate);1115 }1116 return Error::success();1117 }1118 1119 // Check for constant immediates.1120 if (auto *ChildInt = dyn_cast<IntInit>(SrcChild.getLeafValue())) {1121 if (OperandIsImmArg) {1122 // Checks for argument directly in operand list1123 OM.addPredicate<LiteralIntOperandMatcher>(ChildInt->getValue());1124 } else {1125 // Checks for materialized constant1126 OM.addPredicate<ConstantIntOperandMatcher>(ChildInt->getValue());1127 }1128 return Error::success();1129 }1130 1131 // Check for def's like register classes or ComplexPattern's.1132 if (auto *ChildDefInit = dyn_cast<DefInit>(SrcChild.getLeafValue())) {1133 auto *ChildRec = ChildDefInit->getDef();1134 1135 // Check for register classes.1136 if (ChildRec->isSubClassOf("RegisterClassLike") ||1137 ChildRec->isSubClassOf("RegisterOperand")) {1138 OM.addPredicate<RegisterBankOperandMatcher>(1139 Target.getRegisterClass(Target.getInitValueAsRegClass(1140 ChildDefInit,1141 /*AssumeRegClassByHwModeIsDefault=*/true)));1142 return Error::success();1143 }1144 1145 if (ChildRec->isSubClassOf("Register")) {1146 // This just be emitted as a copy to the specific register.1147 ValueTypeByHwMode VT = ChildTypes.front().getValueTypeByHwMode();1148 const CodeGenRegisterClass *RC =1149 CGRegs.getMinimalPhysRegClass(ChildRec, &VT);1150 if (!RC) {1151 return failedImport(1152 "Could not determine physical register class of pattern source");1153 }1154 1155 OM.addPredicate<RegisterBankOperandMatcher>(*RC);1156 return Error::success();1157 }1158 1159 // Check for ValueType.1160 if (ChildRec->isSubClassOf("ValueType")) {1161 // We already added a type check as standard practice so this doesn't need1162 // to do anything.1163 return Error::success();1164 }1165 1166 // Check for ComplexPattern's.1167 if (ChildRec->isSubClassOf("ComplexPattern"))1168 return importComplexPatternOperandMatcher(OM, ChildRec, TempOpIdx);1169 1170 if (ChildRec->isSubClassOf("ImmLeaf")) {1171 return failedImport(1172 "Src pattern child def is an unsupported tablegen class (ImmLeaf)");1173 }1174 1175 // Place holder for SRCVALUE nodes. Nothing to do here.1176 if (ChildRec->getName() == "srcvalue")1177 return Error::success();1178 1179 const bool ImmAllOnesV = ChildRec->getName() == "immAllOnesV";1180 if (ImmAllOnesV || ChildRec->getName() == "immAllZerosV") {1181 auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(1182 InsnMatcher.getRuleMatcher(), SrcChild.getName(), false);1183 InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;1184 1185 ValueTypeByHwMode VTy = ChildTypes.front().getValueTypeByHwMode();1186 1187 const CodeGenInstruction &BuildVector =1188 Target.getInstruction(RK.getDef("G_BUILD_VECTOR"));1189 const CodeGenInstruction &BuildVectorTrunc =1190 Target.getInstruction(RK.getDef("G_BUILD_VECTOR_TRUNC"));1191 1192 // Treat G_BUILD_VECTOR as the canonical opcode, and G_BUILD_VECTOR_TRUNC1193 // as an alternative.1194 InsnOperand.getInsnMatcher().addPredicate<InstructionOpcodeMatcher>(1195 ArrayRef({&BuildVector, &BuildVectorTrunc}));1196 1197 // TODO: Handle both G_BUILD_VECTOR and G_BUILD_VECTOR_TRUNC We could1198 // theoretically not emit any opcode check, but getOpcodeMatcher currently1199 // has to succeed.1200 OperandMatcher &OM =1201 InsnOperand.getInsnMatcher().addOperand(0, "", TempOpIdx);1202 if (auto Error = OM.addTypeCheckPredicate(TypeSetByHwMode(VTy),1203 /*OperandIsAPointer=*/false))1204 return failedImport(toString(std::move(Error)) +1205 " for result of Src pattern operator");1206 1207 InsnOperand.getInsnMatcher().addPredicate<VectorSplatImmPredicateMatcher>(1208 ImmAllOnesV ? VectorSplatImmPredicateMatcher::AllOnes1209 : VectorSplatImmPredicateMatcher::AllZeros);1210 return Error::success();1211 }1212 1213 return failedImport(1214 "Src pattern child def is an unsupported tablegen class");1215 }1216 1217 return failedImport("Src pattern child is an unsupported kind");1218}1219 1220// Equivalent of MatcherGen::EmitResultOfNamedOperand.1221Error GlobalISelEmitter::importNamedNodeRenderer(1222 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N) const {1223 StringRef NodeName = N.getName();1224 1225 if (auto SubOperand = M.getComplexSubOperand(NodeName)) {1226 auto [ComplexPatternRec, RendererID, SubOperandIdx] = *SubOperand;1227 MIBuilder.addRenderer<RenderComplexPatternOperand>(1228 *ComplexPatternRec, NodeName, RendererID, SubOperandIdx);1229 return Error::success();1230 }1231 1232 if (!N.isLeaf()) {1233 StringRef OperatorName = N.getOperator()->getName();1234 1235 if (OperatorName == "imm") {1236 MIBuilder.addRenderer<CopyConstantAsImmRenderer>(NodeName);1237 return Error::success();1238 }1239 1240 if (OperatorName == "fpimm") {1241 MIBuilder.addRenderer<CopyFConstantAsFPImmRenderer>(NodeName);1242 return Error::success();1243 }1244 1245 // TODO: 'imm' and 'fpimm' are the only nodes that need special treatment.1246 // Remove this check and add CopyRenderer unconditionally for other nodes.1247 if (OperatorName == "bb" || OperatorName == "timm" ||1248 OperatorName == "tframeindex") {1249 MIBuilder.addRenderer<CopyRenderer>(NodeName);1250 return Error::success();1251 }1252 1253 return failedImport("node has unsupported operator " + to_string(N));1254 }1255 1256 if (const auto *DI = dyn_cast<DefInit>(N.getLeafValue())) {1257 const Record *R = DI->getDef();1258 1259 if (N.getNumResults() != 1)1260 return failedImport("node does not have one result " + to_string(N));1261 1262 if (R->isSubClassOf("ComplexPattern")) {1263 auto I = ComplexPatternEquivs.find(R);1264 if (I == ComplexPatternEquivs.end())1265 return failedImport("ComplexPattern " + R->getName() +1266 " does not have GISel equivalent");1267 1268 const OperandMatcher &OM = M.getOperandMatcher(NodeName);1269 MIBuilder.addRenderer<RenderComplexPatternOperand>(1270 *I->second, NodeName, OM.getAllocatedTemporariesBaseID());1271 return Error::success();1272 }1273 1274 if (R->isSubClassOf("RegisterOperand") &&1275 !R->isValueUnset("GIZeroRegister")) {1276 MIBuilder.addRenderer<CopyOrAddZeroRegRenderer>(1277 NodeName, R->getValueAsDef("GIZeroRegister"));1278 return Error::success();1279 }1280 1281 // TODO: All special cases are handled above. Remove this check and add1282 // CopyRenderer unconditionally.1283 if (R->isSubClassOf("RegisterClassLike") ||1284 R->isSubClassOf("RegisterOperand") || R->isSubClassOf("ValueType")) {1285 MIBuilder.addRenderer<CopyRenderer>(NodeName);1286 return Error::success();1287 }1288 }1289 1290 // TODO: Change this to assert and move to the beginning of the function.1291 if (!M.hasOperand(NodeName))1292 return failedImport("could not find node $" + NodeName +1293 " in the source DAG");1294 1295 // TODO: Remove this check and add CopyRenderer unconditionally.1296 // TODO: Handle nodes with multiple results (provided they can reach here).1297 if (isa<UnsetInit>(N.getLeafValue())) {1298 MIBuilder.addRenderer<CopyRenderer>(NodeName);1299 return Error::success();1300 }1301 1302 return failedImport("unsupported node " + to_string(N));1303}1304 1305// Equivalent of MatcherGen::EmitResultLeafAsOperand.1306Error GlobalISelEmitter::importLeafNodeRenderer(1307 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N,1308 action_iterator InsertPt) const {1309 if (const auto *II = dyn_cast<IntInit>(N.getLeafValue())) {1310 MIBuilder.addRenderer<ImmRenderer>(II->getValue());1311 return Error::success();1312 }1313 1314 if (const auto *DI = dyn_cast<DefInit>(N.getLeafValue())) {1315 const Record *R = DI->getDef();1316 1317 if (R->isSubClassOf("Register") || R->getName() == "zero_reg") {1318 MIBuilder.addRenderer<AddRegisterRenderer>(Target, R);1319 return Error::success();1320 }1321 1322 if (R->getName() == "undef_tied_input") {1323 std::optional<LLTCodeGen> OpTyOrNone = MVTToLLT(N.getSimpleType(0));1324 if (!OpTyOrNone)1325 return failedImport("unsupported type");1326 1327 unsigned TempRegID = M.allocateTempRegID();1328 M.insertAction<MakeTempRegisterAction>(InsertPt, *OpTyOrNone, TempRegID);1329 1330 auto I = M.insertAction<BuildMIAction>(1331 InsertPt, M.allocateOutputInsnID(),1332 &Target.getInstruction(RK.getDef("IMPLICIT_DEF")));1333 auto &ImpDefBuilder = static_cast<BuildMIAction &>(**I);1334 ImpDefBuilder.addRenderer<TempRegRenderer>(TempRegID, /*IsDef=*/true);1335 1336 MIBuilder.addRenderer<TempRegRenderer>(TempRegID);1337 return Error::success();1338 }1339 1340 if (R->isSubClassOf("SubRegIndex")) {1341 const CodeGenSubRegIndex *SubRegIndex = CGRegs.findSubRegIdx(R);1342 MIBuilder.addRenderer<ImmRenderer>(SubRegIndex->EnumValue);1343 return Error::success();1344 }1345 1346 // There are also RegisterClass / RegisterOperand operands of REG_SEQUENCE /1347 // COPY_TO_REGCLASS, but these instructions are currently handled elsewhere.1348 }1349 1350 return failedImport("unrecognized node " + to_string(N));1351}1352 1353// Equivalent of MatcherGen::EmitResultSDNodeXFormAsOperand.1354Error GlobalISelEmitter::importXFormNodeRenderer(1355 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N) const {1356 const Record *XFormRec = N.getOperator();1357 auto I = SDNodeXFormEquivs.find(XFormRec);1358 if (I == SDNodeXFormEquivs.end())1359 return failedImport("SDNodeXForm " + XFormRec->getName() +1360 " does not have GISel equivalent");1361 1362 // TODO: Fail to import if GISDNodeXForm does not have RendererFn.1363 // This currently results in a fatal error in emitRenderOpcodes.1364 const Record *XFormEquivRec = I->second;1365 1366 // The node to apply the transformation function to.1367 // FIXME: The node may not have a name and may be a leaf. It should be1368 // rendered first, like any other nodes. This may or may not require1369 // introducing a temporary register, and we can't tell that without1370 // inspecting the node (possibly recursively). This is a general drawback1371 // of appending renderers directly to BuildMIAction.1372 const TreePatternNode &Node = N.getChild(0);1373 StringRef NodeName = Node.getName();1374 1375 const Record *XFormOpc = CGP.getSDNodeTransform(XFormRec).first;1376 if (XFormOpc->getName() == "timm") {1377 // If this is a TargetConstant, there won't be a corresponding1378 // instruction to transform. Instead, this will refer directly to an1379 // operand in an instruction's operand list.1380 MIBuilder.addRenderer<CustomOperandRenderer>(*XFormEquivRec, NodeName);1381 } else {1382 MIBuilder.addRenderer<CustomRenderer>(*XFormEquivRec, NodeName);1383 }1384 1385 return Error::success();1386}1387 1388// Equivalent of MatcherGen::EmitResultInstructionAsOperand.1389Error GlobalISelEmitter::importInstructionNodeRenderer(1390 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N,1391 action_iterator &InsertPt) const {1392 Expected<LLTCodeGen> OpTy = getInstResultType(N, Target);1393 if (!OpTy)1394 return OpTy.takeError();1395 1396 // TODO: See the comment in importXFormNodeRenderer. We rely on the node1397 // requiring a temporary register, which prevents us from using this1398 // function on the root of the destination DAG.1399 unsigned TempRegID = M.allocateTempRegID();1400 InsertPt = M.insertAction<MakeTempRegisterAction>(InsertPt, *OpTy, TempRegID);1401 MIBuilder.addRenderer<TempRegRenderer>(TempRegID);1402 1403 auto InsertPtOrError =1404 createAndImportSubInstructionRenderer(++InsertPt, M, N, TempRegID);1405 if (!InsertPtOrError)1406 return InsertPtOrError.takeError();1407 1408 InsertPt = *InsertPtOrError;1409 return Error::success();1410}1411 1412// Equivalent of MatcherGen::EmitResultOperand.1413Error GlobalISelEmitter::importNodeRenderer(RuleMatcher &M,1414 BuildMIAction &MIBuilder,1415 const TreePatternNode &N,1416 action_iterator &InsertPt) const {1417 if (N.hasName())1418 return importNamedNodeRenderer(M, MIBuilder, N);1419 1420 if (N.isLeaf())1421 return importLeafNodeRenderer(M, MIBuilder, N, InsertPt);1422 1423 if (N.getOperator()->isSubClassOf("SDNodeXForm"))1424 return importXFormNodeRenderer(M, MIBuilder, N);1425 1426 if (N.getOperator()->isSubClassOf("Instruction"))1427 return importInstructionNodeRenderer(M, MIBuilder, N, InsertPt);1428 1429 // Should not reach here.1430 return failedImport("unrecognized node " + llvm::to_string(N));1431}1432 1433/// Generates code that builds the resulting instruction(s) from the destination1434/// DAG. Note that to do this we do not and should not need the source DAG.1435/// We do need to know whether a generated instruction defines a result of the1436/// source DAG; this information is available via RuleMatcher::hasOperand.1437Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer(1438 RuleMatcher &M, InstructionMatcher &InsnMatcher,1439 const TreePatternNode &Dst) const {1440 auto InsertPtOrError = createInstructionRenderer(M.actions_end(), M, Dst);1441 if (auto Error = InsertPtOrError.takeError())1442 return std::move(Error);1443 1444 action_iterator InsertPt = InsertPtOrError.get();1445 BuildMIAction &DstMIBuilder = *static_cast<BuildMIAction *>(InsertPt->get());1446 1447 for (auto PhysOp : M.physoperands()) {1448 InsertPt = M.insertAction<BuildMIAction>(1449 InsertPt, M.allocateOutputInsnID(),1450 &Target.getInstruction(RK.getDef("COPY")));1451 BuildMIAction &CopyToPhysRegMIBuilder =1452 *static_cast<BuildMIAction *>(InsertPt->get());1453 CopyToPhysRegMIBuilder.addRenderer<AddRegisterRenderer>(Target,1454 PhysOp.first, true);1455 CopyToPhysRegMIBuilder.addRenderer<CopyPhysRegRenderer>(PhysOp.first);1456 }1457 1458 if (auto Error = importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Dst,1459 /*IsRoot=*/true)1460 .takeError())1461 return std::move(Error);1462 1463 if (auto Error = importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst)1464 .takeError())1465 return std::move(Error);1466 1467 return DstMIBuilder;1468}1469 1470Expected<action_iterator>1471GlobalISelEmitter::createAndImportSubInstructionRenderer(1472 action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst,1473 unsigned TempRegID) const {1474 auto InsertPtOrError = createInstructionRenderer(InsertPt, M, Dst);1475 1476 // TODO: Assert there's exactly one result.1477 1478 if (auto Error = InsertPtOrError.takeError())1479 return std::move(Error);1480 1481 BuildMIAction &DstMIBuilder =1482 *static_cast<BuildMIAction *>(InsertPtOrError.get()->get());1483 1484 // Assign the result to TempReg.1485 DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID, true);1486 1487 // Handle additional (ignored) results.1488 InsertPtOrError = importExplicitDefRenderers(1489 std::prev(*InsertPtOrError), M, DstMIBuilder, Dst, /*IsRoot=*/false);1490 if (auto Error = InsertPtOrError.takeError())1491 return std::move(Error);1492 1493 InsertPtOrError =1494 importExplicitUseRenderers(InsertPtOrError.get(), M, DstMIBuilder, Dst);1495 if (auto Error = InsertPtOrError.takeError())1496 return std::move(Error);1497 1498 if (auto Error =1499 constrainOperands(InsertPt, M, DstMIBuilder.getInsnID(), Dst))1500 return std::move(Error);1501 1502 return InsertPtOrError.get();1503}1504 1505Expected<action_iterator>1506GlobalISelEmitter::createInstructionRenderer(action_iterator InsertPt,1507 RuleMatcher &M,1508 const TreePatternNode &Dst) const {1509 const Record *DstOp = Dst.getOperator();1510 if (!DstOp->isSubClassOf("Instruction")) {1511 if (DstOp->isSubClassOf("ValueType"))1512 return failedImport(1513 "Pattern operator isn't an instruction (it's a ValueType)");1514 return failedImport("Pattern operator isn't an instruction");1515 }1516 const CodeGenInstruction *DstI = &Target.getInstruction(DstOp);1517 1518 // COPY_TO_REGCLASS is just a copy with a ConstrainOperandToRegClassAction1519 // attached. Similarly for EXTRACT_SUBREG except that's a subregister copy.1520 StringRef Name = DstI->getName();1521 if (Name == "COPY_TO_REGCLASS" || Name == "EXTRACT_SUBREG")1522 DstI = &Target.getInstruction(RK.getDef("COPY"));1523 1524 return M.insertAction<BuildMIAction>(InsertPt, M.allocateOutputInsnID(),1525 DstI);1526}1527 1528Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers(1529 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,1530 const TreePatternNode &Dst, bool IsRoot) const {1531 const CodeGenInstruction *DstI = DstMIBuilder.getCGI();1532 1533 // Process explicit defs. The caller may have already handled the first def.1534 for (unsigned I = IsRoot ? 0 : 1, E = DstI->Operands.NumDefs; I != E; ++I) {1535 const CGIOperandList::OperandInfo &OpInfo = DstI->Operands[I];1536 std::string OpName = getMangledRootDefName(OpInfo.Name);1537 1538 // If the def is used in the source DAG, forward it.1539 if (IsRoot && M.hasOperand(OpName)) {1540 // CopyRenderer saves a StringRef, so cannot pass OpName itself -1541 // let's use a string with an appropriate lifetime.1542 StringRef PermanentRef = M.getOperandMatcher(OpName).getSymbolicName();1543 DstMIBuilder.addRenderer<CopyRenderer>(PermanentRef);1544 continue;1545 }1546 1547 // A discarded explicit def may be an optional physical register.1548 // If this is the case, add the default register and mark it as dead.1549 if (OpInfo.Rec->isSubClassOf("OptionalDefOperand")) {1550 for (const TreePatternNode &DefaultOp :1551 make_pointee_range(CGP.getDefaultOperand(OpInfo.Rec).DefaultOps)) {1552 // TODO: Do these checks in ParseDefaultOperands.1553 if (!DefaultOp.isLeaf())1554 return failedImport("optional def is not a leaf");1555 1556 const auto *RegDI = dyn_cast<DefInit>(DefaultOp.getLeafValue());1557 if (!RegDI)1558 return failedImport("optional def is not a record");1559 1560 const Record *Reg = RegDI->getDef();1561 if (!Reg->isSubClassOf("Register") && Reg->getName() != "zero_reg")1562 return failedImport("optional def is not a register");1563 1564 DstMIBuilder.addRenderer<AddRegisterRenderer>(1565 Target, Reg, /*IsDef=*/true, /*IsDead=*/true);1566 }1567 continue;1568 }1569 1570 // The def is discarded, create a dead virtual register for it.1571 const TypeSetByHwMode &ExtTy = Dst.getExtType(I);1572 if (!ExtTy.isMachineValueType())1573 return failedImport("unsupported typeset");1574 1575 auto OpTy = MVTToLLT(ExtTy.getMachineValueType().SimpleTy);1576 if (!OpTy)1577 return failedImport("unsupported type");1578 1579 unsigned TempRegID = M.allocateTempRegID();1580 InsertPt =1581 M.insertAction<MakeTempRegisterAction>(InsertPt, *OpTy, TempRegID);1582 DstMIBuilder.addRenderer<TempRegRenderer>(1583 TempRegID, /*IsDef=*/true, /*SubReg=*/nullptr, /*IsDead=*/true);1584 }1585 1586 // Implicit defs are not currently supported, mark all of them as dead.1587 for (const Record *Reg : DstI->ImplicitDefs) {1588 std::string OpName = getMangledRootDefName(Reg->getName());1589 assert(!M.hasOperand(OpName) && "The pattern should've been rejected");1590 DstMIBuilder.setDeadImplicitDef(Reg);1591 }1592 1593 return InsertPt;1594}1595 1596Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(1597 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,1598 const TreePatternNode &Dst) const {1599 const CodeGenInstruction *DstI = DstMIBuilder.getCGI();1600 const CodeGenInstruction *OrigDstI =1601 &Target.getInstruction(Dst.getOperator());1602 1603 StringRef Name = OrigDstI->getName();1604 unsigned ExpectedDstINumUses = Dst.getNumChildren();1605 1606 // EXTRACT_SUBREG needs to use a subregister COPY.1607 if (Name == "EXTRACT_SUBREG") {1608 if (!Dst.getChild(1).isLeaf())1609 return failedImport("EXTRACT_SUBREG child #1 is not a leaf");1610 const DefInit *SubRegInit =1611 dyn_cast<DefInit>(Dst.getChild(1).getLeafValue());1612 if (!SubRegInit)1613 return failedImport("EXTRACT_SUBREG child #1 is not a subreg index");1614 1615 const CodeGenSubRegIndex *SubIdx =1616 CGRegs.findSubRegIdx(SubRegInit->getDef());1617 const TreePatternNode &ValChild = Dst.getChild(0);1618 if (!ValChild.isLeaf()) {1619 // We really have to handle the source instruction, and then insert a1620 // copy from the subregister.1621 auto ExtractSrcTy = getInstResultType(ValChild, Target);1622 if (!ExtractSrcTy)1623 return ExtractSrcTy.takeError();1624 1625 unsigned TempRegID = M.allocateTempRegID();1626 InsertPt = M.insertAction<MakeTempRegisterAction>(InsertPt, *ExtractSrcTy,1627 TempRegID);1628 1629 auto InsertPtOrError = createAndImportSubInstructionRenderer(1630 ++InsertPt, M, ValChild, TempRegID);1631 if (auto Error = InsertPtOrError.takeError())1632 return std::move(Error);1633 1634 DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID, false, SubIdx);1635 return InsertPt;1636 }1637 1638 // If this is a source operand, this is just a subregister copy.1639 const Record *RCDef =1640 Target.getInitValueAsRegClass(ValChild.getLeafValue());1641 if (!RCDef)1642 return failedImport("EXTRACT_SUBREG child #0 could not "1643 "be coerced to a register class");1644 1645 CodeGenRegisterClass *RC = CGRegs.getRegClass(RCDef);1646 1647 const auto SrcRCDstRCPair =1648 RC->getMatchingSubClassWithSubRegs(CGRegs, SubIdx);1649 if (SrcRCDstRCPair) {1650 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");1651 if (SrcRCDstRCPair->first != RC)1652 return failedImport("EXTRACT_SUBREG requires an additional COPY");1653 }1654 1655 StringRef RegOperandName = Dst.getChild(0).getName();1656 if (const auto &SubOperand = M.getComplexSubOperand(RegOperandName)) {1657 DstMIBuilder.addRenderer<RenderComplexPatternOperand>(1658 *std::get<0>(*SubOperand), RegOperandName, std::get<1>(*SubOperand),1659 std::get<2>(*SubOperand), SubIdx);1660 return InsertPt;1661 }1662 1663 DstMIBuilder.addRenderer<CopySubRegRenderer>(RegOperandName, SubIdx);1664 return InsertPt;1665 }1666 1667 if (Name == "REG_SEQUENCE") {1668 if (!Dst.getChild(0).isLeaf())1669 return failedImport("REG_SEQUENCE child #0 is not a leaf");1670 1671 const Record *RCDef =1672 Target.getInitValueAsRegClass(Dst.getChild(0).getLeafValue());1673 if (!RCDef)1674 return failedImport("REG_SEQUENCE child #0 could not "1675 "be coerced to a register class");1676 1677 if ((ExpectedDstINumUses - 1) % 2 != 0)1678 return failedImport("Malformed REG_SEQUENCE");1679 1680 for (unsigned I = 1; I != ExpectedDstINumUses; I += 2) {1681 const TreePatternNode &ValChild = Dst.getChild(I);1682 const TreePatternNode &SubRegChild = Dst.getChild(I + 1);1683 1684 if (const DefInit *SubRegInit =1685 dyn_cast<DefInit>(SubRegChild.getLeafValue())) {1686 const CodeGenSubRegIndex *SubIdx =1687 CGRegs.findSubRegIdx(SubRegInit->getDef());1688 1689 if (Error Err = importNodeRenderer(M, DstMIBuilder, ValChild, InsertPt))1690 return Err;1691 1692 DstMIBuilder.addRenderer<SubRegIndexRenderer>(SubIdx);1693 }1694 }1695 1696 return InsertPt;1697 }1698 1699 // Render the explicit uses.1700 unsigned DstINumUses = OrigDstI->Operands.size() - OrigDstI->Operands.NumDefs;1701 if (Name == "COPY_TO_REGCLASS") {1702 DstINumUses--; // Ignore the class constraint.1703 ExpectedDstINumUses--;1704 }1705 1706 // NumResults - This is the number of results produced by the instruction in1707 // the "outs" list.1708 unsigned NumResults = OrigDstI->Operands.NumDefs;1709 1710 // Number of operands we know the output instruction must have. If it is1711 // variadic, we could have more operands.1712 unsigned NumFixedOperands = DstI->Operands.size();1713 1714 // Loop over all of the fixed operands of the instruction pattern, emitting1715 // code to fill them all in. The node 'N' usually has number children equal to1716 // the number of input operands of the instruction. However, in cases where1717 // there are predicate operands for an instruction, we need to fill in the1718 // 'execute always' values. Match up the node operands to the instruction1719 // operands to do this.1720 unsigned Child = 0;1721 1722 // Similarly to the code in TreePatternNode::ApplyTypeConstraints, count the1723 // number of operands at the end of the list which have default values.1724 // Those can come from the pattern if it provides enough arguments, or be1725 // filled in with the default if the pattern hasn't provided them. But any1726 // operand with a default value _before_ the last mandatory one will be1727 // filled in with their defaults unconditionally.1728 unsigned NonOverridableOperands = NumFixedOperands;1729 while (NonOverridableOperands > NumResults &&1730 CGP.operandHasDefault(DstI->Operands[NonOverridableOperands - 1].Rec))1731 --NonOverridableOperands;1732 1733 unsigned NumDefaultOps = 0;1734 for (unsigned I = 0; I != DstINumUses; ++I) {1735 unsigned InstOpNo = DstI->Operands.NumDefs + I;1736 1737 // Determine what to emit for this operand.1738 const Record *OperandNode = DstI->Operands[InstOpNo].Rec;1739 1740 // If the operand has default values, introduce them now.1741 if (CGP.operandHasDefault(OperandNode) &&1742 (InstOpNo < NonOverridableOperands || Child >= Dst.getNumChildren())) {1743 // This is a predicate or optional def operand which the pattern has not1744 // overridden, or which we aren't letting it override; emit the 'default1745 // ops' operands.1746 for (const TreePatternNode &OpNode :1747 make_pointee_range(CGP.getDefaultOperand(OperandNode).DefaultOps)) {1748 if (Error Err = importNodeRenderer(M, DstMIBuilder, OpNode, InsertPt))1749 return Err;1750 }1751 1752 ++NumDefaultOps;1753 continue;1754 }1755 1756 if (Error Err =1757 importNodeRenderer(M, DstMIBuilder, Dst.getChild(Child), InsertPt))1758 return Err;1759 1760 ++Child;1761 }1762 1763 if (NumDefaultOps + ExpectedDstINumUses != DstINumUses)1764 return failedImport("Expected " + llvm::to_string(DstINumUses) +1765 " used operands but found " +1766 llvm::to_string(ExpectedDstINumUses) +1767 " explicit ones and " + llvm::to_string(NumDefaultOps) +1768 " default ones");1769 1770 return InsertPt;1771}1772 1773Error GlobalISelEmitter::importImplicitDefRenderers(1774 BuildMIAction &DstMIBuilder, ArrayRef<const Record *> ImplicitDefs) const {1775 if (!ImplicitDefs.empty())1776 return failedImport("Pattern defines a physical register");1777 return Error::success();1778}1779 1780Error GlobalISelEmitter::constrainOperands(action_iterator InsertPt,1781 RuleMatcher &M, unsigned InsnID,1782 const TreePatternNode &Dst) const {1783 const Record *DstOp = Dst.getOperator();1784 const CodeGenInstruction &DstI = Target.getInstruction(DstOp);1785 StringRef DstIName = DstI.getName();1786 1787 if (DstIName == "COPY_TO_REGCLASS") {1788 // COPY_TO_REGCLASS does not provide operand constraints itself but the1789 // result is constrained to the class given by the second child.1790 const Record *DstIOpRec =1791 Target.getInitValueAsRegClass(Dst.getChild(1).getLeafValue());1792 1793 if (DstIOpRec == nullptr)1794 return failedImport("COPY_TO_REGCLASS operand #1 isn't a register class");1795 1796 M.insertAction<ConstrainOperandToRegClassAction>(1797 InsertPt, InsnID, 0, Target.getRegisterClass(DstIOpRec));1798 } else if (DstIName == "EXTRACT_SUBREG") {1799 const CodeGenRegisterClass *SuperClass =1800 inferRegClassFromPattern(Dst.getChild(0));1801 if (!SuperClass)1802 return failedImport(1803 "Cannot infer register class from EXTRACT_SUBREG operand #0");1804 1805 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(Dst.getChild(1));1806 if (!SubIdx)1807 return failedImport("EXTRACT_SUBREG child #1 is not a subreg index");1808 1809 // It would be nice to leave this constraint implicit but we're required1810 // to pick a register class so constrain the result to a register class1811 // that can hold the correct MVT.1812 //1813 // FIXME: This may introduce an extra copy if the chosen class doesn't1814 // actually contain the subregisters.1815 const auto SrcRCDstRCPair =1816 SuperClass->getMatchingSubClassWithSubRegs(CGRegs, SubIdx);1817 if (!SrcRCDstRCPair) {1818 return failedImport("subreg index is incompatible "1819 "with inferred reg class");1820 }1821 1822 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");1823 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, InsnID, 0,1824 *SrcRCDstRCPair->second);1825 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, InsnID, 1,1826 *SrcRCDstRCPair->first);1827 } else if (DstIName == "INSERT_SUBREG") {1828 // We need to constrain the destination, a super regsister source, and a1829 // subregister source.1830 const CodeGenRegisterClass *SubClass =1831 inferRegClassFromPattern(Dst.getChild(1));1832 if (!SubClass)1833 return failedImport(1834 "Cannot infer register class from INSERT_SUBREG operand #1");1835 const CodeGenRegisterClass *SuperClass = inferSuperRegisterClassForNode(1836 Dst.getExtType(0), Dst.getChild(0), Dst.getChild(2));1837 if (!SuperClass)1838 return failedImport(1839 "Cannot infer register class for INSERT_SUBREG operand #0");1840 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, InsnID, 0,1841 *SuperClass);1842 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, InsnID, 1,1843 *SuperClass);1844 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, InsnID, 2,1845 *SubClass);1846 } else if (DstIName == "SUBREG_TO_REG") {1847 // We need to constrain the destination and subregister source.1848 // Attempt to infer the subregister source from the first child. If it has1849 // an explicitly given register class, we'll use that. Otherwise, we will1850 // fail.1851 const CodeGenRegisterClass *SubClass =1852 inferRegClassFromPattern(Dst.getChild(1));1853 if (!SubClass)1854 return failedImport(1855 "Cannot infer register class from SUBREG_TO_REG child #1");1856 // We don't have a child to look at that might have a super register node.1857 const CodeGenRegisterClass *SuperClass =1858 inferSuperRegisterClass(Dst.getExtType(0), Dst.getChild(2));1859 if (!SuperClass)1860 return failedImport(1861 "Cannot infer register class for SUBREG_TO_REG operand #0");1862 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, InsnID, 0,1863 *SuperClass);1864 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, InsnID, 2,1865 *SubClass);1866 } else if (DstIName == "REG_SEQUENCE") {1867 const CodeGenRegisterClass *SuperClass =1868 inferRegClassFromPattern(Dst.getChild(0));1869 1870 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, InsnID, 0,1871 *SuperClass);1872 1873 unsigned Num = Dst.getNumChildren();1874 for (unsigned I = 1; I != Num; I += 2) {1875 const TreePatternNode &SubRegChild = Dst.getChild(I + 1);1876 1877 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(SubRegChild);1878 if (!SubIdx)1879 return failedImport("REG_SEQUENCE child is not a subreg index");1880 1881 const auto SrcRCDstRCPair =1882 SuperClass->getMatchingSubClassWithSubRegs(CGRegs, SubIdx);1883 1884 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, InsnID, I,1885 *SrcRCDstRCPair->second);1886 }1887 } else {1888 M.insertAction<ConstrainOperandsToDefinitionAction>(InsertPt, InsnID);1889 }1890 1891 return Error::success();1892}1893 1894const CodeGenRegisterClass *1895GlobalISelEmitter::getRegClassFromLeaf(const TreePatternNode &Leaf) const {1896 assert(Leaf.isLeaf() && "Expected leaf?");1897 const Record *RCRec = Target.getInitValueAsRegClass(Leaf.getLeafValue());1898 if (!RCRec)1899 return nullptr;1900 return CGRegs.getRegClass(RCRec);1901}1902 1903const CodeGenRegisterClass *1904GlobalISelEmitter::inferRegClassFromPattern(const TreePatternNode &N) const {1905 if (N.isLeaf())1906 return getRegClassFromLeaf(N);1907 1908 // We don't have a leaf node, so we have to try and infer something. Check1909 // that we have an instruction that we can infer something from.1910 1911 // Only handle things that produce at least one value (if multiple values,1912 // just take the first one).1913 if (N.getNumTypes() < 1)1914 return nullptr;1915 const Record *OpRec = N.getOperator();1916 1917 // We only want instructions.1918 if (!OpRec->isSubClassOf("Instruction"))1919 return nullptr;1920 1921 // Don't want to try and infer things when there could potentially be more1922 // than one candidate register class.1923 return inferRegClassFromInstructionPattern(N, /*ResIdx=*/0);1924}1925 1926const CodeGenRegisterClass *1927GlobalISelEmitter::inferRegClassFromRegisterClassLike(1928 const Record *RegClassDef) const {1929 if (RegClassDef->isSubClassOf("RegClassByHwMode")) {1930 // TODO: We only are trying to match the regbank, which we assume is the1931 // same across modes, but we are just picking one class and assuming they1932 // all have the same bank.1933 //1934 // We should verify there is a common regbank among the possible classes,1935 // and return that instead of a concrete class.1936 const HwModeSelect &ModeSelect =1937 Target.getHwModes().getHwModeSelect(RegClassDef);1938 if (ModeSelect.Items.empty())1939 return nullptr;1940 return Target.getRegBank().getRegClass(ModeSelect.Items.front().second);1941 }1942 1943 return &Target.getRegisterClass(RegClassDef);1944}1945 1946const CodeGenRegisterClass *1947GlobalISelEmitter::inferRegClassFromInstructionPattern(const TreePatternNode &N,1948 unsigned ResIdx) const {1949 const CodeGenInstruction &Inst = Target.getInstruction(N.getOperator());1950 assert(ResIdx < Inst.Operands.NumDefs &&1951 "Can only infer register class for explicit defs");1952 1953 // Handle any special-case instructions which we can safely infer register1954 // classes from.1955 StringRef InstName = Inst.getName();1956 if (InstName == "REG_SEQUENCE") {1957 // (outs $super_dst), (ins $dst_regclass, variable_ops)1958 // Destination register class is explicitly specified by the first operand.1959 const TreePatternNode &RCChild = N.getChild(0);1960 if (!RCChild.isLeaf())1961 return nullptr;1962 return getRegClassFromLeaf(RCChild);1963 }1964 1965 if (InstName == "COPY_TO_REGCLASS") {1966 // (outs $dst), (ins $src, $dst_regclass)1967 // Destination register class is explicitly specified by the second operand.1968 const TreePatternNode &RCChild = N.getChild(1);1969 if (!RCChild.isLeaf())1970 return nullptr;1971 return getRegClassFromLeaf(RCChild);1972 }1973 1974 if (InstName == "INSERT_SUBREG") {1975 // (outs $super_dst), (ins $super_src, $sub_src, $sub_idx);1976 // If we can infer the register class for the first operand, use that.1977 // Otherwise, find a register class that supports both the specified1978 // sub-register index and the type of the instruction's result.1979 const TreePatternNode &Child0 = N.getChild(0);1980 assert(Child0.getNumTypes() == 1 && "Unexpected number of types!");1981 return inferSuperRegisterClassForNode(N.getExtType(0), Child0,1982 N.getChild(2));1983 }1984 1985 if (InstName == "EXTRACT_SUBREG") {1986 // (outs $sub_dst), (ins $super_src, $sub_idx)1987 // Find a register class that can be used for a sub-register copy from1988 // the specified source at the specified sub-register index.1989 const CodeGenRegisterClass *SuperRC =1990 inferRegClassFromPattern(N.getChild(0));1991 if (!SuperRC)1992 return nullptr;1993 1994 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(N.getChild(1));1995 if (!SubIdx)1996 return nullptr;1997 1998 const auto SubRCAndSubRegRC =1999 SuperRC->getMatchingSubClassWithSubRegs(CGRegs, SubIdx);2000 if (!SubRCAndSubRegRC)2001 return nullptr;2002 2003 return SubRCAndSubRegRC->second;2004 }2005 2006 if (InstName == "SUBREG_TO_REG") {2007 // (outs $super_dst), (ins $super_src, $sub_src, $sub_idx)2008 // Find a register class that supports both the specified sub-register2009 // index and the type of the instruction's result.2010 return inferSuperRegisterClass(N.getExtType(0), N.getChild(2));2011 }2012 2013 // Handle destination record types that we can safely infer a register class2014 // from.2015 const auto &DstIOperand = Inst.Operands[ResIdx];2016 const Record *DstIOpRec = DstIOperand.Rec;2017 2018 if (DstIOpRec->isSubClassOf("RegisterOperand")) {2019 const Record *RegClassDef = DstIOpRec->getValueAsDef("RegClass");2020 return inferRegClassFromRegisterClassLike(RegClassDef);2021 }2022 2023 if (DstIOpRec->isSubClassOf("RegClassByHwMode"))2024 return inferRegClassFromRegisterClassLike(DstIOpRec);2025 2026 if (DstIOpRec->isSubClassOf("RegisterClass"))2027 return &Target.getRegisterClass(DstIOpRec);2028 2029 return nullptr;2030}2031 2032const CodeGenRegisterClass *GlobalISelEmitter::inferSuperRegisterClass(2033 const TypeSetByHwMode &Ty, const TreePatternNode &SubRegIdxNode) const {2034 // We need a ValueTypeByHwMode for getSuperRegForSubReg.2035 if (!Ty.isValueTypeByHwMode(false))2036 return nullptr;2037 if (!SubRegIdxNode.isLeaf())2038 return nullptr;2039 const DefInit *SubRegInit = dyn_cast<DefInit>(SubRegIdxNode.getLeafValue());2040 if (!SubRegInit)2041 return nullptr;2042 const CodeGenSubRegIndex *SubIdx = CGRegs.findSubRegIdx(SubRegInit->getDef());2043 2044 // Use the information we found above to find a minimal register class which2045 // supports the subregister and type we want.2046 return CGRegs.getSuperRegForSubReg(Ty.getValueTypeByHwMode(), SubIdx,2047 /*MustBeAllocatable=*/true);2048}2049 2050const CodeGenRegisterClass *GlobalISelEmitter::inferSuperRegisterClassForNode(2051 const TypeSetByHwMode &Ty, const TreePatternNode &SuperRegNode,2052 const TreePatternNode &SubRegIdxNode) const {2053 // Check if we already have a defined register class for the super register2054 // node. If we do, then we should preserve that rather than inferring anything2055 // from the subregister index node. We can assume that whoever wrote the2056 // pattern in the first place made sure that the super register and2057 // subregister are compatible.2058 if (const CodeGenRegisterClass *SuperRegisterClass =2059 inferRegClassFromPattern(SuperRegNode))2060 return SuperRegisterClass;2061 return inferSuperRegisterClass(Ty, SubRegIdxNode);2062}2063 2064const CodeGenSubRegIndex *GlobalISelEmitter::inferSubRegIndexForNode(2065 const TreePatternNode &SubRegIdxNode) const {2066 if (!SubRegIdxNode.isLeaf())2067 return nullptr;2068 2069 const DefInit *SubRegInit = dyn_cast<DefInit>(SubRegIdxNode.getLeafValue());2070 if (!SubRegInit)2071 return nullptr;2072 return CGRegs.findSubRegIdx(SubRegInit->getDef());2073}2074 2075Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {2076 // Keep track of the matchers and actions to emit.2077 int Score = P.getPatternComplexity(CGP);2078 RuleMatcher M(P.getSrcRecord()->getLoc());2079 RuleMatcherScores[M.getRuleID()] = Score;2080 M.addAction<DebugCommentAction>(llvm::to_string(P.getSrcPattern()) +2081 " => " +2082 llvm::to_string(P.getDstPattern()));2083 2084 SmallVector<const Record *, 4> Predicates;2085 P.getPredicateRecords(Predicates);2086 if (auto Error = importRulePredicates(M, Predicates))2087 return std::move(Error);2088 2089 if (!P.getHwModeFeatures().empty())2090 M.addHwModeIdx(declareHwModeCheck(P.getHwModeFeatures()));2091 2092 // Next, analyze the pattern operators.2093 TreePatternNode &Src = P.getSrcPattern();2094 TreePatternNode &Dst = P.getDstPattern();2095 2096 // If the root of either pattern isn't a simple operator, ignore it.2097 if (auto Err = isTrivialOperatorNode(Dst))2098 return failedImport("Dst pattern root isn't a trivial operator (" +2099 toString(std::move(Err)) + ")");2100 if (auto Err = isTrivialOperatorNode(Src))2101 return failedImport("Src pattern root isn't a trivial operator (" +2102 toString(std::move(Err)) + ")");2103 2104 // The different predicates and matchers created during2105 // addInstructionMatcher use the RuleMatcher M to set up their2106 // instruction ID (InsnVarID) that are going to be used when2107 // M is going to be emitted.2108 // However, the code doing the emission still relies on the IDs2109 // returned during that process by the RuleMatcher when issuing2110 // the recordInsn opcodes.2111 // Because of that:2112 // 1. The order in which we created the predicates2113 // and such must be the same as the order in which we emit them,2114 // and2115 // 2. We need to reset the generation of the IDs in M somewhere between2116 // addInstructionMatcher and emit2117 //2118 // FIXME: Long term, we don't want to have to rely on this implicit2119 // naming being the same. One possible solution would be to have2120 // explicit operator for operation capture and reference those.2121 // The plus side is that it would expose opportunities to share2122 // the capture accross rules. The downside is that it would2123 // introduce a dependency between predicates (captures must happen2124 // before their first use.)2125 InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(Src.getName());2126 unsigned TempOpIdx = 0;2127 2128 const auto SavedFlags = M.setGISelFlags(P.getSrcRecord());2129 2130 auto InsnMatcherOrError =2131 createAndImportSelDAGMatcher(M, InsnMatcherTemp, Src, TempOpIdx);2132 if (auto Error = InsnMatcherOrError.takeError())2133 return std::move(Error);2134 InstructionMatcher &InsnMatcher = InsnMatcherOrError.get();2135 2136 if (Dst.isLeaf()) {2137 if (const Record *RCDef =2138 Target.getInitValueAsRegClass(Dst.getLeafValue())) {2139 const CodeGenRegisterClass &RC = Target.getRegisterClass(RCDef);2140 2141 // We need to replace the def and all its uses with the specified2142 // operand. However, we must also insert COPY's wherever needed.2143 // For now, emit a copy and let the register allocator clean up.2144 const CodeGenInstruction &DstI = Target.getInstruction(RK.getDef("COPY"));2145 const auto &DstIOperand = DstI.Operands[0];2146 2147 OperandMatcher &OM0 = InsnMatcher.getOperand(0);2148 OM0.setSymbolicName(DstIOperand.Name);2149 M.defineOperand(OM0.getSymbolicName(), OM0);2150 OM0.addPredicate<RegisterBankOperandMatcher>(RC);2151 2152 auto &DstMIBuilder =2153 M.addAction<BuildMIAction>(M.allocateOutputInsnID(), &DstI);2154 DstMIBuilder.addRenderer<CopyRenderer>(DstIOperand.Name);2155 DstMIBuilder.addRenderer<CopyRenderer>(Dst.getName());2156 M.addAction<ConstrainOperandToRegClassAction>(0, 0, RC);2157 2158 // Erase the root.2159 unsigned RootInsnID = M.getInsnVarID(InsnMatcher);2160 M.addAction<EraseInstAction>(RootInsnID);2161 2162 // We're done with this pattern! It's eligible for GISel emission; return2163 // it.2164 ++NumPatternImported;2165 return std::move(M);2166 }2167 2168 return failedImport("Dst pattern root isn't a known leaf");2169 }2170 2171 // Start with the defined operands (i.e., the results of the root operator).2172 const Record *DstOp = Dst.getOperator();2173 if (!DstOp->isSubClassOf("Instruction"))2174 return failedImport("Pattern operator isn't an instruction");2175 2176 const CodeGenInstruction &DstI = Target.getInstruction(DstOp);2177 2178 // Count both implicit and explicit defs in the dst instruction.2179 // This avoids errors importing patterns that have inherent implicit defs.2180 unsigned DstExpDefs = DstI.Operands.NumDefs,2181 DstNumDefs = DstI.ImplicitDefs.size() + DstExpDefs,2182 SrcNumDefs = Src.getExtTypes().size();2183 if (DstNumDefs < SrcNumDefs) {2184 if (DstNumDefs != 0)2185 return failedImport("Src pattern result has more defs than dst MI (" +2186 to_string(SrcNumDefs) + " def(s) vs " +2187 to_string(DstNumDefs) + " def(s))");2188 2189 bool FoundNoUsePred = false;2190 for (const auto &Pred : InsnMatcher.predicates()) {2191 if ((FoundNoUsePred = isa<NoUsePredicateMatcher>(Pred.get())))2192 break;2193 }2194 if (!FoundNoUsePred)2195 return failedImport("Src pattern result has " + to_string(SrcNumDefs) +2196 " def(s) without the HasNoUse predicate set to true "2197 "but Dst MI has no def");2198 }2199 2200 // The root of the match also has constraints on the register bank so that it2201 // matches the result instruction.2202 unsigned N = std::min(DstExpDefs, SrcNumDefs);2203 for (unsigned I = 0; I < N; ++I) {2204 const auto &DstIOperand = DstI.Operands[I];2205 2206 OperandMatcher &OM = InsnMatcher.getOperand(I);2207 // The operand names declared in the DstI instruction are unrelated to2208 // those used in pattern's source and destination DAGs, so mangle the2209 // former to prevent implicitly adding unexpected2210 // GIM_CheckIsSameOperand predicates by the defineOperand method.2211 OM.setSymbolicName(getMangledRootDefName(DstIOperand.Name));2212 M.defineOperand(OM.getSymbolicName(), OM);2213 2214 const CodeGenRegisterClass *RC =2215 inferRegClassFromInstructionPattern(Dst, I);2216 if (!RC)2217 return failedImport("Could not infer register class for result #" +2218 Twine(I) + " from pattern " + to_string(Dst));2219 OM.addPredicate<RegisterBankOperandMatcher>(*RC);2220 }2221 2222 auto DstMIBuilderOrError =2223 createAndImportInstructionRenderer(M, InsnMatcher, Dst);2224 if (auto Error = DstMIBuilderOrError.takeError())2225 return std::move(Error);2226 BuildMIAction &DstMIBuilder = DstMIBuilderOrError.get();2227 2228 // Render the implicit defs.2229 // These are only added to the root of the result.2230 if (auto Error = importImplicitDefRenderers(DstMIBuilder, P.getDstRegs()))2231 return std::move(Error);2232 2233 DstMIBuilder.chooseInsnToMutate(M);2234 2235 // Constrain the registers to classes. This is normally derived from the2236 // emitted instruction but a few instructions require special handling.2237 if (auto Error =2238 constrainOperands(M.actions_end(), M, DstMIBuilder.getInsnID(), Dst))2239 return std::move(Error);2240 2241 // Erase the root.2242 unsigned RootInsnID = M.getInsnVarID(InsnMatcher);2243 M.addAction<EraseInstAction>(RootInsnID);2244 2245 // We're done with this pattern! It's eligible for GISel emission; return it.2246 ++NumPatternImported;2247 return std::move(M);2248}2249 2250MatchTable2251GlobalISelEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules,2252 bool Optimize, bool WithCoverage) {2253 std::vector<Matcher *> InputRules;2254 for (Matcher &Rule : Rules)2255 InputRules.push_back(&Rule);2256 2257 if (!Optimize)2258 return MatchTable::buildTable(InputRules, WithCoverage);2259 2260 unsigned CurrentOrdering = 0;2261 StringMap<unsigned> OpcodeOrder;2262 for (RuleMatcher &Rule : Rules) {2263 const StringRef Opcode = Rule.getOpcode();2264 assert(!Opcode.empty() && "Didn't expect an undefined opcode");2265 if (OpcodeOrder.try_emplace(Opcode, CurrentOrdering).second)2266 ++CurrentOrdering;2267 }2268 2269 llvm::stable_sort(2270 InputRules, [&OpcodeOrder](const Matcher *A, const Matcher *B) {2271 auto *L = static_cast<const RuleMatcher *>(A);2272 auto *R = static_cast<const RuleMatcher *>(B);2273 return std::tuple(OpcodeOrder[L->getOpcode()],2274 L->insnmatchers_front().getNumOperandMatchers()) <2275 std::tuple(OpcodeOrder[R->getOpcode()],2276 R->insnmatchers_front().getNumOperandMatchers());2277 });2278 2279 for (Matcher *Rule : InputRules)2280 Rule->optimize();2281 2282 std::vector<std::unique_ptr<Matcher>> MatcherStorage;2283 std::vector<Matcher *> OptRules =2284 optimizeRules<GroupMatcher>(InputRules, MatcherStorage);2285 2286 for (Matcher *Rule : OptRules)2287 Rule->optimize();2288 2289 OptRules = optimizeRules<SwitchMatcher>(OptRules, MatcherStorage);2290 2291 return MatchTable::buildTable(OptRules, WithCoverage);2292}2293 2294void GlobalISelEmitter::emitAdditionalImpl(raw_ostream &OS) {2295 OS << "bool " << getClassName()2296 << "::selectImpl(MachineInstr &I, CodeGenCoverage "2297 "&CoverageInfo) const {\n"2298 << " const PredicateBitset AvailableFeatures = "2299 "getAvailableFeatures();\n"2300 << " MachineIRBuilder B(I);\n"2301 << " State.MIs.clear();\n"2302 << " State.MIs.push_back(&I);\n\n"2303 << " if (executeMatchTable(*this, State, ExecInfo, B"2304 << ", getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures"2305 << ", &CoverageInfo)) {\n"2306 << " return true;\n"2307 << " }\n\n"2308 << " return false;\n"2309 << "}\n\n";2310}2311 2312void GlobalISelEmitter::emitMIPredicateFns(raw_ostream &OS) {2313 std::vector<const Record *> MatchedRecords;2314 llvm::copy_if(AllPatFrags, std::back_inserter(MatchedRecords),2315 [](const Record *R) {2316 return !R->getValueAsString("GISelPredicateCode").empty();2317 });2318 emitMIPredicateFnsImpl<const Record *>(2319 OS,2320 " const MachineFunction &MF = *MI.getParent()->getParent();\n"2321 " const MachineRegisterInfo &MRI = MF.getRegInfo();\n"2322 " const auto &Operands = State.RecordedOperands;\n"2323 " (void)Operands;\n"2324 " (void)MRI;",2325 ArrayRef<const Record *>(MatchedRecords), &getPatFragPredicateEnumName,2326 [](const Record *R) { return R->getValueAsString("GISelPredicateCode"); },2327 "PatFrag predicates.");2328}2329 2330void GlobalISelEmitter::emitLeafPredicateFns(raw_ostream &OS) {2331 std::vector<const Record *> MatchedRecords;2332 llvm::copy_if(AllPatFrags, std::back_inserter(MatchedRecords),2333 [](const Record *R) {2334 return (!R->getValueAsOptionalString("GISelLeafPredicateCode")2335 .value_or(std::string())2336 .empty());2337 });2338 emitLeafPredicateFnsImpl<const Record *>(2339 OS,2340 " const auto &Operands = State.RecordedOperands;\n"2341 " Register Reg = MO.getReg();\n"2342 " (void)Operands;\n"2343 " (void)Reg;",2344 ArrayRef<const Record *>(MatchedRecords), &getPatFragPredicateEnumName,2345 [](const Record *R) {2346 return R->getValueAsString("GISelLeafPredicateCode");2347 },2348 "PatFrag predicates.");2349}2350 2351void GlobalISelEmitter::emitI64ImmPredicateFns(raw_ostream &OS) {2352 std::vector<const Record *> MatchedRecords;2353 llvm::copy_if(AllPatFrags, std::back_inserter(MatchedRecords),2354 [](const Record *R) {2355 bool Unset;2356 return !R->getValueAsString("ImmediateCode").empty() &&2357 !R->getValueAsBitOrUnset("IsAPFloat", Unset) &&2358 !R->getValueAsBit("IsAPInt");2359 });2360 emitImmPredicateFnsImpl<const Record *>(2361 OS, "I64", "int64_t", ArrayRef<const Record *>(MatchedRecords),2362 &getPatFragPredicateEnumName,2363 [](const Record *R) { return R->getValueAsString("ImmediateCode"); },2364 "PatFrag predicates.");2365}2366 2367void GlobalISelEmitter::emitAPFloatImmPredicateFns(raw_ostream &OS) {2368 std::vector<const Record *> MatchedRecords;2369 llvm::copy_if(AllPatFrags, std::back_inserter(MatchedRecords),2370 [](const Record *R) {2371 bool Unset;2372 return !R->getValueAsString("ImmediateCode").empty() &&2373 R->getValueAsBitOrUnset("IsAPFloat", Unset);2374 });2375 emitImmPredicateFnsImpl<const Record *>(2376 OS, "APFloat", "const APFloat &",2377 ArrayRef<const Record *>(MatchedRecords), &getPatFragPredicateEnumName,2378 [](const Record *R) { return R->getValueAsString("ImmediateCode"); },2379 "PatFrag predicates.");2380}2381 2382void GlobalISelEmitter::emitAPIntImmPredicateFns(raw_ostream &OS) {2383 std::vector<const Record *> MatchedRecords;2384 llvm::copy_if(AllPatFrags, std::back_inserter(MatchedRecords),2385 [](const Record *R) {2386 return !R->getValueAsString("ImmediateCode").empty() &&2387 R->getValueAsBit("IsAPInt");2388 });2389 emitImmPredicateFnsImpl<const Record *>(2390 OS, "APInt", "const APInt &", ArrayRef<const Record *>(MatchedRecords),2391 &getPatFragPredicateEnumName,2392 [](const Record *R) { return R->getValueAsString("ImmediateCode"); },2393 "PatFrag predicates.");2394}2395 2396void GlobalISelEmitter::emitTestSimplePredicate(raw_ostream &OS) {2397 OS << "bool " << getClassName() << "::testSimplePredicate(unsigned) const {\n"2398 << " llvm_unreachable(\"" + getClassName() +2399 " does not support simple predicates!\");\n"2400 << " return false;\n"2401 << "}\n";2402}2403 2404void GlobalISelEmitter::emitRunCustomAction(raw_ostream &OS) {2405 OS << "bool " << getClassName()2406 << "::runCustomAction(unsigned, const MatcherState&, NewMIVector &) const "2407 "{\n"2408 << " llvm_unreachable(\"" + getClassName() +2409 " does not support custom C++ actions!\");\n"2410 << "}\n";2411}2412 2413bool hasBFloatType(const TreePatternNode &Node) {2414 for (unsigned I = 0, E = Node.getNumTypes(); I < E; I++) {2415 auto Ty = Node.getType(I);2416 for (auto T : Ty)2417 if (T.second == MVT::bf16 ||2418 (T.second.isVector() && T.second.getScalarType() == MVT::bf16))2419 return true;2420 }2421 for (const TreePatternNode &C : Node.children())2422 if (hasBFloatType(C))2423 return true;2424 return false;2425}2426 2427void GlobalISelEmitter::run(raw_ostream &OS) {2428 if (!UseCoverageFile.empty()) {2429 RuleCoverage = CodeGenCoverage();2430 auto RuleCoverageBufOrErr = MemoryBuffer::getFile(UseCoverageFile);2431 if (!RuleCoverageBufOrErr) {2432 PrintWarning(SMLoc(), "Missing rule coverage data");2433 RuleCoverage = std::nullopt;2434 } else {2435 if (!RuleCoverage->parse(*RuleCoverageBufOrErr.get(), Target.getName())) {2436 PrintWarning(SMLoc(), "Ignoring invalid or missing rule coverage data");2437 RuleCoverage = std::nullopt;2438 }2439 }2440 }2441 2442 // Track the run-time opcode values2443 gatherOpcodeValues();2444 // Track the run-time LLT ID values2445 gatherTypeIDValues();2446 2447 // Track the GINodeEquiv definitions.2448 gatherNodeEquivs();2449 2450 AllPatFrags = RK.getAllDerivedDefinitions("PatFrags");2451 2452 emitSourceFileHeader(2453 ("Global Instruction Selector for the " + Target.getName() + " target")2454 .str(),2455 OS);2456 std::vector<RuleMatcher> Rules;2457 // Look through the SelectionDAG patterns we found, possibly emitting some.2458 for (const PatternToMatch &Pat : CGP.ptms()) {2459 ++NumPatternTotal;2460 2461 if (Pat.getGISelShouldIgnore())2462 continue; // skip without warning2463 2464 // Skip any patterns containing BF16 types, as GISel cannot currently tell2465 // the difference between fp16 and bf16. FIXME: This can be removed once2466 // BF16 is supported properly.2467 if (hasBFloatType(Pat.getSrcPattern()))2468 continue;2469 2470 auto MatcherOrErr = runOnPattern(Pat);2471 2472 // The pattern analysis can fail, indicating an unsupported pattern.2473 // Report that if we've been asked to do so.2474 if (auto Err = MatcherOrErr.takeError()) {2475 if (WarnOnSkippedPatterns) {2476 PrintWarning(Pat.getSrcRecord()->getLoc(),2477 "Skipped pattern: " + toString(std::move(Err)));2478 } else {2479 consumeError(std::move(Err));2480 }2481 ++NumPatternImportsSkipped;2482 continue;2483 }2484 2485 if (RuleCoverage) {2486 if (RuleCoverage->isCovered(MatcherOrErr->getRuleID()))2487 ++NumPatternsTested;2488 else2489 PrintWarning(Pat.getSrcRecord()->getLoc(),2490 "Pattern is not covered by a test");2491 }2492 Rules.push_back(std::move(MatcherOrErr.get()));2493 }2494 2495 // Comparison function to order records by name.2496 auto OrderByName = [](const Record *A, const Record *B) {2497 return A->getName() < B->getName();2498 };2499 2500 std::vector<const Record *> ComplexPredicates =2501 RK.getAllDerivedDefinitions("GIComplexOperandMatcher");2502 llvm::sort(ComplexPredicates, OrderByName);2503 2504 std::vector<StringRef> CustomRendererFns;2505 transform(RK.getAllDerivedDefinitions("GICustomOperandRenderer"),2506 std::back_inserter(CustomRendererFns), [](const auto &Record) {2507 return Record->getValueAsString("RendererFn");2508 });2509 // Sort and remove duplicates to get a list of unique renderer functions, in2510 // case some were mentioned more than once.2511 llvm::sort(CustomRendererFns);2512 CustomRendererFns.erase(llvm::unique(CustomRendererFns),2513 CustomRendererFns.end());2514 2515 // Create a table containing the LLT objects needed by the matcher and an enum2516 // for the matcher to reference them with.2517 std::vector<LLTCodeGen> TypeObjects;2518 append_range(TypeObjects, KnownTypes);2519 llvm::sort(TypeObjects);2520 2521 // Sort rules.2522 llvm::stable_sort(Rules, [&](const RuleMatcher &A, const RuleMatcher &B) {2523 int ScoreA = RuleMatcherScores[A.getRuleID()];2524 int ScoreB = RuleMatcherScores[B.getRuleID()];2525 if (ScoreA > ScoreB)2526 return true;2527 if (ScoreB > ScoreA)2528 return false;2529 if (A.isHigherPriorityThan(B)) {2530 assert(!B.isHigherPriorityThan(A) && "Cannot be more important "2531 "and less important at "2532 "the same time");2533 return true;2534 }2535 return false;2536 });2537 2538 unsigned MaxTemporaries = 0;2539 for (const auto &Rule : Rules)2540 MaxTemporaries = std::max(MaxTemporaries, Rule.countRendererFns());2541 2542 // Build match table2543 const MatchTable Table =2544 buildMatchTable(Rules, OptimizeMatchTable, GenerateCoverage);2545 2546 emitPredicateBitset(OS, "GET_GLOBALISEL_PREDICATE_BITSET");2547 emitTemporariesDecl(OS, "GET_GLOBALISEL_TEMPORARIES_DECL");2548 emitTemporariesInit(OS, MaxTemporaries, "GET_GLOBALISEL_TEMPORARIES_INIT");2549 emitExecutorImpl(OS, Table, TypeObjects, Rules, ComplexPredicates,2550 CustomRendererFns, "GET_GLOBALISEL_IMPL");2551 emitPredicatesDecl(OS, "GET_GLOBALISEL_PREDICATES_DECL");2552 emitPredicatesInit(OS, "GET_GLOBALISEL_PREDICATES_INIT");2553}2554 2555void GlobalISelEmitter::declareSubtargetFeature(const Record *Predicate) {2556 SubtargetFeatures.try_emplace(Predicate, Predicate, SubtargetFeatures.size());2557}2558 2559unsigned GlobalISelEmitter::declareHwModeCheck(StringRef HwModeFeatures) {2560 return HwModes.emplace(HwModeFeatures.str(), HwModes.size()).first->second;2561}2562 2563//===----------------------------------------------------------------------===//2564 2565static TableGen::Emitter::OptClass<GlobalISelEmitter>2566 X("gen-global-isel", "Generate GlobalISel selector");2567