4282 lines · cpp
1//===- NewGVN.cpp - Global Value Numbering Pass ---------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// This file implements the new LLVM's Global Value Numbering pass.11/// GVN partitions values computed by a function into congruence classes.12/// Values ending up in the same congruence class are guaranteed to be the same13/// for every execution of the program. In that respect, congruency is a14/// compile-time approximation of equivalence of values at runtime.15/// The algorithm implemented here uses a sparse formulation and it's based16/// on the ideas described in the paper:17/// "A Sparse Algorithm for Predicated Global Value Numbering" from18/// Karthik Gargi.19///20/// A brief overview of the algorithm: The algorithm is essentially the same as21/// the standard RPO value numbering algorithm (a good reference is the paper22/// "SCC based value numbering" by L. Taylor Simpson) with one major difference:23/// The RPO algorithm proceeds, on every iteration, to process every reachable24/// block and every instruction in that block. This is because the standard RPO25/// algorithm does not track what things have the same value number, it only26/// tracks what the value number of a given operation is (the mapping is27/// operation -> value number). Thus, when a value number of an operation28/// changes, it must reprocess everything to ensure all uses of a value number29/// get updated properly. In constrast, the sparse algorithm we use *also*30/// tracks what operations have a given value number (IE it also tracks the31/// reverse mapping from value number -> operations with that value number), so32/// that it only needs to reprocess the instructions that are affected when33/// something's value number changes. The vast majority of complexity and code34/// in this file is devoted to tracking what value numbers could change for what35/// instructions when various things happen. The rest of the algorithm is36/// devoted to performing symbolic evaluation, forward propagation, and37/// simplification of operations based on the value numbers deduced so far38///39/// In order to make the GVN mostly-complete, we use a technique derived from40/// "Detection of Redundant Expressions: A Complete and Polynomial-time41/// Algorithm in SSA" by R.R. Pai. The source of incompleteness in most SSA42/// based GVN algorithms is related to their inability to detect equivalence43/// between phi of ops (IE phi(a+b, c+d)) and op of phis (phi(a,c) + phi(b, d)).44/// We resolve this issue by generating the equivalent "phi of ops" form for45/// each op of phis we see, in a way that only takes polynomial time to resolve.46///47/// We also do not perform elimination by using any published algorithm. All48/// published algorithms are O(Instructions). Instead, we use a technique that49/// is O(number of operations with the same value number), enabling us to skip50/// trying to eliminate things that have unique value numbers.51//52//===----------------------------------------------------------------------===//53 54#include "llvm/Transforms/Scalar/NewGVN.h"55#include "llvm/ADT/ArrayRef.h"56#include "llvm/ADT/BitVector.h"57#include "llvm/ADT/DenseMap.h"58#include "llvm/ADT/DenseMapInfo.h"59#include "llvm/ADT/DenseSet.h"60#include "llvm/ADT/DepthFirstIterator.h"61#include "llvm/ADT/GraphTraits.h"62#include "llvm/ADT/Hashing.h"63#include "llvm/ADT/PointerIntPair.h"64#include "llvm/ADT/PostOrderIterator.h"65#include "llvm/ADT/SetOperations.h"66#include "llvm/ADT/SmallPtrSet.h"67#include "llvm/ADT/SmallVector.h"68#include "llvm/ADT/SparseBitVector.h"69#include "llvm/ADT/Statistic.h"70#include "llvm/ADT/iterator_range.h"71#include "llvm/Analysis/AliasAnalysis.h"72#include "llvm/Analysis/AssumptionCache.h"73#include "llvm/Analysis/CFGPrinter.h"74#include "llvm/Analysis/ConstantFolding.h"75#include "llvm/Analysis/GlobalsModRef.h"76#include "llvm/Analysis/InstructionSimplify.h"77#include "llvm/Analysis/MemoryBuiltins.h"78#include "llvm/Analysis/MemorySSA.h"79#include "llvm/Analysis/TargetLibraryInfo.h"80#include "llvm/Analysis/ValueTracking.h"81#include "llvm/IR/Argument.h"82#include "llvm/IR/BasicBlock.h"83#include "llvm/IR/Constant.h"84#include "llvm/IR/Constants.h"85#include "llvm/IR/DebugInfo.h"86#include "llvm/IR/Dominators.h"87#include "llvm/IR/Function.h"88#include "llvm/IR/InstrTypes.h"89#include "llvm/IR/Instruction.h"90#include "llvm/IR/Instructions.h"91#include "llvm/IR/IntrinsicInst.h"92#include "llvm/IR/PatternMatch.h"93#include "llvm/IR/Type.h"94#include "llvm/IR/Use.h"95#include "llvm/IR/User.h"96#include "llvm/IR/Value.h"97#include "llvm/Support/Allocator.h"98#include "llvm/Support/ArrayRecycler.h"99#include "llvm/Support/Casting.h"100#include "llvm/Support/CommandLine.h"101#include "llvm/Support/Debug.h"102#include "llvm/Support/DebugCounter.h"103#include "llvm/Support/ErrorHandling.h"104#include "llvm/Support/PointerLikeTypeTraits.h"105#include "llvm/Support/raw_ostream.h"106#include "llvm/Transforms/Scalar/GVNExpression.h"107#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"108#include "llvm/Transforms/Utils/Local.h"109#include "llvm/Transforms/Utils/PredicateInfo.h"110#include "llvm/Transforms/Utils/VNCoercion.h"111#include <algorithm>112#include <cassert>113#include <cstdint>114#include <iterator>115#include <map>116#include <memory>117#include <set>118#include <string>119#include <tuple>120#include <utility>121#include <vector>122 123using namespace llvm;124using namespace llvm::GVNExpression;125using namespace llvm::VNCoercion;126using namespace llvm::PatternMatch;127 128#define DEBUG_TYPE "newgvn"129 130STATISTIC(NumGVNInstrDeleted, "Number of instructions deleted");131STATISTIC(NumGVNBlocksDeleted, "Number of blocks deleted");132STATISTIC(NumGVNOpsSimplified, "Number of Expressions simplified");133STATISTIC(NumGVNPhisAllSame, "Number of PHIs whos arguments are all the same");134STATISTIC(NumGVNMaxIterations,135 "Maximum Number of iterations it took to converge GVN");136STATISTIC(NumGVNLeaderChanges, "Number of leader changes");137STATISTIC(NumGVNSortedLeaderChanges, "Number of sorted leader changes");138STATISTIC(NumGVNAvoidedSortedLeaderChanges,139 "Number of avoided sorted leader changes");140STATISTIC(NumGVNDeadStores, "Number of redundant/dead stores eliminated");141STATISTIC(NumGVNPHIOfOpsCreated, "Number of PHI of ops created");142STATISTIC(NumGVNPHIOfOpsEliminations,143 "Number of things eliminated using PHI of ops");144DEBUG_COUNTER(VNCounter, "newgvn-vn",145 "Controls which instructions are value numbered");146DEBUG_COUNTER(PHIOfOpsCounter, "newgvn-phi",147 "Controls which instructions we create phi of ops for");148// Currently store defining access refinement is too slow due to basicaa being149// egregiously slow. This flag lets us keep it working while we work on this150// issue.151static cl::opt<bool> EnableStoreRefinement("enable-store-refinement",152 cl::init(false), cl::Hidden);153 154/// Currently, the generation "phi of ops" can result in correctness issues.155static cl::opt<bool> EnablePhiOfOps("enable-phi-of-ops", cl::init(true),156 cl::Hidden);157 158//===----------------------------------------------------------------------===//159// GVN Pass160//===----------------------------------------------------------------------===//161 162// Anchor methods.163Expression::~Expression() = default;164BasicExpression::~BasicExpression() = default;165CallExpression::~CallExpression() = default;166LoadExpression::~LoadExpression() = default;167StoreExpression::~StoreExpression() = default;168AggregateValueExpression::~AggregateValueExpression() = default;169PHIExpression::~PHIExpression() = default;170 171namespace {172 173// Tarjan's SCC finding algorithm with Nuutila's improvements174// SCCIterator is actually fairly complex for the simple thing we want.175// It also wants to hand us SCC's that are unrelated to the phi node we ask176// about, and have us process them there or risk redoing work.177// Graph traits over a filter iterator also doesn't work that well here.178// This SCC finder is specialized to walk use-def chains, and only follows179// instructions,180// not generic values (arguments, etc).181struct TarjanSCC {182 TarjanSCC() : Components(1) {}183 184 void Start(const Instruction *Start) {185 if (Root.lookup(Start) == 0)186 FindSCC(Start);187 }188 189 const SmallPtrSetImpl<const Value *> &getComponentFor(const Value *V) const {190 unsigned ComponentID = ValueToComponent.lookup(V);191 192 assert(ComponentID > 0 &&193 "Asking for a component for a value we never processed");194 return Components[ComponentID];195 }196 197private:198 void FindSCC(const Instruction *I) {199 Root[I] = ++DFSNum;200 // Store the DFS Number we had before it possibly gets incremented.201 unsigned int OurDFS = DFSNum;202 for (const auto &Op : I->operands()) {203 if (auto *InstOp = dyn_cast<Instruction>(Op)) {204 if (Root.lookup(Op) == 0)205 FindSCC(InstOp);206 if (!InComponent.count(Op))207 Root[I] = std::min(Root.lookup(I), Root.lookup(Op));208 }209 }210 // See if we really were the root of a component, by seeing if we still have211 // our DFSNumber. If we do, we are the root of the component, and we have212 // completed a component. If we do not, we are not the root of a component,213 // and belong on the component stack.214 if (Root.lookup(I) == OurDFS) {215 unsigned ComponentID = Components.size();216 Components.resize(Components.size() + 1);217 auto &Component = Components.back();218 Component.insert(I);219 LLVM_DEBUG(dbgs() << "Component root is " << *I << "\n");220 InComponent.insert(I);221 ValueToComponent[I] = ComponentID;222 // Pop a component off the stack and label it.223 while (!Stack.empty() && Root.lookup(Stack.back()) >= OurDFS) {224 auto *Member = Stack.back();225 LLVM_DEBUG(dbgs() << "Component member is " << *Member << "\n");226 Component.insert(Member);227 InComponent.insert(Member);228 ValueToComponent[Member] = ComponentID;229 Stack.pop_back();230 }231 } else {232 // Part of a component, push to stack233 Stack.push_back(I);234 }235 }236 237 unsigned int DFSNum = 1;238 SmallPtrSet<const Value *, 8> InComponent;239 DenseMap<const Value *, unsigned int> Root;240 SmallVector<const Value *, 8> Stack;241 242 // Store the components as vector of ptr sets, because we need the topo order243 // of SCC's, but not individual member order244 SmallVector<SmallPtrSet<const Value *, 8>, 8> Components;245 246 DenseMap<const Value *, unsigned> ValueToComponent;247};248 249// Congruence classes represent the set of expressions/instructions250// that are all the same *during some scope in the function*.251// That is, because of the way we perform equality propagation, and252// because of memory value numbering, it is not correct to assume253// you can willy-nilly replace any member with any other at any254// point in the function.255//256// For any Value in the Member set, it is valid to replace any dominated member257// with that Value.258//259// Every congruence class has a leader, and the leader is used to symbolize260// instructions in a canonical way (IE every operand of an instruction that is a261// member of the same congruence class will always be replaced with leader262// during symbolization). To simplify symbolization, we keep the leader as a263// constant if class can be proved to be a constant value. Otherwise, the264// leader is the member of the value set with the smallest DFS number. Each265// congruence class also has a defining expression, though the expression may be266// null. If it exists, it can be used for forward propagation and reassociation267// of values.268 269// For memory, we also track a representative MemoryAccess, and a set of memory270// members for MemoryPhis (which have no real instructions). Note that for271// memory, it seems tempting to try to split the memory members into a272// MemoryCongruenceClass or something. Unfortunately, this does not work273// easily. The value numbering of a given memory expression depends on the274// leader of the memory congruence class, and the leader of memory congruence275// class depends on the value numbering of a given memory expression. This276// leads to wasted propagation, and in some cases, missed optimization. For277// example: If we had value numbered two stores together before, but now do not,278// we move them to a new value congruence class. This in turn will move at one279// of the memorydefs to a new memory congruence class. Which in turn, affects280// the value numbering of the stores we just value numbered (because the memory281// congruence class is part of the value number). So while theoretically282// possible to split them up, it turns out to be *incredibly* complicated to get283// it to work right, because of the interdependency. While structurally284// slightly messier, it is algorithmically much simpler and faster to do what we285// do here, and track them both at once in the same class.286// Note: The default iterators for this class iterate over values287class CongruenceClass {288public:289 using MemberType = Value;290 using MemberSet = SmallPtrSet<MemberType *, 4>;291 using MemoryMemberType = MemoryPhi;292 using MemoryMemberSet = SmallPtrSet<const MemoryMemberType *, 2>;293 294 explicit CongruenceClass(unsigned ID) : ID(ID) {}295 CongruenceClass(unsigned ID, std::pair<Value *, unsigned int> Leader,296 const Expression *E)297 : ID(ID), RepLeader(Leader), DefiningExpr(E) {}298 299 unsigned getID() const { return ID; }300 301 // True if this class has no members left. This is mainly used for assertion302 // purposes, and for skipping empty classes.303 bool isDead() const {304 // If it's both dead from a value perspective, and dead from a memory305 // perspective, it's really dead.306 return empty() && memory_empty();307 }308 309 // Leader functions310 Value *getLeader() const { return RepLeader.first; }311 void setLeader(std::pair<Value *, unsigned int> Leader) {312 RepLeader = Leader;313 }314 const std::pair<Value *, unsigned int> &getNextLeader() const {315 return NextLeader;316 }317 void resetNextLeader() { NextLeader = {nullptr, ~0}; }318 bool addPossibleLeader(std::pair<Value *, unsigned int> LeaderPair) {319 if (LeaderPair.second < RepLeader.second) {320 NextLeader = RepLeader;321 RepLeader = LeaderPair;322 return true;323 } else if (LeaderPair.second < NextLeader.second) {324 NextLeader = LeaderPair;325 }326 return false;327 }328 329 Value *getStoredValue() const { return RepStoredValue; }330 void setStoredValue(Value *Leader) { RepStoredValue = Leader; }331 const MemoryAccess *getMemoryLeader() const { return RepMemoryAccess; }332 void setMemoryLeader(const MemoryAccess *Leader) { RepMemoryAccess = Leader; }333 334 // Forward propagation info335 const Expression *getDefiningExpr() const { return DefiningExpr; }336 337 // Value member set338 bool empty() const { return Members.empty(); }339 unsigned size() const { return Members.size(); }340 MemberSet::const_iterator begin() const { return Members.begin(); }341 MemberSet::const_iterator end() const { return Members.end(); }342 void insert(MemberType *M) { Members.insert(M); }343 void erase(MemberType *M) { Members.erase(M); }344 void swap(MemberSet &Other) { Members.swap(Other); }345 346 // Memory member set347 bool memory_empty() const { return MemoryMembers.empty(); }348 unsigned memory_size() const { return MemoryMembers.size(); }349 MemoryMemberSet::const_iterator memory_begin() const {350 return MemoryMembers.begin();351 }352 MemoryMemberSet::const_iterator memory_end() const {353 return MemoryMembers.end();354 }355 iterator_range<MemoryMemberSet::const_iterator> memory() const {356 return make_range(memory_begin(), memory_end());357 }358 359 void memory_insert(const MemoryMemberType *M) { MemoryMembers.insert(M); }360 void memory_erase(const MemoryMemberType *M) { MemoryMembers.erase(M); }361 362 // Store count363 unsigned getStoreCount() const { return StoreCount; }364 void incStoreCount() { ++StoreCount; }365 void decStoreCount() {366 assert(StoreCount != 0 && "Store count went negative");367 --StoreCount;368 }369 370 // True if this class has no memory members.371 bool definesNoMemory() const { return StoreCount == 0 && memory_empty(); }372 373 // Return true if two congruence classes are equivalent to each other. This374 // means that every field but the ID number and the dead field are equivalent.375 bool isEquivalentTo(const CongruenceClass *Other) const {376 if (!Other)377 return false;378 if (this == Other)379 return true;380 381 if (std::tie(StoreCount, RepLeader, RepStoredValue, RepMemoryAccess) !=382 std::tie(Other->StoreCount, Other->RepLeader, Other->RepStoredValue,383 Other->RepMemoryAccess))384 return false;385 if (DefiningExpr != Other->DefiningExpr)386 if (!DefiningExpr || !Other->DefiningExpr ||387 *DefiningExpr != *Other->DefiningExpr)388 return false;389 390 if (Members.size() != Other->Members.size())391 return false;392 393 return llvm::set_is_subset(Members, Other->Members);394 }395 396private:397 unsigned ID;398 399 // Representative leader and its corresponding RPO number.400 // The leader must have the lowest RPO number.401 std::pair<Value *, unsigned int> RepLeader = {nullptr, ~0U};402 403 // The most dominating leader after our current leader (given by the RPO404 // number), because the member set is not sorted and is expensive to keep405 // sorted all the time.406 std::pair<Value *, unsigned int> NextLeader = {nullptr, ~0U};407 408 // If this is represented by a store, the value of the store.409 Value *RepStoredValue = nullptr;410 411 // If this class contains MemoryDefs or MemoryPhis, this is the leading memory412 // access.413 const MemoryAccess *RepMemoryAccess = nullptr;414 415 // Defining Expression.416 const Expression *DefiningExpr = nullptr;417 418 // Actual members of this class.419 MemberSet Members;420 421 // This is the set of MemoryPhis that exist in the class. MemoryDefs and422 // MemoryUses have real instructions representing them, so we only need to423 // track MemoryPhis here.424 MemoryMemberSet MemoryMembers;425 426 // Number of stores in this congruence class.427 // This is used so we can detect store equivalence changes properly.428 int StoreCount = 0;429};430 431struct ExactEqualsExpression {432 const Expression &E;433 434 explicit ExactEqualsExpression(const Expression &E) : E(E) {}435 436 hash_code getComputedHash() const { return E.getComputedHash(); }437 438 bool operator==(const Expression &Other) const {439 return E.exactlyEquals(Other);440 }441};442} // end anonymous namespace443 444template <> struct llvm::DenseMapInfo<const Expression *> {445 static const Expression *getEmptyKey() {446 auto Val = static_cast<uintptr_t>(-1);447 Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable;448 return reinterpret_cast<const Expression *>(Val);449 }450 451 static const Expression *getTombstoneKey() {452 auto Val = static_cast<uintptr_t>(~1U);453 Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable;454 return reinterpret_cast<const Expression *>(Val);455 }456 457 static unsigned getHashValue(const Expression *E) {458 return E->getComputedHash();459 }460 461 static unsigned getHashValue(const ExactEqualsExpression &E) {462 return E.getComputedHash();463 }464 465 static bool isEqual(const ExactEqualsExpression &LHS, const Expression *RHS) {466 if (RHS == getTombstoneKey() || RHS == getEmptyKey())467 return false;468 return LHS == *RHS;469 }470 471 static bool isEqual(const Expression *LHS, const Expression *RHS) {472 if (LHS == RHS)473 return true;474 if (LHS == getTombstoneKey() || RHS == getTombstoneKey() ||475 LHS == getEmptyKey() || RHS == getEmptyKey())476 return false;477 // Compare hashes before equality. This is *not* what the hashtable does,478 // since it is computing it modulo the number of buckets, whereas we are479 // using the full hash keyspace. Since the hashes are precomputed, this480 // check is *much* faster than equality.481 if (LHS->getComputedHash() != RHS->getComputedHash())482 return false;483 return *LHS == *RHS;484 }485};486 487namespace {488 489class NewGVN {490 Function &F;491 DominatorTree *DT = nullptr;492 const TargetLibraryInfo *TLI = nullptr;493 AliasAnalysis *AA = nullptr;494 MemorySSA *MSSA = nullptr;495 MemorySSAWalker *MSSAWalker = nullptr;496 AssumptionCache *AC = nullptr;497 const DataLayout &DL;498 499 // These are the only two things the create* functions should have500 // side-effects on due to allocating memory.501 mutable BumpPtrAllocator ExpressionAllocator;502 mutable ArrayRecycler<Value *> ArgRecycler;503 mutable TarjanSCC SCCFinder;504 505 std::unique_ptr<PredicateInfo> PredInfo;506 const SimplifyQuery SQ;507 508 // Number of function arguments, used by ranking509 unsigned int NumFuncArgs = 0;510 511 // RPOOrdering of basic blocks512 DenseMap<const DomTreeNode *, unsigned> RPOOrdering;513 514 // Congruence class info.515 516 // This class is called INITIAL in the paper. It is the class everything517 // startsout in, and represents any value. Being an optimistic analysis,518 // anything in the TOP class has the value TOP, which is indeterminate and519 // equivalent to everything.520 CongruenceClass *TOPClass = nullptr;521 std::vector<CongruenceClass *> CongruenceClasses;522 unsigned NextCongruenceNum = 0;523 524 // Value Mappings.525 DenseMap<Value *, CongruenceClass *> ValueToClass;526 DenseMap<Value *, const Expression *> ValueToExpression;527 528 // Value PHI handling, used to make equivalence between phi(op, op) and529 // op(phi, phi).530 // These mappings just store various data that would normally be part of the531 // IR.532 SmallPtrSet<const Instruction *, 8> PHINodeUses;533 534 // The cached results, in general, are only valid for the specific block where535 // they were computed. The unsigned part of the key is a unique block536 // identifier537 DenseMap<std::pair<const Value *, unsigned>, bool> OpSafeForPHIOfOps;538 unsigned CacheIdx;539 540 // Map a temporary instruction we created to a parent block.541 DenseMap<const Value *, BasicBlock *> TempToBlock;542 543 // Map between the already in-program instructions and the temporary phis we544 // created that they are known equivalent to.545 DenseMap<const Value *, PHINode *> RealToTemp;546 547 // In order to know when we should re-process instructions that have548 // phi-of-ops, we track the set of expressions that they needed as549 // leaders. When we discover new leaders for those expressions, we process the550 // associated phi-of-op instructions again in case they have changed. The551 // other way they may change is if they had leaders, and those leaders552 // disappear. However, at the point they have leaders, there are uses of the553 // relevant operands in the created phi node, and so they will get reprocessed554 // through the normal user marking we perform.555 mutable DenseMap<const Value *, SmallPtrSet<Value *, 2>> AdditionalUsers;556 DenseMap<const Expression *, SmallPtrSet<Instruction *, 2>>557 ExpressionToPhiOfOps;558 559 // Map from temporary operation to MemoryAccess.560 DenseMap<const Instruction *, MemoryUseOrDef *> TempToMemory;561 562 // Set of all temporary instructions we created.563 // Note: This will include instructions that were just created during value564 // numbering. The way to test if something is using them is to check565 // RealToTemp.566 DenseSet<Instruction *> AllTempInstructions;567 568 // This is the set of instructions to revisit on a reachability change. At569 // the end of the main iteration loop it will contain at least all the phi of570 // ops instructions that will be changed to phis, as well as regular phis.571 // During the iteration loop, it may contain other things, such as phi of ops572 // instructions that used edge reachability to reach a result, and so need to573 // be revisited when the edge changes, independent of whether the phi they574 // depended on changes.575 DenseMap<BasicBlock *, SparseBitVector<>> RevisitOnReachabilityChange;576 577 // Mapping from predicate info we used to the instructions we used it with.578 // In order to correctly ensure propagation, we must keep track of what579 // comparisons we used, so that when the values of the comparisons change, we580 // propagate the information to the places we used the comparison.581 mutable DenseMap<const Value *, SmallPtrSet<Instruction *, 2>>582 PredicateToUsers;583 584 // the same reasoning as PredicateToUsers. When we skip MemoryAccesses for585 // stores, we no longer can rely solely on the def-use chains of MemorySSA.586 mutable DenseMap<const MemoryAccess *, SmallPtrSet<MemoryAccess *, 2>>587 MemoryToUsers;588 589 // A table storing which memorydefs/phis represent a memory state provably590 // equivalent to another memory state.591 // We could use the congruence class machinery, but the MemoryAccess's are592 // abstract memory states, so they can only ever be equivalent to each other,593 // and not to constants, etc.594 DenseMap<const MemoryAccess *, CongruenceClass *> MemoryAccessToClass;595 596 // We could, if we wanted, build MemoryPhiExpressions and597 // MemoryVariableExpressions, etc, and value number them the same way we value598 // number phi expressions. For the moment, this seems like overkill. They599 // can only exist in one of three states: they can be TOP (equal to600 // everything), Equivalent to something else, or unique. Because we do not601 // create expressions for them, we need to simulate leader change not just602 // when they change class, but when they change state. Note: We can do the603 // same thing for phis, and avoid having phi expressions if we wanted, We604 // should eventually unify in one direction or the other, so this is a little605 // bit of an experiment in which turns out easier to maintain.606 enum MemoryPhiState { MPS_Invalid, MPS_TOP, MPS_Equivalent, MPS_Unique };607 DenseMap<const MemoryPhi *, MemoryPhiState> MemoryPhiState;608 609 enum InstCycleState { ICS_Unknown, ICS_CycleFree, ICS_Cycle };610 mutable DenseMap<const Instruction *, InstCycleState> InstCycleState;611 612 // Expression to class mapping.613 using ExpressionClassMap = DenseMap<const Expression *, CongruenceClass *>;614 ExpressionClassMap ExpressionToClass;615 616 // We have a single expression that represents currently DeadExpressions.617 // For dead expressions we can prove will stay dead, we mark them with618 // DFS number zero. However, it's possible in the case of phi nodes619 // for us to assume/prove all arguments are dead during fixpointing.620 // We use DeadExpression for that case.621 DeadExpression *SingletonDeadExpression = nullptr;622 623 // Which values have changed as a result of leader changes.624 SmallPtrSet<Value *, 8> LeaderChanges;625 626 // Reachability info.627 using BlockEdge = BasicBlockEdge;628 DenseSet<BlockEdge> ReachableEdges;629 SmallPtrSet<const BasicBlock *, 8> ReachableBlocks;630 631 // This is a bitvector because, on larger functions, we may have632 // thousands of touched instructions at once (entire blocks,633 // instructions with hundreds of uses, etc). Even with optimization634 // for when we mark whole blocks as touched, when this was a635 // SmallPtrSet or DenseSet, for some functions, we spent >20% of all636 // the time in GVN just managing this list. The bitvector, on the637 // other hand, efficiently supports test/set/clear of both638 // individual and ranges, as well as "find next element" This639 // enables us to use it as a worklist with essentially 0 cost.640 BitVector TouchedInstructions;641 642 DenseMap<const BasicBlock *, std::pair<unsigned, unsigned>> BlockInstRange;643 mutable DenseMap<const BitCastInst *, const Value *> PredicateSwapChoice;644 645#ifndef NDEBUG646 // Debugging for how many times each block and instruction got processed.647 DenseMap<const Value *, unsigned> ProcessedCount;648#endif649 650 // DFS info.651 // This contains a mapping from Instructions to DFS numbers.652 // The numbering starts at 1. An instruction with DFS number zero653 // means that the instruction is dead.654 DenseMap<const Value *, unsigned> InstrDFS;655 656 // This contains the mapping DFS numbers to instructions.657 SmallVector<Value *, 32> DFSToInstr;658 659 // Deletion info.660 SmallPtrSet<Instruction *, 8> InstructionsToErase;661 662public:663 NewGVN(Function &F, DominatorTree *DT, AssumptionCache *AC,664 TargetLibraryInfo *TLI, AliasAnalysis *AA, MemorySSA *MSSA,665 const DataLayout &DL)666 : F(F), DT(DT), TLI(TLI), AA(AA), MSSA(MSSA), AC(AC), DL(DL),667 // Reuse ExpressionAllocator for PredicateInfo as well.668 PredInfo(669 std::make_unique<PredicateInfo>(F, *DT, *AC, ExpressionAllocator)),670 SQ(DL, TLI, DT, AC, /*CtxI=*/nullptr, /*UseInstrInfo=*/false,671 /*CanUseUndef=*/false) {}672 673 bool runGVN();674 675private:676 /// Helper struct return a Expression with an optional extra dependency.677 struct ExprResult {678 const Expression *Expr;679 Value *ExtraDep;680 const PredicateBase *PredDep;681 682 ExprResult(const Expression *Expr, Value *ExtraDep = nullptr,683 const PredicateBase *PredDep = nullptr)684 : Expr(Expr), ExtraDep(ExtraDep), PredDep(PredDep) {}685 ExprResult(const ExprResult &) = delete;686 ExprResult(ExprResult &&Other)687 : Expr(Other.Expr), ExtraDep(Other.ExtraDep), PredDep(Other.PredDep) {688 Other.Expr = nullptr;689 Other.ExtraDep = nullptr;690 Other.PredDep = nullptr;691 }692 ExprResult &operator=(const ExprResult &Other) = delete;693 ExprResult &operator=(ExprResult &&Other) = delete;694 695 ~ExprResult() { assert(!ExtraDep && "unhandled ExtraDep"); }696 697 operator bool() const { return Expr; }698 699 static ExprResult none() { return {nullptr, nullptr, nullptr}; }700 static ExprResult some(const Expression *Expr, Value *ExtraDep = nullptr) {701 return {Expr, ExtraDep, nullptr};702 }703 static ExprResult some(const Expression *Expr,704 const PredicateBase *PredDep) {705 return {Expr, nullptr, PredDep};706 }707 static ExprResult some(const Expression *Expr, Value *ExtraDep,708 const PredicateBase *PredDep) {709 return {Expr, ExtraDep, PredDep};710 }711 };712 713 // Expression handling.714 ExprResult createExpression(Instruction *) const;715 const Expression *createBinaryExpression(unsigned, Type *, Value *, Value *,716 Instruction *) const;717 718 // Our canonical form for phi arguments is a pair of incoming value, incoming719 // basic block.720 using ValPair = std::pair<Value *, BasicBlock *>;721 722 PHIExpression *createPHIExpression(ArrayRef<ValPair>, const Instruction *,723 BasicBlock *, bool &HasBackEdge,724 bool &OriginalOpsConstant) const;725 const DeadExpression *createDeadExpression() const;726 const VariableExpression *createVariableExpression(Value *) const;727 const ConstantExpression *createConstantExpression(Constant *) const;728 const Expression *createVariableOrConstant(Value *V) const;729 const UnknownExpression *createUnknownExpression(Instruction *) const;730 const StoreExpression *createStoreExpression(StoreInst *,731 const MemoryAccess *) const;732 LoadExpression *createLoadExpression(Type *, Value *, LoadInst *,733 const MemoryAccess *) const;734 const CallExpression *createCallExpression(CallInst *,735 const MemoryAccess *) const;736 const AggregateValueExpression *737 createAggregateValueExpression(Instruction *) const;738 bool setBasicExpressionInfo(Instruction *, BasicExpression *) const;739 740 // Congruence class handling.741 CongruenceClass *createCongruenceClass(Value *Leader, const Expression *E) {742 // Set RPO to 0 for values that are always available (constants and function743 // args). These should always be made leader.744 unsigned LeaderDFS = 0;745 746 // If Leader is not specified, either we have a memory class or the leader747 // will be set later. Otherwise, if Leader is an Instruction, set LeaderDFS748 // to its RPO number.749 if (!Leader)750 LeaderDFS = ~0;751 else if (auto *I = dyn_cast<Instruction>(Leader))752 LeaderDFS = InstrToDFSNum(I);753 auto *result =754 new CongruenceClass(NextCongruenceNum++, {Leader, LeaderDFS}, E);755 CongruenceClasses.emplace_back(result);756 return result;757 }758 759 CongruenceClass *createMemoryClass(MemoryAccess *MA) {760 auto *CC = createCongruenceClass(nullptr, nullptr);761 CC->setMemoryLeader(MA);762 return CC;763 }764 765 CongruenceClass *ensureLeaderOfMemoryClass(MemoryAccess *MA) {766 auto *CC = getMemoryClass(MA);767 if (CC->getMemoryLeader() != MA)768 CC = createMemoryClass(MA);769 return CC;770 }771 772 CongruenceClass *createSingletonCongruenceClass(Value *Member) {773 CongruenceClass *CClass = createCongruenceClass(Member, nullptr);774 CClass->insert(Member);775 ValueToClass[Member] = CClass;776 return CClass;777 }778 779 void initializeCongruenceClasses(Function &F);780 const Expression *makePossiblePHIOfOps(Instruction *,781 SmallPtrSetImpl<Value *> &);782 Value *findLeaderForInst(Instruction *ValueOp,783 SmallPtrSetImpl<Value *> &Visited,784 MemoryAccess *MemAccess, Instruction *OrigInst,785 BasicBlock *PredBB);786 bool OpIsSafeForPHIOfOps(Value *Op, const BasicBlock *PHIBlock,787 SmallPtrSetImpl<const Value *> &);788 void addPhiOfOps(PHINode *Op, BasicBlock *BB, Instruction *ExistingValue);789 void removePhiOfOps(Instruction *I, PHINode *PHITemp);790 791 // Value number an Instruction or MemoryPhi.792 void valueNumberMemoryPhi(MemoryPhi *);793 void valueNumberInstruction(Instruction *);794 795 // Symbolic evaluation.796 ExprResult checkExprResults(Expression *, Instruction *, Value *) const;797 ExprResult performSymbolicEvaluation(Instruction *,798 SmallPtrSetImpl<Value *> &) const;799 const Expression *performSymbolicLoadCoercion(Type *, Value *, LoadInst *,800 Instruction *,801 MemoryAccess *) const;802 const Expression *performSymbolicLoadEvaluation(Instruction *) const;803 const Expression *performSymbolicStoreEvaluation(Instruction *) const;804 ExprResult performSymbolicCallEvaluation(Instruction *) const;805 void sortPHIOps(MutableArrayRef<ValPair> Ops) const;806 const Expression *performSymbolicPHIEvaluation(ArrayRef<ValPair>,807 Instruction *I,808 BasicBlock *PHIBlock) const;809 const Expression *performSymbolicAggrValueEvaluation(Instruction *) const;810 ExprResult performSymbolicCmpEvaluation(Instruction *) const;811 ExprResult performSymbolicPredicateInfoEvaluation(BitCastInst *) const;812 813 // Congruence finding.814 bool someEquivalentDominates(const Instruction *, const Instruction *) const;815 Value *lookupOperandLeader(Value *) const;816 CongruenceClass *getClassForExpression(const Expression *E) const;817 void performCongruenceFinding(Instruction *, const Expression *);818 void moveValueToNewCongruenceClass(Instruction *, const Expression *,819 CongruenceClass *, CongruenceClass *);820 void moveMemoryToNewCongruenceClass(Instruction *, MemoryAccess *,821 CongruenceClass *, CongruenceClass *);822 Value *getNextValueLeader(CongruenceClass *) const;823 const MemoryAccess *getNextMemoryLeader(CongruenceClass *) const;824 bool setMemoryClass(const MemoryAccess *From, CongruenceClass *To);825 CongruenceClass *getMemoryClass(const MemoryAccess *MA) const;826 const MemoryAccess *lookupMemoryLeader(const MemoryAccess *) const;827 bool isMemoryAccessTOP(const MemoryAccess *) const;828 829 // Ranking830 unsigned int getRank(const Value *) const;831 bool shouldSwapOperands(const Value *, const Value *) const;832 bool shouldSwapOperandsForPredicate(const Value *, const Value *,833 const BitCastInst *I) const;834 835 // Reachability handling.836 void updateReachableEdge(BasicBlock *, BasicBlock *);837 void processOutgoingEdges(Instruction *, BasicBlock *);838 Value *findConditionEquivalence(Value *) const;839 840 // Elimination.841 struct ValueDFS;842 void convertClassToDFSOrdered(const CongruenceClass &,843 SmallVectorImpl<ValueDFS> &,844 DenseMap<const Value *, unsigned int> &,845 SmallPtrSetImpl<Instruction *> &) const;846 void convertClassToLoadsAndStores(const CongruenceClass &,847 SmallVectorImpl<ValueDFS> &) const;848 849 bool eliminateInstructions(Function &);850 void replaceInstruction(Instruction *, Value *);851 void markInstructionForDeletion(Instruction *);852 void deleteInstructionsInBlock(BasicBlock *);853 Value *findPHIOfOpsLeader(const Expression *, const Instruction *,854 const BasicBlock *) const;855 856 // Various instruction touch utilities857 template <typename Map, typename KeyType>858 void touchAndErase(Map &, const KeyType &);859 void markUsersTouched(Value *);860 void markMemoryUsersTouched(const MemoryAccess *);861 void markMemoryDefTouched(const MemoryAccess *);862 void markPredicateUsersTouched(Instruction *);863 void markValueLeaderChangeTouched(CongruenceClass *CC);864 void markMemoryLeaderChangeTouched(CongruenceClass *CC);865 void markPhiOfOpsChanged(const Expression *E);866 void addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const;867 void addAdditionalUsers(Value *To, Value *User) const;868 void addAdditionalUsers(ExprResult &Res, Instruction *User) const;869 870 // Main loop of value numbering871 void iterateTouchedInstructions();872 873 // Utilities.874 void cleanupTables();875 std::pair<unsigned, unsigned> assignDFSNumbers(BasicBlock *, unsigned);876 void updateProcessedCount(const Value *V);877 void verifyMemoryCongruency() const;878 void verifyIterationSettled(Function &F);879 void verifyStoreExpressions() const;880 bool singleReachablePHIPath(SmallPtrSet<const MemoryAccess *, 8> &,881 const MemoryAccess *, const MemoryAccess *) const;882 BasicBlock *getBlockForValue(Value *V) const;883 void deleteExpression(const Expression *E) const;884 MemoryUseOrDef *getMemoryAccess(const Instruction *) const;885 MemoryPhi *getMemoryAccess(const BasicBlock *) const;886 template <class T, class Range> T *getMinDFSOfRange(const Range &) const;887 888 unsigned InstrToDFSNum(const Value *V) const {889 assert(isa<Instruction>(V) && "This should not be used for MemoryAccesses");890 return InstrDFS.lookup(V);891 }892 893 unsigned InstrToDFSNum(const MemoryAccess *MA) const {894 return MemoryToDFSNum(MA);895 }896 897 Value *InstrFromDFSNum(unsigned DFSNum) { return DFSToInstr[DFSNum]; }898 899 // Given a MemoryAccess, return the relevant instruction DFS number. Note:900 // This deliberately takes a value so it can be used with Use's, which will901 // auto-convert to Value's but not to MemoryAccess's.902 unsigned MemoryToDFSNum(const Value *MA) const {903 assert(isa<MemoryAccess>(MA) &&904 "This should not be used with instructions");905 return isa<MemoryUseOrDef>(MA)906 ? InstrToDFSNum(cast<MemoryUseOrDef>(MA)->getMemoryInst())907 : InstrDFS.lookup(MA);908 }909 910 bool isCycleFree(const Instruction *) const;911 bool isBackedge(BasicBlock *From, BasicBlock *To) const;912 913 // Debug counter info. When verifying, we have to reset the value numbering914 // debug counter to the same state it started in to get the same results.915 DebugCounter::CounterState StartingVNCounter;916};917 918} // end anonymous namespace919 920template <typename T>921static bool equalsLoadStoreHelper(const T &LHS, const Expression &RHS) {922 if (!isa<LoadExpression>(RHS) && !isa<StoreExpression>(RHS))923 return false;924 return LHS.MemoryExpression::equals(RHS);925}926 927bool LoadExpression::equals(const Expression &Other) const {928 return equalsLoadStoreHelper(*this, Other);929}930 931bool StoreExpression::equals(const Expression &Other) const {932 if (!equalsLoadStoreHelper(*this, Other))933 return false;934 // Make sure that store vs store includes the value operand.935 if (const auto *S = dyn_cast<StoreExpression>(&Other))936 if (getStoredValue() != S->getStoredValue())937 return false;938 return true;939}940 941bool CallExpression::equals(const Expression &Other) const {942 if (!MemoryExpression::equals(Other))943 return false;944 945 if (auto *RHS = dyn_cast<CallExpression>(&Other))946 return Call->getAttributes()947 .intersectWith(Call->getContext(), RHS->Call->getAttributes())948 .has_value();949 950 return false;951}952 953// Determine if the edge From->To is a backedge954bool NewGVN::isBackedge(BasicBlock *From, BasicBlock *To) const {955 return From == To ||956 RPOOrdering.lookup(DT->getNode(From)) >=957 RPOOrdering.lookup(DT->getNode(To));958}959 960#ifndef NDEBUG961static std::string getBlockName(const BasicBlock *B) {962 return DOTGraphTraits<DOTFuncInfo *>::getSimpleNodeLabel(B, nullptr);963}964#endif965 966// Get a MemoryAccess for an instruction, fake or real.967MemoryUseOrDef *NewGVN::getMemoryAccess(const Instruction *I) const {968 auto *Result = MSSA->getMemoryAccess(I);969 return Result ? Result : TempToMemory.lookup(I);970}971 972// Get a MemoryPhi for a basic block. These are all real.973MemoryPhi *NewGVN::getMemoryAccess(const BasicBlock *BB) const {974 return MSSA->getMemoryAccess(BB);975}976 977// Get the basic block from an instruction/memory value.978BasicBlock *NewGVN::getBlockForValue(Value *V) const {979 if (auto *I = dyn_cast<Instruction>(V)) {980 auto *Parent = I->getParent();981 if (Parent)982 return Parent;983 Parent = TempToBlock.lookup(V);984 assert(Parent && "Every fake instruction should have a block");985 return Parent;986 }987 988 auto *MP = dyn_cast<MemoryPhi>(V);989 assert(MP && "Should have been an instruction or a MemoryPhi");990 return MP->getBlock();991}992 993// Delete a definitely dead expression, so it can be reused by the expression994// allocator. Some of these are not in creation functions, so we have to accept995// const versions.996void NewGVN::deleteExpression(const Expression *E) const {997 assert(isa<BasicExpression>(E));998 auto *BE = cast<BasicExpression>(E);999 const_cast<BasicExpression *>(BE)->deallocateOperands(ArgRecycler);1000 ExpressionAllocator.Deallocate(E);1001}1002 1003// If V is a predicateinfo copy, get the thing it is a copy of.1004static Value *getCopyOf(const Value *V) {1005 if (auto *BC = dyn_cast<BitCastInst>(V))1006 if (BC->getType() == BC->getOperand(0)->getType())1007 return BC->getOperand(0);1008 return nullptr;1009}1010 1011// Return true if V is really PN, even accounting for predicateinfo copies.1012static bool isCopyOfPHI(const Value *V, const PHINode *PN) {1013 return V == PN || getCopyOf(V) == PN;1014}1015 1016static bool isCopyOfAPHI(const Value *V) {1017 auto *CO = getCopyOf(V);1018 return CO && isa<PHINode>(CO);1019}1020 1021// Sort PHI Operands into a canonical order. What we use here is an RPO1022// order. The BlockInstRange numbers are generated in an RPO walk of the basic1023// blocks.1024void NewGVN::sortPHIOps(MutableArrayRef<ValPair> Ops) const {1025 llvm::sort(Ops, [&](const ValPair &P1, const ValPair &P2) {1026 return BlockInstRange.lookup(P1.second).first <1027 BlockInstRange.lookup(P2.second).first;1028 });1029}1030 1031// Return true if V is a value that will always be available (IE can1032// be placed anywhere) in the function. We don't do globals here1033// because they are often worse to put in place.1034static bool alwaysAvailable(Value *V) {1035 return isa<Constant>(V) || isa<Argument>(V);1036}1037 1038// Create a PHIExpression from an array of {incoming edge, value} pairs. I is1039// the original instruction we are creating a PHIExpression for (but may not be1040// a phi node). We require, as an invariant, that all the PHIOperands in the1041// same block are sorted the same way. sortPHIOps will sort them into a1042// canonical order.1043PHIExpression *NewGVN::createPHIExpression(ArrayRef<ValPair> PHIOperands,1044 const Instruction *I,1045 BasicBlock *PHIBlock,1046 bool &HasBackedge,1047 bool &OriginalOpsConstant) const {1048 unsigned NumOps = PHIOperands.size();1049 auto *E = new (ExpressionAllocator) PHIExpression(NumOps, PHIBlock);1050 1051 E->allocateOperands(ArgRecycler, ExpressionAllocator);1052 E->setType(PHIOperands.begin()->first->getType());1053 E->setOpcode(Instruction::PHI);1054 1055 // Filter out unreachable phi operands.1056 auto Filtered = make_filter_range(PHIOperands, [&](const ValPair &P) {1057 auto *BB = P.second;1058 if (auto *PHIOp = dyn_cast<PHINode>(I))1059 if (isCopyOfPHI(P.first, PHIOp))1060 return false;1061 if (!ReachableEdges.count({BB, PHIBlock}))1062 return false;1063 // Things in TOPClass are equivalent to everything.1064 if (ValueToClass.lookup(P.first) == TOPClass)1065 return false;1066 OriginalOpsConstant = OriginalOpsConstant && isa<Constant>(P.first);1067 HasBackedge = HasBackedge || isBackedge(BB, PHIBlock);1068 return lookupOperandLeader(P.first) != I;1069 });1070 llvm::transform(Filtered, op_inserter(E), [&](const ValPair &P) -> Value * {1071 return lookupOperandLeader(P.first);1072 });1073 return E;1074}1075 1076// Set basic expression info (Arguments, type, opcode) for Expression1077// E from Instruction I in block B.1078bool NewGVN::setBasicExpressionInfo(Instruction *I, BasicExpression *E) const {1079 bool AllConstant = true;1080 if (auto *GEP = dyn_cast<GetElementPtrInst>(I))1081 E->setType(GEP->getSourceElementType());1082 else1083 E->setType(I->getType());1084 E->setOpcode(I->getOpcode());1085 E->allocateOperands(ArgRecycler, ExpressionAllocator);1086 1087 // Transform the operand array into an operand leader array, and keep track of1088 // whether all members are constant.1089 std::transform(I->op_begin(), I->op_end(), op_inserter(E), [&](Value *O) {1090 auto Operand = lookupOperandLeader(O);1091 AllConstant = AllConstant && isa<Constant>(Operand);1092 return Operand;1093 });1094 1095 return AllConstant;1096}1097 1098const Expression *NewGVN::createBinaryExpression(unsigned Opcode, Type *T,1099 Value *Arg1, Value *Arg2,1100 Instruction *I) const {1101 auto *E = new (ExpressionAllocator) BasicExpression(2);1102 // TODO: we need to remove context instruction after Value Tracking1103 // can run without context instruction1104 const SimplifyQuery Q = SQ.getWithInstruction(I);1105 1106 E->setType(T);1107 E->setOpcode(Opcode);1108 E->allocateOperands(ArgRecycler, ExpressionAllocator);1109 if (Instruction::isCommutative(Opcode)) {1110 // Ensure that commutative instructions that only differ by a permutation1111 // of their operands get the same value number by sorting the operand value1112 // numbers. Since all commutative instructions have two operands it is more1113 // efficient to sort by hand rather than using, say, std::sort.1114 if (shouldSwapOperands(Arg1, Arg2))1115 std::swap(Arg1, Arg2);1116 }1117 E->op_push_back(lookupOperandLeader(Arg1));1118 E->op_push_back(lookupOperandLeader(Arg2));1119 1120 Value *V = simplifyBinOp(Opcode, E->getOperand(0), E->getOperand(1), Q);1121 if (auto Simplified = checkExprResults(E, I, V)) {1122 addAdditionalUsers(Simplified, I);1123 return Simplified.Expr;1124 }1125 return E;1126}1127 1128// Take a Value returned by simplification of Expression E/Instruction1129// I, and see if it resulted in a simpler expression. If so, return1130// that expression.1131NewGVN::ExprResult NewGVN::checkExprResults(Expression *E, Instruction *I,1132 Value *V) const {1133 if (!V)1134 return ExprResult::none();1135 1136 if (auto *C = dyn_cast<Constant>(V)) {1137 if (I)1138 LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "1139 << " constant " << *C << "\n");1140 NumGVNOpsSimplified++;1141 assert(isa<BasicExpression>(E) &&1142 "We should always have had a basic expression here");1143 deleteExpression(E);1144 return ExprResult::some(createConstantExpression(C));1145 } else if (isa<Argument>(V) || isa<GlobalVariable>(V)) {1146 if (I)1147 LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "1148 << " variable " << *V << "\n");1149 deleteExpression(E);1150 return ExprResult::some(createVariableExpression(V));1151 }1152 1153 CongruenceClass *CC = ValueToClass.lookup(V);1154 if (CC) {1155 if (CC->getLeader() && CC->getLeader() != I) {1156 return ExprResult::some(createVariableOrConstant(CC->getLeader()), V);1157 }1158 if (CC->getDefiningExpr()) {1159 if (I)1160 LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "1161 << " expression " << *CC->getDefiningExpr() << "\n");1162 NumGVNOpsSimplified++;1163 deleteExpression(E);1164 return ExprResult::some(CC->getDefiningExpr(), V);1165 }1166 }1167 1168 return ExprResult::none();1169}1170 1171// Create a value expression from the instruction I, replacing operands with1172// their leaders.1173 1174NewGVN::ExprResult NewGVN::createExpression(Instruction *I) const {1175 auto *E = new (ExpressionAllocator) BasicExpression(I->getNumOperands());1176 // TODO: we need to remove context instruction after Value Tracking1177 // can run without context instruction1178 const SimplifyQuery Q = SQ.getWithInstruction(I);1179 1180 bool AllConstant = setBasicExpressionInfo(I, E);1181 1182 if (I->isCommutative()) {1183 // Ensure that commutative instructions that only differ by a permutation1184 // of their operands get the same value number by sorting the operand value1185 // numbers. Since all commutative instructions have two operands it is more1186 // efficient to sort by hand rather than using, say, std::sort.1187 assert(I->getNumOperands() == 2 && "Unsupported commutative instruction!");1188 if (shouldSwapOperands(E->getOperand(0), E->getOperand(1)))1189 E->swapOperands(0, 1);1190 }1191 // Perform simplification.1192 if (auto *CI = dyn_cast<CmpInst>(I)) {1193 // Sort the operand value numbers so x<y and y>x get the same value1194 // number.1195 CmpInst::Predicate Predicate = CI->getPredicate();1196 if (shouldSwapOperands(E->getOperand(0), E->getOperand(1))) {1197 E->swapOperands(0, 1);1198 Predicate = CmpInst::getSwappedPredicate(Predicate);1199 }1200 E->setOpcode((CI->getOpcode() << 8) | Predicate);1201 // TODO: 25% of our time is spent in simplifyCmpInst with pointer operands1202 assert(I->getOperand(0)->getType() == I->getOperand(1)->getType() &&1203 "Wrong types on cmp instruction");1204 assert((E->getOperand(0)->getType() == I->getOperand(0)->getType() &&1205 E->getOperand(1)->getType() == I->getOperand(1)->getType()));1206 Value *V =1207 simplifyCmpInst(Predicate, E->getOperand(0), E->getOperand(1), Q);1208 if (auto Simplified = checkExprResults(E, I, V))1209 return Simplified;1210 } else if (isa<SelectInst>(I)) {1211 if (isa<Constant>(E->getOperand(0)) ||1212 E->getOperand(1) == E->getOperand(2)) {1213 assert(E->getOperand(1)->getType() == I->getOperand(1)->getType() &&1214 E->getOperand(2)->getType() == I->getOperand(2)->getType());1215 Value *V = simplifySelectInst(E->getOperand(0), E->getOperand(1),1216 E->getOperand(2), Q);1217 if (auto Simplified = checkExprResults(E, I, V))1218 return Simplified;1219 }1220 } else if (I->isBinaryOp()) {1221 Value *V =1222 simplifyBinOp(E->getOpcode(), E->getOperand(0), E->getOperand(1), Q);1223 if (auto Simplified = checkExprResults(E, I, V))1224 return Simplified;1225 } else if (auto *CI = dyn_cast<CastInst>(I)) {1226 Value *V =1227 simplifyCastInst(CI->getOpcode(), E->getOperand(0), CI->getType(), Q);1228 if (auto Simplified = checkExprResults(E, I, V))1229 return Simplified;1230 } else if (auto *GEPI = dyn_cast<GetElementPtrInst>(I)) {1231 Value *V = simplifyGEPInst(GEPI->getSourceElementType(), *E->op_begin(),1232 ArrayRef(std::next(E->op_begin()), E->op_end()),1233 GEPI->getNoWrapFlags(), Q);1234 if (auto Simplified = checkExprResults(E, I, V))1235 return Simplified;1236 } else if (AllConstant) {1237 // We don't bother trying to simplify unless all of the operands1238 // were constant.1239 // TODO: There are a lot of Simplify*'s we could call here, if we1240 // wanted to. The original motivating case for this code was a1241 // zext i1 false to i8, which we don't have an interface to1242 // simplify (IE there is no SimplifyZExt).1243 1244 SmallVector<Constant *, 8> C;1245 for (Value *Arg : E->operands())1246 C.emplace_back(cast<Constant>(Arg));1247 1248 if (Value *V = ConstantFoldInstOperands(I, C, DL, TLI))1249 if (auto Simplified = checkExprResults(E, I, V))1250 return Simplified;1251 }1252 return ExprResult::some(E);1253}1254 1255const AggregateValueExpression *1256NewGVN::createAggregateValueExpression(Instruction *I) const {1257 if (auto *II = dyn_cast<InsertValueInst>(I)) {1258 auto *E = new (ExpressionAllocator)1259 AggregateValueExpression(I->getNumOperands(), II->getNumIndices());1260 setBasicExpressionInfo(I, E);1261 E->allocateIntOperands(ExpressionAllocator);1262 llvm::copy(II->indices(), int_op_inserter(E));1263 return E;1264 } else if (auto *EI = dyn_cast<ExtractValueInst>(I)) {1265 auto *E = new (ExpressionAllocator)1266 AggregateValueExpression(I->getNumOperands(), EI->getNumIndices());1267 setBasicExpressionInfo(EI, E);1268 E->allocateIntOperands(ExpressionAllocator);1269 llvm::copy(EI->indices(), int_op_inserter(E));1270 return E;1271 }1272 llvm_unreachable("Unhandled type of aggregate value operation");1273}1274 1275const DeadExpression *NewGVN::createDeadExpression() const {1276 // DeadExpression has no arguments and all DeadExpression's are the same,1277 // so we only need one of them.1278 return SingletonDeadExpression;1279}1280 1281const VariableExpression *NewGVN::createVariableExpression(Value *V) const {1282 auto *E = new (ExpressionAllocator) VariableExpression(V);1283 E->setOpcode(V->getValueID());1284 return E;1285}1286 1287const Expression *NewGVN::createVariableOrConstant(Value *V) const {1288 if (auto *C = dyn_cast<Constant>(V))1289 return createConstantExpression(C);1290 return createVariableExpression(V);1291}1292 1293const ConstantExpression *NewGVN::createConstantExpression(Constant *C) const {1294 auto *E = new (ExpressionAllocator) ConstantExpression(C);1295 E->setOpcode(C->getValueID());1296 return E;1297}1298 1299const UnknownExpression *NewGVN::createUnknownExpression(Instruction *I) const {1300 auto *E = new (ExpressionAllocator) UnknownExpression(I);1301 E->setOpcode(I->getOpcode());1302 return E;1303}1304 1305const CallExpression *1306NewGVN::createCallExpression(CallInst *CI, const MemoryAccess *MA) const {1307 // FIXME: Add operand bundles for calls.1308 auto *E =1309 new (ExpressionAllocator) CallExpression(CI->getNumOperands(), CI, MA);1310 setBasicExpressionInfo(CI, E);1311 if (CI->isCommutative()) {1312 // Ensure that commutative intrinsics that only differ by a permutation1313 // of their operands get the same value number by sorting the operand value1314 // numbers.1315 assert(CI->getNumOperands() >= 2 && "Unsupported commutative intrinsic!");1316 if (shouldSwapOperands(E->getOperand(0), E->getOperand(1)))1317 E->swapOperands(0, 1);1318 }1319 return E;1320}1321 1322// Return true if some equivalent of instruction Inst dominates instruction U.1323bool NewGVN::someEquivalentDominates(const Instruction *Inst,1324 const Instruction *U) const {1325 auto *CC = ValueToClass.lookup(Inst);1326 // This must be an instruction because we are only called from phi nodes1327 // in the case that the value it needs to check against is an instruction.1328 1329 // The most likely candidates for dominance are the leader and the next leader.1330 // The leader or nextleader will dominate in all cases where there is an1331 // equivalent that is higher up in the dom tree.1332 // We can't *only* check them, however, because the1333 // dominator tree could have an infinite number of non-dominating siblings1334 // with instructions that are in the right congruence class.1335 // A1336 // B C D E F G1337 // |1338 // H1339 // Instruction U could be in H, with equivalents in every other sibling.1340 // Depending on the rpo order picked, the leader could be the equivalent in1341 // any of these siblings.1342 if (!CC)1343 return false;1344 if (alwaysAvailable(CC->getLeader()))1345 return true;1346 if (DT->dominates(cast<Instruction>(CC->getLeader()), U))1347 return true;1348 if (CC->getNextLeader().first &&1349 DT->dominates(cast<Instruction>(CC->getNextLeader().first), U))1350 return true;1351 return llvm::any_of(*CC, [&](const Value *Member) {1352 return Member != CC->getLeader() &&1353 DT->dominates(cast<Instruction>(Member), U);1354 });1355}1356 1357// See if we have a congruence class and leader for this operand, and if so,1358// return it. Otherwise, return the operand itself.1359Value *NewGVN::lookupOperandLeader(Value *V) const {1360 CongruenceClass *CC = ValueToClass.lookup(V);1361 if (CC) {1362 // Everything in TOP is represented by poison, as it can be any value.1363 // We do have to make sure we get the type right though, so we can't set the1364 // RepLeader to poison.1365 if (CC == TOPClass)1366 return PoisonValue::get(V->getType());1367 return CC->getStoredValue() ? CC->getStoredValue() : CC->getLeader();1368 }1369 1370 return V;1371}1372 1373const MemoryAccess *NewGVN::lookupMemoryLeader(const MemoryAccess *MA) const {1374 auto *CC = getMemoryClass(MA);1375 assert(CC->getMemoryLeader() &&1376 "Every MemoryAccess should be mapped to a congruence class with a "1377 "representative memory access");1378 return CC->getMemoryLeader();1379}1380 1381// Return true if the MemoryAccess is really equivalent to everything. This is1382// equivalent to the lattice value "TOP" in most lattices. This is the initial1383// state of all MemoryAccesses.1384bool NewGVN::isMemoryAccessTOP(const MemoryAccess *MA) const {1385 return getMemoryClass(MA) == TOPClass;1386}1387 1388LoadExpression *NewGVN::createLoadExpression(Type *LoadType, Value *PointerOp,1389 LoadInst *LI,1390 const MemoryAccess *MA) const {1391 auto *E =1392 new (ExpressionAllocator) LoadExpression(1, LI, lookupMemoryLeader(MA));1393 E->allocateOperands(ArgRecycler, ExpressionAllocator);1394 E->setType(LoadType);1395 1396 // Give store and loads same opcode so they value number together.1397 E->setOpcode(0);1398 E->op_push_back(PointerOp);1399 1400 // TODO: Value number heap versions. We may be able to discover1401 // things alias analysis can't on it's own (IE that a store and a1402 // load have the same value, and thus, it isn't clobbering the load).1403 return E;1404}1405 1406const StoreExpression *1407NewGVN::createStoreExpression(StoreInst *SI, const MemoryAccess *MA) const {1408 auto *StoredValueLeader = lookupOperandLeader(SI->getValueOperand());1409 auto *E = new (ExpressionAllocator)1410 StoreExpression(SI->getNumOperands(), SI, StoredValueLeader, MA);1411 E->allocateOperands(ArgRecycler, ExpressionAllocator);1412 E->setType(SI->getValueOperand()->getType());1413 1414 // Give store and loads same opcode so they value number together.1415 E->setOpcode(0);1416 E->op_push_back(lookupOperandLeader(SI->getPointerOperand()));1417 1418 // TODO: Value number heap versions. We may be able to discover1419 // things alias analysis can't on it's own (IE that a store and a1420 // load have the same value, and thus, it isn't clobbering the load).1421 return E;1422}1423 1424const Expression *NewGVN::performSymbolicStoreEvaluation(Instruction *I) const {1425 // Unlike loads, we never try to eliminate stores, so we do not check if they1426 // are simple and avoid value numbering them.1427 auto *SI = cast<StoreInst>(I);1428 auto *StoreAccess = getMemoryAccess(SI);1429 // Get the expression, if any, for the RHS of the MemoryDef.1430 const MemoryAccess *StoreRHS = StoreAccess->getDefiningAccess();1431 if (EnableStoreRefinement)1432 StoreRHS = MSSAWalker->getClobberingMemoryAccess(StoreAccess);1433 // If we bypassed the use-def chains, make sure we add a use.1434 StoreRHS = lookupMemoryLeader(StoreRHS);1435 if (StoreRHS != StoreAccess->getDefiningAccess())1436 addMemoryUsers(StoreRHS, StoreAccess);1437 // If we are defined by ourselves, use the live on entry def.1438 if (StoreRHS == StoreAccess)1439 StoreRHS = MSSA->getLiveOnEntryDef();1440 1441 if (SI->isSimple()) {1442 // See if we are defined by a previous store expression, it already has a1443 // value, and it's the same value as our current store. FIXME: Right now, we1444 // only do this for simple stores, we should expand to cover memcpys, etc.1445 const auto *LastStore = createStoreExpression(SI, StoreRHS);1446 const auto *LastCC = ExpressionToClass.lookup(LastStore);1447 // We really want to check whether the expression we matched was a store. No1448 // easy way to do that. However, we can check that the class we found has a1449 // store, which, assuming the value numbering state is not corrupt, is1450 // sufficient, because we must also be equivalent to that store's expression1451 // for it to be in the same class as the load.1452 if (LastCC && LastCC->getStoredValue() == LastStore->getStoredValue())1453 return LastStore;1454 // Also check if our value operand is defined by a load of the same memory1455 // location, and the memory state is the same as it was then (otherwise, it1456 // could have been overwritten later. See test32 in1457 // transforms/DeadStoreElimination/simple.ll).1458 if (auto *LI = dyn_cast<LoadInst>(LastStore->getStoredValue()))1459 if ((lookupOperandLeader(LI->getPointerOperand()) ==1460 LastStore->getOperand(0)) &&1461 (lookupMemoryLeader(getMemoryAccess(LI)->getDefiningAccess()) ==1462 StoreRHS))1463 return LastStore;1464 deleteExpression(LastStore);1465 }1466 1467 // If the store is not equivalent to anything, value number it as a store that1468 // produces a unique memory state (instead of using it's MemoryUse, we use1469 // it's MemoryDef).1470 return createStoreExpression(SI, StoreAccess);1471}1472 1473// See if we can extract the value of a loaded pointer from a load, a store, or1474// a memory instruction.1475const Expression *1476NewGVN::performSymbolicLoadCoercion(Type *LoadType, Value *LoadPtr,1477 LoadInst *LI, Instruction *DepInst,1478 MemoryAccess *DefiningAccess) const {1479 assert((!LI || LI->isSimple()) && "Not a simple load");1480 if (auto *DepSI = dyn_cast<StoreInst>(DepInst)) {1481 // Can't forward from non-atomic to atomic without violating memory model.1482 // Also don't need to coerce if they are the same type, we will just1483 // propagate.1484 if (LI->isAtomic() > DepSI->isAtomic() ||1485 LoadType == DepSI->getValueOperand()->getType())1486 return nullptr;1487 int Offset = analyzeLoadFromClobberingStore(LoadType, LoadPtr, DepSI, DL);1488 if (Offset >= 0) {1489 if (auto *C = dyn_cast<Constant>(1490 lookupOperandLeader(DepSI->getValueOperand()))) {1491 if (Constant *Res = getConstantValueForLoad(C, Offset, LoadType, DL)) {1492 LLVM_DEBUG(dbgs() << "Coercing load from store " << *DepSI1493 << " to constant " << *Res << "\n");1494 return createConstantExpression(Res);1495 }1496 }1497 }1498 } else if (auto *DepLI = dyn_cast<LoadInst>(DepInst)) {1499 // Can't forward from non-atomic to atomic without violating memory model.1500 if (LI->isAtomic() > DepLI->isAtomic())1501 return nullptr;1502 int Offset = analyzeLoadFromClobberingLoad(LoadType, LoadPtr, DepLI, DL);1503 if (Offset >= 0) {1504 // We can coerce a constant load into a load.1505 if (auto *C = dyn_cast<Constant>(lookupOperandLeader(DepLI)))1506 if (auto *PossibleConstant =1507 getConstantValueForLoad(C, Offset, LoadType, DL)) {1508 LLVM_DEBUG(dbgs() << "Coercing load from load " << *LI1509 << " to constant " << *PossibleConstant << "\n");1510 return createConstantExpression(PossibleConstant);1511 }1512 }1513 } else if (auto *DepMI = dyn_cast<MemIntrinsic>(DepInst)) {1514 int Offset = analyzeLoadFromClobberingMemInst(LoadType, LoadPtr, DepMI, DL);1515 if (Offset >= 0) {1516 if (auto *PossibleConstant =1517 getConstantMemInstValueForLoad(DepMI, Offset, LoadType, DL)) {1518 LLVM_DEBUG(dbgs() << "Coercing load from meminst " << *DepMI1519 << " to constant " << *PossibleConstant << "\n");1520 return createConstantExpression(PossibleConstant);1521 }1522 }1523 }1524 1525 if (auto *II = dyn_cast<IntrinsicInst>(DepInst)) {1526 if (II->getIntrinsicID() == Intrinsic::lifetime_start) {1527 auto *LifetimePtr = II->getOperand(0);1528 if (LoadPtr == lookupOperandLeader(LifetimePtr) ||1529 AA->isMustAlias(LoadPtr, LifetimePtr))1530 return createConstantExpression(UndefValue::get(LoadType));1531 }1532 }1533 1534 // All of the below are only true if the loaded pointer is produced1535 // by the dependent instruction.1536 if (!DepInst->getType()->isPointerTy() ||1537 (LoadPtr != lookupOperandLeader(DepInst) &&1538 !AA->isMustAlias(LoadPtr, DepInst)))1539 return nullptr;1540 // If this load really doesn't depend on anything, then we must be loading an1541 // undef value. This can happen when loading for a fresh allocation with no1542 // intervening stores, for example. Note that this is only true in the case1543 // that the result of the allocation is pointer equal to the load ptr.1544 if (isa<AllocaInst>(DepInst)) {1545 return createConstantExpression(UndefValue::get(LoadType));1546 } else if (auto *InitVal =1547 getInitialValueOfAllocation(DepInst, TLI, LoadType))1548 return createConstantExpression(InitVal);1549 1550 return nullptr;1551}1552 1553const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const {1554 auto *LI = cast<LoadInst>(I);1555 1556 // We can eliminate in favor of non-simple loads, but we won't be able to1557 // eliminate the loads themselves.1558 if (!LI->isSimple())1559 return nullptr;1560 1561 Value *LoadAddressLeader = lookupOperandLeader(LI->getPointerOperand());1562 // Load of undef is UB.1563 if (isa<UndefValue>(LoadAddressLeader))1564 return createConstantExpression(PoisonValue::get(LI->getType()));1565 MemoryAccess *OriginalAccess = getMemoryAccess(I);1566 MemoryAccess *DefiningAccess =1567 MSSAWalker->getClobberingMemoryAccess(OriginalAccess);1568 1569 if (!MSSA->isLiveOnEntryDef(DefiningAccess)) {1570 if (auto *MD = dyn_cast<MemoryDef>(DefiningAccess)) {1571 Instruction *DefiningInst = MD->getMemoryInst();1572 // If the defining instruction is not reachable, replace with poison.1573 if (!ReachableBlocks.count(DefiningInst->getParent()))1574 return createConstantExpression(PoisonValue::get(LI->getType()));1575 // This will handle stores and memory insts. We only do if it the1576 // defining access has a different type, or it is a pointer produced by1577 // certain memory operations that cause the memory to have a fixed value1578 // (IE things like calloc).1579 if (const auto *CoercionResult =1580 performSymbolicLoadCoercion(LI->getType(), LoadAddressLeader, LI,1581 DefiningInst, DefiningAccess))1582 return CoercionResult;1583 }1584 }1585 1586 const auto *LE = createLoadExpression(LI->getType(), LoadAddressLeader, LI,1587 DefiningAccess);1588 // If our MemoryLeader is not our defining access, add a use to the1589 // MemoryLeader, so that we get reprocessed when it changes.1590 if (LE->getMemoryLeader() != DefiningAccess)1591 addMemoryUsers(LE->getMemoryLeader(), OriginalAccess);1592 return LE;1593}1594 1595NewGVN::ExprResult1596NewGVN::performSymbolicPredicateInfoEvaluation(BitCastInst *I) const {1597 auto *PI = PredInfo->getPredicateInfoFor(I);1598 if (!PI)1599 return ExprResult::none();1600 1601 LLVM_DEBUG(dbgs() << "Found predicate info from instruction !\n");1602 1603 const std::optional<PredicateConstraint> &Constraint = PI->getConstraint();1604 if (!Constraint)1605 return ExprResult::none();1606 1607 CmpInst::Predicate Predicate = Constraint->Predicate;1608 Value *CmpOp0 = I->getOperand(0);1609 Value *CmpOp1 = Constraint->OtherOp;1610 1611 Value *FirstOp = lookupOperandLeader(CmpOp0);1612 Value *SecondOp = lookupOperandLeader(CmpOp1);1613 Value *AdditionallyUsedValue = CmpOp0;1614 1615 // Sort the ops.1616 if (shouldSwapOperandsForPredicate(FirstOp, SecondOp, I)) {1617 std::swap(FirstOp, SecondOp);1618 Predicate = CmpInst::getSwappedPredicate(Predicate);1619 AdditionallyUsedValue = CmpOp1;1620 }1621 1622 if (Predicate == CmpInst::ICMP_EQ)1623 return ExprResult::some(createVariableOrConstant(FirstOp),1624 AdditionallyUsedValue, PI);1625 1626 // Handle the special case of floating point.1627 if (Predicate == CmpInst::FCMP_OEQ && isa<ConstantFP>(FirstOp) &&1628 !cast<ConstantFP>(FirstOp)->isZero())1629 return ExprResult::some(createConstantExpression(cast<Constant>(FirstOp)),1630 AdditionallyUsedValue, PI);1631 1632 return ExprResult::none();1633}1634 1635// Evaluate read only and pure calls, and create an expression result.1636NewGVN::ExprResult NewGVN::performSymbolicCallEvaluation(Instruction *I) const {1637 auto *CI = cast<CallInst>(I);1638 1639 // FIXME: Currently the calls which may access the thread id may1640 // be considered as not accessing the memory. But this is1641 // problematic for coroutines, since coroutines may resume in a1642 // different thread. So we disable the optimization here for the1643 // correctness. However, it may block many other correct1644 // optimizations. Revert this one when we detect the memory1645 // accessing kind more precisely.1646 if (CI->getFunction()->isPresplitCoroutine())1647 return ExprResult::none();1648 1649 // Do not combine convergent calls since they implicitly depend on the set of1650 // threads that is currently executing, and they might be in different basic1651 // blocks.1652 if (CI->isConvergent())1653 return ExprResult::none();1654 1655 if (AA->doesNotAccessMemory(CI)) {1656 return ExprResult::some(1657 createCallExpression(CI, TOPClass->getMemoryLeader()));1658 } else if (AA->onlyReadsMemory(CI)) {1659 if (auto *MA = MSSA->getMemoryAccess(CI)) {1660 auto *DefiningAccess = MSSAWalker->getClobberingMemoryAccess(MA);1661 return ExprResult::some(createCallExpression(CI, DefiningAccess));1662 } else // MSSA determined that CI does not access memory.1663 return ExprResult::some(1664 createCallExpression(CI, TOPClass->getMemoryLeader()));1665 }1666 return ExprResult::none();1667}1668 1669// Retrieve the memory class for a given MemoryAccess.1670CongruenceClass *NewGVN::getMemoryClass(const MemoryAccess *MA) const {1671 auto *Result = MemoryAccessToClass.lookup(MA);1672 assert(Result && "Should have found memory class");1673 return Result;1674}1675 1676// Update the MemoryAccess equivalence table to say that From is equal to To,1677// and return true if this is different from what already existed in the table.1678bool NewGVN::setMemoryClass(const MemoryAccess *From,1679 CongruenceClass *NewClass) {1680 assert(NewClass &&1681 "Every MemoryAccess should be getting mapped to a non-null class");1682 LLVM_DEBUG(dbgs() << "Setting " << *From);1683 LLVM_DEBUG(dbgs() << " equivalent to congruence class ");1684 LLVM_DEBUG(dbgs() << NewClass->getID()1685 << " with current MemoryAccess leader ");1686 LLVM_DEBUG(dbgs() << *NewClass->getMemoryLeader() << "\n");1687 1688 auto LookupResult = MemoryAccessToClass.find(From);1689 bool Changed = false;1690 // If it's already in the table, see if the value changed.1691 if (LookupResult != MemoryAccessToClass.end()) {1692 auto *OldClass = LookupResult->second;1693 if (OldClass != NewClass) {1694 // If this is a phi, we have to handle memory member updates.1695 if (auto *MP = dyn_cast<MemoryPhi>(From)) {1696 OldClass->memory_erase(MP);1697 NewClass->memory_insert(MP);1698 // This may have killed the class if it had no non-memory members1699 if (OldClass->getMemoryLeader() == From) {1700 if (OldClass->definesNoMemory()) {1701 OldClass->setMemoryLeader(nullptr);1702 } else {1703 OldClass->setMemoryLeader(getNextMemoryLeader(OldClass));1704 LLVM_DEBUG(dbgs() << "Memory class leader change for class "1705 << OldClass->getID() << " to "1706 << *OldClass->getMemoryLeader()1707 << " due to removal of a memory member " << *From1708 << "\n");1709 markMemoryLeaderChangeTouched(OldClass);1710 }1711 }1712 }1713 // It wasn't equivalent before, and now it is.1714 LookupResult->second = NewClass;1715 Changed = true;1716 }1717 }1718 1719 return Changed;1720}1721 1722// Determine if a instruction is cycle-free. That means the values in the1723// instruction don't depend on any expressions that can change value as a result1724// of the instruction. For example, a non-cycle free instruction would be v =1725// phi(0, v+1).1726bool NewGVN::isCycleFree(const Instruction *I) const {1727 // In order to compute cycle-freeness, we do SCC finding on the instruction,1728 // and see what kind of SCC it ends up in. If it is a singleton, it is1729 // cycle-free. If it is not in a singleton, it is only cycle free if the1730 // other members are all phi nodes (as they do not compute anything, they are1731 // copies).1732 auto ICS = InstCycleState.lookup(I);1733 if (ICS == ICS_Unknown) {1734 SCCFinder.Start(I);1735 auto &SCC = SCCFinder.getComponentFor(I);1736 // It's cycle free if it's size 1 or the SCC is *only* phi nodes.1737 if (SCC.size() == 1)1738 InstCycleState.insert({I, ICS_CycleFree});1739 else {1740 bool AllPhis = llvm::all_of(SCC, [](const Value *V) {1741 return isa<PHINode>(V) || isCopyOfAPHI(V);1742 });1743 ICS = AllPhis ? ICS_CycleFree : ICS_Cycle;1744 for (const auto *Member : SCC)1745 if (auto *MemberPhi = dyn_cast<PHINode>(Member))1746 InstCycleState.insert({MemberPhi, ICS});1747 }1748 }1749 if (ICS == ICS_Cycle)1750 return false;1751 return true;1752}1753 1754// Evaluate PHI nodes symbolically and create an expression result.1755const Expression *1756NewGVN::performSymbolicPHIEvaluation(ArrayRef<ValPair> PHIOps,1757 Instruction *I,1758 BasicBlock *PHIBlock) const {1759 // True if one of the incoming phi edges is a backedge.1760 bool HasBackedge = false;1761 // All constant tracks the state of whether all the *original* phi operands1762 // This is really shorthand for "this phi cannot cycle due to forward1763 // change in value of the phi is guaranteed not to later change the value of1764 // the phi. IE it can't be v = phi(undef, v+1)1765 bool OriginalOpsConstant = true;1766 auto *E = cast<PHIExpression>(createPHIExpression(1767 PHIOps, I, PHIBlock, HasBackedge, OriginalOpsConstant));1768 // We match the semantics of SimplifyPhiNode from InstructionSimplify here.1769 // See if all arguments are the same.1770 // We track if any were undef because they need special handling.1771 bool HasUndef = false, HasPoison = false;1772 auto Filtered = make_filter_range(E->operands(), [&](Value *Arg) {1773 if (isa<PoisonValue>(Arg)) {1774 HasPoison = true;1775 return false;1776 }1777 if (isa<UndefValue>(Arg)) {1778 HasUndef = true;1779 return false;1780 }1781 return true;1782 });1783 // If we are left with no operands, it's dead.1784 if (Filtered.empty()) {1785 // If it has undef or poison at this point, it means there are no-non-undef1786 // arguments, and thus, the value of the phi node must be undef.1787 if (HasUndef) {1788 LLVM_DEBUG(1789 dbgs() << "PHI Node " << *I1790 << " has no non-undef arguments, valuing it as undef\n");1791 return createConstantExpression(UndefValue::get(I->getType()));1792 }1793 if (HasPoison) {1794 LLVM_DEBUG(1795 dbgs() << "PHI Node " << *I1796 << " has no non-poison arguments, valuing it as poison\n");1797 return createConstantExpression(PoisonValue::get(I->getType()));1798 }1799 1800 LLVM_DEBUG(dbgs() << "No arguments of PHI node " << *I << " are live\n");1801 deleteExpression(E);1802 return createDeadExpression();1803 }1804 Value *AllSameValue = *(Filtered.begin());1805 ++Filtered.begin();1806 // Can't use std::equal here, sadly, because filter.begin moves.1807 if (llvm::all_of(Filtered, [&](Value *Arg) { return Arg == AllSameValue; })) {1808 // Can't fold phi(undef, X) -> X unless X can't be poison (thus X is undef1809 // in the worst case).1810 if (HasUndef && !isGuaranteedNotToBePoison(AllSameValue, AC, nullptr, DT))1811 return E;1812 1813 // In LLVM's non-standard representation of phi nodes, it's possible to have1814 // phi nodes with cycles (IE dependent on other phis that are .... dependent1815 // on the original phi node), especially in weird CFG's where some arguments1816 // are unreachable, or uninitialized along certain paths. This can cause1817 // infinite loops during evaluation. We work around this by not trying to1818 // really evaluate them independently, but instead using a variable1819 // expression to say if one is equivalent to the other.1820 // We also special case undef/poison, so that if we have an undef, we can't1821 // use the common value unless it dominates the phi block.1822 if (HasPoison || HasUndef) {1823 // If we have undef and at least one other value, this is really a1824 // multivalued phi, and we need to know if it's cycle free in order to1825 // evaluate whether we can ignore the undef. The other parts of this are1826 // just shortcuts. If there is no backedge, or all operands are1827 // constants, it also must be cycle free.1828 if (HasBackedge && !OriginalOpsConstant &&1829 !isa<UndefValue>(AllSameValue) && !isCycleFree(I))1830 return E;1831 1832 // Only have to check for instructions1833 if (auto *AllSameInst = dyn_cast<Instruction>(AllSameValue))1834 if (!someEquivalentDominates(AllSameInst, I))1835 return E;1836 }1837 // Can't simplify to something that comes later in the iteration.1838 // Otherwise, when and if it changes congruence class, we will never catch1839 // up. We will always be a class behind it.1840 if (isa<Instruction>(AllSameValue) &&1841 InstrToDFSNum(AllSameValue) > InstrToDFSNum(I))1842 return E;1843 NumGVNPhisAllSame++;1844 LLVM_DEBUG(dbgs() << "Simplified PHI node " << *I << " to " << *AllSameValue1845 << "\n");1846 deleteExpression(E);1847 return createVariableOrConstant(AllSameValue);1848 }1849 return E;1850}1851 1852const Expression *1853NewGVN::performSymbolicAggrValueEvaluation(Instruction *I) const {1854 if (auto *EI = dyn_cast<ExtractValueInst>(I)) {1855 auto *WO = dyn_cast<WithOverflowInst>(EI->getAggregateOperand());1856 if (WO && EI->getNumIndices() == 1 && *EI->idx_begin() == 0)1857 // EI is an extract from one of our with.overflow intrinsics. Synthesize1858 // a semantically equivalent expression instead of an extract value1859 // expression.1860 return createBinaryExpression(WO->getBinaryOp(), EI->getType(),1861 WO->getLHS(), WO->getRHS(), I);1862 }1863 1864 return createAggregateValueExpression(I);1865}1866 1867NewGVN::ExprResult NewGVN::performSymbolicCmpEvaluation(Instruction *I) const {1868 assert(isa<CmpInst>(I) && "Expected a cmp instruction.");1869 1870 auto *CI = cast<CmpInst>(I);1871 // See if our operands are equal to those of a previous predicate, and if so,1872 // if it implies true or false.1873 auto Op0 = lookupOperandLeader(CI->getOperand(0));1874 auto Op1 = lookupOperandLeader(CI->getOperand(1));1875 auto OurPredicate = CI->getPredicate();1876 if (shouldSwapOperands(Op0, Op1)) {1877 std::swap(Op0, Op1);1878 OurPredicate = CI->getSwappedPredicate();1879 }1880 1881 // Avoid processing the same info twice.1882 const PredicateBase *LastPredInfo = nullptr;1883 // See if we know something about the comparison itself, like it is the target1884 // of an assume.1885 auto *CmpPI = PredInfo->getPredicateInfoFor(I);1886 if (isa_and_nonnull<PredicateAssume>(CmpPI))1887 return ExprResult::some(1888 createConstantExpression(ConstantInt::getTrue(CI->getType())));1889 1890 if (Op0 == Op1) {1891 // This condition does not depend on predicates, no need to add users1892 if (CI->isTrueWhenEqual())1893 return ExprResult::some(1894 createConstantExpression(ConstantInt::getTrue(CI->getType())));1895 else if (CI->isFalseWhenEqual())1896 return ExprResult::some(1897 createConstantExpression(ConstantInt::getFalse(CI->getType())));1898 }1899 1900 // NOTE: Because we are comparing both operands here and below, and using1901 // previous comparisons, we rely on fact that predicateinfo knows to mark1902 // comparisons that use renamed operands as users of the earlier comparisons.1903 // It is *not* enough to just mark predicateinfo renamed operands as users of1904 // the earlier comparisons, because the *other* operand may have changed in a1905 // previous iteration.1906 // Example:1907 // icmp slt %a, %b1908 // %b.0 = ssa.copy(%b)1909 // false branch:1910 // icmp slt %c, %b.01911 1912 // %c and %a may start out equal, and thus, the code below will say the second1913 // %icmp is false. c may become equal to something else, and in that case the1914 // %second icmp *must* be reexamined, but would not if only the renamed1915 // %operands are considered users of the icmp.1916 1917 // *Currently* we only check one level of comparisons back, and only mark one1918 // level back as touched when changes happen. If you modify this code to look1919 // back farther through comparisons, you *must* mark the appropriate1920 // comparisons as users in PredicateInfo.cpp, or you will cause bugs. See if1921 // we know something just from the operands themselves1922 1923 // See if our operands have predicate info, so that we may be able to derive1924 // something from a previous comparison.1925 for (const auto &Op : CI->operands()) {1926 auto *PI = PredInfo->getPredicateInfoFor(Op);1927 if (const auto *PBranch = dyn_cast_or_null<PredicateBranch>(PI)) {1928 if (PI == LastPredInfo)1929 continue;1930 LastPredInfo = PI;1931 // In phi of ops cases, we may have predicate info that we are evaluating1932 // in a different context.1933 if (!DT->dominates(PBranch->To, I->getParent()))1934 continue;1935 // TODO: Along the false edge, we may know more things too, like1936 // icmp of1937 // same operands is false.1938 // TODO: We only handle actual comparison conditions below, not1939 // and/or.1940 auto *BranchCond = dyn_cast<CmpInst>(PBranch->Condition);1941 if (!BranchCond)1942 continue;1943 auto *BranchOp0 = lookupOperandLeader(BranchCond->getOperand(0));1944 auto *BranchOp1 = lookupOperandLeader(BranchCond->getOperand(1));1945 auto BranchPredicate = BranchCond->getPredicate();1946 if (shouldSwapOperands(BranchOp0, BranchOp1)) {1947 std::swap(BranchOp0, BranchOp1);1948 BranchPredicate = BranchCond->getSwappedPredicate();1949 }1950 if (BranchOp0 == Op0 && BranchOp1 == Op1) {1951 if (PBranch->TrueEdge) {1952 // If we know the previous predicate is true and we are in the true1953 // edge then we may be implied true or false.1954 if (auto R = ICmpInst::isImpliedByMatchingCmp(BranchPredicate,1955 OurPredicate)) {1956 auto *C = ConstantInt::getBool(CI->getType(), *R);1957 return ExprResult::some(createConstantExpression(C), PI);1958 }1959 } else {1960 // Just handle the ne and eq cases, where if we have the same1961 // operands, we may know something.1962 if (BranchPredicate == OurPredicate) {1963 // Same predicate, same ops,we know it was false, so this is false.1964 return ExprResult::some(1965 createConstantExpression(ConstantInt::getFalse(CI->getType())),1966 PI);1967 } else if (BranchPredicate ==1968 CmpInst::getInversePredicate(OurPredicate)) {1969 // Inverse predicate, we know the other was false, so this is true.1970 return ExprResult::some(1971 createConstantExpression(ConstantInt::getTrue(CI->getType())),1972 PI);1973 }1974 }1975 }1976 }1977 }1978 // Create expression will take care of simplifyCmpInst1979 return createExpression(I);1980}1981 1982// Substitute and symbolize the instruction before value numbering.1983NewGVN::ExprResult1984NewGVN::performSymbolicEvaluation(Instruction *I,1985 SmallPtrSetImpl<Value *> &Visited) const {1986 1987 const Expression *E = nullptr;1988 // TODO: memory intrinsics.1989 // TODO: Some day, we should do the forward propagation and reassociation1990 // parts of the algorithm.1991 switch (I->getOpcode()) {1992 case Instruction::ExtractValue:1993 case Instruction::InsertValue:1994 E = performSymbolicAggrValueEvaluation(I);1995 break;1996 case Instruction::PHI: {1997 SmallVector<ValPair, 3> Ops;1998 auto *PN = cast<PHINode>(I);1999 for (unsigned i = 0; i < PN->getNumOperands(); ++i)2000 Ops.push_back({PN->getIncomingValue(i), PN->getIncomingBlock(i)});2001 // Sort to ensure the invariant createPHIExpression requires is met.2002 sortPHIOps(Ops);2003 E = performSymbolicPHIEvaluation(Ops, I, getBlockForValue(I));2004 } break;2005 case Instruction::Call:2006 return performSymbolicCallEvaluation(I);2007 break;2008 case Instruction::Store:2009 E = performSymbolicStoreEvaluation(I);2010 break;2011 case Instruction::Load:2012 E = performSymbolicLoadEvaluation(I);2013 break;2014 case Instruction::BitCast:2015 // Intrinsics with the returned attribute are copies of arguments.2016 if (I->getType() == I->getOperand(0)->getType())2017 if (auto Res =2018 performSymbolicPredicateInfoEvaluation(cast<BitCastInst>(I)))2019 return Res;2020 [[fallthrough]];2021 case Instruction::AddrSpaceCast:2022 case Instruction::Freeze:2023 return createExpression(I);2024 break;2025 case Instruction::ICmp:2026 case Instruction::FCmp:2027 return performSymbolicCmpEvaluation(I);2028 break;2029 case Instruction::FNeg:2030 case Instruction::Add:2031 case Instruction::FAdd:2032 case Instruction::Sub:2033 case Instruction::FSub:2034 case Instruction::Mul:2035 case Instruction::FMul:2036 case Instruction::UDiv:2037 case Instruction::SDiv:2038 case Instruction::FDiv:2039 case Instruction::URem:2040 case Instruction::SRem:2041 case Instruction::FRem:2042 case Instruction::Shl:2043 case Instruction::LShr:2044 case Instruction::AShr:2045 case Instruction::And:2046 case Instruction::Or:2047 case Instruction::Xor:2048 case Instruction::Trunc:2049 case Instruction::ZExt:2050 case Instruction::SExt:2051 case Instruction::FPToUI:2052 case Instruction::FPToSI:2053 case Instruction::UIToFP:2054 case Instruction::SIToFP:2055 case Instruction::FPTrunc:2056 case Instruction::FPExt:2057 case Instruction::PtrToInt:2058 case Instruction::PtrToAddr:2059 case Instruction::IntToPtr:2060 case Instruction::Select:2061 case Instruction::ExtractElement:2062 case Instruction::InsertElement:2063 case Instruction::GetElementPtr:2064 return createExpression(I);2065 break;2066 case Instruction::ShuffleVector:2067 // FIXME: Add support for shufflevector to createExpression.2068 return ExprResult::none();2069 default:2070 return ExprResult::none();2071 }2072 return ExprResult::some(E);2073}2074 2075// Look up a container of values/instructions in a map, and touch all the2076// instructions in the container. Then erase value from the map.2077template <typename Map, typename KeyType>2078void NewGVN::touchAndErase(Map &M, const KeyType &Key) {2079 const auto Result = M.find_as(Key);2080 if (Result != M.end()) {2081 for (const typename Map::mapped_type::value_type Mapped : Result->second)2082 TouchedInstructions.set(InstrToDFSNum(Mapped));2083 M.erase(Result);2084 }2085}2086 2087void NewGVN::addAdditionalUsers(Value *To, Value *User) const {2088 assert(User && To != User);2089 if (isa<Instruction>(To))2090 AdditionalUsers[To].insert(User);2091}2092 2093void NewGVN::addAdditionalUsers(ExprResult &Res, Instruction *User) const {2094 if (Res.ExtraDep && Res.ExtraDep != User)2095 addAdditionalUsers(Res.ExtraDep, User);2096 Res.ExtraDep = nullptr;2097 2098 if (Res.PredDep) {2099 if (const auto *PBranch = dyn_cast<PredicateBranch>(Res.PredDep))2100 PredicateToUsers[PBranch->Condition].insert(User);2101 else if (const auto *PAssume = dyn_cast<PredicateAssume>(Res.PredDep))2102 PredicateToUsers[PAssume->Condition].insert(User);2103 }2104 Res.PredDep = nullptr;2105}2106 2107void NewGVN::markUsersTouched(Value *V) {2108 // Now mark the users as touched.2109 for (auto *User : V->users()) {2110 assert(isa<Instruction>(User) && "Use of value not within an instruction?");2111 TouchedInstructions.set(InstrToDFSNum(User));2112 }2113 touchAndErase(AdditionalUsers, V);2114}2115 2116void NewGVN::addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const {2117 LLVM_DEBUG(dbgs() << "Adding memory user " << *U << " to " << *To << "\n");2118 MemoryToUsers[To].insert(U);2119}2120 2121void NewGVN::markMemoryDefTouched(const MemoryAccess *MA) {2122 TouchedInstructions.set(MemoryToDFSNum(MA));2123}2124 2125void NewGVN::markMemoryUsersTouched(const MemoryAccess *MA) {2126 if (isa<MemoryUse>(MA))2127 return;2128 for (const auto *U : MA->users())2129 TouchedInstructions.set(MemoryToDFSNum(U));2130 touchAndErase(MemoryToUsers, MA);2131}2132 2133// Touch all the predicates that depend on this instruction.2134void NewGVN::markPredicateUsersTouched(Instruction *I) {2135 touchAndErase(PredicateToUsers, I);2136}2137 2138// Mark users affected by a memory leader change.2139void NewGVN::markMemoryLeaderChangeTouched(CongruenceClass *CC) {2140 for (const auto *M : CC->memory())2141 markMemoryDefTouched(M);2142}2143 2144// Touch the instructions that need to be updated after a congruence class has a2145// leader change, and mark changed values.2146void NewGVN::markValueLeaderChangeTouched(CongruenceClass *CC) {2147 for (auto *M : *CC) {2148 if (auto *I = dyn_cast<Instruction>(M))2149 TouchedInstructions.set(InstrToDFSNum(I));2150 LeaderChanges.insert(M);2151 }2152}2153 2154// Give a range of things that have instruction DFS numbers, this will return2155// the member of the range with the smallest dfs number.2156template <class T, class Range>2157T *NewGVN::getMinDFSOfRange(const Range &R) const {2158 std::pair<T *, unsigned> MinDFS = {nullptr, ~0U};2159 for (const auto X : R) {2160 auto DFSNum = InstrToDFSNum(X);2161 if (DFSNum < MinDFS.second)2162 MinDFS = {X, DFSNum};2163 }2164 return MinDFS.first;2165}2166 2167// This function returns the MemoryAccess that should be the next leader of2168// congruence class CC, under the assumption that the current leader is going to2169// disappear.2170const MemoryAccess *NewGVN::getNextMemoryLeader(CongruenceClass *CC) const {2171 // TODO: If this ends up to slow, we can maintain a next memory leader like we2172 // do for regular leaders.2173 // Make sure there will be a leader to find.2174 assert(!CC->definesNoMemory() && "Can't get next leader if there is none");2175 if (CC->getStoreCount() > 0) {2176 if (auto *NL = dyn_cast_or_null<StoreInst>(CC->getNextLeader().first))2177 return getMemoryAccess(NL);2178 // Find the store with the minimum DFS number.2179 auto *V = getMinDFSOfRange<Value>(make_filter_range(2180 *CC, [&](const Value *V) { return isa<StoreInst>(V); }));2181 return getMemoryAccess(cast<StoreInst>(V));2182 }2183 assert(CC->getStoreCount() == 0);2184 2185 // Given our assertion, hitting this part must mean2186 // !OldClass->memory_empty()2187 if (CC->memory_size() == 1)2188 return *CC->memory_begin();2189 return getMinDFSOfRange<const MemoryPhi>(CC->memory());2190}2191 2192// This function returns the next value leader of a congruence class, under the2193// assumption that the current leader is going away. This should end up being2194// the next most dominating member.2195Value *NewGVN::getNextValueLeader(CongruenceClass *CC) const {2196 // We don't need to sort members if there is only 1, and we don't care about2197 // sorting the TOP class because everything either gets out of it or is2198 // unreachable.2199 2200 if (CC->size() == 1 || CC == TOPClass) {2201 return *(CC->begin());2202 } else if (CC->getNextLeader().first) {2203 ++NumGVNAvoidedSortedLeaderChanges;2204 return CC->getNextLeader().first;2205 } else {2206 ++NumGVNSortedLeaderChanges;2207 // NOTE: If this ends up to slow, we can maintain a dual structure for2208 // member testing/insertion, or keep things mostly sorted, and sort only2209 // here, or use SparseBitVector or ....2210 return getMinDFSOfRange<Value>(*CC);2211 }2212}2213 2214// Move a MemoryAccess, currently in OldClass, to NewClass, including updates to2215// the memory members, etc for the move.2216//2217// The invariants of this function are:2218//2219// - I must be moving to NewClass from OldClass2220// - The StoreCount of OldClass and NewClass is expected to have been updated2221// for I already if it is a store.2222// - The OldClass memory leader has not been updated yet if I was the leader.2223void NewGVN::moveMemoryToNewCongruenceClass(Instruction *I,2224 MemoryAccess *InstMA,2225 CongruenceClass *OldClass,2226 CongruenceClass *NewClass) {2227 // If the leader is I, and we had a representative MemoryAccess, it should2228 // be the MemoryAccess of OldClass.2229 assert((!InstMA || !OldClass->getMemoryLeader() ||2230 OldClass->getLeader() != I ||2231 MemoryAccessToClass.lookup(OldClass->getMemoryLeader()) ==2232 MemoryAccessToClass.lookup(InstMA)) &&2233 "Representative MemoryAccess mismatch");2234 // First, see what happens to the new class2235 if (!NewClass->getMemoryLeader()) {2236 // Should be a new class, or a store becoming a leader of a new class.2237 assert(NewClass->size() == 1 ||2238 (isa<StoreInst>(I) && NewClass->getStoreCount() == 1));2239 NewClass->setMemoryLeader(InstMA);2240 // Mark it touched if we didn't just create a singleton2241 LLVM_DEBUG(dbgs() << "Memory class leader change for class "2242 << NewClass->getID()2243 << " due to new memory instruction becoming leader\n");2244 markMemoryLeaderChangeTouched(NewClass);2245 }2246 setMemoryClass(InstMA, NewClass);2247 // Now, fixup the old class if necessary2248 if (OldClass->getMemoryLeader() == InstMA) {2249 if (!OldClass->definesNoMemory()) {2250 OldClass->setMemoryLeader(getNextMemoryLeader(OldClass));2251 LLVM_DEBUG(dbgs() << "Memory class leader change for class "2252 << OldClass->getID() << " to "2253 << *OldClass->getMemoryLeader()2254 << " due to removal of old leader " << *InstMA << "\n");2255 markMemoryLeaderChangeTouched(OldClass);2256 } else2257 OldClass->setMemoryLeader(nullptr);2258 }2259}2260 2261// Move a value, currently in OldClass, to be part of NewClass2262// Update OldClass and NewClass for the move (including changing leaders, etc).2263void NewGVN::moveValueToNewCongruenceClass(Instruction *I, const Expression *E,2264 CongruenceClass *OldClass,2265 CongruenceClass *NewClass) {2266 if (I == OldClass->getNextLeader().first)2267 OldClass->resetNextLeader();2268 2269 OldClass->erase(I);2270 NewClass->insert(I);2271 2272 // Ensure that the leader has the lowest RPO. If the leader changed notify all2273 // members of the class.2274 if (NewClass->getLeader() != I &&2275 NewClass->addPossibleLeader({I, InstrToDFSNum(I)})) {2276 markValueLeaderChangeTouched(NewClass);2277 }2278 2279 // Handle our special casing of stores.2280 if (auto *SI = dyn_cast<StoreInst>(I)) {2281 OldClass->decStoreCount();2282 // Okay, so when do we want to make a store a leader of a class?2283 // If we have a store defined by an earlier load, we want the earlier load2284 // to lead the class.2285 // If we have a store defined by something else, we want the store to lead2286 // the class so everything else gets the "something else" as a value.2287 // If we have a store as the single member of the class, we want the store2288 // as the leader2289 if (NewClass->getStoreCount() == 0 && !NewClass->getStoredValue()) {2290 // If it's a store expression we are using, it means we are not equivalent2291 // to something earlier.2292 if (auto *SE = dyn_cast<StoreExpression>(E)) {2293 NewClass->setStoredValue(SE->getStoredValue());2294 markValueLeaderChangeTouched(NewClass);2295 // Shift the new class leader to be the store2296 LLVM_DEBUG(dbgs() << "Changing leader of congruence class "2297 << NewClass->getID() << " from "2298 << *NewClass->getLeader() << " to " << *SI2299 << " because store joined class\n");2300 // If we changed the leader, we have to mark it changed because we don't2301 // know what it will do to symbolic evaluation.2302 NewClass->setLeader({SI, InstrToDFSNum(SI)});2303 }2304 // We rely on the code below handling the MemoryAccess change.2305 }2306 NewClass->incStoreCount();2307 }2308 // True if there is no memory instructions left in a class that had memory2309 // instructions before.2310 2311 // If it's not a memory use, set the MemoryAccess equivalence2312 auto *InstMA = dyn_cast_or_null<MemoryDef>(getMemoryAccess(I));2313 if (InstMA)2314 moveMemoryToNewCongruenceClass(I, InstMA, OldClass, NewClass);2315 ValueToClass[I] = NewClass;2316 // See if we destroyed the class or need to swap leaders.2317 if (OldClass->empty() && OldClass != TOPClass) {2318 if (OldClass->getDefiningExpr()) {2319 LLVM_DEBUG(dbgs() << "Erasing expression " << *OldClass->getDefiningExpr()2320 << " from table\n");2321 // We erase it as an exact expression to make sure we don't just erase an2322 // equivalent one.2323 auto Iter = ExpressionToClass.find_as(2324 ExactEqualsExpression(*OldClass->getDefiningExpr()));2325 if (Iter != ExpressionToClass.end())2326 ExpressionToClass.erase(Iter);2327#ifdef EXPENSIVE_CHECKS2328 assert(2329 (*OldClass->getDefiningExpr() != *E || ExpressionToClass.lookup(E)) &&2330 "We erased the expression we just inserted, which should not happen");2331#endif2332 }2333 } else if (OldClass->getLeader() == I) {2334 // When the leader changes, the value numbering of2335 // everything may change due to symbolization changes, so we need to2336 // reprocess.2337 LLVM_DEBUG(dbgs() << "Value class leader change for class "2338 << OldClass->getID() << "\n");2339 ++NumGVNLeaderChanges;2340 // Destroy the stored value if there are no more stores to represent it.2341 // Note that this is basically clean up for the expression removal that2342 // happens below. If we remove stores from a class, we may leave it as a2343 // class of equivalent memory phis.2344 if (OldClass->getStoreCount() == 0) {2345 if (OldClass->getStoredValue())2346 OldClass->setStoredValue(nullptr);2347 }2348 OldClass->setLeader({getNextValueLeader(OldClass),2349 InstrToDFSNum(getNextValueLeader(OldClass))});2350 OldClass->resetNextLeader();2351 markValueLeaderChangeTouched(OldClass);2352 }2353}2354 2355// For a given expression, mark the phi of ops instructions that could have2356// changed as a result.2357void NewGVN::markPhiOfOpsChanged(const Expression *E) {2358 touchAndErase(ExpressionToPhiOfOps, E);2359}2360 2361// Perform congruence finding on a given value numbering expression.2362void NewGVN::performCongruenceFinding(Instruction *I, const Expression *E) {2363 // This is guaranteed to return something, since it will at least find2364 // TOP.2365 2366 CongruenceClass *IClass = ValueToClass.lookup(I);2367 assert(IClass && "Should have found a IClass");2368 // Dead classes should have been eliminated from the mapping.2369 assert(!IClass->isDead() && "Found a dead class");2370 2371 CongruenceClass *EClass = nullptr;2372 if (const auto *VE = dyn_cast<VariableExpression>(E)) {2373 EClass = ValueToClass.lookup(VE->getVariableValue());2374 } else if (isa<DeadExpression>(E)) {2375 EClass = TOPClass;2376 }2377 if (!EClass) {2378 auto lookupResult = ExpressionToClass.try_emplace(E);2379 2380 // If it's not in the value table, create a new congruence class.2381 if (lookupResult.second) {2382 CongruenceClass *NewClass = createCongruenceClass(nullptr, E);2383 auto place = lookupResult.first;2384 place->second = NewClass;2385 2386 // Constants and variables should always be made the leader.2387 if (const auto *CE = dyn_cast<ConstantExpression>(E)) {2388 NewClass->setLeader({CE->getConstantValue(), 0});2389 } else if (const auto *SE = dyn_cast<StoreExpression>(E)) {2390 StoreInst *SI = SE->getStoreInst();2391 NewClass->setLeader({SI, InstrToDFSNum(SI)});2392 NewClass->setStoredValue(SE->getStoredValue());2393 // The RepMemoryAccess field will be filled in properly by the2394 // moveValueToNewCongruenceClass call.2395 } else {2396 NewClass->setLeader({I, InstrToDFSNum(I)});2397 }2398 assert(!isa<VariableExpression>(E) &&2399 "VariableExpression should have been handled already");2400 2401 EClass = NewClass;2402 LLVM_DEBUG(dbgs() << "Created new congruence class for " << *I2403 << " using expression " << *E << " at "2404 << NewClass->getID() << " and leader "2405 << *(NewClass->getLeader()));2406 if (NewClass->getStoredValue())2407 LLVM_DEBUG(dbgs() << " and stored value "2408 << *(NewClass->getStoredValue()));2409 LLVM_DEBUG(dbgs() << "\n");2410 } else {2411 EClass = lookupResult.first->second;2412 if (isa<ConstantExpression>(E))2413 assert((isa<Constant>(EClass->getLeader()) ||2414 (EClass->getStoredValue() &&2415 isa<Constant>(EClass->getStoredValue()))) &&2416 "Any class with a constant expression should have a "2417 "constant leader");2418 2419 assert(EClass && "Somehow don't have an eclass");2420 2421 assert(!EClass->isDead() && "We accidentally looked up a dead class");2422 }2423 }2424 bool ClassChanged = IClass != EClass;2425 bool LeaderChanged = LeaderChanges.erase(I);2426 if (ClassChanged || LeaderChanged) {2427 LLVM_DEBUG(dbgs() << "New class " << EClass->getID() << " for expression "2428 << *E << "\n");2429 if (ClassChanged) {2430 moveValueToNewCongruenceClass(I, E, IClass, EClass);2431 markPhiOfOpsChanged(E);2432 }2433 2434 markUsersTouched(I);2435 if (MemoryAccess *MA = getMemoryAccess(I))2436 markMemoryUsersTouched(MA);2437 if (auto *CI = dyn_cast<CmpInst>(I))2438 markPredicateUsersTouched(CI);2439 }2440 // If we changed the class of the store, we want to ensure nothing finds the2441 // old store expression. In particular, loads do not compare against stored2442 // value, so they will find old store expressions (and associated class2443 // mappings) if we leave them in the table.2444 if (ClassChanged && isa<StoreInst>(I)) {2445 auto *OldE = ValueToExpression.lookup(I);2446 // It could just be that the old class died. We don't want to erase it if we2447 // just moved classes.2448 if (OldE && isa<StoreExpression>(OldE) && *E != *OldE) {2449 // Erase this as an exact expression to ensure we don't erase expressions2450 // equivalent to it.2451 auto Iter = ExpressionToClass.find_as(ExactEqualsExpression(*OldE));2452 if (Iter != ExpressionToClass.end())2453 ExpressionToClass.erase(Iter);2454 }2455 }2456 ValueToExpression[I] = E;2457}2458 2459// Process the fact that Edge (from, to) is reachable, including marking2460// any newly reachable blocks and instructions for processing.2461void NewGVN::updateReachableEdge(BasicBlock *From, BasicBlock *To) {2462 // Check if the Edge was reachable before.2463 if (ReachableEdges.insert({From, To}).second) {2464 // If this block wasn't reachable before, all instructions are touched.2465 if (ReachableBlocks.insert(To).second) {2466 LLVM_DEBUG(dbgs() << "Block " << getBlockName(To)2467 << " marked reachable\n");2468 const auto &InstRange = BlockInstRange.lookup(To);2469 TouchedInstructions.set(InstRange.first, InstRange.second);2470 } else {2471 LLVM_DEBUG(dbgs() << "Block " << getBlockName(To)2472 << " was reachable, but new edge {"2473 << getBlockName(From) << "," << getBlockName(To)2474 << "} to it found\n");2475 2476 // We've made an edge reachable to an existing block, which may2477 // impact predicates. Otherwise, only mark the phi nodes as touched, as2478 // they are the only thing that depend on new edges. Anything using their2479 // values will get propagated to if necessary.2480 if (MemoryAccess *MemPhi = getMemoryAccess(To))2481 TouchedInstructions.set(InstrToDFSNum(MemPhi));2482 2483 // FIXME: We should just add a union op on a Bitvector and2484 // SparseBitVector. We can do it word by word faster than we are doing it2485 // here.2486 for (auto InstNum : RevisitOnReachabilityChange[To])2487 TouchedInstructions.set(InstNum);2488 }2489 }2490}2491 2492// Given a predicate condition (from a switch, cmp, or whatever) and a block,2493// see if we know some constant value for it already.2494Value *NewGVN::findConditionEquivalence(Value *Cond) const {2495 auto Result = lookupOperandLeader(Cond);2496 return isa<Constant>(Result) ? Result : nullptr;2497}2498 2499// Process the outgoing edges of a block for reachability.2500void NewGVN::processOutgoingEdges(Instruction *TI, BasicBlock *B) {2501 // Evaluate reachability of terminator instruction.2502 Value *Cond;2503 BasicBlock *TrueSucc, *FalseSucc;2504 if (match(TI, m_Br(m_Value(Cond), TrueSucc, FalseSucc))) {2505 Value *CondEvaluated = findConditionEquivalence(Cond);2506 if (!CondEvaluated) {2507 if (auto *I = dyn_cast<Instruction>(Cond)) {2508 SmallPtrSet<Value *, 4> Visited;2509 auto Res = performSymbolicEvaluation(I, Visited);2510 if (const auto *CE = dyn_cast_or_null<ConstantExpression>(Res.Expr)) {2511 CondEvaluated = CE->getConstantValue();2512 addAdditionalUsers(Res, I);2513 } else {2514 // Did not use simplification result, no need to add the extra2515 // dependency.2516 Res.ExtraDep = nullptr;2517 }2518 } else if (isa<ConstantInt>(Cond)) {2519 CondEvaluated = Cond;2520 }2521 }2522 ConstantInt *CI;2523 if (CondEvaluated && (CI = dyn_cast<ConstantInt>(CondEvaluated))) {2524 if (CI->isOne()) {2525 LLVM_DEBUG(dbgs() << "Condition for Terminator " << *TI2526 << " evaluated to true\n");2527 updateReachableEdge(B, TrueSucc);2528 } else if (CI->isZero()) {2529 LLVM_DEBUG(dbgs() << "Condition for Terminator " << *TI2530 << " evaluated to false\n");2531 updateReachableEdge(B, FalseSucc);2532 }2533 } else {2534 updateReachableEdge(B, TrueSucc);2535 updateReachableEdge(B, FalseSucc);2536 }2537 } else if (auto *SI = dyn_cast<SwitchInst>(TI)) {2538 // For switches, propagate the case values into the case2539 // destinations.2540 2541 Value *SwitchCond = SI->getCondition();2542 Value *CondEvaluated = findConditionEquivalence(SwitchCond);2543 // See if we were able to turn this switch statement into a constant.2544 if (CondEvaluated && isa<ConstantInt>(CondEvaluated)) {2545 auto *CondVal = cast<ConstantInt>(CondEvaluated);2546 // We should be able to get case value for this.2547 auto Case = *SI->findCaseValue(CondVal);2548 if (Case.getCaseSuccessor() == SI->getDefaultDest()) {2549 // We proved the value is outside of the range of the case.2550 // We can't do anything other than mark the default dest as reachable,2551 // and go home.2552 updateReachableEdge(B, SI->getDefaultDest());2553 return;2554 }2555 // Now get where it goes and mark it reachable.2556 BasicBlock *TargetBlock = Case.getCaseSuccessor();2557 updateReachableEdge(B, TargetBlock);2558 } else {2559 for (BasicBlock *TargetBlock : successors(SI->getParent()))2560 updateReachableEdge(B, TargetBlock);2561 }2562 } else {2563 // Otherwise this is either unconditional, or a type we have no2564 // idea about. Just mark successors as reachable.2565 for (BasicBlock *TargetBlock : successors(TI->getParent()))2566 updateReachableEdge(B, TargetBlock);2567 2568 // This also may be a memory defining terminator, in which case, set it2569 // equivalent only to itself.2570 //2571 auto *MA = getMemoryAccess(TI);2572 if (MA && !isa<MemoryUse>(MA)) {2573 auto *CC = ensureLeaderOfMemoryClass(MA);2574 if (setMemoryClass(MA, CC))2575 markMemoryUsersTouched(MA);2576 }2577 }2578}2579 2580// Remove the PHI of Ops PHI for I2581void NewGVN::removePhiOfOps(Instruction *I, PHINode *PHITemp) {2582 InstrDFS.erase(PHITemp);2583 // It's still a temp instruction. We keep it in the array so it gets erased.2584 // However, it's no longer used by I, or in the block2585 TempToBlock.erase(PHITemp);2586 RealToTemp.erase(I);2587 // We don't remove the users from the phi node uses. This wastes a little2588 // time, but such is life. We could use two sets to track which were there2589 // are the start of NewGVN, and which were added, but right nowt he cost of2590 // tracking is more than the cost of checking for more phi of ops.2591}2592 2593// Add PHI Op in BB as a PHI of operations version of ExistingValue.2594void NewGVN::addPhiOfOps(PHINode *Op, BasicBlock *BB,2595 Instruction *ExistingValue) {2596 InstrDFS[Op] = InstrToDFSNum(ExistingValue);2597 AllTempInstructions.insert(Op);2598 TempToBlock[Op] = BB;2599 RealToTemp[ExistingValue] = Op;2600 // Add all users to phi node use, as they are now uses of the phi of ops phis2601 // and may themselves be phi of ops.2602 for (auto *U : ExistingValue->users())2603 if (auto *UI = dyn_cast<Instruction>(U))2604 PHINodeUses.insert(UI);2605}2606 2607static bool okayForPHIOfOps(const Instruction *I) {2608 if (!EnablePhiOfOps)2609 return false;2610 return isa<BinaryOperator>(I) || isa<SelectInst>(I) || isa<CmpInst>(I) ||2611 isa<LoadInst>(I);2612}2613 2614// Return true if this operand will be safe to use for phi of ops.2615//2616// The reason some operands are unsafe is that we are not trying to recursively2617// translate everything back through phi nodes. We actually expect some lookups2618// of expressions to fail. In particular, a lookup where the expression cannot2619// exist in the predecessor. This is true even if the expression, as shown, can2620// be determined to be constant.2621bool NewGVN::OpIsSafeForPHIOfOps(Value *V, const BasicBlock *PHIBlock,2622 SmallPtrSetImpl<const Value *> &Visited) {2623 SmallVector<Value *, 4> Worklist;2624 Worklist.push_back(V);2625 while (!Worklist.empty()) {2626 auto *I = Worklist.pop_back_val();2627 if (!isa<Instruction>(I))2628 continue;2629 2630 auto OISIt = OpSafeForPHIOfOps.find({I, CacheIdx});2631 if (OISIt != OpSafeForPHIOfOps.end())2632 return OISIt->second;2633 2634 // Keep walking until we either dominate the phi block, or hit a phi, or run2635 // out of things to check.2636 if (DT->properlyDominates(getBlockForValue(I), PHIBlock)) {2637 OpSafeForPHIOfOps.insert({{I, CacheIdx}, true});2638 continue;2639 }2640 // PHI in the same block.2641 if (isa<PHINode>(I) && getBlockForValue(I) == PHIBlock) {2642 OpSafeForPHIOfOps.insert({{I, CacheIdx}, false});2643 return false;2644 }2645 2646 auto *OrigI = cast<Instruction>(I);2647 // When we hit an instruction that reads memory (load, call, etc), we must2648 // consider any store that may happen in the loop. For now, we assume the2649 // worst: there is a store in the loop that alias with this read.2650 // The case where the load is outside the loop is already covered by the2651 // dominator check above.2652 // TODO: relax this condition2653 if (OrigI->mayReadFromMemory())2654 return false;2655 2656 // Check the operands of the current instruction.2657 for (auto *Op : OrigI->operand_values()) {2658 if (!isa<Instruction>(Op))2659 continue;2660 // Stop now if we find an unsafe operand.2661 auto OISIt = OpSafeForPHIOfOps.find({OrigI, CacheIdx});2662 if (OISIt != OpSafeForPHIOfOps.end()) {2663 if (!OISIt->second) {2664 OpSafeForPHIOfOps.insert({{I, CacheIdx}, false});2665 return false;2666 }2667 continue;2668 }2669 if (!Visited.insert(Op).second)2670 continue;2671 Worklist.push_back(cast<Instruction>(Op));2672 }2673 }2674 OpSafeForPHIOfOps.insert({{V, CacheIdx}, true});2675 return true;2676}2677 2678// Try to find a leader for instruction TransInst, which is a phi translated2679// version of something in our original program. Visited is used to ensure we2680// don't infinite loop during translations of cycles. OrigInst is the2681// instruction in the original program, and PredBB is the predecessor we2682// translated it through.2683Value *NewGVN::findLeaderForInst(Instruction *TransInst,2684 SmallPtrSetImpl<Value *> &Visited,2685 MemoryAccess *MemAccess, Instruction *OrigInst,2686 BasicBlock *PredBB) {2687 unsigned IDFSNum = InstrToDFSNum(OrigInst);2688 // Make sure it's marked as a temporary instruction.2689 AllTempInstructions.insert(TransInst);2690 // and make sure anything that tries to add it's DFS number is2691 // redirected to the instruction we are making a phi of ops2692 // for.2693 TempToBlock.insert({TransInst, PredBB});2694 InstrDFS.insert({TransInst, IDFSNum});2695 2696 auto Res = performSymbolicEvaluation(TransInst, Visited);2697 const Expression *E = Res.Expr;2698 addAdditionalUsers(Res, OrigInst);2699 InstrDFS.erase(TransInst);2700 AllTempInstructions.erase(TransInst);2701 TempToBlock.erase(TransInst);2702 if (MemAccess)2703 TempToMemory.erase(TransInst);2704 if (!E)2705 return nullptr;2706 auto *FoundVal = findPHIOfOpsLeader(E, OrigInst, PredBB);2707 if (!FoundVal) {2708 ExpressionToPhiOfOps[E].insert(OrigInst);2709 LLVM_DEBUG(dbgs() << "Cannot find phi of ops operand for " << *TransInst2710 << " in block " << getBlockName(PredBB) << "\n");2711 return nullptr;2712 }2713 if (auto *SI = dyn_cast<StoreInst>(FoundVal))2714 FoundVal = SI->getValueOperand();2715 return FoundVal;2716}2717 2718// When we see an instruction that is an op of phis, generate the equivalent phi2719// of ops form.2720const Expression *2721NewGVN::makePossiblePHIOfOps(Instruction *I,2722 SmallPtrSetImpl<Value *> &Visited) {2723 if (!okayForPHIOfOps(I))2724 return nullptr;2725 2726 if (!Visited.insert(I).second)2727 return nullptr;2728 // For now, we require the instruction be cycle free because we don't2729 // *always* create a phi of ops for instructions that could be done as phi2730 // of ops, we only do it if we think it is useful. If we did do it all the2731 // time, we could remove the cycle free check.2732 if (!isCycleFree(I))2733 return nullptr;2734 2735 // TODO: We don't do phi translation on memory accesses because it's2736 // complicated. For a load, we'd need to be able to simulate a new memoryuse,2737 // which we don't have a good way of doing ATM.2738 auto *MemAccess = getMemoryAccess(I);2739 // If the memory operation is defined by a memory operation this block that2740 // isn't a MemoryPhi, transforming the pointer backwards through a scalar phi2741 // can't help, as it would still be killed by that memory operation.2742 if (MemAccess && !isa<MemoryPhi>(MemAccess->getDefiningAccess()) &&2743 MemAccess->getDefiningAccess()->getBlock() == I->getParent())2744 return nullptr;2745 2746 // Convert op of phis to phi of ops2747 SmallPtrSet<const Value *, 10> VisitedOps;2748 SmallVector<Value *, 4> Ops(I->operand_values());2749 BasicBlock *SamePHIBlock = nullptr;2750 PHINode *OpPHI = nullptr;2751 if (!DebugCounter::shouldExecute(PHIOfOpsCounter))2752 return nullptr;2753 for (auto *Op : Ops) {2754 if (!isa<PHINode>(Op)) {2755 auto *ValuePHI = RealToTemp.lookup(Op);2756 if (!ValuePHI)2757 continue;2758 LLVM_DEBUG(dbgs() << "Found possible dependent phi of ops\n");2759 Op = ValuePHI;2760 }2761 OpPHI = cast<PHINode>(Op);2762 if (!SamePHIBlock) {2763 SamePHIBlock = getBlockForValue(OpPHI);2764 } else if (SamePHIBlock != getBlockForValue(OpPHI)) {2765 LLVM_DEBUG(2766 dbgs()2767 << "PHIs for operands are not all in the same block, aborting\n");2768 return nullptr;2769 }2770 // No point in doing this for one-operand phis.2771 // Since all PHIs for operands must be in the same block, then they must2772 // have the same number of operands so we can just abort.2773 if (OpPHI->getNumOperands() == 1)2774 return nullptr;2775 }2776 2777 if (!OpPHI)2778 return nullptr;2779 2780 SmallVector<ValPair, 4> PHIOps;2781 SmallPtrSet<Value *, 4> Deps;2782 auto *PHIBlock = getBlockForValue(OpPHI);2783 RevisitOnReachabilityChange[PHIBlock].reset(InstrToDFSNum(I));2784 for (unsigned PredNum = 0; PredNum < OpPHI->getNumOperands(); ++PredNum) {2785 auto *PredBB = OpPHI->getIncomingBlock(PredNum);2786 Value *FoundVal = nullptr;2787 SmallPtrSet<Value *, 4> CurrentDeps;2788 // We could just skip unreachable edges entirely but it's tricky to do2789 // with rewriting existing phi nodes.2790 if (ReachableEdges.count({PredBB, PHIBlock})) {2791 // Clone the instruction, create an expression from it that is2792 // translated back into the predecessor, and see if we have a leader.2793 Instruction *ValueOp = I->clone();2794 // Emit the temporal instruction in the predecessor basic block where the2795 // corresponding value is defined.2796 ValueOp->insertBefore(PredBB->getTerminator()->getIterator());2797 if (MemAccess)2798 TempToMemory.insert({ValueOp, MemAccess});2799 bool SafeForPHIOfOps = true;2800 VisitedOps.clear();2801 for (auto &Op : ValueOp->operands()) {2802 auto *OrigOp = &*Op;2803 // When these operand changes, it could change whether there is a2804 // leader for us or not, so we have to add additional users.2805 if (isa<PHINode>(Op)) {2806 Op = Op->DoPHITranslation(PHIBlock, PredBB);2807 if (Op != OrigOp && Op != I)2808 CurrentDeps.insert(Op);2809 } else if (auto *ValuePHI = RealToTemp.lookup(Op)) {2810 if (getBlockForValue(ValuePHI) == PHIBlock)2811 Op = ValuePHI->getIncomingValueForBlock(PredBB);2812 }2813 // If we phi-translated the op, it must be safe.2814 SafeForPHIOfOps =2815 SafeForPHIOfOps &&2816 (Op != OrigOp || OpIsSafeForPHIOfOps(Op, PHIBlock, VisitedOps));2817 }2818 // FIXME: For those things that are not safe we could generate2819 // expressions all the way down, and see if this comes out to a2820 // constant. For anything where that is true, and unsafe, we should2821 // have made a phi-of-ops (or value numbered it equivalent to something)2822 // for the pieces already.2823 FoundVal = !SafeForPHIOfOps ? nullptr2824 : findLeaderForInst(ValueOp, Visited,2825 MemAccess, I, PredBB);2826 ValueOp->eraseFromParent();2827 if (!FoundVal) {2828 // We failed to find a leader for the current ValueOp, but this might2829 // change in case of the translated operands change.2830 if (SafeForPHIOfOps)2831 for (auto *Dep : CurrentDeps)2832 addAdditionalUsers(Dep, I);2833 2834 return nullptr;2835 }2836 Deps.insert_range(CurrentDeps);2837 } else {2838 LLVM_DEBUG(dbgs() << "Skipping phi of ops operand for incoming block "2839 << getBlockName(PredBB)2840 << " because the block is unreachable\n");2841 FoundVal = PoisonValue::get(I->getType());2842 RevisitOnReachabilityChange[PHIBlock].set(InstrToDFSNum(I));2843 }2844 2845 PHIOps.push_back({FoundVal, PredBB});2846 LLVM_DEBUG(dbgs() << "Found phi of ops operand " << *FoundVal << " in "2847 << getBlockName(PredBB) << "\n");2848 }2849 for (auto *Dep : Deps)2850 addAdditionalUsers(Dep, I);2851 sortPHIOps(PHIOps);2852 auto *E = performSymbolicPHIEvaluation(PHIOps, I, PHIBlock);2853 if (isa<ConstantExpression>(E) || isa<VariableExpression>(E)) {2854 LLVM_DEBUG(2855 dbgs()2856 << "Not creating real PHI of ops because it simplified to existing "2857 "value or constant\n");2858 // We have leaders for all operands, but do not create a real PHI node with2859 // those leaders as operands, so the link between the operands and the2860 // PHI-of-ops is not materialized in the IR. If any of those leaders2861 // changes, the PHI-of-op may change also, so we need to add the operands as2862 // additional users.2863 for (auto &O : PHIOps)2864 addAdditionalUsers(O.first, I);2865 2866 return E;2867 }2868 auto *ValuePHI = RealToTemp.lookup(I);2869 bool NewPHI = false;2870 if (!ValuePHI) {2871 ValuePHI =2872 PHINode::Create(I->getType(), OpPHI->getNumOperands(), "phiofops");2873 addPhiOfOps(ValuePHI, PHIBlock, I);2874 NewPHI = true;2875 NumGVNPHIOfOpsCreated++;2876 }2877 if (NewPHI) {2878 for (auto PHIOp : PHIOps)2879 ValuePHI->addIncoming(PHIOp.first, PHIOp.second);2880 } else {2881 TempToBlock[ValuePHI] = PHIBlock;2882 unsigned int i = 0;2883 for (auto PHIOp : PHIOps) {2884 ValuePHI->setIncomingValue(i, PHIOp.first);2885 ValuePHI->setIncomingBlock(i, PHIOp.second);2886 ++i;2887 }2888 }2889 RevisitOnReachabilityChange[PHIBlock].set(InstrToDFSNum(I));2890 LLVM_DEBUG(dbgs() << "Created phi of ops " << *ValuePHI << " for " << *I2891 << "\n");2892 2893 return E;2894}2895 2896// The algorithm initially places the values of the routine in the TOP2897// congruence class. The leader of TOP is the undetermined value `poison`.2898// When the algorithm has finished, values still in TOP are unreachable.2899void NewGVN::initializeCongruenceClasses(Function &F) {2900 NextCongruenceNum = 0;2901 2902 // Note that even though we use the live on entry def as a representative2903 // MemoryAccess, it is *not* the same as the actual live on entry def. We2904 // have no real equivalent to poison for MemoryAccesses, and so we really2905 // should be checking whether the MemoryAccess is top if we want to know if it2906 // is equivalent to everything. Otherwise, what this really signifies is that2907 // the access "it reaches all the way back to the beginning of the function"2908 2909 // Initialize all other instructions to be in TOP class.2910 TOPClass = createCongruenceClass(nullptr, nullptr);2911 TOPClass->setMemoryLeader(MSSA->getLiveOnEntryDef());2912 // The live on entry def gets put into it's own class2913 MemoryAccessToClass[MSSA->getLiveOnEntryDef()] =2914 createMemoryClass(MSSA->getLiveOnEntryDef());2915 2916 for (auto *DTN : nodes(DT)) {2917 BasicBlock *BB = DTN->getBlock();2918 // All MemoryAccesses are equivalent to live on entry to start. They must2919 // be initialized to something so that initial changes are noticed. For2920 // the maximal answer, we initialize them all to be the same as2921 // liveOnEntry.2922 auto *MemoryBlockDefs = MSSA->getBlockDefs(BB);2923 if (MemoryBlockDefs)2924 for (const auto &Def : *MemoryBlockDefs) {2925 MemoryAccessToClass[&Def] = TOPClass;2926 auto *MD = dyn_cast<MemoryDef>(&Def);2927 // Insert the memory phis into the member list.2928 if (!MD) {2929 const MemoryPhi *MP = cast<MemoryPhi>(&Def);2930 TOPClass->memory_insert(MP);2931 MemoryPhiState.insert({MP, MPS_TOP});2932 }2933 2934 if (MD && isa<StoreInst>(MD->getMemoryInst()))2935 TOPClass->incStoreCount();2936 }2937 2938 // FIXME: This is trying to discover which instructions are uses of phi2939 // nodes. We should move this into one of the myriad of places that walk2940 // all the operands already.2941 for (auto &I : *BB) {2942 if (isa<PHINode>(&I))2943 for (auto *U : I.users())2944 if (auto *UInst = dyn_cast<Instruction>(U))2945 if (InstrToDFSNum(UInst) != 0 && okayForPHIOfOps(UInst))2946 PHINodeUses.insert(UInst);2947 // Don't insert void terminators into the class. We don't value number2948 // them, and they just end up sitting in TOP.2949 if (I.isTerminator() && I.getType()->isVoidTy())2950 continue;2951 TOPClass->insert(&I);2952 ValueToClass[&I] = TOPClass;2953 }2954 }2955 2956 // Initialize arguments to be in their own unique congruence classes2957 for (auto &FA : F.args())2958 createSingletonCongruenceClass(&FA);2959}2960 2961void NewGVN::cleanupTables() {2962 for (CongruenceClass *&CC : CongruenceClasses) {2963 LLVM_DEBUG(dbgs() << "Congruence class " << CC->getID() << " has "2964 << CC->size() << " members\n");2965 // Make sure we delete the congruence class (probably worth switching to2966 // a unique_ptr at some point.2967 delete CC;2968 CC = nullptr;2969 }2970 2971 // Destroy the value expressions2972 SmallVector<Instruction *, 8> TempInst(AllTempInstructions.begin(),2973 AllTempInstructions.end());2974 AllTempInstructions.clear();2975 2976 // We have to drop all references for everything first, so there are no uses2977 // left as we delete them.2978 for (auto *I : TempInst) {2979 I->dropAllReferences();2980 }2981 2982 while (!TempInst.empty()) {2983 auto *I = TempInst.pop_back_val();2984 I->deleteValue();2985 }2986 2987 ValueToClass.clear();2988 ArgRecycler.clear(ExpressionAllocator);2989 ExpressionAllocator.Reset();2990 CongruenceClasses.clear();2991 ExpressionToClass.clear();2992 ValueToExpression.clear();2993 RealToTemp.clear();2994 AdditionalUsers.clear();2995 ExpressionToPhiOfOps.clear();2996 TempToBlock.clear();2997 TempToMemory.clear();2998 PHINodeUses.clear();2999 OpSafeForPHIOfOps.clear();3000 ReachableBlocks.clear();3001 ReachableEdges.clear();3002#ifndef NDEBUG3003 ProcessedCount.clear();3004#endif3005 InstrDFS.clear();3006 InstructionsToErase.clear();3007 DFSToInstr.clear();3008 BlockInstRange.clear();3009 TouchedInstructions.clear();3010 MemoryAccessToClass.clear();3011 PredicateToUsers.clear();3012 MemoryToUsers.clear();3013 RevisitOnReachabilityChange.clear();3014 PredicateSwapChoice.clear();3015}3016 3017// Assign local DFS number mapping to instructions, and leave space for Value3018// PHI's.3019std::pair<unsigned, unsigned> NewGVN::assignDFSNumbers(BasicBlock *B,3020 unsigned Start) {3021 unsigned End = Start;3022 if (MemoryAccess *MemPhi = getMemoryAccess(B)) {3023 InstrDFS[MemPhi] = End++;3024 DFSToInstr.emplace_back(MemPhi);3025 }3026 3027 // Then the real block goes next.3028 for (auto &I : *B) {3029 // There's no need to call isInstructionTriviallyDead more than once on3030 // an instruction. Therefore, once we know that an instruction is dead3031 // we change its DFS number so that it doesn't get value numbered.3032 if (isInstructionTriviallyDead(&I, TLI)) {3033 InstrDFS[&I] = 0;3034 LLVM_DEBUG(dbgs() << "Skipping trivially dead instruction " << I << "\n");3035 salvageDebugInfo(I);3036 markInstructionForDeletion(&I);3037 continue;3038 }3039 if (isa<PHINode>(&I))3040 RevisitOnReachabilityChange[B].set(End);3041 InstrDFS[&I] = End++;3042 DFSToInstr.emplace_back(&I);3043 }3044 3045 // All of the range functions taken half-open ranges (open on the end side).3046 // So we do not subtract one from count, because at this point it is one3047 // greater than the last instruction.3048 return std::make_pair(Start, End);3049}3050 3051void NewGVN::updateProcessedCount(const Value *V) {3052#ifndef NDEBUG3053 assert(++ProcessedCount[V] < 100 &&3054 "Seem to have processed the same Value a lot");3055#endif3056}3057 3058// Evaluate MemoryPhi nodes symbolically, just like PHI nodes3059void NewGVN::valueNumberMemoryPhi(MemoryPhi *MP) {3060 // If all the arguments are the same, the MemoryPhi has the same value as the3061 // argument. Filter out unreachable blocks and self phis from our operands.3062 // TODO: We could do cycle-checking on the memory phis to allow valueizing for3063 // self-phi checking.3064 const BasicBlock *PHIBlock = MP->getBlock();3065 auto Filtered = make_filter_range(MP->operands(), [&](const Use &U) {3066 return cast<MemoryAccess>(U) != MP &&3067 !isMemoryAccessTOP(cast<MemoryAccess>(U)) &&3068 ReachableEdges.count({MP->getIncomingBlock(U), PHIBlock});3069 });3070 // If all that is left is nothing, our memoryphi is poison. We keep it as3071 // InitialClass. Note: The only case this should happen is if we have at3072 // least one self-argument.3073 if (Filtered.begin() == Filtered.end()) {3074 if (setMemoryClass(MP, TOPClass))3075 markMemoryUsersTouched(MP);3076 return;3077 }3078 3079 // Transform the remaining operands into operand leaders.3080 // FIXME: mapped_iterator should have a range version.3081 auto LookupFunc = [&](const Use &U) {3082 return lookupMemoryLeader(cast<MemoryAccess>(U));3083 };3084 auto MappedBegin = map_iterator(Filtered.begin(), LookupFunc);3085 auto MappedEnd = map_iterator(Filtered.end(), LookupFunc);3086 3087 // and now check if all the elements are equal.3088 // Sadly, we can't use std::equals since these are random access iterators.3089 const auto *AllSameValue = *MappedBegin;3090 ++MappedBegin;3091 bool AllEqual = std::all_of(3092 MappedBegin, MappedEnd,3093 [&AllSameValue](const MemoryAccess *V) { return V == AllSameValue; });3094 3095 if (AllEqual)3096 LLVM_DEBUG(dbgs() << "Memory Phi value numbered to " << *AllSameValue3097 << "\n");3098 else3099 LLVM_DEBUG(dbgs() << "Memory Phi value numbered to itself\n");3100 // If it's equal to something, it's in that class. Otherwise, it has to be in3101 // a class where it is the leader (other things may be equivalent to it, but3102 // it needs to start off in its own class, which means it must have been the3103 // leader, and it can't have stopped being the leader because it was never3104 // removed).3105 CongruenceClass *CC =3106 AllEqual ? getMemoryClass(AllSameValue) : ensureLeaderOfMemoryClass(MP);3107 auto OldState = MemoryPhiState.lookup(MP);3108 assert(OldState != MPS_Invalid && "Invalid memory phi state");3109 auto NewState = AllEqual ? MPS_Equivalent : MPS_Unique;3110 MemoryPhiState[MP] = NewState;3111 if (setMemoryClass(MP, CC) || OldState != NewState)3112 markMemoryUsersTouched(MP);3113}3114 3115// Value number a single instruction, symbolically evaluating, performing3116// congruence finding, and updating mappings.3117void NewGVN::valueNumberInstruction(Instruction *I) {3118 LLVM_DEBUG(dbgs() << "Processing instruction " << *I << "\n");3119 if (!I->isTerminator()) {3120 const Expression *Symbolized = nullptr;3121 SmallPtrSet<Value *, 2> Visited;3122 if (DebugCounter::shouldExecute(VNCounter)) {3123 auto Res = performSymbolicEvaluation(I, Visited);3124 Symbolized = Res.Expr;3125 addAdditionalUsers(Res, I);3126 3127 // Make a phi of ops if necessary3128 if (Symbolized && !isa<ConstantExpression>(Symbolized) &&3129 !isa<VariableExpression>(Symbolized) && PHINodeUses.count(I)) {3130 auto *PHIE = makePossiblePHIOfOps(I, Visited);3131 // If we created a phi of ops, use it.3132 // If we couldn't create one, make sure we don't leave one lying around3133 if (PHIE) {3134 Symbolized = PHIE;3135 } else if (auto *Op = RealToTemp.lookup(I)) {3136 removePhiOfOps(I, Op);3137 }3138 }3139 } else {3140 // Mark the instruction as unused so we don't value number it again.3141 InstrDFS[I] = 0;3142 }3143 // If we couldn't come up with a symbolic expression, use the unknown3144 // expression3145 if (Symbolized == nullptr)3146 Symbolized = createUnknownExpression(I);3147 performCongruenceFinding(I, Symbolized);3148 } else {3149 // Handle terminators that return values. All of them produce values we3150 // don't currently understand. We don't place non-value producing3151 // terminators in a class.3152 if (!I->getType()->isVoidTy()) {3153 auto *Symbolized = createUnknownExpression(I);3154 performCongruenceFinding(I, Symbolized);3155 }3156 processOutgoingEdges(I, I->getParent());3157 }3158}3159 3160// Check if there is a path, using single or equal argument phi nodes, from3161// First to Second.3162bool NewGVN::singleReachablePHIPath(3163 SmallPtrSet<const MemoryAccess *, 8> &Visited, const MemoryAccess *First,3164 const MemoryAccess *Second) const {3165 if (First == Second)3166 return true;3167 if (MSSA->isLiveOnEntryDef(First))3168 return false;3169 3170 // This is not perfect, but as we're just verifying here, we can live with3171 // the loss of precision. The real solution would be that of doing strongly3172 // connected component finding in this routine, and it's probably not worth3173 // the complexity for the time being. So, we just keep a set of visited3174 // MemoryAccess and return true when we hit a cycle.3175 if (!Visited.insert(First).second)3176 return true;3177 3178 const auto *EndDef = First;3179 for (const auto *ChainDef : optimized_def_chain(First)) {3180 if (ChainDef == Second)3181 return true;3182 if (MSSA->isLiveOnEntryDef(ChainDef))3183 return false;3184 EndDef = ChainDef;3185 }3186 auto *MP = cast<MemoryPhi>(EndDef);3187 auto ReachableOperandPred = [&](const Use &U) {3188 return ReachableEdges.count({MP->getIncomingBlock(U), MP->getBlock()});3189 };3190 auto FilteredPhiArgs =3191 make_filter_range(MP->operands(), ReachableOperandPred);3192 SmallVector<const Value *, 32> OperandList(FilteredPhiArgs);3193 bool Okay = all_equal(OperandList);3194 if (Okay)3195 return singleReachablePHIPath(Visited, cast<MemoryAccess>(OperandList[0]),3196 Second);3197 return false;3198}3199 3200// Verify the that the memory equivalence table makes sense relative to the3201// congruence classes. Note that this checking is not perfect, and is currently3202// subject to very rare false negatives. It is only useful for3203// testing/debugging.3204void NewGVN::verifyMemoryCongruency() const {3205#ifndef NDEBUG3206 // Verify that the memory table equivalence and memory member set match3207 for (const auto *CC : CongruenceClasses) {3208 if (CC == TOPClass || CC->isDead())3209 continue;3210 if (CC->getStoreCount() != 0) {3211 assert((CC->getStoredValue() || !isa<StoreInst>(CC->getLeader())) &&3212 "Any class with a store as a leader should have a "3213 "representative stored value");3214 assert(CC->getMemoryLeader() &&3215 "Any congruence class with a store should have a "3216 "representative access");3217 }3218 3219 if (CC->getMemoryLeader())3220 assert(MemoryAccessToClass.lookup(CC->getMemoryLeader()) == CC &&3221 "Representative MemoryAccess does not appear to be reverse "3222 "mapped properly");3223 for (const auto *M : CC->memory())3224 assert(MemoryAccessToClass.lookup(M) == CC &&3225 "Memory member does not appear to be reverse mapped properly");3226 }3227 3228 // Anything equivalent in the MemoryAccess table should be in the same3229 // congruence class.3230 3231 // Filter out the unreachable and trivially dead entries, because they may3232 // never have been updated if the instructions were not processed.3233 auto ReachableAccessPred =3234 [&](const std::pair<const MemoryAccess *, CongruenceClass *> Pair) {3235 bool Result = ReachableBlocks.count(Pair.first->getBlock());3236 if (!Result || MSSA->isLiveOnEntryDef(Pair.first) ||3237 MemoryToDFSNum(Pair.first) == 0)3238 return false;3239 if (auto *MemDef = dyn_cast<MemoryDef>(Pair.first))3240 return !isInstructionTriviallyDead(MemDef->getMemoryInst());3241 3242 // We could have phi nodes which operands are all trivially dead,3243 // so we don't process them.3244 if (auto *MemPHI = dyn_cast<MemoryPhi>(Pair.first)) {3245 for (const auto &U : MemPHI->incoming_values()) {3246 if (auto *I = dyn_cast<Instruction>(&*U)) {3247 if (!isInstructionTriviallyDead(I))3248 return true;3249 }3250 }3251 return false;3252 }3253 3254 return true;3255 };3256 3257 auto Filtered = make_filter_range(MemoryAccessToClass, ReachableAccessPred);3258 for (auto KV : Filtered) {3259 if (auto *FirstMUD = dyn_cast<MemoryUseOrDef>(KV.first)) {3260 auto *SecondMUD = dyn_cast<MemoryUseOrDef>(KV.second->getMemoryLeader());3261 if (FirstMUD && SecondMUD) {3262 SmallPtrSet<const MemoryAccess *, 8> VisitedMAS;3263 assert((singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) ||3264 ValueToClass.lookup(FirstMUD->getMemoryInst()) ==3265 ValueToClass.lookup(SecondMUD->getMemoryInst())) &&3266 "The instructions for these memory operations should have "3267 "been in the same congruence class or reachable through"3268 "a single argument phi");3269 }3270 } else if (auto *FirstMP = dyn_cast<MemoryPhi>(KV.first)) {3271 // We can only sanely verify that MemoryDefs in the operand list all have3272 // the same class.3273 auto ReachableOperandPred = [&](const Use &U) {3274 return ReachableEdges.count(3275 {FirstMP->getIncomingBlock(U), FirstMP->getBlock()}) &&3276 isa<MemoryDef>(U);3277 };3278 // All arguments should in the same class, ignoring unreachable arguments3279 auto FilteredPhiArgs =3280 make_filter_range(FirstMP->operands(), ReachableOperandPred);3281 SmallVector<const CongruenceClass *, 16> PhiOpClasses;3282 std::transform(FilteredPhiArgs.begin(), FilteredPhiArgs.end(),3283 std::back_inserter(PhiOpClasses), [&](const Use &U) {3284 const MemoryDef *MD = cast<MemoryDef>(U);3285 return ValueToClass.lookup(MD->getMemoryInst());3286 });3287 assert(all_equal(PhiOpClasses) &&3288 "All MemoryPhi arguments should be in the same class");3289 }3290 }3291#endif3292}3293 3294// Verify that the sparse propagation we did actually found the maximal fixpoint3295// We do this by storing the value to class mapping, touching all instructions,3296// and redoing the iteration to see if anything changed.3297void NewGVN::verifyIterationSettled(Function &F) {3298#ifndef NDEBUG3299 LLVM_DEBUG(dbgs() << "Beginning iteration verification\n");3300 if (DebugCounter::isCounterSet(VNCounter))3301 DebugCounter::setCounterState(VNCounter, StartingVNCounter);3302 3303 // Note that we have to store the actual classes, as we may change existing3304 // classes during iteration. This is because our memory iteration propagation3305 // is not perfect, and so may waste a little work. But it should generate3306 // exactly the same congruence classes we have now, with different IDs.3307 std::map<const Value *, CongruenceClass> BeforeIteration;3308 3309 for (auto &KV : ValueToClass) {3310 if (auto *I = dyn_cast<Instruction>(KV.first))3311 // Skip unused/dead instructions.3312 if (InstrToDFSNum(I) == 0)3313 continue;3314 BeforeIteration.insert({KV.first, *KV.second});3315 }3316 3317 TouchedInstructions.set();3318 TouchedInstructions.reset(0);3319 OpSafeForPHIOfOps.clear();3320 CacheIdx = 0;3321 iterateTouchedInstructions();3322 DenseSet<std::pair<const CongruenceClass *, const CongruenceClass *>>3323 EqualClasses;3324 for (const auto &KV : ValueToClass) {3325 if (auto *I = dyn_cast<Instruction>(KV.first))3326 // Skip unused/dead instructions.3327 if (InstrToDFSNum(I) == 0)3328 continue;3329 // We could sink these uses, but i think this adds a bit of clarity here as3330 // to what we are comparing.3331 auto *BeforeCC = &BeforeIteration.find(KV.first)->second;3332 auto *AfterCC = KV.second;3333 // Note that the classes can't change at this point, so we memoize the set3334 // that are equal.3335 if (!EqualClasses.count({BeforeCC, AfterCC})) {3336 assert(BeforeCC->isEquivalentTo(AfterCC) &&3337 "Value number changed after main loop completed!");3338 EqualClasses.insert({BeforeCC, AfterCC});3339 }3340 }3341#endif3342}3343 3344// Verify that for each store expression in the expression to class mapping,3345// only the latest appears, and multiple ones do not appear.3346// Because loads do not use the stored value when doing equality with stores,3347// if we don't erase the old store expressions from the table, a load can find3348// a no-longer valid StoreExpression.3349void NewGVN::verifyStoreExpressions() const {3350#ifndef NDEBUG3351 // This is the only use of this, and it's not worth defining a complicated3352 // densemapinfo hash/equality function for it.3353 std::set<3354 std::pair<const Value *,3355 std::tuple<const Value *, const CongruenceClass *, Value *>>>3356 StoreExpressionSet;3357 for (const auto &KV : ExpressionToClass) {3358 if (auto *SE = dyn_cast<StoreExpression>(KV.first)) {3359 // Make sure a version that will conflict with loads is not already there3360 auto Res = StoreExpressionSet.insert(3361 {SE->getOperand(0), std::make_tuple(SE->getMemoryLeader(), KV.second,3362 SE->getStoredValue())});3363 bool Okay = Res.second;3364 // It's okay to have the same expression already in there if it is3365 // identical in nature.3366 // This can happen when the leader of the stored value changes over time.3367 if (!Okay)3368 Okay = (std::get<1>(Res.first->second) == KV.second) &&3369 (lookupOperandLeader(std::get<2>(Res.first->second)) ==3370 lookupOperandLeader(SE->getStoredValue()));3371 assert(Okay && "Stored expression conflict exists in expression table");3372 auto *ValueExpr = ValueToExpression.lookup(SE->getStoreInst());3373 assert(ValueExpr && ValueExpr->equals(*SE) &&3374 "StoreExpression in ExpressionToClass is not latest "3375 "StoreExpression for value");3376 }3377 }3378#endif3379}3380 3381// This is the main value numbering loop, it iterates over the initial touched3382// instruction set, propagating value numbers, marking things touched, etc,3383// until the set of touched instructions is completely empty.3384void NewGVN::iterateTouchedInstructions() {3385 uint64_t Iterations = 0;3386 // Figure out where touchedinstructions starts3387 int FirstInstr = TouchedInstructions.find_first();3388 // Nothing set, nothing to iterate, just return.3389 if (FirstInstr == -1)3390 return;3391 const BasicBlock *LastBlock = getBlockForValue(InstrFromDFSNum(FirstInstr));3392 while (TouchedInstructions.any()) {3393 ++Iterations;3394 // Walk through all the instructions in all the blocks in RPO.3395 // TODO: As we hit a new block, we should push and pop equalities into a3396 // table lookupOperandLeader can use, to catch things PredicateInfo3397 // might miss, like edge-only equivalences.3398 for (unsigned InstrNum : TouchedInstructions.set_bits()) {3399 3400 // This instruction was found to be dead. We don't bother looking3401 // at it again.3402 if (InstrNum == 0) {3403 TouchedInstructions.reset(InstrNum);3404 continue;3405 }3406 3407 Value *V = InstrFromDFSNum(InstrNum);3408 const BasicBlock *CurrBlock = getBlockForValue(V);3409 3410 // If we hit a new block, do reachability processing.3411 if (CurrBlock != LastBlock) {3412 LastBlock = CurrBlock;3413 bool BlockReachable = ReachableBlocks.count(CurrBlock);3414 const auto &CurrInstRange = BlockInstRange.lookup(CurrBlock);3415 3416 // If it's not reachable, erase any touched instructions and move on.3417 if (!BlockReachable) {3418 TouchedInstructions.reset(CurrInstRange.first, CurrInstRange.second);3419 LLVM_DEBUG(dbgs() << "Skipping instructions in block "3420 << getBlockName(CurrBlock)3421 << " because it is unreachable\n");3422 continue;3423 }3424 // Use the appropriate cache for "OpIsSafeForPHIOfOps".3425 CacheIdx = RPOOrdering.lookup(DT->getNode(CurrBlock)) - 1;3426 updateProcessedCount(CurrBlock);3427 }3428 // Reset after processing (because we may mark ourselves as touched when3429 // we propagate equalities).3430 TouchedInstructions.reset(InstrNum);3431 3432 if (auto *MP = dyn_cast<MemoryPhi>(V)) {3433 LLVM_DEBUG(dbgs() << "Processing MemoryPhi " << *MP << "\n");3434 valueNumberMemoryPhi(MP);3435 } else if (auto *I = dyn_cast<Instruction>(V)) {3436 valueNumberInstruction(I);3437 } else {3438 llvm_unreachable("Should have been a MemoryPhi or Instruction");3439 }3440 updateProcessedCount(V);3441 }3442 }3443 NumGVNMaxIterations = std::max(NumGVNMaxIterations.getValue(), Iterations);3444}3445 3446// This is the main transformation entry point.3447bool NewGVN::runGVN() {3448 if (DebugCounter::isCounterSet(VNCounter))3449 StartingVNCounter = DebugCounter::getCounterState(VNCounter);3450 bool Changed = false;3451 NumFuncArgs = F.arg_size();3452 MSSAWalker = MSSA->getWalker();3453 SingletonDeadExpression = new (ExpressionAllocator) DeadExpression();3454 3455 // Count number of instructions for sizing of hash tables, and come3456 // up with a global dfs numbering for instructions.3457 unsigned ICount = 1;3458 // Add an empty instruction to account for the fact that we start at 13459 DFSToInstr.emplace_back(nullptr);3460 // Note: We want ideal RPO traversal of the blocks, which is not quite the3461 // same as dominator tree order, particularly with regard whether backedges3462 // get visited first or second, given a block with multiple successors.3463 // If we visit in the wrong order, we will end up performing N times as many3464 // iterations.3465 // The dominator tree does guarantee that, for a given dom tree node, it's3466 // parent must occur before it in the RPO ordering. Thus, we only need to sort3467 // the siblings.3468 ReversePostOrderTraversal<Function *> RPOT(&F);3469 unsigned Counter = 0;3470 for (auto &B : RPOT) {3471 auto *Node = DT->getNode(B);3472 assert(Node && "RPO and Dominator tree should have same reachability");3473 RPOOrdering[Node] = ++Counter;3474 }3475 // Sort dominator tree children arrays into RPO.3476 for (auto &B : RPOT) {3477 auto *Node = DT->getNode(B);3478 if (Node->getNumChildren() > 1)3479 llvm::sort(*Node, [&](const DomTreeNode *A, const DomTreeNode *B) {3480 return RPOOrdering[A] < RPOOrdering[B];3481 });3482 }3483 3484 // Now a standard depth first ordering of the domtree is equivalent to RPO.3485 for (auto *DTN : depth_first(DT->getRootNode())) {3486 BasicBlock *B = DTN->getBlock();3487 const auto &BlockRange = assignDFSNumbers(B, ICount);3488 BlockInstRange.insert({B, BlockRange});3489 ICount += BlockRange.second - BlockRange.first;3490 }3491 initializeCongruenceClasses(F);3492 3493 TouchedInstructions.resize(ICount);3494 // Ensure we don't end up resizing the expressionToClass map, as3495 // that can be quite expensive. At most, we have one expression per3496 // instruction.3497 ExpressionToClass.reserve(ICount);3498 3499 // Initialize the touched instructions to include the entry block.3500 const auto &InstRange = BlockInstRange.lookup(&F.getEntryBlock());3501 TouchedInstructions.set(InstRange.first, InstRange.second);3502 LLVM_DEBUG(dbgs() << "Block " << getBlockName(&F.getEntryBlock())3503 << " marked reachable\n");3504 ReachableBlocks.insert(&F.getEntryBlock());3505 // Use index corresponding to entry block.3506 CacheIdx = 0;3507 3508 iterateTouchedInstructions();3509 verifyMemoryCongruency();3510 verifyIterationSettled(F);3511 verifyStoreExpressions();3512 3513 Changed |= eliminateInstructions(F);3514 3515 // Delete all instructions marked for deletion.3516 for (Instruction *ToErase : InstructionsToErase) {3517 if (!ToErase->use_empty())3518 ToErase->replaceAllUsesWith(PoisonValue::get(ToErase->getType()));3519 3520 assert(ToErase->getParent() &&3521 "BB containing ToErase deleted unexpectedly!");3522 ToErase->eraseFromParent();3523 }3524 Changed |= !InstructionsToErase.empty();3525 3526 // Delete all unreachable blocks.3527 auto UnreachableBlockPred = [&](const BasicBlock &BB) {3528 return !ReachableBlocks.count(&BB);3529 };3530 3531 for (auto &BB : make_filter_range(F, UnreachableBlockPred)) {3532 LLVM_DEBUG(dbgs() << "We believe block " << getBlockName(&BB)3533 << " is unreachable\n");3534 deleteInstructionsInBlock(&BB);3535 Changed = true;3536 }3537 3538 cleanupTables();3539 return Changed;3540}3541 3542struct NewGVN::ValueDFS {3543 int DFSIn = 0;3544 int DFSOut = 0;3545 int LocalNum = 0;3546 3547 // Only one of Def and U will be set.3548 // The bool in the Def tells us whether the Def is the stored value of a3549 // store.3550 PointerIntPair<Value *, 1, bool> Def;3551 Use *U = nullptr;3552 3553 bool operator<(const ValueDFS &Other) const {3554 // It's not enough that any given field be less than - we have sets3555 // of fields that need to be evaluated together to give a proper ordering.3556 // For example, if you have;3557 // DFS (1, 3)3558 // Val 03559 // DFS (1, 2)3560 // Val 503561 // We want the second to be less than the first, but if we just go field3562 // by field, we will get to Val 0 < Val 50 and say the first is less than3563 // the second. We only want it to be less than if the DFS orders are equal.3564 //3565 // Each LLVM instruction only produces one value, and thus the lowest-level3566 // differentiator that really matters for the stack (and what we use as a3567 // replacement) is the local dfs number.3568 // Everything else in the structure is instruction level, and only affects3569 // the order in which we will replace operands of a given instruction.3570 //3571 // For a given instruction (IE things with equal dfsin, dfsout, localnum),3572 // the order of replacement of uses does not matter.3573 // IE given,3574 // a = 53575 // b = a + a3576 // When you hit b, you will have two valuedfs with the same dfsin, out, and3577 // localnum.3578 // The .val will be the same as well.3579 // The .u's will be different.3580 // You will replace both, and it does not matter what order you replace them3581 // in (IE whether you replace operand 2, then operand 1, or operand 1, then3582 // operand 2).3583 // Similarly for the case of same dfsin, dfsout, localnum, but different3584 // .val's3585 // a = 53586 // b = 63587 // c = a + b3588 // in c, we will a valuedfs for a, and one for b,with everything the same3589 // but .val and .u.3590 // It does not matter what order we replace these operands in.3591 // You will always end up with the same IR, and this is guaranteed.3592 return std::tie(DFSIn, DFSOut, LocalNum, Def, U) <3593 std::tie(Other.DFSIn, Other.DFSOut, Other.LocalNum, Other.Def,3594 Other.U);3595 }3596};3597 3598// This function converts the set of members for a congruence class from values,3599// to sets of defs and uses with associated DFS info. The total number of3600// reachable uses for each value is stored in UseCount, and instructions that3601// seem3602// dead (have no non-dead uses) are stored in ProbablyDead.3603void NewGVN::convertClassToDFSOrdered(3604 const CongruenceClass &Dense, SmallVectorImpl<ValueDFS> &DFSOrderedSet,3605 DenseMap<const Value *, unsigned int> &UseCounts,3606 SmallPtrSetImpl<Instruction *> &ProbablyDead) const {3607 for (auto *D : Dense) {3608 // First add the value.3609 BasicBlock *BB = getBlockForValue(D);3610 // Constants are handled prior to ever calling this function, so3611 // we should only be left with instructions as members.3612 assert(BB && "Should have figured out a basic block for value");3613 ValueDFS VDDef;3614 DomTreeNode *DomNode = DT->getNode(BB);3615 VDDef.DFSIn = DomNode->getDFSNumIn();3616 VDDef.DFSOut = DomNode->getDFSNumOut();3617 // If it's a store, use the leader of the value operand, if it's always3618 // available, or the value operand. TODO: We could do dominance checks to3619 // find a dominating leader, but not worth it ATM.3620 if (auto *SI = dyn_cast<StoreInst>(D)) {3621 auto Leader = lookupOperandLeader(SI->getValueOperand());3622 if (alwaysAvailable(Leader)) {3623 VDDef.Def.setPointer(Leader);3624 } else {3625 VDDef.Def.setPointer(SI->getValueOperand());3626 VDDef.Def.setInt(true);3627 }3628 } else {3629 VDDef.Def.setPointer(D);3630 }3631 assert(isa<Instruction>(D) &&3632 "The dense set member should always be an instruction");3633 Instruction *Def = cast<Instruction>(D);3634 VDDef.LocalNum = InstrToDFSNum(D);3635 DFSOrderedSet.push_back(VDDef);3636 // If there is a phi node equivalent, add it3637 if (auto *PN = RealToTemp.lookup(Def)) {3638 auto *PHIE =3639 dyn_cast_or_null<PHIExpression>(ValueToExpression.lookup(Def));3640 if (PHIE) {3641 VDDef.Def.setInt(false);3642 VDDef.Def.setPointer(PN);3643 VDDef.LocalNum = 0;3644 DFSOrderedSet.push_back(VDDef);3645 }3646 }3647 3648 unsigned int UseCount = 0;3649 // Now add the uses.3650 for (auto &U : Def->uses()) {3651 if (auto *I = dyn_cast<Instruction>(U.getUser())) {3652 // Don't try to replace into dead uses3653 if (InstructionsToErase.count(I))3654 continue;3655 ValueDFS VDUse;3656 // Put the phi node uses in the incoming block.3657 BasicBlock *IBlock;3658 if (auto *P = dyn_cast<PHINode>(I)) {3659 IBlock = P->getIncomingBlock(U);3660 // Make phi node users appear last in the incoming block3661 // they are from.3662 VDUse.LocalNum = InstrDFS.size() + 1;3663 } else {3664 IBlock = getBlockForValue(I);3665 VDUse.LocalNum = InstrToDFSNum(I);3666 }3667 3668 // Skip uses in unreachable blocks, as we're going3669 // to delete them.3670 if (!ReachableBlocks.contains(IBlock))3671 continue;3672 3673 DomTreeNode *DomNode = DT->getNode(IBlock);3674 VDUse.DFSIn = DomNode->getDFSNumIn();3675 VDUse.DFSOut = DomNode->getDFSNumOut();3676 VDUse.U = &U;3677 ++UseCount;3678 DFSOrderedSet.emplace_back(VDUse);3679 }3680 }3681 3682 // If there are no uses, it's probably dead (but it may have side-effects,3683 // so not definitely dead. Otherwise, store the number of uses so we can3684 // track if it becomes dead later).3685 if (UseCount == 0)3686 ProbablyDead.insert(Def);3687 else3688 UseCounts[Def] = UseCount;3689 }3690}3691 3692// This function converts the set of members for a congruence class from values,3693// to the set of defs for loads and stores, with associated DFS info.3694void NewGVN::convertClassToLoadsAndStores(3695 const CongruenceClass &Dense,3696 SmallVectorImpl<ValueDFS> &LoadsAndStores) const {3697 for (auto *D : Dense) {3698 if (!isa<LoadInst>(D) && !isa<StoreInst>(D))3699 continue;3700 3701 BasicBlock *BB = getBlockForValue(D);3702 ValueDFS VD;3703 DomTreeNode *DomNode = DT->getNode(BB);3704 VD.DFSIn = DomNode->getDFSNumIn();3705 VD.DFSOut = DomNode->getDFSNumOut();3706 VD.Def.setPointer(D);3707 3708 // If it's an instruction, use the real local dfs number.3709 if (auto *I = dyn_cast<Instruction>(D))3710 VD.LocalNum = InstrToDFSNum(I);3711 else3712 llvm_unreachable("Should have been an instruction");3713 3714 LoadsAndStores.emplace_back(VD);3715 }3716}3717 3718static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) {3719 patchReplacementInstruction(I, Repl);3720 I->replaceAllUsesWith(Repl);3721}3722 3723void NewGVN::deleteInstructionsInBlock(BasicBlock *BB) {3724 LLVM_DEBUG(dbgs() << " BasicBlock Dead:" << *BB);3725 ++NumGVNBlocksDeleted;3726 3727 // Delete the instructions backwards, as it has a reduced likelihood of having3728 // to update as many def-use and use-def chains. Start after the terminator.3729 auto StartPoint = BB->rbegin();3730 ++StartPoint;3731 // Note that we explicitly recalculate BB->rend() on each iteration,3732 // as it may change when we remove the first instruction.3733 for (BasicBlock::reverse_iterator I(StartPoint); I != BB->rend();) {3734 Instruction &Inst = *I++;3735 if (!Inst.use_empty())3736 Inst.replaceAllUsesWith(PoisonValue::get(Inst.getType()));3737 if (isa<LandingPadInst>(Inst))3738 continue;3739 salvageKnowledge(&Inst, AC);3740 3741 Inst.eraseFromParent();3742 ++NumGVNInstrDeleted;3743 }3744 // Now insert something that simplifycfg will turn into an unreachable.3745 Type *Int8Ty = Type::getInt8Ty(BB->getContext());3746 new StoreInst(3747 PoisonValue::get(Int8Ty),3748 Constant::getNullValue(PointerType::getUnqual(BB->getContext())),3749 BB->getTerminator()->getIterator());3750}3751 3752void NewGVN::markInstructionForDeletion(Instruction *I) {3753 LLVM_DEBUG(dbgs() << "Marking " << *I << " for deletion\n");3754 InstructionsToErase.insert(I);3755}3756 3757void NewGVN::replaceInstruction(Instruction *I, Value *V) {3758 LLVM_DEBUG(dbgs() << "Replacing " << *I << " with " << *V << "\n");3759 patchAndReplaceAllUsesWith(I, V);3760 // We save the actual erasing to avoid invalidating memory3761 // dependencies until we are done with everything.3762 markInstructionForDeletion(I);3763}3764 3765namespace {3766 3767// This is a stack that contains both the value and dfs info of where3768// that value is valid.3769class ValueDFSStack {3770public:3771 Value *back() const { return ValueStack.back(); }3772 std::pair<int, int> dfs_back() const { return DFSStack.back(); }3773 3774 void push_back(Value *V, int DFSIn, int DFSOut) {3775 ValueStack.emplace_back(V);3776 DFSStack.emplace_back(DFSIn, DFSOut);3777 }3778 3779 bool empty() const { return DFSStack.empty(); }3780 3781 bool isInScope(int DFSIn, int DFSOut) const {3782 if (empty())3783 return false;3784 return DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second;3785 }3786 3787 void popUntilDFSScope(int DFSIn, int DFSOut) {3788 3789 // These two should always be in sync at this point.3790 assert(ValueStack.size() == DFSStack.size() &&3791 "Mismatch between ValueStack and DFSStack");3792 while (3793 !DFSStack.empty() &&3794 !(DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second)) {3795 DFSStack.pop_back();3796 ValueStack.pop_back();3797 }3798 }3799 3800private:3801 SmallVector<Value *, 8> ValueStack;3802 SmallVector<std::pair<int, int>, 8> DFSStack;3803};3804 3805} // end anonymous namespace3806 3807// Given an expression, get the congruence class for it.3808CongruenceClass *NewGVN::getClassForExpression(const Expression *E) const {3809 if (auto *VE = dyn_cast<VariableExpression>(E))3810 return ValueToClass.lookup(VE->getVariableValue());3811 else if (isa<DeadExpression>(E))3812 return TOPClass;3813 return ExpressionToClass.lookup(E);3814}3815 3816// Given a value and a basic block we are trying to see if it is available in,3817// see if the value has a leader available in that block.3818Value *NewGVN::findPHIOfOpsLeader(const Expression *E,3819 const Instruction *OrigInst,3820 const BasicBlock *BB) const {3821 // It would already be constant if we could make it constant3822 if (auto *CE = dyn_cast<ConstantExpression>(E))3823 return CE->getConstantValue();3824 if (auto *VE = dyn_cast<VariableExpression>(E)) {3825 auto *V = VE->getVariableValue();3826 if (alwaysAvailable(V) || DT->dominates(getBlockForValue(V), BB))3827 return VE->getVariableValue();3828 }3829 3830 auto *CC = getClassForExpression(E);3831 if (!CC)3832 return nullptr;3833 if (alwaysAvailable(CC->getLeader()))3834 return CC->getLeader();3835 3836 for (auto *Member : *CC) {3837 auto *MemberInst = dyn_cast<Instruction>(Member);3838 if (MemberInst == OrigInst)3839 continue;3840 // Anything that isn't an instruction is always available.3841 if (!MemberInst)3842 return Member;3843 if (DT->dominates(getBlockForValue(MemberInst), BB))3844 return Member;3845 }3846 return nullptr;3847}3848 3849bool NewGVN::eliminateInstructions(Function &F) {3850 // This is a non-standard eliminator. The normal way to eliminate is3851 // to walk the dominator tree in order, keeping track of available3852 // values, and eliminating them. However, this is mildly3853 // pointless. It requires doing lookups on every instruction,3854 // regardless of whether we will ever eliminate it. For3855 // instructions part of most singleton congruence classes, we know we3856 // will never eliminate them.3857 3858 // Instead, this eliminator looks at the congruence classes directly, sorts3859 // them into a DFS ordering of the dominator tree, and then we just3860 // perform elimination straight on the sets by walking the congruence3861 // class member uses in order, and eliminate the ones dominated by the3862 // last member. This is worst case O(E log E) where E = number of3863 // instructions in a single congruence class. In theory, this is all3864 // instructions. In practice, it is much faster, as most instructions are3865 // either in singleton congruence classes or can't possibly be eliminated3866 // anyway (if there are no overlapping DFS ranges in class).3867 // When we find something not dominated, it becomes the new leader3868 // for elimination purposes.3869 // TODO: If we wanted to be faster, We could remove any members with no3870 // overlapping ranges while sorting, as we will never eliminate anything3871 // with those members, as they don't dominate anything else in our set.3872 3873 bool AnythingReplaced = false;3874 3875 // Since we are going to walk the domtree anyway, and we can't guarantee the3876 // DFS numbers are updated, we compute some ourselves.3877 DT->updateDFSNumbers();3878 3879 // Go through all of our phi nodes, and kill the arguments associated with3880 // unreachable edges.3881 auto ReplaceUnreachablePHIArgs = [&](PHINode *PHI, BasicBlock *BB) {3882 for (auto &Operand : PHI->incoming_values())3883 if (!ReachableEdges.count({PHI->getIncomingBlock(Operand), BB})) {3884 LLVM_DEBUG(dbgs() << "Replacing incoming value of " << PHI3885 << " for block "3886 << getBlockName(PHI->getIncomingBlock(Operand))3887 << " with poison due to it being unreachable\n");3888 Operand.set(PoisonValue::get(PHI->getType()));3889 }3890 };3891 // Replace unreachable phi arguments.3892 // At this point, RevisitOnReachabilityChange only contains:3893 //3894 // 1. PHIs3895 // 2. Temporaries that will convert to PHIs3896 // 3. Operations that are affected by an unreachable edge but do not fit into3897 // 1 or 2 (rare).3898 // So it is a slight overshoot of what we want. We could make it exact by3899 // using two SparseBitVectors per block.3900 DenseMap<const BasicBlock *, unsigned> ReachablePredCount;3901 for (auto &KV : ReachableEdges)3902 ReachablePredCount[KV.getEnd()]++;3903 for (auto &BBPair : RevisitOnReachabilityChange) {3904 for (auto InstNum : BBPair.second) {3905 auto *Inst = InstrFromDFSNum(InstNum);3906 auto *PHI = dyn_cast<PHINode>(Inst);3907 PHI = PHI ? PHI : dyn_cast_or_null<PHINode>(RealToTemp.lookup(Inst));3908 if (!PHI)3909 continue;3910 auto *BB = BBPair.first;3911 if (ReachablePredCount.lookup(BB) != PHI->getNumIncomingValues())3912 ReplaceUnreachablePHIArgs(PHI, BB);3913 }3914 }3915 3916 // Map to store the use counts3917 DenseMap<const Value *, unsigned int> UseCounts;3918 for (auto *CC : reverse(CongruenceClasses)) {3919 LLVM_DEBUG(dbgs() << "Eliminating in congruence class " << CC->getID()3920 << "\n");3921 // Track the equivalent store info so we can decide whether to try3922 // dead store elimination.3923 SmallVector<ValueDFS, 8> PossibleDeadStores;3924 SmallPtrSet<Instruction *, 8> ProbablyDead;3925 if (CC->isDead() || CC->empty())3926 continue;3927 // Everything still in the TOP class is unreachable or dead.3928 if (CC == TOPClass) {3929 for (auto *M : *CC) {3930 auto *VTE = ValueToExpression.lookup(M);3931 if (VTE && isa<DeadExpression>(VTE))3932 markInstructionForDeletion(cast<Instruction>(M));3933 assert((!ReachableBlocks.count(cast<Instruction>(M)->getParent()) ||3934 InstructionsToErase.count(cast<Instruction>(M))) &&3935 "Everything in TOP should be unreachable or dead at this "3936 "point");3937 }3938 continue;3939 }3940 3941 assert(CC->getLeader() && "We should have had a leader");3942 // If this is a leader that is always available, and it's a3943 // constant or has no equivalences, just replace everything with3944 // it. We then update the congruence class with whatever members3945 // are left.3946 Value *Leader =3947 CC->getStoredValue() ? CC->getStoredValue() : CC->getLeader();3948 if (alwaysAvailable(Leader)) {3949 CongruenceClass::MemberSet MembersLeft;3950 for (auto *M : *CC) {3951 Value *Member = M;3952 // Void things have no uses we can replace.3953 if (Member == Leader || !isa<Instruction>(Member) ||3954 Member->getType()->isVoidTy()) {3955 MembersLeft.insert(Member);3956 continue;3957 }3958 3959 LLVM_DEBUG(dbgs() << "Found replacement " << *(Leader) << " for "3960 << *Member << "\n");3961 auto *I = cast<Instruction>(Member);3962 assert(Leader != I && "About to accidentally remove our leader");3963 replaceInstruction(I, Leader);3964 AnythingReplaced = true;3965 }3966 CC->swap(MembersLeft);3967 } else {3968 // If this is a singleton, we can skip it.3969 if (CC->size() != 1 || RealToTemp.count(Leader)) {3970 // This is a stack because equality replacement/etc may place3971 // constants in the middle of the member list, and we want to use3972 // those constant values in preference to the current leader, over3973 // the scope of those constants.3974 ValueDFSStack EliminationStack;3975 3976 // Convert the members to DFS ordered sets and then merge them.3977 SmallVector<ValueDFS, 8> DFSOrderedSet;3978 convertClassToDFSOrdered(*CC, DFSOrderedSet, UseCounts, ProbablyDead);3979 3980 // Sort the whole thing.3981 llvm::sort(DFSOrderedSet);3982 for (auto &VD : DFSOrderedSet) {3983 int MemberDFSIn = VD.DFSIn;3984 int MemberDFSOut = VD.DFSOut;3985 Value *Def = VD.Def.getPointer();3986 bool FromStore = VD.Def.getInt();3987 Use *U = VD.U;3988 // We ignore void things because we can't get a value from them.3989 if (Def && Def->getType()->isVoidTy())3990 continue;3991 auto *DefInst = dyn_cast_or_null<Instruction>(Def);3992 if (DefInst && AllTempInstructions.count(DefInst)) {3993 auto *PN = cast<PHINode>(DefInst);3994 3995 // If this is a value phi and that's the expression we used, insert3996 // it into the program3997 // remove from temp instruction list.3998 AllTempInstructions.erase(PN);3999 auto *DefBlock = getBlockForValue(Def);4000 LLVM_DEBUG(dbgs() << "Inserting fully real phi of ops" << *Def4001 << " into block "4002 << getBlockName(getBlockForValue(Def)) << "\n");4003 PN->insertBefore(DefBlock->begin());4004 Def = PN;4005 NumGVNPHIOfOpsEliminations++;4006 }4007 4008 if (EliminationStack.empty()) {4009 LLVM_DEBUG(dbgs() << "Elimination Stack is empty\n");4010 } else {4011 LLVM_DEBUG(dbgs() << "Elimination Stack Top DFS numbers are ("4012 << EliminationStack.dfs_back().first << ","4013 << EliminationStack.dfs_back().second << ")\n");4014 }4015 4016 LLVM_DEBUG(dbgs() << "Current DFS numbers are (" << MemberDFSIn << ","4017 << MemberDFSOut << ")\n");4018 // First, we see if we are out of scope or empty. If so,4019 // and there equivalences, we try to replace the top of4020 // stack with equivalences (if it's on the stack, it must4021 // not have been eliminated yet).4022 // Then we synchronize to our current scope, by4023 // popping until we are back within a DFS scope that4024 // dominates the current member.4025 // Then, what happens depends on a few factors4026 // If the stack is now empty, we need to push4027 // If we have a constant or a local equivalence we want to4028 // start using, we also push.4029 // Otherwise, we walk along, processing members who are4030 // dominated by this scope, and eliminate them.4031 bool ShouldPush = Def && EliminationStack.empty();4032 bool OutOfScope =4033 !EliminationStack.isInScope(MemberDFSIn, MemberDFSOut);4034 4035 if (OutOfScope || ShouldPush) {4036 // Sync to our current scope.4037 EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut);4038 bool ShouldPush = Def && EliminationStack.empty();4039 if (ShouldPush) {4040 EliminationStack.push_back(Def, MemberDFSIn, MemberDFSOut);4041 }4042 }4043 4044 // Skip the Def's, we only want to eliminate on their uses. But mark4045 // dominated defs as dead.4046 if (Def) {4047 // For anything in this case, what and how we value number4048 // guarantees that any side-effects that would have occurred (ie4049 // throwing, etc) can be proven to either still occur (because it's4050 // dominated by something that has the same side-effects), or never4051 // occur. Otherwise, we would not have been able to prove it value4052 // equivalent to something else. For these things, we can just mark4053 // it all dead. Note that this is different from the "ProbablyDead"4054 // set, which may not be dominated by anything, and thus, are only4055 // easy to prove dead if they are also side-effect free. Note that4056 // because stores are put in terms of the stored value, we skip4057 // stored values here. If the stored value is really dead, it will4058 // still be marked for deletion when we process it in its own class.4059 auto *DefI = dyn_cast<Instruction>(Def);4060 if (!EliminationStack.empty() && DefI && !FromStore) {4061 Value *DominatingLeader = EliminationStack.back();4062 if (DominatingLeader != Def) {4063 // Even if the instruction is removed, we still need to update4064 // flags/metadata due to downstreams users of the leader.4065 patchReplacementInstruction(DefI, DominatingLeader);4066 4067 SmallVector<DbgVariableRecord *> DVRUsers;4068 findDbgUsers(DefI, DVRUsers);4069 4070 for (auto *DVR : DVRUsers)4071 DVR->replaceVariableLocationOp(DefI, DominatingLeader);4072 4073 markInstructionForDeletion(DefI);4074 }4075 }4076 continue;4077 }4078 // At this point, we know it is a Use we are trying to possibly4079 // replace.4080 4081 assert(isa<Instruction>(U->get()) &&4082 "Current def should have been an instruction");4083 assert(isa<Instruction>(U->getUser()) &&4084 "Current user should have been an instruction");4085 4086 // If the thing we are replacing into is already marked to be dead,4087 // this use is dead. Note that this is true regardless of whether4088 // we have anything dominating the use or not. We do this here4089 // because we are already walking all the uses anyway.4090 Instruction *InstUse = cast<Instruction>(U->getUser());4091 if (InstructionsToErase.count(InstUse)) {4092 auto &UseCount = UseCounts[U->get()];4093 if (--UseCount == 0) {4094 ProbablyDead.insert(cast<Instruction>(U->get()));4095 }4096 }4097 4098 // If we get to this point, and the stack is empty we must have a use4099 // with nothing we can use to eliminate this use, so just skip it.4100 if (EliminationStack.empty())4101 continue;4102 4103 Value *DominatingLeader = EliminationStack.back();4104 4105 Instruction *SSACopy = nullptr;4106 if (auto *BC = dyn_cast<BitCastInst>(DominatingLeader)) {4107 if (BC->getType() == BC->getOperand(0)->getType() &&4108 PredInfo->getPredicateInfoFor(DominatingLeader)) {4109 SSACopy = BC;4110 DominatingLeader = BC->getOperand(0);4111 }4112 }4113 4114 // Don't replace our existing users with ourselves.4115 if (U->get() == DominatingLeader)4116 continue;4117 4118 // If we replaced something in an instruction, handle the patching of4119 // metadata. Skip this if we are replacing predicateinfo with its4120 // original operand, as we already know we can just drop it.4121 auto *ReplacedInst = cast<Instruction>(U->get());4122 auto *PI = PredInfo->getPredicateInfoFor(ReplacedInst);4123 if (!PI || DominatingLeader != PI->OriginalOp)4124 patchReplacementInstruction(ReplacedInst, DominatingLeader);4125 4126 LLVM_DEBUG(dbgs()4127 << "Found replacement " << *DominatingLeader << " for "4128 << *U->get() << " in " << *(U->getUser()) << "\n");4129 U->set(DominatingLeader);4130 // This is now a use of the dominating leader, which means if the4131 // dominating leader was dead, it's now live!4132 auto &LeaderUseCount = UseCounts[DominatingLeader];4133 // It's about to be alive again.4134 if (LeaderUseCount == 0 && isa<Instruction>(DominatingLeader))4135 ProbablyDead.erase(cast<Instruction>(DominatingLeader));4136 // For copy instructions, we use their operand as a leader,4137 // which means we remove a user of the copy and it may become dead.4138 if (SSACopy) {4139 auto It = UseCounts.find(SSACopy);4140 if (It != UseCounts.end()) {4141 unsigned &IIUseCount = It->second;4142 if (--IIUseCount == 0)4143 ProbablyDead.insert(SSACopy);4144 }4145 }4146 ++LeaderUseCount;4147 AnythingReplaced = true;4148 }4149 }4150 }4151 4152 // At this point, anything still in the ProbablyDead set is actually dead if4153 // would be trivially dead.4154 for (auto *I : ProbablyDead)4155 if (wouldInstructionBeTriviallyDead(I))4156 markInstructionForDeletion(I);4157 4158 // Cleanup the congruence class.4159 CongruenceClass::MemberSet MembersLeft;4160 for (auto *Member : *CC)4161 if (!isa<Instruction>(Member) ||4162 !InstructionsToErase.count(cast<Instruction>(Member)))4163 MembersLeft.insert(Member);4164 CC->swap(MembersLeft);4165 4166 // If we have possible dead stores to look at, try to eliminate them.4167 if (CC->getStoreCount() > 0) {4168 convertClassToLoadsAndStores(*CC, PossibleDeadStores);4169 llvm::sort(PossibleDeadStores);4170 ValueDFSStack EliminationStack;4171 for (auto &VD : PossibleDeadStores) {4172 int MemberDFSIn = VD.DFSIn;4173 int MemberDFSOut = VD.DFSOut;4174 Instruction *Member = cast<Instruction>(VD.Def.getPointer());4175 if (EliminationStack.empty() ||4176 !EliminationStack.isInScope(MemberDFSIn, MemberDFSOut)) {4177 // Sync to our current scope.4178 EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut);4179 if (EliminationStack.empty()) {4180 EliminationStack.push_back(Member, MemberDFSIn, MemberDFSOut);4181 continue;4182 }4183 }4184 // We already did load elimination, so nothing to do here.4185 if (isa<LoadInst>(Member))4186 continue;4187 assert(!EliminationStack.empty());4188 Instruction *Leader = cast<Instruction>(EliminationStack.back());4189 (void)Leader;4190 assert(DT->dominates(Leader->getParent(), Member->getParent()));4191 // Member is dominater by Leader, and thus dead4192 LLVM_DEBUG(dbgs() << "Marking dead store " << *Member4193 << " that is dominated by " << *Leader << "\n");4194 markInstructionForDeletion(Member);4195 CC->erase(Member);4196 ++NumGVNDeadStores;4197 }4198 }4199 }4200 return AnythingReplaced;4201}4202 4203// This function provides global ranking of operations so that we can place them4204// in a canonical order. Note that rank alone is not necessarily enough for a4205// complete ordering, as constants all have the same rank. However, generally,4206// we will simplify an operation with all constants so that it doesn't matter4207// what order they appear in.4208unsigned int NewGVN::getRank(const Value *V) const {4209 // Prefer constants to undef to anything else4210 // Undef is a constant, have to check it first.4211 // Prefer poison to undef as it's less defined.4212 // Prefer smaller constants to constantexprs4213 // Note that the order here matters because of class inheritance4214 if (isa<ConstantExpr>(V))4215 return 3;4216 if (isa<PoisonValue>(V))4217 return 1;4218 if (isa<UndefValue>(V))4219 return 2;4220 if (isa<Constant>(V))4221 return 0;4222 if (auto *A = dyn_cast<Argument>(V))4223 return 4 + A->getArgNo();4224 4225 // Need to shift the instruction DFS by number of arguments + 5 to account for4226 // the constant and argument ranking above.4227 unsigned Result = InstrToDFSNum(V);4228 if (Result > 0)4229 return 5 + NumFuncArgs + Result;4230 // Unreachable or something else, just return a really large number.4231 return ~0;4232}4233 4234// This is a function that says whether two commutative operations should4235// have their order swapped when canonicalizing.4236bool NewGVN::shouldSwapOperands(const Value *A, const Value *B) const {4237 // Because we only care about a total ordering, and don't rewrite expressions4238 // in this order, we order by rank, which will give a strict weak ordering to4239 // everything but constants, and then we order by pointer address.4240 return std::make_pair(getRank(A), A) > std::make_pair(getRank(B), B);4241}4242 4243bool NewGVN::shouldSwapOperandsForPredicate(const Value *A, const Value *B,4244 const BitCastInst *I) const {4245 if (shouldSwapOperands(A, B)) {4246 PredicateSwapChoice[I] = B;4247 return true;4248 }4249 4250 auto LookupResult = PredicateSwapChoice.find(I);4251 if (LookupResult != PredicateSwapChoice.end()) {4252 auto *SeenPredicate = LookupResult->second;4253 if (SeenPredicate) {4254 // We previously decided to swap B to the left. Keep that choice.4255 if (SeenPredicate == B)4256 return true;4257 else4258 LookupResult->second = nullptr;4259 }4260 }4261 return false;4262}4263 4264PreservedAnalyses NewGVNPass::run(Function &F, AnalysisManager<Function> &AM) {4265 // Apparently the order in which we get these results matter for4266 // the old GVN (see Chandler's comment in GVN.cpp). I'll keep4267 // the same order here, just in case.4268 auto &AC = AM.getResult<AssumptionAnalysis>(F);4269 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);4270 auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);4271 auto &AA = AM.getResult<AAManager>(F);4272 auto &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA();4273 bool Changed =4274 NewGVN(F, &DT, &AC, &TLI, &AA, &MSSA, F.getDataLayout())4275 .runGVN();4276 if (!Changed)4277 return PreservedAnalyses::all();4278 PreservedAnalyses PA;4279 PA.preserve<DominatorTreeAnalysis>();4280 return PA;4281}4282