582 lines · cpp
1//===- AliasSetTracker.cpp - Alias Sets Tracker implementation-------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the AliasSetTracker and AliasSet classes.10//11//===----------------------------------------------------------------------===//12 13#include "llvm/Analysis/AliasSetTracker.h"14#include "llvm/ADT/SetVector.h"15#include "llvm/ADT/StringExtras.h"16#include "llvm/Analysis/AliasAnalysis.h"17#include "llvm/Analysis/GuardUtils.h"18#include "llvm/Analysis/MemoryLocation.h"19#include "llvm/Config/llvm-config.h"20#include "llvm/IR/Function.h"21#include "llvm/IR/InstIterator.h"22#include "llvm/IR/Instructions.h"23#include "llvm/IR/IntrinsicInst.h"24#include "llvm/IR/PassManager.h"25#include "llvm/IR/PatternMatch.h"26#include "llvm/IR/Value.h"27#include "llvm/Pass.h"28#include "llvm/Support/AtomicOrdering.h"29#include "llvm/Support/CommandLine.h"30#include "llvm/Support/Compiler.h"31#include "llvm/Support/Debug.h"32#include "llvm/Support/ErrorHandling.h"33#include "llvm/Support/raw_ostream.h"34 35using namespace llvm;36 37static cl::opt<unsigned> SaturationThreshold(38 "alias-set-saturation-threshold", cl::Hidden, cl::init(250),39 cl::desc("The maximum total number of memory locations alias "40 "sets may contain before degradation"));41 42/// mergeSetIn - Merge the specified alias set into this alias set.43void AliasSet::mergeSetIn(AliasSet &AS, AliasSetTracker &AST,44 BatchAAResults &BatchAA) {45 assert(!AS.Forward && "Alias set is already forwarding!");46 assert(!Forward && "This set is a forwarding set!!");47 48 // Update the alias and access types of this set...49 Access |= AS.Access;50 Alias |= AS.Alias;51 52 if (Alias == SetMustAlias) {53 // Check that these two merged sets really are must aliases. If we cannot54 // find a must-alias pair between them, this set becomes a may alias.55 if (!any_of(MemoryLocs, [&](const MemoryLocation &MemLoc) {56 return any_of(AS.MemoryLocs, [&](const MemoryLocation &ASMemLoc) {57 return BatchAA.isMustAlias(MemLoc, ASMemLoc);58 });59 }))60 Alias = SetMayAlias;61 }62 63 // Merge the list of constituent memory locations...64 if (MemoryLocs.empty()) {65 std::swap(MemoryLocs, AS.MemoryLocs);66 } else {67 append_range(MemoryLocs, AS.MemoryLocs);68 AS.MemoryLocs.clear();69 }70 71 bool ASHadUnknownInsts = !AS.UnknownInsts.empty();72 if (UnknownInsts.empty()) { // Merge call sites...73 if (ASHadUnknownInsts) {74 std::swap(UnknownInsts, AS.UnknownInsts);75 addRef();76 }77 } else if (ASHadUnknownInsts) {78 llvm::append_range(UnknownInsts, AS.UnknownInsts);79 AS.UnknownInsts.clear();80 }81 82 AS.Forward = this; // Forward across AS now...83 addRef(); // AS is now pointing to us...84 85 if (ASHadUnknownInsts)86 AS.dropRef(AST);87}88 89void AliasSetTracker::removeAliasSet(AliasSet *AS) {90 if (AliasSet *Fwd = AS->Forward) {91 Fwd->dropRef(*this);92 AS->Forward = nullptr;93 } else // Update TotalAliasSetSize only if not forwarding.94 TotalAliasSetSize -= AS->size();95 96 AliasSets.erase(AS);97 // If we've removed the saturated alias set, set saturated marker back to98 // nullptr and ensure this tracker is empty.99 if (AS == AliasAnyAS) {100 AliasAnyAS = nullptr;101 assert(AliasSets.empty() && "Tracker not empty");102 }103}104 105void AliasSet::removeFromTracker(AliasSetTracker &AST) {106 assert(RefCount == 0 && "Cannot remove non-dead alias set from tracker!");107 AST.removeAliasSet(this);108}109 110void AliasSet::addMemoryLocation(AliasSetTracker &AST,111 const MemoryLocation &MemLoc,112 bool KnownMustAlias) {113 if (isMustAlias() && !KnownMustAlias) {114 // If we cannot find a must-alias with any of the existing MemoryLocs, we115 // must downgrade to may-alias.116 if (!any_of(MemoryLocs, [&](const MemoryLocation &ASMemLoc) {117 return AST.getAliasAnalysis().isMustAlias(MemLoc, ASMemLoc);118 }))119 Alias = SetMayAlias;120 }121 122 // Add it to the end of the list...123 MemoryLocs.push_back(MemLoc);124 125 AST.TotalAliasSetSize++;126}127 128void AliasSet::addUnknownInst(Instruction *I, BatchAAResults &AA) {129 if (UnknownInsts.empty())130 addRef();131 UnknownInsts.emplace_back(I);132 133 // Guards are marked as modifying memory for control flow modelling purposes,134 // but don't actually modify any specific memory location.135 using namespace PatternMatch;136 bool MayWriteMemory = I->mayWriteToMemory() && !isGuard(I) &&137 !(I->use_empty() && match(I, m_Intrinsic<Intrinsic::invariant_start>()));138 if (!MayWriteMemory) {139 Alias = SetMayAlias;140 Access |= RefAccess;141 return;142 }143 144 // FIXME: This should use mod/ref information to make this not suck so bad145 Alias = SetMayAlias;146 Access = ModRefAccess;147}148 149/// aliasesMemoryLocation - If the specified memory location "may" (or must)150/// alias one of the members in the set return the appropriate AliasResult.151/// Otherwise return NoAlias.152///153AliasResult AliasSet::aliasesMemoryLocation(const MemoryLocation &MemLoc,154 BatchAAResults &AA) const {155 if (AliasAny)156 return AliasResult::MayAlias;157 158 // Check all of the memory locations in the set...159 for (const auto &ASMemLoc : MemoryLocs) {160 AliasResult AR = AA.alias(MemLoc, ASMemLoc);161 if (AR != AliasResult::NoAlias)162 return AR;163 }164 165 // Check the unknown instructions...166 for (Instruction *Inst : UnknownInsts)167 if (isModOrRefSet(AA.getModRefInfo(Inst, MemLoc)))168 return AliasResult::MayAlias;169 170 return AliasResult::NoAlias;171}172 173ModRefInfo AliasSet::aliasesUnknownInst(const Instruction *Inst,174 BatchAAResults &AA) const {175 176 if (AliasAny)177 return ModRefInfo::ModRef;178 179 if (!Inst->mayReadOrWriteMemory())180 return ModRefInfo::NoModRef;181 182 for (Instruction *UnknownInst : UnknownInsts) {183 const auto *C1 = dyn_cast<CallBase>(UnknownInst);184 const auto *C2 = dyn_cast<CallBase>(Inst);185 if (!C1 || !C2 || isModOrRefSet(AA.getModRefInfo(C1, C2)) ||186 isModOrRefSet(AA.getModRefInfo(C2, C1))) {187 // TODO: Could be more precise, but not really useful right now.188 return ModRefInfo::ModRef;189 }190 }191 192 ModRefInfo MR = ModRefInfo::NoModRef;193 for (const auto &ASMemLoc : MemoryLocs) {194 MR |= AA.getModRefInfo(Inst, ASMemLoc);195 if (isModAndRefSet(MR))196 return MR;197 }198 199 return MR;200}201 202AliasSet::PointerVector AliasSet::getPointers() const {203 SmallSetVector<const Value *, 8> Pointers;204 for (const MemoryLocation &MemLoc : MemoryLocs)205 Pointers.insert(MemLoc.Ptr);206 return Pointers.takeVector();207}208 209void AliasSetTracker::clear() {210 PointerMap.clear();211 AliasSets.clear();212}213 214/// mergeAliasSetsForMemoryLocation - Given a memory location, merge all alias215/// sets that may alias it. Return the unified set, or nullptr if no aliasing216/// set was found. A known existing alias set for the pointer value of the217/// memory location can be passed in (or nullptr if not available). MustAliasAll218/// is updated to true/false if the memory location is found to MustAlias all219/// the sets it merged.220AliasSet *AliasSetTracker::mergeAliasSetsForMemoryLocation(221 const MemoryLocation &MemLoc, AliasSet *PtrAS, bool &MustAliasAll) {222 AliasSet *FoundSet = nullptr;223 MustAliasAll = true;224 for (AliasSet &AS : llvm::make_early_inc_range(*this)) {225 if (AS.Forward)226 continue;227 228 // An alias set that already contains a memory location with the same229 // pointer value is directly assumed to MustAlias; we bypass the AA query in230 // this case.231 // Note: it is not guaranteed that AA would always provide the same result;232 // a known exception are undef pointer values, where alias(undef, undef) is233 // NoAlias, while we treat it as MustAlias.234 if (&AS != PtrAS) {235 AliasResult AR = AS.aliasesMemoryLocation(MemLoc, AA);236 if (AR == AliasResult::NoAlias)237 continue;238 239 if (AR != AliasResult::MustAlias)240 MustAliasAll = false;241 }242 243 if (!FoundSet) {244 // If this is the first alias set ptr can go into, remember it.245 FoundSet = &AS;246 } else {247 // Otherwise, we must merge the sets.248 FoundSet->mergeSetIn(AS, *this, AA);249 }250 }251 252 return FoundSet;253}254 255AliasSet *AliasSetTracker::findAliasSetForUnknownInst(Instruction *Inst) {256 AliasSet *FoundSet = nullptr;257 for (AliasSet &AS : llvm::make_early_inc_range(*this)) {258 if (AS.Forward || !isModOrRefSet(AS.aliasesUnknownInst(Inst, AA)))259 continue;260 if (!FoundSet) {261 // If this is the first alias set ptr can go into, remember it.262 FoundSet = &AS;263 } else {264 // Otherwise, we must merge the sets.265 FoundSet->mergeSetIn(AS, *this, AA);266 }267 }268 return FoundSet;269}270 271AliasSet &AliasSetTracker::getAliasSetFor(const MemoryLocation &MemLoc) {272 // The alias sets are indexed with a map from the memory locations' pointer273 // values. If the memory location is already registered, we can find it in the274 // alias set associated with its pointer.275 AliasSet *&MapEntry = PointerMap[MemLoc.Ptr];276 if (MapEntry) {277 collapseForwardingIn(MapEntry);278 if (is_contained(MapEntry->MemoryLocs, MemLoc))279 return *MapEntry;280 }281 282 AliasSet *AS;283 bool MustAliasAll = false;284 if (AliasAnyAS) {285 // At this point, the AST is saturated, so we only have one active alias286 // set. That means we already know which alias set we want to return, and287 // just need to add the memory location to that set to keep the data288 // structure consistent.289 // This, of course, means that we will never need a merge here.290 AS = AliasAnyAS;291 } else if (AliasSet *AliasAS = mergeAliasSetsForMemoryLocation(292 MemLoc, MapEntry, MustAliasAll)) {293 // Add it to the alias set it aliases.294 AS = AliasAS;295 } else {296 // Otherwise create a new alias set to hold the new memory location.297 AliasSets.push_back(AS = new AliasSet());298 MustAliasAll = true;299 }300 301 // Register memory location in selected alias set.302 AS->addMemoryLocation(*this, MemLoc, MustAliasAll);303 // Register selected alias set in pointer map (or ensure it is consistent with304 // earlier map entry after taking into account new merging).305 if (MapEntry) {306 collapseForwardingIn(MapEntry);307 assert(MapEntry == AS && "Memory locations with same pointer value cannot "308 "be in different alias sets");309 } else {310 AS->addRef();311 MapEntry = AS;312 }313 return *AS;314}315 316void AliasSetTracker::add(const MemoryLocation &Loc) {317 addMemoryLocation(Loc, AliasSet::NoAccess);318}319 320void AliasSetTracker::add(LoadInst *LI) {321 if (isStrongerThanMonotonic(LI->getOrdering()))322 return addUnknown(LI);323 addMemoryLocation(MemoryLocation::get(LI), AliasSet::RefAccess);324}325 326void AliasSetTracker::add(StoreInst *SI) {327 if (isStrongerThanMonotonic(SI->getOrdering()))328 return addUnknown(SI);329 addMemoryLocation(MemoryLocation::get(SI), AliasSet::ModAccess);330}331 332void AliasSetTracker::add(VAArgInst *VAAI) {333 addMemoryLocation(MemoryLocation::get(VAAI), AliasSet::ModRefAccess);334}335 336void AliasSetTracker::add(AnyMemSetInst *MSI) {337 addMemoryLocation(MemoryLocation::getForDest(MSI), AliasSet::ModAccess);338}339 340void AliasSetTracker::add(AnyMemTransferInst *MTI) {341 addMemoryLocation(MemoryLocation::getForDest(MTI), AliasSet::ModAccess);342 addMemoryLocation(MemoryLocation::getForSource(MTI), AliasSet::RefAccess);343}344 345void AliasSetTracker::addUnknown(Instruction *Inst) {346 if (auto *II = dyn_cast<IntrinsicInst>(Inst)) {347 // These intrinsics will show up as affecting memory, but they are just348 // markers.349 switch (II->getIntrinsicID()) {350 default:351 break;352 // FIXME: Add lifetime/invariant intrinsics (See: PR30807).353 case Intrinsic::allow_runtime_check:354 case Intrinsic::allow_ubsan_check:355 case Intrinsic::assume:356 case Intrinsic::experimental_noalias_scope_decl:357 case Intrinsic::sideeffect:358 case Intrinsic::pseudoprobe:359 return;360 }361 }362 if (!Inst->mayReadOrWriteMemory())363 return; // doesn't alias anything364 365 if (AliasSet *AS = findAliasSetForUnknownInst(Inst)) {366 AS->addUnknownInst(Inst, AA);367 return;368 }369 AliasSets.push_back(new AliasSet());370 AliasSets.back().addUnknownInst(Inst, AA);371}372 373void AliasSetTracker::add(Instruction *I) {374 // Dispatch to one of the other add methods.375 if (LoadInst *LI = dyn_cast<LoadInst>(I))376 return add(LI);377 if (StoreInst *SI = dyn_cast<StoreInst>(I))378 return add(SI);379 if (VAArgInst *VAAI = dyn_cast<VAArgInst>(I))380 return add(VAAI);381 if (AnyMemSetInst *MSI = dyn_cast<AnyMemSetInst>(I))382 return add(MSI);383 if (AnyMemTransferInst *MTI = dyn_cast<AnyMemTransferInst>(I))384 return add(MTI);385 386 // Handle all calls with known mod/ref sets genericall387 if (auto *Call = dyn_cast<CallBase>(I))388 if (Call->onlyAccessesArgMemory()) {389 auto getAccessFromModRef = [](ModRefInfo MRI) {390 if (isRefSet(MRI) && isModSet(MRI))391 return AliasSet::ModRefAccess;392 else if (isModSet(MRI))393 return AliasSet::ModAccess;394 else if (isRefSet(MRI))395 return AliasSet::RefAccess;396 else397 return AliasSet::NoAccess;398 };399 400 ModRefInfo CallMask = AA.getMemoryEffects(Call).getModRef();401 402 // Some intrinsics are marked as modifying memory for control flow403 // modelling purposes, but don't actually modify any specific memory404 // location.405 using namespace PatternMatch;406 if (Call->use_empty() &&407 match(Call, m_Intrinsic<Intrinsic::invariant_start>()))408 CallMask &= ModRefInfo::Ref;409 410 for (auto IdxArgPair : enumerate(Call->args())) {411 int ArgIdx = IdxArgPair.index();412 const Value *Arg = IdxArgPair.value();413 if (!Arg->getType()->isPointerTy())414 continue;415 MemoryLocation ArgLoc =416 MemoryLocation::getForArgument(Call, ArgIdx, nullptr);417 ModRefInfo ArgMask = AA.getArgModRefInfo(Call, ArgIdx);418 ArgMask &= CallMask;419 if (!isNoModRef(ArgMask))420 addMemoryLocation(ArgLoc, getAccessFromModRef(ArgMask));421 }422 return;423 }424 425 return addUnknown(I);426}427 428void AliasSetTracker::add(BasicBlock &BB) {429 for (auto &I : BB)430 add(&I);431}432 433void AliasSetTracker::add(const AliasSetTracker &AST) {434 assert(&AA == &AST.AA &&435 "Merging AliasSetTracker objects with different Alias Analyses!");436 437 // Loop over all of the alias sets in AST, adding the members contained438 // therein into the current alias sets. This can cause alias sets to be439 // merged together in the current AST.440 for (const AliasSet &AS : AST) {441 if (AS.Forward)442 continue; // Ignore forwarding alias sets443 444 // If there are any call sites in the alias set, add them to this AST.445 for (Instruction *Inst : AS.UnknownInsts)446 add(Inst);447 448 // Loop over all of the memory locations in this alias set.449 for (const MemoryLocation &ASMemLoc : AS.MemoryLocs)450 addMemoryLocation(ASMemLoc, (AliasSet::AccessLattice)AS.Access);451 }452}453 454AliasSet &AliasSetTracker::mergeAllAliasSets() {455 assert(!AliasAnyAS && (TotalAliasSetSize > SaturationThreshold) &&456 "Full merge should happen once, when the saturation threshold is "457 "reached");458 459 // Collect all alias sets, so that we can drop references with impunity460 // without worrying about iterator invalidation.461 std::vector<AliasSet *> ASVector;462 ASVector.reserve(SaturationThreshold);463 for (AliasSet &AS : *this)464 ASVector.push_back(&AS);465 466 // Copy all instructions and memory locations into a new set, and forward all467 // other sets to it.468 AliasSets.push_back(new AliasSet());469 AliasAnyAS = &AliasSets.back();470 AliasAnyAS->Alias = AliasSet::SetMayAlias;471 AliasAnyAS->Access = AliasSet::ModRefAccess;472 AliasAnyAS->AliasAny = true;473 474 for (auto *Cur : ASVector) {475 // If Cur was already forwarding, just forward to the new AS instead.476 AliasSet *FwdTo = Cur->Forward;477 if (FwdTo) {478 Cur->Forward = AliasAnyAS;479 AliasAnyAS->addRef();480 FwdTo->dropRef(*this);481 continue;482 }483 484 // Otherwise, perform the actual merge.485 AliasAnyAS->mergeSetIn(*Cur, *this, AA);486 }487 488 return *AliasAnyAS;489}490 491AliasSet &AliasSetTracker::addMemoryLocation(MemoryLocation Loc,492 AliasSet::AccessLattice E) {493 AliasSet &AS = getAliasSetFor(Loc);494 AS.Access |= E;495 496 if (!AliasAnyAS && (TotalAliasSetSize > SaturationThreshold)) {497 // The AST is now saturated. From here on, we conservatively consider all498 // elements to alias each-other.499 return mergeAllAliasSets();500 }501 502 return AS;503}504 505//===----------------------------------------------------------------------===//506// AliasSet/AliasSetTracker Printing Support507//===----------------------------------------------------------------------===//508 509void AliasSet::print(raw_ostream &OS) const {510 OS << " AliasSet[" << (const void*)this << ", " << RefCount << "] ";511 OS << (Alias == SetMustAlias ? "must" : "may") << " alias, ";512 switch (Access) {513 case NoAccess: OS << "No access "; break;514 case RefAccess: OS << "Ref "; break;515 case ModAccess: OS << "Mod "; break;516 case ModRefAccess: OS << "Mod/Ref "; break;517 default: llvm_unreachable("Bad value for Access!");518 }519 if (Forward)520 OS << " forwarding to " << (void*)Forward;521 522 if (!MemoryLocs.empty()) {523 ListSeparator LS;524 OS << "Memory locations: ";525 for (const MemoryLocation &MemLoc : MemoryLocs) {526 OS << LS;527 MemLoc.Ptr->printAsOperand(OS << "(");528 if (MemLoc.Size == LocationSize::afterPointer())529 OS << ", unknown after)";530 else if (MemLoc.Size == LocationSize::beforeOrAfterPointer())531 OS << ", unknown before-or-after)";532 else533 OS << ", " << MemLoc.Size << ")";534 }535 }536 if (!UnknownInsts.empty()) {537 ListSeparator LS;538 OS << "\n " << UnknownInsts.size() << " Unknown instructions: ";539 for (Instruction *I : UnknownInsts) {540 OS << LS;541 if (I->hasName())542 I->printAsOperand(OS);543 else544 I->print(OS);545 }546 }547 OS << "\n";548}549 550void AliasSetTracker::print(raw_ostream &OS) const {551 OS << "Alias Set Tracker: " << AliasSets.size();552 if (AliasAnyAS)553 OS << " (Saturated)";554 OS << " alias sets for " << PointerMap.size() << " pointer values.\n";555 for (const AliasSet &AS : *this)556 AS.print(OS);557 OS << "\n";558}559 560#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)561LLVM_DUMP_METHOD void AliasSet::dump() const { print(dbgs()); }562LLVM_DUMP_METHOD void AliasSetTracker::dump() const { print(dbgs()); }563#endif564 565//===----------------------------------------------------------------------===//566// AliasSetPrinter Pass567//===----------------------------------------------------------------------===//568 569AliasSetsPrinterPass::AliasSetsPrinterPass(raw_ostream &OS) : OS(OS) {}570 571PreservedAnalyses AliasSetsPrinterPass::run(Function &F,572 FunctionAnalysisManager &AM) {573 auto &AA = AM.getResult<AAManager>(F);574 BatchAAResults BatchAA(AA);575 AliasSetTracker Tracker(BatchAA);576 OS << "Alias sets for function '" << F.getName() << "':\n";577 for (Instruction &I : instructions(F))578 Tracker.add(&I);579 Tracker.print(OS);580 return PreservedAnalyses::all();581}582