267 lines · cpp
1//=========- MemTransferLowerTest.cpp - MemTransferLower unit tests -=========//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/Analysis/CGSCCPassManager.h"10#include "llvm/Analysis/ScalarEvolution.h"11#include "llvm/Analysis/TargetTransformInfo.h"12#include "llvm/AsmParser/Parser.h"13#include "llvm/IR/BasicBlock.h"14#include "llvm/IR/Function.h"15#include "llvm/IR/Instructions.h"16#include "llvm/IR/IntrinsicInst.h"17#include "llvm/IR/LLVMContext.h"18#include "llvm/IR/Module.h"19#include "llvm/IR/PassManager.h"20#include "llvm/Passes/PassBuilder.h"21#include "llvm/Support/Debug.h"22#include "llvm/Support/SourceMgr.h"23#include "llvm/Testing/Support/Error.h"24#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"25#include "llvm/Transforms/Vectorize/LoopVectorize.h"26 27#include "gtest/gtest-spi.h"28#include "gtest/gtest.h"29 30using namespace llvm;31 32namespace {33struct ForwardingPass : public PassInfoMixin<ForwardingPass> {34 template <typename T> ForwardingPass(T &&Arg) : Func(std::forward<T>(Arg)) {}35 36 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM) {37 return Func(F, FAM);38 }39 40 std::function<PreservedAnalyses(Function &, FunctionAnalysisManager &)> Func;41};42 43struct MemTransferLowerTest : public testing::Test {44 PassBuilder PB;45 LoopAnalysisManager LAM;46 FunctionAnalysisManager FAM;47 CGSCCAnalysisManager CGAM;48 ModuleAnalysisManager MAM;49 ModulePassManager MPM;50 LLVMContext Context;51 std::unique_ptr<Module> M;52 53 MemTransferLowerTest() {54 // Register all the basic analyses with the managers.55 PB.registerModuleAnalyses(MAM);56 PB.registerCGSCCAnalyses(CGAM);57 PB.registerFunctionAnalyses(FAM);58 PB.registerLoopAnalyses(LAM);59 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);60 }61 62 BasicBlock *getBasicBlockByName(Function &F, StringRef Name) const {63 for (BasicBlock &BB : F) {64 if (BB.getName() == Name)65 return &BB;66 }67 return nullptr;68 }69 70 Instruction *getInstructionByOpcode(BasicBlock &BB, unsigned Opcode,71 unsigned Number) const {72 unsigned CurrNumber = 0;73 for (Instruction &I : BB)74 if (I.getOpcode() == Opcode) {75 ++CurrNumber;76 if (CurrNumber == Number)77 return &I;78 }79 return nullptr;80 }81 82 void ParseAssembly(const char *IR) {83 SMDiagnostic Error;84 M = parseAssemblyString(IR, Error, Context);85 std::string errMsg;86 raw_string_ostream os(errMsg);87 Error.print("", os);88 89 // A failure here means that the test itself is buggy.90 if (!M)91 report_fatal_error(errMsg.c_str());92 }93};94 95// By semantics source and destination of llvm.memcpy.* intrinsic96// are either equal or don't overlap. Once the intrinsic is lowered97// to a loop it can be hard or impossible to reason about these facts.98// For that reason expandMemCpyAsLoop is expected to explicitly mark99// loads from source and stores to destination as not aliasing.100TEST_F(MemTransferLowerTest, MemCpyKnownLength) {101 ParseAssembly("declare void @llvm.memcpy.p0i8.p0i8.i64(ptr, ptr, i64, i1)\n"102 "define void @foo(ptr %dst, ptr %src, i64 %n) optsize {\n"103 "entry:\n"104 " %is_not_equal = icmp ne ptr %dst, %src\n"105 " br i1 %is_not_equal, label %memcpy, label %exit\n"106 "memcpy:\n"107 " call void @llvm.memcpy.p0i8.p0i8.i64(ptr %dst, ptr %src, "108 "i64 1024, i1 false)\n"109 " br label %exit\n"110 "exit:\n"111 " ret void\n"112 "}\n");113 114 FunctionPassManager FPM;115 FPM.addPass(ForwardingPass(116 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {117 TargetTransformInfo TTI(M->getDataLayout());118 auto *MemCpyBB = getBasicBlockByName(F, "memcpy");119 Instruction *Inst = &MemCpyBB->front();120 MemCpyInst *MemCpyI = cast<MemCpyInst>(Inst);121 auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);122 expandMemCpyAsLoop(MemCpyI, TTI, &SE);123 auto *CopyLoopBB = getBasicBlockByName(F, "load-store-loop");124 Instruction *LoadInst =125 getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1);126 EXPECT_NE(nullptr, LoadInst->getMetadata(LLVMContext::MD_alias_scope));127 Instruction *StoreInst =128 getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1);129 EXPECT_NE(nullptr, StoreInst->getMetadata(LLVMContext::MD_noalias));130 return PreservedAnalyses::none();131 }));132 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));133 134 MPM.run(*M, MAM);135}136 137// This test indirectly checks that loads and stores (generated as a result of138// llvm.memcpy lowering) doesn't alias by making sure the loop can be139// successfully vectorized without additional runtime checks.140TEST_F(MemTransferLowerTest, VecMemCpyKnownLength) {141 ParseAssembly("declare void @llvm.memcpy.p0i8.p0i8.i64(ptr, ptr, i64, i1)\n"142 "define void @foo(ptr %dst, ptr %src, i64 %n) optsize {\n"143 "entry:\n"144 " %is_not_equal = icmp ne ptr %dst, %src\n"145 " br i1 %is_not_equal, label %memcpy, label %exit\n"146 "memcpy:\n"147 " call void @llvm.memcpy.p0i8.p0i8.i64(ptr %dst, ptr %src, "148 "i64 1024, i1 false)\n"149 " br label %exit\n"150 "exit:\n"151 " ret void\n"152 "}\n");153 154 FunctionPassManager FPM;155 FPM.addPass(ForwardingPass(156 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {157 TargetTransformInfo TTI(M->getDataLayout());158 auto *MemCpyBB = getBasicBlockByName(F, "memcpy");159 Instruction *Inst = &MemCpyBB->front();160 MemCpyInst *MemCpyI = cast<MemCpyInst>(Inst);161 auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);162 expandMemCpyAsLoop(MemCpyI, TTI, &SE);163 return PreservedAnalyses::none();164 }));165 FPM.addPass(LoopVectorizePass(LoopVectorizeOptions()));166 FPM.addPass(ForwardingPass(167 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {168 auto *TargetBB = getBasicBlockByName(F, "vector.body");169 EXPECT_NE(nullptr, TargetBB);170 return PreservedAnalyses::all();171 }));172 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));173 174 MPM.run(*M, MAM);175}176 177TEST_F(MemTransferLowerTest, AtomicMemCpyKnownLength) {178 ParseAssembly("declare void "179 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(ptr, "180 "i32 *, i64, i32)\n"181 "define void @foo(ptr %dst, ptr %src, i64 %n) optsize {\n"182 "entry:\n"183 " %is_not_equal = icmp ne ptr %dst, %src\n"184 " br i1 %is_not_equal, label %memcpy, label %exit\n"185 "memcpy:\n"186 " call void "187 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(ptr "188 "%dst, ptr %src, "189 "i64 1024, i32 4)\n"190 " br label %exit\n"191 "exit:\n"192 " ret void\n"193 "}\n");194 195 FunctionPassManager FPM;196 FPM.addPass(ForwardingPass(197 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {198 TargetTransformInfo TTI(M->getDataLayout());199 auto *MemCpyBB = getBasicBlockByName(F, "memcpy");200 Instruction *Inst = &MemCpyBB->front();201 assert(isa<AnyMemCpyInst>(Inst) &&202 "Expecting llvm.memcpy.p0i8.i64 instructon");203 AnyMemCpyInst *MemCpyI = cast<AnyMemCpyInst>(Inst);204 auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);205 expandAtomicMemCpyAsLoop(MemCpyI, TTI, &SE);206 auto *CopyLoopBB = getBasicBlockByName(F, "load-store-loop");207 Instruction *LoadInst =208 getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1);209 EXPECT_TRUE(LoadInst->isAtomic());210 EXPECT_NE(LoadInst->getMetadata(LLVMContext::MD_alias_scope), nullptr);211 Instruction *StoreInst =212 getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1);213 EXPECT_TRUE(StoreInst->isAtomic());214 EXPECT_NE(StoreInst->getMetadata(LLVMContext::MD_noalias), nullptr);215 return PreservedAnalyses::none();216 }));217 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));218 219 MPM.run(*M, MAM);220}221 222TEST_F(MemTransferLowerTest, AtomicMemCpyUnKnownLength) {223 ParseAssembly("declare void "224 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(ptr, "225 "i32 *, i64, i32)\n"226 "define void @foo(ptr %dst, ptr %src, i64 %n) optsize {\n"227 "entry:\n"228 " %is_not_equal = icmp ne ptr %dst, %src\n"229 " br i1 %is_not_equal, label %memcpy, label %exit\n"230 "memcpy:\n"231 " call void "232 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(ptr "233 "%dst, ptr %src, "234 "i64 %n, i32 4)\n"235 " br label %exit\n"236 "exit:\n"237 " ret void\n"238 "}\n");239 240 FunctionPassManager FPM;241 FPM.addPass(ForwardingPass(242 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {243 TargetTransformInfo TTI(M->getDataLayout());244 auto *MemCpyBB = getBasicBlockByName(F, "memcpy");245 Instruction *Inst = &MemCpyBB->front();246 assert(isa<AnyMemCpyInst>(Inst) &&247 "Expecting llvm.memcpy.p0i8.i64 instructon");248 auto *MemCpyI = cast<AnyMemCpyInst>(Inst);249 auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);250 expandAtomicMemCpyAsLoop(MemCpyI, TTI, &SE);251 auto *CopyLoopBB = getBasicBlockByName(F, "loop-memcpy-expansion");252 Instruction *LoadInst =253 getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1);254 EXPECT_TRUE(LoadInst->isAtomic());255 EXPECT_NE(LoadInst->getMetadata(LLVMContext::MD_alias_scope), nullptr);256 Instruction *StoreInst =257 getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1);258 EXPECT_TRUE(StoreInst->isAtomic());259 EXPECT_NE(StoreInst->getMetadata(LLVMContext::MD_noalias), nullptr);260 return PreservedAnalyses::none();261 }));262 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));263 264 MPM.run(*M, MAM);265}266} // namespace267