448 lines · cpp
1//===-- Assembler.cpp -------------------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "Assembler.h"10 11#include "SnippetRepetitor.h"12#include "SubprocessMemory.h"13#include "Target.h"14#include "llvm/Analysis/TargetLibraryInfo.h"15#include "llvm/CodeGen/FunctionLoweringInfo.h"16#include "llvm/CodeGen/GlobalISel/CallLowering.h"17#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"18#include "llvm/CodeGen/MachineInstrBuilder.h"19#include "llvm/CodeGen/MachineModuleInfo.h"20#include "llvm/CodeGen/MachineRegisterInfo.h"21#include "llvm/CodeGen/TargetInstrInfo.h"22#include "llvm/CodeGen/TargetLowering.h"23#include "llvm/CodeGen/TargetPassConfig.h"24#include "llvm/CodeGen/TargetSubtargetInfo.h"25#include "llvm/ExecutionEngine/Orc/LLJIT.h"26#include "llvm/IR/BasicBlock.h"27#include "llvm/IR/Instructions.h"28#include "llvm/IR/LegacyPassManager.h"29#include "llvm/MC/MCInstrInfo.h"30#include "llvm/Object/SymbolSize.h"31#include "llvm/Support/Alignment.h"32#include "llvm/Support/Error.h"33#include "llvm/Support/MemoryBuffer.h"34#include "llvm/Support/raw_ostream.h"35 36#ifdef HAVE_LIBPFM37#include "perfmon/perf_event.h"38#endif // HAVE_LIBPFM39 40#ifdef __linux__41#include <unistd.h>42#endif43 44namespace llvm {45namespace exegesis {46 47static constexpr char ModuleID[] = "ExegesisInfoTest";48static constexpr char FunctionID[] = "foo";49static const Align kFunctionAlignment(4096);50 51// Fills the given basic block with register setup code, and returns true if52// all registers could be setup correctly.53static bool generateSnippetSetupCode(const ExegesisTarget &ET,54 const MCSubtargetInfo *const MSI,55 BasicBlockFiller &BBF,56 const BenchmarkKey &Key,57 bool GenerateMemoryInstructions) {58 bool IsSnippetSetupComplete = true;59 if (GenerateMemoryInstructions) {60 BBF.addInstructions(ET.generateMemoryInitialSetup());61 for (const MemoryMapping &MM : Key.MemoryMappings) {62#ifdef __linux__63 // The frontend that generates that parses the memory mapping information64 // from the user should validate that the requested address is a multiple65 // of the page size. Assert that this is true here.66 assert(MM.Address % getpagesize() == 0 &&67 "Memory mappings need to be aligned to page boundaries.");68#endif69 const MemoryValue &MemVal = Key.MemoryValues.at(MM.MemoryValueName);70 BBF.addInstructions(ET.generateMmap(71 MM.Address, MemVal.SizeBytes,72 ET.getAuxiliaryMemoryStartAddress() +73 sizeof(int) *74 (MemVal.Index + SubprocessMemory::AuxiliaryMemoryOffset)));75 }76 BBF.addInstructions(ET.setStackRegisterToAuxMem());77 }78 Register StackPointerRegister = BBF.MF.getSubtarget()79 .getTargetLowering()80 ->getStackPointerRegisterToSaveRestore();81 for (const RegisterValue &RV : Key.RegisterInitialValues) {82 if (GenerateMemoryInstructions) {83 // If we're generating memory instructions, don't load in the value for84 // the register with the stack pointer as it will be used later to finish85 // the setup.86 if (Register(RV.Register) == StackPointerRegister)87 continue;88 }89 // Load a constant in the register.90 const auto SetRegisterCode = ET.setRegTo(*MSI, RV.Register, RV.Value);91 if (SetRegisterCode.empty())92 IsSnippetSetupComplete = false;93 BBF.addInstructions(SetRegisterCode);94 }95 if (GenerateMemoryInstructions) {96#ifdef HAVE_LIBPFM97 BBF.addInstructions(ET.configurePerfCounter(PERF_EVENT_IOC_RESET, true));98#endif // HAVE_LIBPFM99 for (const RegisterValue &RV : Key.RegisterInitialValues) {100 // Load in the stack register now as we're done using it elsewhere101 // and need to set the value in preparation for executing the102 // snippet.103 if (Register(RV.Register) != StackPointerRegister)104 continue;105 const auto SetRegisterCode = ET.setRegTo(*MSI, RV.Register, RV.Value);106 if (SetRegisterCode.empty())107 IsSnippetSetupComplete = false;108 BBF.addInstructions(SetRegisterCode);109 break;110 }111 }112 return IsSnippetSetupComplete;113}114 115// Small utility function to add named passes.116static bool addPass(PassManagerBase &PM, StringRef PassName,117 TargetPassConfig &TPC) {118 const PassRegistry *PR = PassRegistry::getPassRegistry();119 const PassInfo *PI = PR->getPassInfo(PassName);120 if (!PI) {121 errs() << " run-pass " << PassName << " is not registered.\n";122 return true;123 }124 125 if (!PI->getNormalCtor()) {126 errs() << " cannot create pass: " << PI->getPassName() << "\n";127 return true;128 }129 Pass *P = PI->getNormalCtor()();130 std::string Banner = std::string("After ") + std::string(P->getPassName());131 PM.add(P);132 TPC.printAndVerify(Banner);133 134 return false;135}136 137MachineFunction &createVoidVoidPtrMachineFunction(StringRef FunctionName,138 Module *Module,139 MachineModuleInfo *MMI) {140 Type *const ReturnType = Type::getInt32Ty(Module->getContext());141 Type *const MemParamType =142 PointerType::get(Module->getContext(), 0 /*default address space*/);143 FunctionType *FunctionType =144 FunctionType::get(ReturnType, {MemParamType}, false);145 Function *const F = Function::Create(146 FunctionType, GlobalValue::ExternalLinkage, FunctionName, Module);147 BasicBlock *BB = BasicBlock::Create(Module->getContext(), "", F);148 new UnreachableInst(Module->getContext(), BB);149 return MMI->getOrCreateMachineFunction(*F);150}151 152BasicBlockFiller::BasicBlockFiller(MachineFunction &MF, MachineBasicBlock *MBB,153 const MCInstrInfo *MCII)154 : MF(MF), MBB(MBB), MCII(MCII) {}155 156void BasicBlockFiller::addInstruction(const MCInst &Inst, const DebugLoc &DL) {157 const unsigned Opcode = Inst.getOpcode();158 const MCInstrDesc &MCID = MCII->get(Opcode);159 MachineInstrBuilder Builder = BuildMI(MBB, DL, MCID);160 for (unsigned OpIndex = 0, E = Inst.getNumOperands(); OpIndex < E;161 ++OpIndex) {162 const MCOperand &Op = Inst.getOperand(OpIndex);163 if (Op.isReg()) {164 const bool IsDef = OpIndex < MCID.getNumDefs();165 unsigned Flags = 0;166 const MCOperandInfo &OpInfo = MCID.operands().begin()[OpIndex];167 if (IsDef && !OpInfo.isOptionalDef())168 Flags |= RegState::Define;169 Builder.addReg(Op.getReg(), Flags);170 } else if (Op.isImm()) {171 Builder.addImm(Op.getImm());172 } else if (!Op.isValid()) {173 llvm_unreachable("Operand is not set");174 } else {175 llvm_unreachable("Not yet implemented");176 }177 }178}179 180void BasicBlockFiller::addInstructions(ArrayRef<MCInst> Insts,181 const DebugLoc &DL) {182 for (const MCInst &Inst : Insts)183 addInstruction(Inst, DL);184}185 186void BasicBlockFiller::addReturn(const ExegesisTarget &ET,187 bool SubprocessCleanup, const DebugLoc &DL) {188 // Insert cleanup code189 if (SubprocessCleanup) {190#ifdef HAVE_LIBPFM191 addInstructions(ET.configurePerfCounter(PERF_EVENT_IOC_DISABLE, false));192#endif // HAVE_LIBPFM193#ifdef __linux__194 addInstructions(ET.generateExitSyscall(0));195#endif // __linux__196 }197 // Insert the return code.198 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();199 if (TII->getReturnOpcode() < TII->getNumOpcodes()) {200 BuildMI(MBB, DL, TII->get(TII->getReturnOpcode()));201 } else {202 MachineIRBuilder MIB(MF);203 MIB.setMBB(*MBB);204 205 FunctionLoweringInfo FuncInfo;206 FuncInfo.CanLowerReturn = true;207 MF.getSubtarget().getCallLowering()->lowerReturn(MIB, nullptr, {}, FuncInfo,208 0);209 }210}211 212FunctionFiller::FunctionFiller(MachineFunction &MF,213 std::vector<MCRegister> RegistersSetUp)214 : MF(MF), MCII(MF.getTarget().getMCInstrInfo()), Entry(addBasicBlock()),215 RegistersSetUp(std::move(RegistersSetUp)) {}216 217BasicBlockFiller FunctionFiller::addBasicBlock() {218 MachineBasicBlock *MBB = MF.CreateMachineBasicBlock();219 MF.push_back(MBB);220 return BasicBlockFiller(MF, MBB, MCII);221}222 223ArrayRef<MCRegister> FunctionFiller::getRegistersSetUp() const {224 return RegistersSetUp;225}226 227static std::unique_ptr<Module>228createModule(const std::unique_ptr<LLVMContext> &Context, const DataLayout &DL) {229 auto Mod = std::make_unique<Module>(ModuleID, *Context);230 Mod->setDataLayout(DL);231 return Mod;232}233 234BitVector getFunctionReservedRegs(const TargetMachine &TM) {235 std::unique_ptr<LLVMContext> Context = std::make_unique<LLVMContext>();236 std::unique_ptr<Module> Module = createModule(Context, TM.createDataLayout());237 auto MMIWP = std::make_unique<MachineModuleInfoWrapperPass>(&TM);238 MachineFunction &MF = createVoidVoidPtrMachineFunction(239 FunctionID, Module.get(), &MMIWP->getMMI());240 // Saving reserved registers for client.241 return MF.getSubtarget().getRegisterInfo()->getReservedRegs(MF);242}243 244Error assembleToStream(const ExegesisTarget &ET,245 std::unique_ptr<TargetMachine> TM,246 ArrayRef<MCRegister> LiveIns, const FillFunction &Fill,247 raw_pwrite_stream &AsmStream, const BenchmarkKey &Key,248 bool GenerateMemoryInstructions) {249 auto Context = std::make_unique<LLVMContext>();250 std::unique_ptr<Module> Module =251 createModule(Context, TM->createDataLayout());252 auto MMIWP = std::make_unique<MachineModuleInfoWrapperPass>(TM.get());253 MachineFunction &MF = createVoidVoidPtrMachineFunction(254 FunctionID, Module.get(), &MMIWP.get()->getMMI());255 MF.ensureAlignment(kFunctionAlignment);256 257 // We need to instruct the passes that we're done with SSA and virtual258 // registers.259 auto &Properties = MF.getProperties();260 Properties.setNoVRegs().resetIsSSA().setNoPHIs();261 262 for (const MCRegister Reg : LiveIns)263 MF.getRegInfo().addLiveIn(Reg);264 265 if (GenerateMemoryInstructions) {266 for (const MCRegister Reg : ET.getArgumentRegisters())267 MF.getRegInfo().addLiveIn(Reg);268 // Add a live in for registers that need saving so that the machine verifier269 // doesn't fail if the register is never defined.270 for (const MCRegister Reg : ET.getRegistersNeedSaving())271 MF.getRegInfo().addLiveIn(Reg);272 }273 274 std::vector<MCRegister> RegistersSetUp;275 RegistersSetUp.reserve(Key.RegisterInitialValues.size());276 for (const auto &InitValue : Key.RegisterInitialValues) {277 RegistersSetUp.push_back(InitValue.Register);278 }279 FunctionFiller Sink(MF, std::move(RegistersSetUp));280 auto Entry = Sink.getEntry();281 282 for (const MCRegister Reg : LiveIns)283 Entry.MBB->addLiveIn(Reg);284 285 if (GenerateMemoryInstructions) {286 for (const MCRegister Reg : ET.getArgumentRegisters())287 Entry.MBB->addLiveIn(Reg);288 // Add a live in for registers that need saving so that the machine verifier289 // doesn't fail if the register is never defined.290 for (const MCRegister Reg : ET.getRegistersNeedSaving())291 Entry.MBB->addLiveIn(Reg);292 }293 294 const bool IsSnippetSetupComplete = generateSnippetSetupCode(295 ET, TM->getMCSubtargetInfo(), Entry, Key, GenerateMemoryInstructions);296 297 // If the snippet setup is not complete, we disable liveliness tracking. This298 // means that we won't know what values are in the registers.299 // FIXME: this should probably be an assertion.300 if (!IsSnippetSetupComplete)301 Properties.resetTracksLiveness();302 303 Fill(Sink);304 305 // prologue/epilogue pass needs the reserved registers to be frozen, this306 // is usually done by the SelectionDAGISel pass.307 MF.getRegInfo().freezeReservedRegs();308 309 // We create the pass manager, run the passes to populate AsmBuffer.310 MCContext &MCContext = MMIWP->getMMI().getContext();311 legacy::PassManager PM;312 313 TargetLibraryInfoImpl TLII(Module->getTargetTriple());314 PM.add(new TargetLibraryInfoWrapperPass(TLII));315 316 TargetPassConfig *TPC = TM->createPassConfig(PM);317 PM.add(TPC);318 PM.add(MMIWP.release());319 TPC->printAndVerify("MachineFunctionGenerator::assemble");320 // Add target-specific passes.321 ET.addTargetSpecificPasses(PM);322 TPC->printAndVerify("After ExegesisTarget::addTargetSpecificPasses");323 // Adding the following passes:324 // - postrapseudos: expands pseudo return instructions used on some targets.325 // - prologepilog: saves and restore callee saved registers.326 for (const char *PassName : {"postrapseudos", "prologepilog"})327 if (addPass(PM, PassName, *TPC))328 return make_error<Failure>("Unable to add a mandatory pass");329 TPC->setInitialized();330 331 // AsmPrinter is responsible for generating the assembly into AsmBuffer.332 if (TM->addAsmPrinter(PM, AsmStream, nullptr, CodeGenFileType::ObjectFile,333 MCContext))334 return make_error<Failure>("Cannot add AsmPrinter passes");335 336 PM.run(*Module); // Run all the passes337 bool MFWellFormed =338 MF.verify(nullptr, "llvm-exegesis Assembly", &outs(), false);339 if (!MFWellFormed)340 return make_error<Failure>("The machine function failed verification.");341 return Error::success();342}343 344object::OwningBinary<object::ObjectFile>345getObjectFromBuffer(StringRef InputData) {346 // Storing the generated assembly into a MemoryBuffer that owns the memory.347 std::unique_ptr<MemoryBuffer> Buffer =348 MemoryBuffer::getMemBufferCopy(InputData);349 // Create the ObjectFile from the MemoryBuffer.350 std::unique_ptr<object::ObjectFile> Obj =351 cantFail(object::ObjectFile::createObjectFile(Buffer->getMemBufferRef()));352 // Returning both the MemoryBuffer and the ObjectFile.353 return object::OwningBinary<object::ObjectFile>(std::move(Obj),354 std::move(Buffer));355}356 357object::OwningBinary<object::ObjectFile> getObjectFromFile(StringRef Filename) {358 return cantFail(object::ObjectFile::createObjectFile(Filename));359}360 361Expected<ExecutableFunction> ExecutableFunction::create(362 std::unique_ptr<TargetMachine> TM,363 object::OwningBinary<object::ObjectFile> &&ObjectFileHolder) {364 assert(ObjectFileHolder.getBinary() && "cannot create object file");365 std::unique_ptr<LLVMContext> Ctx = std::make_unique<LLVMContext>();366 367 auto SymbolSizes = object::computeSymbolSizes(*ObjectFileHolder.getBinary());368 // Get the size of the function that we want to call into (with the name of369 // FunctionID).370 auto SymbolIt = find_if(SymbolSizes, [&](const auto &Pair) {371 auto SymbolName = Pair.first.getName();372 if (SymbolName)373 return *SymbolName == FunctionID;374 // We should always succeed in finding the FunctionID, hence we suppress375 // the error here and assert later on the search result, rather than376 // propagating the Expected<> error back to the caller.377 consumeError(SymbolName.takeError());378 return false;379 });380 assert(SymbolIt != SymbolSizes.end() &&381 "Cannot find the symbol for FunctionID");382 uintptr_t CodeSize = SymbolIt->second;383 384 auto EJITOrErr = orc::LLJITBuilder().create();385 if (!EJITOrErr)386 return EJITOrErr.takeError();387 388 auto EJIT = std::move(*EJITOrErr);389 390 if (auto ObjErr =391 EJIT->addObjectFile(std::get<1>(ObjectFileHolder.takeBinary())))392 return std::move(ObjErr);393 394 auto FunctionAddressOrErr = EJIT->lookup(FunctionID);395 if (!FunctionAddressOrErr)396 return FunctionAddressOrErr.takeError();397 398 const uint64_t FunctionAddress = FunctionAddressOrErr->getValue();399 400 assert(isAligned(kFunctionAlignment, FunctionAddress) &&401 "function is not properly aligned");402 403 StringRef FBytes =404 StringRef(reinterpret_cast<const char *>(FunctionAddress), CodeSize);405 return ExecutableFunction(std::move(Ctx), std::move(EJIT), FBytes);406}407 408ExecutableFunction::ExecutableFunction(std::unique_ptr<LLVMContext> Ctx,409 std::unique_ptr<orc::LLJIT> EJIT,410 StringRef FB)411 : FunctionBytes(FB), Context(std::move(Ctx)), ExecJIT(std::move(EJIT)) {}412 413Error getBenchmarkFunctionBytes(const StringRef InputData,414 std::vector<uint8_t> &Bytes) {415 const auto Holder = getObjectFromBuffer(InputData);416 const auto *Obj = Holder.getBinary();417 // See RuntimeDyldImpl::loadObjectImpl(Obj) for much more complete418 // implementation.419 420 // Find the only function in the object file.421 SmallVector<object::SymbolRef, 1> Functions;422 for (auto &Sym : Obj->symbols()) {423 auto SymType = Sym.getType();424 if (SymType && *SymType == object::SymbolRef::Type::ST_Function)425 Functions.push_back(Sym);426 }427 if (Functions.size() != 1)428 return make_error<Failure>("Exactly one function expected");429 430 // Find the containing section - it is assumed to contain only this function.431 auto SectionOrErr = Functions.front().getSection();432 if (!SectionOrErr || *SectionOrErr == Obj->section_end())433 return make_error<Failure>("Section not found");434 435 auto Address = Functions.front().getAddress();436 if (!Address || *Address != SectionOrErr.get()->getAddress())437 return make_error<Failure>("Unexpected layout");438 439 auto ContentsOrErr = SectionOrErr.get()->getContents();440 if (!ContentsOrErr)441 return ContentsOrErr.takeError();442 Bytes.assign(ContentsOrErr->begin(), ContentsOrErr->end());443 return Error::success();444}445 446} // namespace exegesis447} // namespace llvm448