324 lines · cpp
1//===- GlobalHandler.cpp - Target independent global & env. var handling --===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Target independent global handler and environment manager.10//11//===----------------------------------------------------------------------===//12 13#include "GlobalHandler.h"14#include "PluginInterface.h"15#include "Utils/ELF.h"16 17#include "Shared/Utils.h"18 19#include "llvm/ProfileData/InstrProfData.inc"20#include "llvm/Support/Error.h"21 22#include <cstring>23#include <string>24 25using namespace llvm;26using namespace omp;27using namespace target;28using namespace plugin;29using namespace error;30 31Expected<std::unique_ptr<ObjectFile>>32GenericGlobalHandlerTy::getELFObjectFile(DeviceImageTy &Image) {33 assert(utils::elf::isELF(Image.getMemoryBuffer().getBuffer()) &&34 "Input is not an ELF file");35 36 auto Expected =37 ELFObjectFileBase::createELFObjectFile(Image.getMemoryBuffer());38 if (!Expected) {39 return Plugin::error(ErrorCode::INVALID_BINARY, Expected.takeError(),40 "error parsing binary");41 }42 return Expected;43}44 45Error GenericGlobalHandlerTy::moveGlobalBetweenDeviceAndHost(46 GenericDeviceTy &Device, DeviceImageTy &Image, const GlobalTy &HostGlobal,47 bool Device2Host) {48 49 GlobalTy DeviceGlobal(HostGlobal.getName(), HostGlobal.getSize());50 51 // Get the metadata from the global on the device.52 if (auto Err = getGlobalMetadataFromDevice(Device, Image, DeviceGlobal))53 return Err;54 55 // Perform the actual transfer.56 return moveGlobalBetweenDeviceAndHost(Device, HostGlobal, DeviceGlobal,57 Device2Host);58}59 60/// Actually move memory between host and device. See readGlobalFromDevice and61/// writeGlobalToDevice for the interface description.62Error GenericGlobalHandlerTy::moveGlobalBetweenDeviceAndHost(63 GenericDeviceTy &Device, const GlobalTy &HostGlobal,64 const GlobalTy &DeviceGlobal, bool Device2Host) {65 66 // Transfer the data from the source to the destination.67 if (Device2Host) {68 if (auto Err =69 Device.dataRetrieve(HostGlobal.getPtr(), DeviceGlobal.getPtr(),70 HostGlobal.getSize(), nullptr))71 return Err;72 } else {73 if (auto Err = Device.dataSubmit(DeviceGlobal.getPtr(), HostGlobal.getPtr(),74 HostGlobal.getSize(), nullptr))75 return Err;76 }77 78 DP("Successfully %s %u bytes associated with global symbol '%s' %s the "79 "device "80 "(%p -> %p).\n",81 Device2Host ? "read" : "write", HostGlobal.getSize(),82 HostGlobal.getName().data(), Device2Host ? "from" : "to",83 DeviceGlobal.getPtr(), HostGlobal.getPtr());84 85 return Plugin::success();86}87 88bool GenericGlobalHandlerTy::isSymbolInImage(GenericDeviceTy &Device,89 DeviceImageTy &Image,90 StringRef SymName) {91 // Get the ELF object file for the image. Notice the ELF object may already92 // be created in previous calls, so we can reuse it. If this is unsuccessful93 // just return false as we couldn't find it.94 auto ELFObjOrErr = getELFObjectFile(Image);95 if (!ELFObjOrErr) {96 consumeError(ELFObjOrErr.takeError());97 return false;98 }99 100 // Search the ELF symbol using the symbol name.101 auto SymOrErr = utils::elf::getSymbol(**ELFObjOrErr, SymName);102 if (!SymOrErr) {103 consumeError(SymOrErr.takeError());104 return false;105 }106 107 return SymOrErr->has_value();108}109 110Error GenericGlobalHandlerTy::getGlobalMetadataFromImage(111 GenericDeviceTy &Device, DeviceImageTy &Image, GlobalTy &ImageGlobal) {112 113 // Get the ELF object file for the image. Notice the ELF object may already114 // be created in previous calls, so we can reuse it.115 auto ELFObj = getELFObjectFile(Image);116 if (!ELFObj)117 return ELFObj.takeError();118 119 // Search the ELF symbol using the symbol name.120 auto SymOrErr = utils::elf::getSymbol(**ELFObj, ImageGlobal.getName());121 if (!SymOrErr)122 return Plugin::error(123 ErrorCode::NOT_FOUND, "failed ELF lookup of global '%s': %s",124 ImageGlobal.getName().data(), toString(SymOrErr.takeError()).data());125 126 if (!SymOrErr->has_value())127 return Plugin::error(ErrorCode::NOT_FOUND,128 "failed to find global symbol '%s' in the ELF image",129 ImageGlobal.getName().data());130 131 auto AddrOrErr = utils::elf::getSymbolAddress(**SymOrErr);132 // Get the section to which the symbol belongs.133 if (!AddrOrErr)134 return Plugin::error(135 ErrorCode::NOT_FOUND, "failed to get ELF symbol from global '%s': %s",136 ImageGlobal.getName().data(), toString(AddrOrErr.takeError()).data());137 138 // Setup the global symbol's address and size.139 ImageGlobal.setPtr(const_cast<void *>(*AddrOrErr));140 ImageGlobal.setSize((*SymOrErr)->getSize());141 142 return Plugin::success();143}144 145Error GenericGlobalHandlerTy::readGlobalFromImage(GenericDeviceTy &Device,146 DeviceImageTy &Image,147 const GlobalTy &HostGlobal) {148 149 GlobalTy ImageGlobal(HostGlobal.getName(), -1);150 if (auto Err = getGlobalMetadataFromImage(Device, Image, ImageGlobal))151 return Err;152 153 if (ImageGlobal.getSize() != HostGlobal.getSize())154 return Plugin::error(ErrorCode::INVALID_BINARY,155 "transfer failed because global symbol '%s' has "156 "%u bytes in the ELF image but %u bytes on the host",157 HostGlobal.getName().data(), ImageGlobal.getSize(),158 HostGlobal.getSize());159 160 DP("Global symbol '%s' was found in the ELF image and %u bytes will copied "161 "from %p to %p.\n",162 HostGlobal.getName().data(), HostGlobal.getSize(), ImageGlobal.getPtr(),163 HostGlobal.getPtr());164 165 assert(Image.getStart() <= ImageGlobal.getPtr() &&166 utils::advancePtr(ImageGlobal.getPtr(), ImageGlobal.getSize()) <167 utils::advancePtr(Image.getStart(), Image.getSize()) &&168 "Attempting to read outside the image!");169 170 // Perform the copy from the image to the host memory.171 std::memcpy(HostGlobal.getPtr(), ImageGlobal.getPtr(), HostGlobal.getSize());172 173 return Plugin::success();174}175 176Expected<GPUProfGlobals>177GenericGlobalHandlerTy::readProfilingGlobals(GenericDeviceTy &Device,178 DeviceImageTy &Image) {179 GPUProfGlobals DeviceProfileData;180 auto ObjFile = getELFObjectFile(Image);181 if (!ObjFile)182 return ObjFile.takeError();183 184 std::unique_ptr<ELFObjectFileBase> ELFObj(185 static_cast<ELFObjectFileBase *>(ObjFile->release()));186 DeviceProfileData.TargetTriple = ELFObj->makeTriple();187 188 // Iterate through elf symbols189 for (auto &Sym : ELFObj->symbols()) {190 auto NameOrErr = Sym.getName();191 if (!NameOrErr)192 return NameOrErr.takeError();193 194 // Check if given current global is a profiling global based195 // on name196 if (*NameOrErr == getInstrProfNamesVarName()) {197 // Read in profiled function names from ELF198 auto SectionOrErr = Sym.getSection();199 if (!SectionOrErr)200 return SectionOrErr.takeError();201 202 auto ContentsOrErr = (*SectionOrErr)->getContents();203 if (!ContentsOrErr)204 return ContentsOrErr.takeError();205 206 SmallVector<uint8_t> NameBytes(ContentsOrErr->bytes());207 DeviceProfileData.NamesData = NameBytes;208 } else if (NameOrErr->starts_with(getInstrProfCountersVarPrefix())) {209 // Read global variable profiling counts210 SmallVector<int64_t> Counts(Sym.getSize() / sizeof(int64_t), 0);211 GlobalTy CountGlobal(NameOrErr->str(), Sym.getSize(), Counts.data());212 if (auto Err = readGlobalFromDevice(Device, Image, CountGlobal))213 return Err;214 DeviceProfileData.Counts.append(std::move(Counts));215 } else if (NameOrErr->starts_with(getInstrProfDataVarPrefix())) {216 // Read profiling data for this global variable217 __llvm_profile_data Data{};218 GlobalTy DataGlobal(NameOrErr->str(), Sym.getSize(), &Data);219 if (auto Err = readGlobalFromDevice(Device, Image, DataGlobal))220 return Err;221 DeviceProfileData.Data.push_back(std::move(Data));222 } else if (*NameOrErr == INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)) {223 uint64_t RawVersionData;224 GlobalTy RawVersionGlobal(NameOrErr->str(), Sym.getSize(),225 &RawVersionData);226 if (auto Err = readGlobalFromDevice(Device, Image, RawVersionGlobal))227 return Err;228 DeviceProfileData.Version = RawVersionData;229 }230 }231 return DeviceProfileData;232}233 234void GPUProfGlobals::dump() const {235 outs() << "======= GPU Profile =======\nTarget: " << TargetTriple.str()236 << "\n";237 238 outs() << "======== Counters =========\n";239 for (size_t i = 0; i < Counts.size(); i++) {240 if (i > 0 && i % 10 == 0)241 outs() << "\n";242 else if (i != 0)243 outs() << " ";244 outs() << Counts[i];245 }246 outs() << "\n";247 248 outs() << "========== Data ===========\n";249 for (const auto &ProfData : Data) {250 outs() << "{ ";251// The ProfData.Name maybe array, eg: NumValueSites[IPVK_Last+1] .252// If we print out it directly, we are accessing out of bound data.253// Skip dumping the array for now.254#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \255 if (sizeof(#Name) > 2 && #Name[sizeof(#Name) - 2] == ']') { \256 outs() << "[...] "; \257 } else { \258 outs() << ProfData.Name << " "; \259 }260#include "llvm/ProfileData/InstrProfData.inc"261 outs() << "}\n";262 }263 264 outs() << "======== Functions ========\n";265 std::string s;266 s.reserve(NamesData.size());267 for (uint8_t Name : NamesData) {268 s.push_back((char)Name);269 }270 271 InstrProfSymtab Symtab;272 if (Error Err = Symtab.create(StringRef(s))) {273 consumeError(std::move(Err));274 }275 Symtab.dumpNames(outs());276 outs() << "===========================\n";277}278 279Error GPUProfGlobals::write() const {280 if (!__llvm_write_custom_profile)281 return Plugin::error(ErrorCode::INVALID_BINARY,282 "could not find symbol __llvm_write_custom_profile. "283 "The compiler-rt profiling library must be linked for "284 "GPU PGO to work.");285 286 size_t DataSize = Data.size() * sizeof(__llvm_profile_data),287 CountsSize = Counts.size() * sizeof(int64_t);288 __llvm_profile_data *DataBegin, *DataEnd;289 char *CountersBegin, *CountersEnd, *NamesBegin, *NamesEnd;290 291 // Initialize array of contiguous data. We need to make sure each section is292 // contiguous so that the PGO library can compute deltas properly293 SmallVector<uint8_t> ContiguousData(NamesData.size() + DataSize + CountsSize);294 295 // Compute region pointers296 DataBegin = (__llvm_profile_data *)(ContiguousData.data() + CountsSize);297 DataEnd =298 (__llvm_profile_data *)(ContiguousData.data() + CountsSize + DataSize);299 CountersBegin = (char *)ContiguousData.data();300 CountersEnd = (char *)(ContiguousData.data() + CountsSize);301 NamesBegin = (char *)(ContiguousData.data() + CountsSize + DataSize);302 NamesEnd = (char *)(ContiguousData.data() + CountsSize + DataSize +303 NamesData.size());304 305 // Copy data to contiguous buffer306 memcpy(DataBegin, Data.data(), DataSize);307 memcpy(CountersBegin, Counts.data(), CountsSize);308 memcpy(NamesBegin, NamesData.data(), NamesData.size());309 310 // Invoke compiler-rt entrypoint311 int result = __llvm_write_custom_profile(312 TargetTriple.str().c_str(), DataBegin, DataEnd, CountersBegin,313 CountersEnd, NamesBegin, NamesEnd, &Version);314 if (result != 0)315 return Plugin::error(ErrorCode::HOST_IO,316 "error writing GPU PGO data to file");317 318 return Plugin::success();319}320 321bool GPUProfGlobals::empty() const {322 return Counts.empty() && Data.empty() && NamesData.empty();323}324