1048 lines · cpp
1//===-- AMDGPUTargetStreamer.cpp - Mips Target Streamer Methods -----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file provides AMDGPU specific target streamer methods.10//11//===----------------------------------------------------------------------===//12 13#include "AMDGPUTargetStreamer.h"14#include "AMDGPUMCExpr.h"15#include "AMDGPUMCKernelDescriptor.h"16#include "AMDGPUPTNote.h"17#include "Utils/AMDGPUBaseInfo.h"18#include "Utils/AMDKernelCodeTUtils.h"19#include "llvm/BinaryFormat/AMDGPUMetadataVerifier.h"20#include "llvm/BinaryFormat/ELF.h"21#include "llvm/MC/MCAsmInfo.h"22#include "llvm/MC/MCAssembler.h"23#include "llvm/MC/MCContext.h"24#include "llvm/MC/MCELFObjectWriter.h"25#include "llvm/MC/MCELFStreamer.h"26#include "llvm/MC/MCSubtargetInfo.h"27#include "llvm/Support/AMDGPUMetadata.h"28#include "llvm/Support/AMDHSAKernelDescriptor.h"29#include "llvm/Support/CommandLine.h"30#include "llvm/Support/FormattedStream.h"31#include "llvm/TargetParser/TargetParser.h"32 33using namespace llvm;34using namespace llvm::AMDGPU;35 36//===----------------------------------------------------------------------===//37// AMDGPUTargetStreamer38//===----------------------------------------------------------------------===//39 40static cl::opt<unsigned>41 ForceGenericVersion("amdgpu-force-generic-version",42 cl::desc("Force a specific generic_v<N> flag to be "43 "added. For testing purposes only."),44 cl::ReallyHidden, cl::init(0));45 46bool AMDGPUTargetStreamer::EmitHSAMetadataV3(StringRef HSAMetadataString) {47 msgpack::Document HSAMetadataDoc;48 if (!HSAMetadataDoc.fromYAML(HSAMetadataString))49 return false;50 return EmitHSAMetadata(HSAMetadataDoc, false);51}52 53StringRef AMDGPUTargetStreamer::getArchNameFromElfMach(unsigned ElfMach) {54 AMDGPU::GPUKind AK;55 56 // clang-format off57 switch (ElfMach) {58 case ELF::EF_AMDGPU_MACH_R600_R600: AK = GK_R600; break;59 case ELF::EF_AMDGPU_MACH_R600_R630: AK = GK_R630; break;60 case ELF::EF_AMDGPU_MACH_R600_RS880: AK = GK_RS880; break;61 case ELF::EF_AMDGPU_MACH_R600_RV670: AK = GK_RV670; break;62 case ELF::EF_AMDGPU_MACH_R600_RV710: AK = GK_RV710; break;63 case ELF::EF_AMDGPU_MACH_R600_RV730: AK = GK_RV730; break;64 case ELF::EF_AMDGPU_MACH_R600_RV770: AK = GK_RV770; break;65 case ELF::EF_AMDGPU_MACH_R600_CEDAR: AK = GK_CEDAR; break;66 case ELF::EF_AMDGPU_MACH_R600_CYPRESS: AK = GK_CYPRESS; break;67 case ELF::EF_AMDGPU_MACH_R600_JUNIPER: AK = GK_JUNIPER; break;68 case ELF::EF_AMDGPU_MACH_R600_REDWOOD: AK = GK_REDWOOD; break;69 case ELF::EF_AMDGPU_MACH_R600_SUMO: AK = GK_SUMO; break;70 case ELF::EF_AMDGPU_MACH_R600_BARTS: AK = GK_BARTS; break;71 case ELF::EF_AMDGPU_MACH_R600_CAICOS: AK = GK_CAICOS; break;72 case ELF::EF_AMDGPU_MACH_R600_CAYMAN: AK = GK_CAYMAN; break;73 case ELF::EF_AMDGPU_MACH_R600_TURKS: AK = GK_TURKS; break;74 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX600: AK = GK_GFX600; break;75 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX601: AK = GK_GFX601; break;76 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX602: AK = GK_GFX602; break;77 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX700: AK = GK_GFX700; break;78 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX701: AK = GK_GFX701; break;79 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX702: AK = GK_GFX702; break;80 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX703: AK = GK_GFX703; break;81 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX704: AK = GK_GFX704; break;82 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX705: AK = GK_GFX705; break;83 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX801: AK = GK_GFX801; break;84 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX802: AK = GK_GFX802; break;85 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX803: AK = GK_GFX803; break;86 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX805: AK = GK_GFX805; break;87 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX810: AK = GK_GFX810; break;88 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX900: AK = GK_GFX900; break;89 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX902: AK = GK_GFX902; break;90 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX904: AK = GK_GFX904; break;91 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX906: AK = GK_GFX906; break;92 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX908: AK = GK_GFX908; break;93 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX909: AK = GK_GFX909; break;94 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX90A: AK = GK_GFX90A; break;95 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX90C: AK = GK_GFX90C; break;96 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX942: AK = GK_GFX942; break;97 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX950: AK = GK_GFX950; break;98 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1010: AK = GK_GFX1010; break;99 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1011: AK = GK_GFX1011; break;100 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1012: AK = GK_GFX1012; break;101 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1013: AK = GK_GFX1013; break;102 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1030: AK = GK_GFX1030; break;103 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1031: AK = GK_GFX1031; break;104 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1032: AK = GK_GFX1032; break;105 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1033: AK = GK_GFX1033; break;106 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1034: AK = GK_GFX1034; break;107 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1035: AK = GK_GFX1035; break;108 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1036: AK = GK_GFX1036; break;109 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1100: AK = GK_GFX1100; break;110 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1101: AK = GK_GFX1101; break;111 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1102: AK = GK_GFX1102; break;112 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1103: AK = GK_GFX1103; break;113 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1150: AK = GK_GFX1150; break;114 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1151: AK = GK_GFX1151; break;115 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1152: AK = GK_GFX1152; break;116 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1153: AK = GK_GFX1153; break;117 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200: AK = GK_GFX1200; break;118 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1201: AK = GK_GFX1201; break;119 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1250: AK = GK_GFX1250; break;120 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1251: AK = GK_GFX1251; break;121 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC: AK = GK_GFX9_GENERIC; break;122 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_4_GENERIC: AK = GK_GFX9_4_GENERIC; break;123 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX10_1_GENERIC: AK = GK_GFX10_1_GENERIC; break;124 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX10_3_GENERIC: AK = GK_GFX10_3_GENERIC; break;125 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX11_GENERIC: AK = GK_GFX11_GENERIC; break;126 case ELF::EF_AMDGPU_MACH_AMDGCN_GFX12_GENERIC: AK = GK_GFX12_GENERIC; break;127 case ELF::EF_AMDGPU_MACH_NONE: AK = GK_NONE; break;128 default: AK = GK_NONE; break;129 }130 // clang-format on131 132 StringRef GPUName = getArchNameAMDGCN(AK);133 if (GPUName != "")134 return GPUName;135 return getArchNameR600(AK);136}137 138unsigned AMDGPUTargetStreamer::getElfMach(StringRef GPU) {139 AMDGPU::GPUKind AK = parseArchAMDGCN(GPU);140 if (AK == AMDGPU::GPUKind::GK_NONE)141 AK = parseArchR600(GPU);142 143 // clang-format off144 switch (AK) {145 case GK_R600: return ELF::EF_AMDGPU_MACH_R600_R600;146 case GK_R630: return ELF::EF_AMDGPU_MACH_R600_R630;147 case GK_RS880: return ELF::EF_AMDGPU_MACH_R600_RS880;148 case GK_RV670: return ELF::EF_AMDGPU_MACH_R600_RV670;149 case GK_RV710: return ELF::EF_AMDGPU_MACH_R600_RV710;150 case GK_RV730: return ELF::EF_AMDGPU_MACH_R600_RV730;151 case GK_RV770: return ELF::EF_AMDGPU_MACH_R600_RV770;152 case GK_CEDAR: return ELF::EF_AMDGPU_MACH_R600_CEDAR;153 case GK_CYPRESS: return ELF::EF_AMDGPU_MACH_R600_CYPRESS;154 case GK_JUNIPER: return ELF::EF_AMDGPU_MACH_R600_JUNIPER;155 case GK_REDWOOD: return ELF::EF_AMDGPU_MACH_R600_REDWOOD;156 case GK_SUMO: return ELF::EF_AMDGPU_MACH_R600_SUMO;157 case GK_BARTS: return ELF::EF_AMDGPU_MACH_R600_BARTS;158 case GK_CAICOS: return ELF::EF_AMDGPU_MACH_R600_CAICOS;159 case GK_CAYMAN: return ELF::EF_AMDGPU_MACH_R600_CAYMAN;160 case GK_TURKS: return ELF::EF_AMDGPU_MACH_R600_TURKS;161 case GK_GFX600: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX600;162 case GK_GFX601: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX601;163 case GK_GFX602: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX602;164 case GK_GFX700: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX700;165 case GK_GFX701: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX701;166 case GK_GFX702: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX702;167 case GK_GFX703: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX703;168 case GK_GFX704: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX704;169 case GK_GFX705: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX705;170 case GK_GFX801: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX801;171 case GK_GFX802: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX802;172 case GK_GFX803: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX803;173 case GK_GFX805: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX805;174 case GK_GFX810: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX810;175 case GK_GFX900: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX900;176 case GK_GFX902: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX902;177 case GK_GFX904: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX904;178 case GK_GFX906: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX906;179 case GK_GFX908: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX908;180 case GK_GFX909: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX909;181 case GK_GFX90A: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX90A;182 case GK_GFX90C: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX90C;183 case GK_GFX942: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX942;184 case GK_GFX950: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX950;185 case GK_GFX1010: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1010;186 case GK_GFX1011: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1011;187 case GK_GFX1012: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1012;188 case GK_GFX1013: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1013;189 case GK_GFX1030: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1030;190 case GK_GFX1031: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1031;191 case GK_GFX1032: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1032;192 case GK_GFX1033: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1033;193 case GK_GFX1034: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1034;194 case GK_GFX1035: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1035;195 case GK_GFX1036: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1036;196 case GK_GFX1100: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1100;197 case GK_GFX1101: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1101;198 case GK_GFX1102: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1102;199 case GK_GFX1103: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1103;200 case GK_GFX1150: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1150;201 case GK_GFX1151: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1151;202 case GK_GFX1152: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1152;203 case GK_GFX1153: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1153;204 case GK_GFX1200: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200;205 case GK_GFX1201: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1201;206 case GK_GFX1250: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1250;207 case GK_GFX1251: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1251;208 case GK_GFX9_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC;209 case GK_GFX9_4_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_4_GENERIC;210 case GK_GFX10_1_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX10_1_GENERIC;211 case GK_GFX10_3_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX10_3_GENERIC;212 case GK_GFX11_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX11_GENERIC;213 case GK_GFX12_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX12_GENERIC;214 case GK_NONE: return ELF::EF_AMDGPU_MACH_NONE;215 }216 // clang-format on217 218 llvm_unreachable("unknown GPU");219}220 221//===----------------------------------------------------------------------===//222// AMDGPUTargetAsmStreamer223//===----------------------------------------------------------------------===//224 225AMDGPUTargetAsmStreamer::AMDGPUTargetAsmStreamer(MCStreamer &S,226 formatted_raw_ostream &OS)227 : AMDGPUTargetStreamer(S), OS(OS) { }228 229// A hook for emitting stuff at the end.230// We use it for emitting the accumulated PAL metadata as directives.231// The PAL metadata is reset after it is emitted.232void AMDGPUTargetAsmStreamer::finish() {233 std::string S;234 getPALMetadata()->toString(S);235 OS << S;236 237 // Reset the pal metadata so its data will not affect a compilation that238 // reuses this object.239 getPALMetadata()->reset();240}241 242void AMDGPUTargetAsmStreamer::EmitDirectiveAMDGCNTarget() {243 OS << "\t.amdgcn_target \"" << getTargetID()->toString() << "\"\n";244}245 246void AMDGPUTargetAsmStreamer::EmitDirectiveAMDHSACodeObjectVersion(247 unsigned COV) {248 AMDGPUTargetStreamer::EmitDirectiveAMDHSACodeObjectVersion(COV);249 OS << "\t.amdhsa_code_object_version " << COV << '\n';250}251 252void AMDGPUTargetAsmStreamer::EmitAMDKernelCodeT(AMDGPUMCKernelCodeT &Header) {253 auto FoldAndPrint = [&](const MCExpr *Expr, raw_ostream &OS,254 const MCAsmInfo *MAI) {255 printAMDGPUMCExpr(foldAMDGPUMCExpr(Expr, getContext()), OS, MAI);256 };257 258 OS << "\t.amd_kernel_code_t\n";259 Header.EmitKernelCodeT(OS, getContext(), FoldAndPrint);260 OS << "\t.end_amd_kernel_code_t\n";261}262 263void AMDGPUTargetAsmStreamer::EmitAMDGPUSymbolType(StringRef SymbolName,264 unsigned Type) {265 switch (Type) {266 default: llvm_unreachable("Invalid AMDGPU symbol type");267 case ELF::STT_AMDGPU_HSA_KERNEL:268 OS << "\t.amdgpu_hsa_kernel " << SymbolName << '\n' ;269 break;270 }271}272 273void AMDGPUTargetAsmStreamer::emitAMDGPULDS(MCSymbol *Symbol, unsigned Size,274 Align Alignment) {275 OS << "\t.amdgpu_lds " << Symbol->getName() << ", " << Size << ", "276 << Alignment.value() << '\n';277}278 279void AMDGPUTargetAsmStreamer::EmitMCResourceInfo(280 const MCSymbol *NumVGPR, const MCSymbol *NumAGPR,281 const MCSymbol *NumExplicitSGPR, const MCSymbol *NumNamedBarrier,282 const MCSymbol *PrivateSegmentSize, const MCSymbol *UsesVCC,283 const MCSymbol *UsesFlatScratch, const MCSymbol *HasDynamicallySizedStack,284 const MCSymbol *HasRecursion, const MCSymbol *HasIndirectCall) {285#define PRINT_RES_INFO(ARG) \286 OS << "\t.set "; \287 ARG->print(OS, getContext().getAsmInfo()); \288 OS << ", "; \289 getContext().getAsmInfo()->printExpr(OS, *ARG->getVariableValue()); \290 Streamer.addBlankLine();291 292 PRINT_RES_INFO(NumVGPR);293 PRINT_RES_INFO(NumAGPR);294 PRINT_RES_INFO(NumExplicitSGPR);295 PRINT_RES_INFO(NumNamedBarrier);296 PRINT_RES_INFO(PrivateSegmentSize);297 PRINT_RES_INFO(UsesVCC);298 PRINT_RES_INFO(UsesFlatScratch);299 PRINT_RES_INFO(HasDynamicallySizedStack);300 PRINT_RES_INFO(HasRecursion);301 PRINT_RES_INFO(HasIndirectCall);302#undef PRINT_RES_INFO303}304 305void AMDGPUTargetAsmStreamer::EmitMCResourceMaximums(const MCSymbol *MaxVGPR,306 const MCSymbol *MaxAGPR,307 const MCSymbol *MaxSGPR) {308#define PRINT_RES_INFO(ARG) \309 OS << "\t.set "; \310 ARG->print(OS, getContext().getAsmInfo()); \311 OS << ", "; \312 getContext().getAsmInfo()->printExpr(OS, *ARG->getVariableValue()); \313 Streamer.addBlankLine();314 315 PRINT_RES_INFO(MaxVGPR);316 PRINT_RES_INFO(MaxAGPR);317 PRINT_RES_INFO(MaxSGPR);318#undef PRINT_RES_INFO319}320 321bool AMDGPUTargetAsmStreamer::EmitISAVersion() {322 OS << "\t.amd_amdgpu_isa \"" << getTargetID()->toString() << "\"\n";323 return true;324}325 326bool AMDGPUTargetAsmStreamer::EmitHSAMetadata(327 msgpack::Document &HSAMetadataDoc, bool Strict) {328 HSAMD::V3::MetadataVerifier Verifier(Strict);329 if (!Verifier.verify(HSAMetadataDoc.getRoot()))330 return false;331 332 std::string HSAMetadataString;333 raw_string_ostream StrOS(HSAMetadataString);334 HSAMetadataDoc.toYAML(StrOS);335 336 OS << '\t' << HSAMD::V3::AssemblerDirectiveBegin << '\n';337 OS << StrOS.str() << '\n';338 OS << '\t' << HSAMD::V3::AssemblerDirectiveEnd << '\n';339 return true;340}341 342bool AMDGPUTargetAsmStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) {343 const uint32_t Encoded_s_code_end = 0xbf9f0000;344 const uint32_t Encoded_s_nop = 0xbf800000;345 uint32_t Encoded_pad = Encoded_s_code_end;346 347 // Instruction cache line size in bytes.348 const unsigned Log2CacheLineSize = AMDGPU::isGFX11Plus(STI) ? 7 : 6;349 const unsigned CacheLineSize = 1u << Log2CacheLineSize;350 351 // Extra padding amount in bytes to support prefetch mode 3.352 unsigned FillSize = 3 * CacheLineSize;353 354 if (AMDGPU::isGFX90A(STI)) {355 Encoded_pad = Encoded_s_nop;356 FillSize = 16 * CacheLineSize;357 }358 359 OS << "\t.p2alignl " << Log2CacheLineSize << ", " << Encoded_pad << '\n';360 OS << "\t.fill " << (FillSize / 4) << ", 4, " << Encoded_pad << '\n';361 return true;362}363 364void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(365 const MCSubtargetInfo &STI, StringRef KernelName,366 const MCKernelDescriptor &KD, const MCExpr *NextVGPR,367 const MCExpr *NextSGPR, const MCExpr *ReserveVCC,368 const MCExpr *ReserveFlatScr) {369 IsaVersion IVersion = getIsaVersion(STI.getCPU());370 const MCAsmInfo *MAI = getContext().getAsmInfo();371 372 OS << "\t.amdhsa_kernel " << KernelName << '\n';373 374 auto PrintField = [&](const MCExpr *Expr, uint32_t Shift, uint32_t Mask,375 StringRef Directive) {376 OS << "\t\t" << Directive << ' ';377 const MCExpr *ShiftedAndMaskedExpr =378 MCKernelDescriptor::bits_get(Expr, Shift, Mask, getContext());379 const MCExpr *New = foldAMDGPUMCExpr(ShiftedAndMaskedExpr, getContext());380 printAMDGPUMCExpr(New, OS, MAI);381 OS << '\n';382 };383 384 auto EmitMCExpr = [&](const MCExpr *Value) {385 const MCExpr *NewExpr = foldAMDGPUMCExpr(Value, getContext());386 printAMDGPUMCExpr(NewExpr, OS, MAI);387 };388 389 OS << "\t\t.amdhsa_group_segment_fixed_size ";390 EmitMCExpr(KD.group_segment_fixed_size);391 OS << '\n';392 393 OS << "\t\t.amdhsa_private_segment_fixed_size ";394 EmitMCExpr(KD.private_segment_fixed_size);395 OS << '\n';396 397 OS << "\t\t.amdhsa_kernarg_size ";398 EmitMCExpr(KD.kernarg_size);399 OS << '\n';400 401 if (isGFX1250(STI)) {402 PrintField(KD.compute_pgm_rsrc2,403 amdhsa::COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,404 amdhsa::COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT,405 ".amdhsa_user_sgpr_count");406 } else {407 PrintField(KD.compute_pgm_rsrc2,408 amdhsa::COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,409 amdhsa::COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT,410 ".amdhsa_user_sgpr_count");411 }412 413 if (!hasArchitectedFlatScratch(STI))414 PrintField(415 KD.kernel_code_properties,416 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER_SHIFT,417 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,418 ".amdhsa_user_sgpr_private_segment_buffer");419 PrintField(KD.kernel_code_properties,420 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR_SHIFT,421 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR,422 ".amdhsa_user_sgpr_dispatch_ptr");423 PrintField(KD.kernel_code_properties,424 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR_SHIFT,425 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR,426 ".amdhsa_user_sgpr_queue_ptr");427 PrintField(KD.kernel_code_properties,428 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR_SHIFT,429 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,430 ".amdhsa_user_sgpr_kernarg_segment_ptr");431 PrintField(KD.kernel_code_properties,432 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID_SHIFT,433 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID,434 ".amdhsa_user_sgpr_dispatch_id");435 if (!hasArchitectedFlatScratch(STI))436 PrintField(KD.kernel_code_properties,437 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT_SHIFT,438 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,439 ".amdhsa_user_sgpr_flat_scratch_init");440 if (hasKernargPreload(STI)) {441 PrintField(KD.kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_LENGTH_SHIFT,442 amdhsa::KERNARG_PRELOAD_SPEC_LENGTH,443 ".amdhsa_user_sgpr_kernarg_preload_length");444 PrintField(KD.kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_OFFSET_SHIFT,445 amdhsa::KERNARG_PRELOAD_SPEC_OFFSET,446 ".amdhsa_user_sgpr_kernarg_preload_offset");447 }448 PrintField(449 KD.kernel_code_properties,450 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE_SHIFT,451 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,452 ".amdhsa_user_sgpr_private_segment_size");453 if (IVersion.Major >= 10)454 PrintField(KD.kernel_code_properties,455 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT,456 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32,457 ".amdhsa_wavefront_size32");458 if (CodeObjectVersion >= AMDGPU::AMDHSA_COV5)459 PrintField(KD.kernel_code_properties,460 amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK_SHIFT,461 amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK,462 ".amdhsa_uses_dynamic_stack");463 PrintField(KD.compute_pgm_rsrc2,464 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT_SHIFT,465 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT,466 (hasArchitectedFlatScratch(STI)467 ? ".amdhsa_enable_private_segment"468 : ".amdhsa_system_sgpr_private_segment_wavefront_offset"));469 PrintField(KD.compute_pgm_rsrc2,470 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X_SHIFT,471 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X,472 ".amdhsa_system_sgpr_workgroup_id_x");473 PrintField(KD.compute_pgm_rsrc2,474 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y_SHIFT,475 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y,476 ".amdhsa_system_sgpr_workgroup_id_y");477 PrintField(KD.compute_pgm_rsrc2,478 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z_SHIFT,479 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z,480 ".amdhsa_system_sgpr_workgroup_id_z");481 PrintField(KD.compute_pgm_rsrc2,482 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO_SHIFT,483 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO,484 ".amdhsa_system_sgpr_workgroup_info");485 PrintField(KD.compute_pgm_rsrc2,486 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID_SHIFT,487 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID,488 ".amdhsa_system_vgpr_workitem_id");489 490 // These directives are required.491 OS << "\t\t.amdhsa_next_free_vgpr ";492 EmitMCExpr(NextVGPR);493 OS << '\n';494 495 OS << "\t\t.amdhsa_next_free_sgpr ";496 EmitMCExpr(NextSGPR);497 OS << '\n';498 499 if (AMDGPU::isGFX90A(STI)) {500 // MCExpr equivalent of taking the (accum_offset + 1) * 4.501 const MCExpr *accum_bits = MCKernelDescriptor::bits_get(502 KD.compute_pgm_rsrc3,503 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,504 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, getContext());505 accum_bits = MCBinaryExpr::createAdd(506 accum_bits, MCConstantExpr::create(1, getContext()), getContext());507 accum_bits = MCBinaryExpr::createMul(508 accum_bits, MCConstantExpr::create(4, getContext()), getContext());509 OS << "\t\t.amdhsa_accum_offset ";510 const MCExpr *New = foldAMDGPUMCExpr(accum_bits, getContext());511 printAMDGPUMCExpr(New, OS, MAI);512 OS << '\n';513 }514 515 if (AMDGPU::isGFX1250(STI))516 PrintField(KD.compute_pgm_rsrc3,517 amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,518 amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,519 ".amdhsa_named_barrier_count");520 521 OS << "\t\t.amdhsa_reserve_vcc ";522 EmitMCExpr(ReserveVCC);523 OS << '\n';524 525 if (IVersion.Major >= 7 && !hasArchitectedFlatScratch(STI)) {526 OS << "\t\t.amdhsa_reserve_flat_scratch ";527 EmitMCExpr(ReserveFlatScr);528 OS << '\n';529 }530 531 switch (CodeObjectVersion) {532 default:533 break;534 case AMDGPU::AMDHSA_COV4:535 case AMDGPU::AMDHSA_COV5:536 if (getTargetID()->isXnackSupported())537 OS << "\t\t.amdhsa_reserve_xnack_mask " << getTargetID()->isXnackOnOrAny() << '\n';538 break;539 }540 541 PrintField(KD.compute_pgm_rsrc1,542 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32_SHIFT,543 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32,544 ".amdhsa_float_round_mode_32");545 PrintField(KD.compute_pgm_rsrc1,546 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64_SHIFT,547 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64,548 ".amdhsa_float_round_mode_16_64");549 PrintField(KD.compute_pgm_rsrc1,550 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32_SHIFT,551 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32,552 ".amdhsa_float_denorm_mode_32");553 PrintField(KD.compute_pgm_rsrc1,554 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64_SHIFT,555 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64,556 ".amdhsa_float_denorm_mode_16_64");557 if (IVersion.Major < 12) {558 PrintField(KD.compute_pgm_rsrc1,559 amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP_SHIFT,560 amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP,561 ".amdhsa_dx10_clamp");562 PrintField(KD.compute_pgm_rsrc1,563 amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE_SHIFT,564 amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE,565 ".amdhsa_ieee_mode");566 }567 if (IVersion.Major >= 9) {568 PrintField(KD.compute_pgm_rsrc1,569 amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL_SHIFT,570 amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL,571 ".amdhsa_fp16_overflow");572 }573 if (AMDGPU::isGFX90A(STI))574 PrintField(KD.compute_pgm_rsrc3,575 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT,576 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, ".amdhsa_tg_split");577 if (AMDGPU::supportsWGP(STI))578 PrintField(KD.compute_pgm_rsrc1,579 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE_SHIFT,580 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE,581 ".amdhsa_workgroup_processor_mode");582 if (IVersion.Major >= 10) {583 PrintField(KD.compute_pgm_rsrc1,584 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED_SHIFT,585 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED,586 ".amdhsa_memory_ordered");587 PrintField(KD.compute_pgm_rsrc1,588 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS_SHIFT,589 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS,590 ".amdhsa_forward_progress");591 }592 if (IVersion.Major >= 10 && IVersion.Major < 12) {593 PrintField(KD.compute_pgm_rsrc3,594 amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT_SHIFT,595 amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT,596 ".amdhsa_shared_vgpr_count");597 }598 if (IVersion.Major == 11) {599 PrintField(KD.compute_pgm_rsrc3,600 amdhsa::COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE_SHIFT,601 amdhsa::COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE,602 ".amdhsa_inst_pref_size");603 }604 if (IVersion.Major >= 12) {605 PrintField(KD.compute_pgm_rsrc3,606 amdhsa::COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE_SHIFT,607 amdhsa::COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE,608 ".amdhsa_inst_pref_size");609 PrintField(KD.compute_pgm_rsrc1,610 amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN_SHIFT,611 amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN,612 ".amdhsa_round_robin_scheduling");613 }614 PrintField(615 KD.compute_pgm_rsrc2,616 amdhsa::617 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION_SHIFT,618 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,619 ".amdhsa_exception_fp_ieee_invalid_op");620 PrintField(621 KD.compute_pgm_rsrc2,622 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE_SHIFT,623 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,624 ".amdhsa_exception_fp_denorm_src");625 PrintField(626 KD.compute_pgm_rsrc2,627 amdhsa::628 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO_SHIFT,629 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,630 ".amdhsa_exception_fp_ieee_div_zero");631 PrintField(632 KD.compute_pgm_rsrc2,633 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW_SHIFT,634 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,635 ".amdhsa_exception_fp_ieee_overflow");636 PrintField(637 KD.compute_pgm_rsrc2,638 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW_SHIFT,639 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,640 ".amdhsa_exception_fp_ieee_underflow");641 PrintField(642 KD.compute_pgm_rsrc2,643 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT_SHIFT,644 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,645 ".amdhsa_exception_fp_ieee_inexact");646 PrintField(647 KD.compute_pgm_rsrc2,648 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO_SHIFT,649 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,650 ".amdhsa_exception_int_div_zero");651 652 OS << "\t.end_amdhsa_kernel\n";653}654 655//===----------------------------------------------------------------------===//656// AMDGPUTargetELFStreamer657//===----------------------------------------------------------------------===//658 659AMDGPUTargetELFStreamer::AMDGPUTargetELFStreamer(MCStreamer &S,660 const MCSubtargetInfo &STI)661 : AMDGPUTargetStreamer(S), STI(STI), Streamer(S) {}662 663MCELFStreamer &AMDGPUTargetELFStreamer::getStreamer() {664 return static_cast<MCELFStreamer &>(Streamer);665}666 667// A hook for emitting stuff at the end.668// We use it for emitting the accumulated PAL metadata as a .note record.669// The PAL metadata is reset after it is emitted.670void AMDGPUTargetELFStreamer::finish() {671 ELFObjectWriter &W = getStreamer().getWriter();672 W.setELFHeaderEFlags(getEFlags());673 W.setOverrideABIVersion(674 getELFABIVersion(STI.getTargetTriple(), CodeObjectVersion));675 676 std::string Blob;677 const char *Vendor = getPALMetadata()->getVendor();678 unsigned Type = getPALMetadata()->getType();679 getPALMetadata()->toBlob(Type, Blob);680 if (Blob.empty())681 return;682 EmitNote(Vendor, MCConstantExpr::create(Blob.size(), getContext()), Type,683 [&](MCELFStreamer &OS) { OS.emitBytes(Blob); });684 685 // Reset the pal metadata so its data will not affect a compilation that686 // reuses this object.687 getPALMetadata()->reset();688}689 690void AMDGPUTargetELFStreamer::EmitNote(691 StringRef Name, const MCExpr *DescSZ, unsigned NoteType,692 function_ref<void(MCELFStreamer &)> EmitDesc) {693 auto &S = getStreamer();694 auto &Context = S.getContext();695 696 auto NameSZ = Name.size() + 1;697 698 unsigned NoteFlags = 0;699 // TODO Apparently, this is currently needed for OpenCL as mentioned in700 // https://reviews.llvm.org/D74995701 if (isHsaAbi(STI))702 NoteFlags = ELF::SHF_ALLOC;703 704 S.pushSection();705 S.switchSection(706 Context.getELFSection(ElfNote::SectionName, ELF::SHT_NOTE, NoteFlags));707 S.emitInt32(NameSZ); // namesz708 S.emitValue(DescSZ, 4); // descz709 S.emitInt32(NoteType); // type710 S.emitBytes(Name); // name711 S.emitValueToAlignment(Align(4), 0, 1, 0); // padding 0712 EmitDesc(S); // desc713 S.emitValueToAlignment(Align(4), 0, 1, 0); // padding 0714 S.popSection();715}716 717unsigned AMDGPUTargetELFStreamer::getEFlags() {718 switch (STI.getTargetTriple().getArch()) {719 default:720 llvm_unreachable("Unsupported Arch");721 case Triple::r600:722 return getEFlagsR600();723 case Triple::amdgcn:724 return getEFlagsAMDGCN();725 }726}727 728unsigned AMDGPUTargetELFStreamer::getEFlagsR600() {729 assert(STI.getTargetTriple().getArch() == Triple::r600);730 731 return getElfMach(STI.getCPU());732}733 734unsigned AMDGPUTargetELFStreamer::getEFlagsAMDGCN() {735 assert(STI.getTargetTriple().isAMDGCN());736 737 switch (STI.getTargetTriple().getOS()) {738 default:739 // TODO: Why are some tests have "mingw" listed as OS?740 // llvm_unreachable("Unsupported OS");741 case Triple::UnknownOS:742 return getEFlagsUnknownOS();743 case Triple::AMDHSA:744 return getEFlagsAMDHSA();745 case Triple::AMDPAL:746 return getEFlagsAMDPAL();747 case Triple::Mesa3D:748 return getEFlagsMesa3D();749 }750}751 752unsigned AMDGPUTargetELFStreamer::getEFlagsUnknownOS() {753 // TODO: Why are some tests have "mingw" listed as OS?754 // assert(STI.getTargetTriple().getOS() == Triple::UnknownOS);755 756 return getEFlagsV3();757}758 759unsigned AMDGPUTargetELFStreamer::getEFlagsAMDHSA() {760 assert(isHsaAbi(STI));761 762 if (CodeObjectVersion >= 6)763 return getEFlagsV6();764 return getEFlagsV4();765}766 767unsigned AMDGPUTargetELFStreamer::getEFlagsAMDPAL() {768 assert(STI.getTargetTriple().getOS() == Triple::AMDPAL);769 770 return getEFlagsV3();771}772 773unsigned AMDGPUTargetELFStreamer::getEFlagsMesa3D() {774 assert(STI.getTargetTriple().getOS() == Triple::Mesa3D);775 776 return getEFlagsV3();777}778 779unsigned AMDGPUTargetELFStreamer::getEFlagsV3() {780 unsigned EFlagsV3 = 0;781 782 // mach.783 EFlagsV3 |= getElfMach(STI.getCPU());784 785 // xnack.786 if (getTargetID()->isXnackOnOrAny())787 EFlagsV3 |= ELF::EF_AMDGPU_FEATURE_XNACK_V3;788 // sramecc.789 if (getTargetID()->isSramEccOnOrAny())790 EFlagsV3 |= ELF::EF_AMDGPU_FEATURE_SRAMECC_V3;791 792 return EFlagsV3;793}794 795unsigned AMDGPUTargetELFStreamer::getEFlagsV4() {796 unsigned EFlagsV4 = 0;797 798 // mach.799 EFlagsV4 |= getElfMach(STI.getCPU());800 801 // xnack.802 switch (getTargetID()->getXnackSetting()) {803 case AMDGPU::IsaInfo::TargetIDSetting::Unsupported:804 EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_XNACK_UNSUPPORTED_V4;805 break;806 case AMDGPU::IsaInfo::TargetIDSetting::Any:807 EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_XNACK_ANY_V4;808 break;809 case AMDGPU::IsaInfo::TargetIDSetting::Off:810 EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_XNACK_OFF_V4;811 break;812 case AMDGPU::IsaInfo::TargetIDSetting::On:813 EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_XNACK_ON_V4;814 break;815 }816 // sramecc.817 switch (getTargetID()->getSramEccSetting()) {818 case AMDGPU::IsaInfo::TargetIDSetting::Unsupported:819 EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_SRAMECC_UNSUPPORTED_V4;820 break;821 case AMDGPU::IsaInfo::TargetIDSetting::Any:822 EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_SRAMECC_ANY_V4;823 break;824 case AMDGPU::IsaInfo::TargetIDSetting::Off:825 EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_SRAMECC_OFF_V4;826 break;827 case AMDGPU::IsaInfo::TargetIDSetting::On:828 EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_SRAMECC_ON_V4;829 break;830 }831 832 return EFlagsV4;833}834 835unsigned AMDGPUTargetELFStreamer::getEFlagsV6() {836 unsigned Flags = getEFlagsV4();837 838 unsigned Version = ForceGenericVersion;839 if (!Version) {840 switch (parseArchAMDGCN(STI.getCPU())) {841 case AMDGPU::GK_GFX9_GENERIC:842 Version = GenericVersion::GFX9;843 break;844 case AMDGPU::GK_GFX9_4_GENERIC:845 Version = GenericVersion::GFX9_4;846 break;847 case AMDGPU::GK_GFX10_1_GENERIC:848 Version = GenericVersion::GFX10_1;849 break;850 case AMDGPU::GK_GFX10_3_GENERIC:851 Version = GenericVersion::GFX10_3;852 break;853 case AMDGPU::GK_GFX11_GENERIC:854 Version = GenericVersion::GFX11;855 break;856 case AMDGPU::GK_GFX12_GENERIC:857 Version = GenericVersion::GFX12;858 break;859 default:860 break;861 }862 }863 864 // Versions start at 1.865 if (Version) {866 if (Version > ELF::EF_AMDGPU_GENERIC_VERSION_MAX)867 report_fatal_error("Cannot encode generic code object version " +868 Twine(Version) +869 " - no ELF flag can represent this version!");870 Flags |= (Version << ELF::EF_AMDGPU_GENERIC_VERSION_OFFSET);871 }872 873 return Flags;874}875 876void AMDGPUTargetELFStreamer::EmitDirectiveAMDGCNTarget() {}877 878void AMDGPUTargetELFStreamer::EmitAMDKernelCodeT(AMDGPUMCKernelCodeT &Header) {879 MCStreamer &OS = getStreamer();880 OS.pushSection();881 Header.EmitKernelCodeT(OS, getContext());882 OS.popSection();883}884 885void AMDGPUTargetELFStreamer::EmitAMDGPUSymbolType(StringRef SymbolName,886 unsigned Type) {887 auto *Symbol = static_cast<MCSymbolELF *>(888 getStreamer().getContext().getOrCreateSymbol(SymbolName));889 Symbol->setType(Type);890}891 892void AMDGPUTargetELFStreamer::emitAMDGPULDS(MCSymbol *Symbol, unsigned Size,893 Align Alignment) {894 auto *SymbolELF = static_cast<MCSymbolELF *>(Symbol);895 SymbolELF->setType(ELF::STT_OBJECT);896 897 if (!SymbolELF->isBindingSet())898 SymbolELF->setBinding(ELF::STB_GLOBAL);899 900 if (SymbolELF->declareCommon(Size, Alignment)) {901 report_fatal_error("Symbol: " + Symbol->getName() +902 " redeclared as different type");903 }904 905 SymbolELF->setIndex(ELF::SHN_AMDGPU_LDS);906 SymbolELF->setSize(MCConstantExpr::create(Size, getContext()));907}908 909bool AMDGPUTargetELFStreamer::EmitISAVersion() {910 // Create two labels to mark the beginning and end of the desc field911 // and a MCExpr to calculate the size of the desc field.912 auto &Context = getContext();913 auto *DescBegin = Context.createTempSymbol();914 auto *DescEnd = Context.createTempSymbol();915 auto *DescSZ = MCBinaryExpr::createSub(916 MCSymbolRefExpr::create(DescEnd, Context),917 MCSymbolRefExpr::create(DescBegin, Context), Context);918 919 EmitNote(ElfNote::NoteNameV2, DescSZ, ELF::NT_AMD_HSA_ISA_NAME,920 [&](MCELFStreamer &OS) {921 OS.emitLabel(DescBegin);922 OS.emitBytes(getTargetID()->toString());923 OS.emitLabel(DescEnd);924 });925 return true;926}927 928bool AMDGPUTargetELFStreamer::EmitHSAMetadata(msgpack::Document &HSAMetadataDoc,929 bool Strict) {930 HSAMD::V3::MetadataVerifier Verifier(Strict);931 if (!Verifier.verify(HSAMetadataDoc.getRoot()))932 return false;933 934 std::string HSAMetadataString;935 HSAMetadataDoc.writeToBlob(HSAMetadataString);936 937 // Create two labels to mark the beginning and end of the desc field938 // and a MCExpr to calculate the size of the desc field.939 auto &Context = getContext();940 auto *DescBegin = Context.createTempSymbol();941 auto *DescEnd = Context.createTempSymbol();942 auto *DescSZ = MCBinaryExpr::createSub(943 MCSymbolRefExpr::create(DescEnd, Context),944 MCSymbolRefExpr::create(DescBegin, Context), Context);945 946 EmitNote(ElfNote::NoteNameV3, DescSZ, ELF::NT_AMDGPU_METADATA,947 [&](MCELFStreamer &OS) {948 OS.emitLabel(DescBegin);949 OS.emitBytes(HSAMetadataString);950 OS.emitLabel(DescEnd);951 });952 return true;953}954 955bool AMDGPUTargetELFStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) {956 const uint32_t Encoded_s_code_end = 0xbf9f0000;957 const uint32_t Encoded_s_nop = 0xbf800000;958 uint32_t Encoded_pad = Encoded_s_code_end;959 960 // Instruction cache line size in bytes.961 const unsigned Log2CacheLineSize = AMDGPU::isGFX11Plus(STI) ? 7 : 6;962 const unsigned CacheLineSize = 1u << Log2CacheLineSize;963 964 // Extra padding amount in bytes to support prefetch mode 3.965 unsigned FillSize = 3 * CacheLineSize;966 967 if (AMDGPU::isGFX90A(STI)) {968 Encoded_pad = Encoded_s_nop;969 FillSize = 16 * CacheLineSize;970 }971 972 MCStreamer &OS = getStreamer();973 OS.pushSection();974 OS.emitValueToAlignment(Align(CacheLineSize), Encoded_pad, 4);975 for (unsigned I = 0; I < FillSize; I += 4)976 OS.emitInt32(Encoded_pad);977 OS.popSection();978 return true;979}980 981void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor(982 const MCSubtargetInfo &STI, StringRef KernelName,983 const MCKernelDescriptor &KernelDescriptor, const MCExpr *NextVGPR,984 const MCExpr *NextSGPR, const MCExpr *ReserveVCC,985 const MCExpr *ReserveFlatScr) {986 auto &Streamer = getStreamer();987 auto &Context = Streamer.getContext();988 989 auto *KernelCodeSymbol =990 static_cast<MCSymbolELF *>(Context.getOrCreateSymbol(Twine(KernelName)));991 auto *KernelDescriptorSymbol = static_cast<MCSymbolELF *>(992 Context.getOrCreateSymbol(Twine(KernelName) + Twine(".kd")));993 994 // Copy kernel descriptor symbol's binding, other and visibility from the995 // kernel code symbol.996 KernelDescriptorSymbol->setBinding(KernelCodeSymbol->getBinding());997 KernelDescriptorSymbol->setOther(KernelCodeSymbol->getOther());998 KernelDescriptorSymbol->setVisibility(KernelCodeSymbol->getVisibility());999 // Kernel descriptor symbol's type and size are fixed.1000 KernelDescriptorSymbol->setType(ELF::STT_OBJECT);1001 KernelDescriptorSymbol->setSize(1002 MCConstantExpr::create(sizeof(amdhsa::kernel_descriptor_t), Context));1003 1004 // The visibility of the kernel code symbol must be protected or less to allow1005 // static relocations from the kernel descriptor to be used.1006 if (KernelCodeSymbol->getVisibility() == ELF::STV_DEFAULT)1007 KernelCodeSymbol->setVisibility(ELF::STV_PROTECTED);1008 1009 Streamer.emitLabel(KernelDescriptorSymbol);1010 Streamer.emitValue(1011 KernelDescriptor.group_segment_fixed_size,1012 sizeof(amdhsa::kernel_descriptor_t::group_segment_fixed_size));1013 Streamer.emitValue(1014 KernelDescriptor.private_segment_fixed_size,1015 sizeof(amdhsa::kernel_descriptor_t::private_segment_fixed_size));1016 Streamer.emitValue(KernelDescriptor.kernarg_size,1017 sizeof(amdhsa::kernel_descriptor_t::kernarg_size));1018 1019 for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved0); ++i)1020 Streamer.emitInt8(0u);1021 1022 // FIXME: Remove the use of VK_AMDGPU_REL64 in the expression below. The1023 // expression being created is:1024 // (start of kernel code) - (start of kernel descriptor)1025 // It implies R_AMDGPU_REL64, but ends up being R_AMDGPU_ABS64.1026 Streamer.emitValue(1027 MCBinaryExpr::createSub(1028 MCSymbolRefExpr::create(KernelCodeSymbol, AMDGPUMCExpr::S_REL64,1029 Context),1030 MCSymbolRefExpr::create(KernelDescriptorSymbol, Context), Context),1031 sizeof(amdhsa::kernel_descriptor_t::kernel_code_entry_byte_offset));1032 for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved1); ++i)1033 Streamer.emitInt8(0u);1034 Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc3,1035 sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc3));1036 Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc1,1037 sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc1));1038 Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc2,1039 sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc2));1040 Streamer.emitValue(1041 KernelDescriptor.kernel_code_properties,1042 sizeof(amdhsa::kernel_descriptor_t::kernel_code_properties));1043 Streamer.emitValue(KernelDescriptor.kernarg_preload,1044 sizeof(amdhsa::kernel_descriptor_t::kernarg_preload));1045 for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved3); ++i)1046 Streamer.emitInt8(0u);1047}1048