368 lines · cpp
1//===AMDGPUAsanInstrumentation.cpp - ASAN related helper functions===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===-------------------------------------------------------------===//8 9#include "AMDGPUAsanInstrumentation.h"10 11#define DEBUG_TYPE "amdgpu-asan-instrumentation"12 13using namespace llvm;14 15namespace llvm {16namespace AMDGPU {17 18static uint64_t getRedzoneSizeForScale(int AsanScale) {19 // Redzone used for stack and globals is at least 32 bytes.20 // For scales 6 and 7, the redzone has to be 64 and 128 bytes respectively.21 return std::max(32U, 1U << AsanScale);22}23 24static uint64_t getMinRedzoneSizeForGlobal(int AsanScale) {25 return getRedzoneSizeForScale(AsanScale);26}27 28uint64_t getRedzoneSizeForGlobal(int AsanScale, uint64_t SizeInBytes) {29 constexpr uint64_t kMaxRZ = 1 << 18;30 const uint64_t MinRZ = getMinRedzoneSizeForGlobal(AsanScale);31 32 uint64_t RZ = 0;33 if (SizeInBytes <= MinRZ / 2) {34 // Reduce redzone size for small size objects, e.g. int, char[1]. MinRZ is35 // at least 32 bytes, optimize when SizeInBytes is less than or equal to36 // half of MinRZ.37 RZ = MinRZ - SizeInBytes;38 } else {39 // Calculate RZ, where MinRZ <= RZ <= MaxRZ, and RZ ~ 1/4 * SizeInBytes.40 RZ = std::clamp((SizeInBytes / MinRZ / 4) * MinRZ, MinRZ, kMaxRZ);41 42 // Round up to multiple of MinRZ.43 if (SizeInBytes % MinRZ)44 RZ += MinRZ - (SizeInBytes % MinRZ);45 }46 47 assert((RZ + SizeInBytes) % MinRZ == 0);48 49 return RZ;50}51 52static size_t TypeStoreSizeToSizeIndex(uint32_t TypeSize) {53 size_t Res = llvm::countr_zero(TypeSize / 8);54 return Res;55}56 57static Instruction *genAMDGPUReportBlock(Module &M, IRBuilder<> &IRB,58 Value *Cond, bool Recover) {59 Value *ReportCond = Cond;60 if (!Recover) {61 auto *Ballot =62 IRB.CreateIntrinsic(Intrinsic::amdgcn_ballot, IRB.getInt64Ty(), {Cond});63 ReportCond = IRB.CreateIsNotNull(Ballot);64 }65 66 auto *Trm = SplitBlockAndInsertIfThen(67 ReportCond, &*IRB.GetInsertPoint(), false,68 MDBuilder(M.getContext()).createUnlikelyBranchWeights());69 Trm->getParent()->setName("asan.report");70 71 if (Recover)72 return Trm;73 74 Trm = SplitBlockAndInsertIfThen(Cond, Trm, false);75 IRB.SetInsertPoint(Trm);76 return IRB.CreateIntrinsic(Intrinsic::amdgcn_unreachable, {});77}78 79static Value *createSlowPathCmp(Module &M, IRBuilder<> &IRB, Type *IntptrTy,80 Value *AddrLong, Value *ShadowValue,81 uint32_t TypeStoreSize, int AsanScale) {82 uint64_t Granularity = static_cast<uint64_t>(1) << AsanScale;83 // Addr & (Granularity - 1)84 Value *LastAccessedByte =85 IRB.CreateAnd(AddrLong, ConstantInt::get(IntptrTy, Granularity - 1));86 // (Addr & (Granularity - 1)) + size - 187 if (TypeStoreSize / 8 > 1)88 LastAccessedByte = IRB.CreateAdd(89 LastAccessedByte, ConstantInt::get(IntptrTy, TypeStoreSize / 8 - 1));90 // (uint8_t) ((Addr & (Granularity-1)) + size - 1)91 LastAccessedByte =92 IRB.CreateIntCast(LastAccessedByte, ShadowValue->getType(), false);93 // ((uint8_t) ((Addr & (Granularity-1)) + size - 1)) >= ShadowValue94 return IRB.CreateICmpSGE(LastAccessedByte, ShadowValue);95}96 97static Instruction *generateCrashCode(Module &M, IRBuilder<> &IRB,98 Type *IntptrTy, Instruction *InsertBefore,99 Value *Addr, bool IsWrite,100 size_t AccessSizeIndex,101 Value *SizeArgument, bool Recover) {102 IRB.SetInsertPoint(InsertBefore);103 CallInst *Call = nullptr;104 SmallString<128> kAsanReportErrorTemplate{"__asan_report_"};105 SmallString<64> TypeStr{IsWrite ? "store" : "load"};106 SmallString<64> EndingStr{Recover ? "_noabort" : ""};107 108 SmallString<128> AsanErrorCallbackSizedString;109 raw_svector_ostream AsanErrorCallbackSizedOS(AsanErrorCallbackSizedString);110 AsanErrorCallbackSizedOS << kAsanReportErrorTemplate << TypeStr << "_n"111 << EndingStr;112 113 SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};114 AttributeList AL2;115 FunctionCallee AsanErrorCallbackSized = M.getOrInsertFunction(116 AsanErrorCallbackSizedOS.str(),117 FunctionType::get(IRB.getVoidTy(), Args2, false), AL2);118 SmallVector<Type *, 2> Args1{1, IntptrTy};119 AttributeList AL1;120 121 SmallString<128> AsanErrorCallbackString;122 raw_svector_ostream AsanErrorCallbackOS(AsanErrorCallbackString);123 AsanErrorCallbackOS << kAsanReportErrorTemplate << TypeStr124 << (1ULL << AccessSizeIndex) << EndingStr;125 126 FunctionCallee AsanErrorCallback = M.getOrInsertFunction(127 AsanErrorCallbackOS.str(),128 FunctionType::get(IRB.getVoidTy(), Args1, false), AL1);129 if (SizeArgument) {130 Call = IRB.CreateCall(AsanErrorCallbackSized, {Addr, SizeArgument});131 } else {132 Call = IRB.CreateCall(AsanErrorCallback, Addr);133 }134 135 Call->setCannotMerge();136 return Call;137}138 139static Value *memToShadow(Module &M, IRBuilder<> &IRB, Type *IntptrTy,140 Value *Shadow, int AsanScale, uint32_t AsanOffset) {141 // Shadow >> scale142 Shadow = IRB.CreateLShr(Shadow, AsanScale);143 if (AsanOffset == 0)144 return Shadow;145 // (Shadow >> scale) | offset146 Value *ShadowBase = ConstantInt::get(IntptrTy, AsanOffset);147 return IRB.CreateAdd(Shadow, ShadowBase);148}149 150static void instrumentAddressImpl(Module &M, IRBuilder<> &IRB,151 Instruction *OrigIns,152 Instruction *InsertBefore, Value *Addr,153 Align Alignment, uint32_t TypeStoreSize,154 bool IsWrite, Value *SizeArgument,155 bool UseCalls, bool Recover, int AsanScale,156 int AsanOffset) {157 Type *AddrTy = Addr->getType();158 Type *IntptrTy = M.getDataLayout().getIntPtrType(159 M.getContext(), AddrTy->getPointerAddressSpace());160 IRB.SetInsertPoint(InsertBefore);161 size_t AccessSizeIndex = TypeStoreSizeToSizeIndex(TypeStoreSize);162 Type *ShadowTy = IntegerType::get(M.getContext(),163 std::max(8U, TypeStoreSize >> AsanScale));164 Type *ShadowPtrTy = PointerType::get(M.getContext(), 0);165 Value *AddrLong = IRB.CreatePtrToInt(Addr, IntptrTy);166 Value *ShadowPtr =167 memToShadow(M, IRB, IntptrTy, AddrLong, AsanScale, AsanOffset);168 const uint64_t ShadowAlign =169 std::max<uint64_t>(Alignment.value() >> AsanScale, 1);170 Value *ShadowValue = IRB.CreateAlignedLoad(171 ShadowTy, IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy), Align(ShadowAlign));172 Value *Cmp = IRB.CreateIsNotNull(ShadowValue);173 auto *Cmp2 = createSlowPathCmp(M, IRB, IntptrTy, AddrLong, ShadowValue,174 TypeStoreSize, AsanScale);175 Cmp = IRB.CreateAnd(Cmp, Cmp2);176 Instruction *CrashTerm = genAMDGPUReportBlock(M, IRB, Cmp, Recover);177 Instruction *Crash =178 generateCrashCode(M, IRB, IntptrTy, CrashTerm, AddrLong, IsWrite,179 AccessSizeIndex, SizeArgument, Recover);180 Crash->setDebugLoc(OrigIns->getDebugLoc());181}182 183void instrumentAddress(Module &M, IRBuilder<> &IRB, Instruction *OrigIns,184 Instruction *InsertBefore, Value *Addr, Align Alignment,185 TypeSize TypeStoreSize, bool IsWrite,186 Value *SizeArgument, bool UseCalls, bool Recover,187 int AsanScale, int AsanOffset) {188 if (!TypeStoreSize.isScalable()) {189 unsigned Granularity = 1 << AsanScale;190 const auto FixedSize = TypeStoreSize.getFixedValue();191 switch (FixedSize) {192 case 8:193 case 16:194 case 32:195 case 64:196 case 128:197 if (Alignment.value() >= Granularity ||198 Alignment.value() >= FixedSize / 8)199 return instrumentAddressImpl(200 M, IRB, OrigIns, InsertBefore, Addr, Alignment, FixedSize, IsWrite,201 SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);202 }203 }204 // Instrument unusual size or unusual alignment.205 IRB.SetInsertPoint(InsertBefore);206 Type *AddrTy = Addr->getType();207 Type *IntptrTy = M.getDataLayout().getIntPtrType(AddrTy);208 Value *NumBits = IRB.CreateTypeSize(IntptrTy, TypeStoreSize);209 Value *Size = IRB.CreateLShr(NumBits, ConstantInt::get(IntptrTy, 3));210 Value *AddrLong = IRB.CreatePtrToInt(Addr, IntptrTy);211 Value *SizeMinusOne = IRB.CreateAdd(Size, ConstantInt::get(IntptrTy, -1));212 Value *LastByte =213 IRB.CreateIntToPtr(IRB.CreateAdd(AddrLong, SizeMinusOne), AddrTy);214 instrumentAddressImpl(M, IRB, OrigIns, InsertBefore, Addr, {}, 8, IsWrite,215 SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);216 instrumentAddressImpl(M, IRB, OrigIns, InsertBefore, LastByte, {}, 8, IsWrite,217 SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);218}219 220void getInterestingMemoryOperands(221 Module &M, Instruction *I,222 SmallVectorImpl<InterestingMemoryOperand> &Interesting) {223 const DataLayout &DL = M.getDataLayout();224 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {225 Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,226 LI->getType(), LI->getAlign());227 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {228 Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,229 SI->getValueOperand()->getType(), SI->getAlign());230 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {231 Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,232 RMW->getValOperand()->getType(), std::nullopt);233 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {234 Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,235 XCHG->getCompareOperand()->getType(),236 std::nullopt);237 } else if (auto *CI = dyn_cast<CallInst>(I)) {238 switch (CI->getIntrinsicID()) {239 case Intrinsic::masked_load:240 case Intrinsic::masked_store:241 case Intrinsic::masked_gather:242 case Intrinsic::masked_scatter: {243 bool IsWrite = CI->getType()->isVoidTy();244 // Masked store has an initial operand for the value.245 unsigned OpOffset = IsWrite ? 1 : 0;246 Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();247 MaybeAlign Alignment = CI->getParamAlign(OpOffset);248 Value *Mask = CI->getOperand(1 + OpOffset);249 Interesting.emplace_back(I, OpOffset, IsWrite, Ty, Alignment, Mask);250 break;251 }252 case Intrinsic::masked_expandload:253 case Intrinsic::masked_compressstore: {254 bool IsWrite = CI->getIntrinsicID() == Intrinsic::masked_compressstore;255 unsigned OpOffset = IsWrite ? 1 : 0;256 auto *BasePtr = CI->getOperand(OpOffset);257 MaybeAlign Alignment = BasePtr->getPointerAlignment(DL);258 Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();259 IRBuilder<> IB(I);260 Value *Mask = CI->getOperand(1 + OpOffset);261 Type *IntptrTy = M.getDataLayout().getIntPtrType(262 M.getContext(), BasePtr->getType()->getPointerAddressSpace());263 // Use the popcount of Mask as the effective vector length.264 Type *ExtTy = VectorType::get(IntptrTy, cast<VectorType>(Ty));265 Value *ExtMask = IB.CreateZExt(Mask, ExtTy);266 Value *EVL = IB.CreateAddReduce(ExtMask);267 Value *TrueMask = ConstantInt::get(Mask->getType(), 1);268 Interesting.emplace_back(I, OpOffset, IsWrite, Ty, Alignment, TrueMask,269 EVL);270 break;271 }272 case Intrinsic::vp_load:273 case Intrinsic::vp_store:274 case Intrinsic::experimental_vp_strided_load:275 case Intrinsic::experimental_vp_strided_store: {276 auto *VPI = cast<VPIntrinsic>(CI);277 unsigned IID = CI->getIntrinsicID();278 bool IsWrite = CI->getType()->isVoidTy();279 unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);280 Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();281 MaybeAlign Alignment = VPI->getOperand(PtrOpNo)->getPointerAlignment(DL);282 Value *Stride = nullptr;283 if (IID == Intrinsic::experimental_vp_strided_store ||284 IID == Intrinsic::experimental_vp_strided_load) {285 Stride = VPI->getOperand(PtrOpNo + 1);286 // Use the pointer alignment as the element alignment if the stride is a287 // mutiple of the pointer alignment. Otherwise, the element alignment288 // should be Align(1).289 unsigned PointerAlign = Alignment.valueOrOne().value();290 if (!isa<ConstantInt>(Stride) ||291 cast<ConstantInt>(Stride)->getZExtValue() % PointerAlign != 0)292 Alignment = Align(1);293 }294 Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment,295 VPI->getMaskParam(), VPI->getVectorLengthParam(),296 Stride);297 break;298 }299 case Intrinsic::vp_gather:300 case Intrinsic::vp_scatter: {301 auto *VPI = cast<VPIntrinsic>(CI);302 unsigned IID = CI->getIntrinsicID();303 bool IsWrite = IID == Intrinsic::vp_scatter;304 unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);305 Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();306 MaybeAlign Alignment = VPI->getPointerAlignment();307 Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment,308 VPI->getMaskParam(),309 VPI->getVectorLengthParam());310 break;311 }312 case Intrinsic::amdgcn_raw_buffer_load:313 case Intrinsic::amdgcn_raw_ptr_buffer_load:314 case Intrinsic::amdgcn_raw_buffer_load_format:315 case Intrinsic::amdgcn_raw_ptr_buffer_load_format:316 case Intrinsic::amdgcn_raw_tbuffer_load:317 case Intrinsic::amdgcn_raw_ptr_tbuffer_load:318 case Intrinsic::amdgcn_struct_buffer_load:319 case Intrinsic::amdgcn_struct_ptr_buffer_load:320 case Intrinsic::amdgcn_struct_buffer_load_format:321 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:322 case Intrinsic::amdgcn_struct_tbuffer_load:323 case Intrinsic::amdgcn_struct_ptr_tbuffer_load:324 case Intrinsic::amdgcn_s_buffer_load:325 case Intrinsic::amdgcn_global_load_tr_b64:326 case Intrinsic::amdgcn_global_load_tr_b128: {327 unsigned PtrOpNo = 0;328 bool IsWrite = false;329 Type *Ty = CI->getType();330 Value *Ptr = CI->getArgOperand(PtrOpNo);331 MaybeAlign Alignment = Ptr->getPointerAlignment(DL);332 Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment);333 break;334 }335 case Intrinsic::amdgcn_raw_tbuffer_store:336 case Intrinsic::amdgcn_raw_ptr_tbuffer_store:337 case Intrinsic::amdgcn_raw_buffer_store:338 case Intrinsic::amdgcn_raw_ptr_buffer_store:339 case Intrinsic::amdgcn_raw_buffer_store_format:340 case Intrinsic::amdgcn_raw_ptr_buffer_store_format:341 case Intrinsic::amdgcn_struct_buffer_store:342 case Intrinsic::amdgcn_struct_ptr_buffer_store:343 case Intrinsic::amdgcn_struct_buffer_store_format:344 case Intrinsic::amdgcn_struct_ptr_buffer_store_format:345 case Intrinsic::amdgcn_struct_tbuffer_store:346 case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {347 unsigned PtrOpNo = 1;348 bool IsWrite = true;349 Value *Ptr = CI->getArgOperand(PtrOpNo);350 Type *Ty = Ptr->getType();351 MaybeAlign Alignment = Ptr->getPointerAlignment(DL);352 Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment);353 break;354 }355 default:356 for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {357 if (Type *Ty = CI->getParamByRefType(ArgNo)) {358 Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));359 } else if (Type *Ty = CI->getParamByValType(ArgNo)) {360 Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));361 }362 }363 }364 }365}366} // end namespace AMDGPU367} // end namespace llvm368