481 lines · cpp
1//===-------- cfi.cpp -----------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the runtime support for the cross-DSO CFI.10//11//===----------------------------------------------------------------------===//12 13#include <assert.h>14#include <elf.h>15 16#include "sanitizer_common/sanitizer_common.h"17#if SANITIZER_FREEBSD18#include <sys/link_elf.h>19#endif20#include <link.h>21#include <string.h>22#include <stdlib.h>23#include <sys/mman.h>24 25#if SANITIZER_LINUX26typedef ElfW(Phdr) Elf_Phdr;27typedef ElfW(Ehdr) Elf_Ehdr;28typedef ElfW(Addr) Elf_Addr;29typedef ElfW(Sym) Elf_Sym;30typedef ElfW(Dyn) Elf_Dyn;31#elif SANITIZER_FREEBSD32#if SANITIZER_WORDSIZE == 6433#define ElfW64_Dyn Elf_Dyn34#define ElfW64_Sym Elf_Sym35#else36#define ElfW32_Dyn Elf_Dyn37#define ElfW32_Sym Elf_Sym38#endif39#endif40 41#include "interception/interception.h"42#include "sanitizer_common/sanitizer_flag_parser.h"43#include "ubsan/ubsan_init.h"44#include "ubsan/ubsan_flags.h"45 46#ifdef CFI_ENABLE_DIAG47#include "ubsan/ubsan_handlers.h"48#endif49 50using namespace __sanitizer;51 52namespace __cfi {53 54#if SANITIZER_LOONGARCH6455#define kCfiShadowLimitsStorageSize 16384 // 16KiB on loongarch64 per page56#else57#define kCfiShadowLimitsStorageSize 4096 // 1 page58#endif59// Lets hope that the data segment is mapped with 4K pages.60// The pointer to the cfi shadow region is stored at the start of this page.61// The rest of the page is unused and re-mapped read-only.62static union {63 char space[kCfiShadowLimitsStorageSize];64 struct {65 uptr start;66 uptr size;67 } limits;68} cfi_shadow_limits_storage69 __attribute__((aligned(kCfiShadowLimitsStorageSize)));70static constexpr uptr kShadowGranularity = 12;71static constexpr uptr kShadowAlign = 1UL << kShadowGranularity; // 409672 73static constexpr uint16_t kInvalidShadow = 0;74static constexpr uint16_t kUncheckedShadow = 0xFFFFU;75 76// Get the start address of the CFI shadow region.77uptr GetShadow() {78 return cfi_shadow_limits_storage.limits.start;79}80 81uptr GetShadowSize() {82 return cfi_shadow_limits_storage.limits.size;83}84 85// This will only work while the shadow is not allocated.86void SetShadowSize(uptr size) {87 cfi_shadow_limits_storage.limits.size = size;88}89 90uptr MemToShadowOffset(uptr x) {91 return (x >> kShadowGranularity) << 1;92}93 94uint16_t *MemToShadow(uptr x, uptr shadow_base) {95 return (uint16_t *)(shadow_base + MemToShadowOffset(x));96}97 98typedef int (*CFICheckFn)(u64, void *, void *);99 100// This class reads and decodes the shadow contents.101class ShadowValue {102 uptr addr;103 uint16_t v;104 explicit ShadowValue(uptr addr, uint16_t v) : addr(addr), v(v) {}105 106public:107 bool is_invalid() const { return v == kInvalidShadow; }108 109 bool is_unchecked() const { return v == kUncheckedShadow; }110 111 CFICheckFn get_cfi_check() const {112 assert(!is_invalid() && !is_unchecked());113 uptr aligned_addr = addr & ~(kShadowAlign - 1);114 uptr p = aligned_addr - (((uptr)v - 1) << kShadowGranularity);115 return reinterpret_cast<CFICheckFn>(p);116 }117 118 // Load a shadow value for the given application memory address.119 static const ShadowValue load(uptr addr) {120 uptr shadow_base = GetShadow();121 uptr shadow_offset = MemToShadowOffset(addr);122 if (shadow_offset > GetShadowSize())123 return ShadowValue(addr, kInvalidShadow);124 else125 return ShadowValue(126 addr, *reinterpret_cast<uint16_t *>(shadow_base + shadow_offset));127 }128};129 130class ShadowBuilder {131 uptr shadow_;132 133public:134 // Allocate a new empty shadow (for the entire address space) on the side.135 void Start();136 // Mark the given address range as unchecked.137 // This is used for uninstrumented libraries like libc.138 // Any CFI check with a target in that range will pass.139 void AddUnchecked(uptr begin, uptr end);140 // Mark the given address range as belonging to a library with the given141 // cfi_check function.142 void Add(uptr begin, uptr end, uptr cfi_check);143 // Finish shadow construction. Atomically switch the current active shadow144 // region with the newly constructed one and deallocate the former.145 void Install();146};147 148void ShadowBuilder::Start() {149 shadow_ = (uptr)MmapNoReserveOrDie(GetShadowSize(), "CFI shadow");150 VReport(1, "CFI: shadow at %zx .. %zx\n", shadow_, shadow_ + GetShadowSize());151}152 153void ShadowBuilder::AddUnchecked(uptr begin, uptr end) {154 uint16_t *shadow_begin = MemToShadow(begin, shadow_);155 uint16_t *shadow_end = MemToShadow(end - 1, shadow_) + 1;156 // memset takes a byte, so our unchecked shadow value requires both bytes to157 // be the same. Make sure we're ok during compilation.158 static_assert((kUncheckedShadow & 0xff) == ((kUncheckedShadow >> 8) & 0xff),159 "Both bytes of the 16-bit value must be the same!");160 memset(shadow_begin, kUncheckedShadow & 0xff,161 (shadow_end - shadow_begin) * sizeof(*shadow_begin));162}163 164void ShadowBuilder::Add(uptr begin, uptr end, uptr cfi_check) {165 assert((cfi_check & (kShadowAlign - 1)) == 0);166 167 // Don't fill anything below cfi_check. We can not represent those addresses168 // in the shadow, and must make sure at codegen to place all valid call169 // targets above cfi_check.170 begin = Max(begin, cfi_check);171 uint16_t *s = MemToShadow(begin, shadow_);172 uint16_t *s_end = MemToShadow(end - 1, shadow_) + 1;173 uint16_t sv = ((begin - cfi_check) >> kShadowGranularity) + 1;174 for (; s < s_end; s++, sv++)175 *s = sv;176}177 178#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD179void ShadowBuilder::Install() {180 MprotectReadOnly(shadow_, GetShadowSize());181 uptr main_shadow = GetShadow();182 if (main_shadow) {183 // Update.184#if SANITIZER_LINUX185 void *res = mremap((void *)shadow_, GetShadowSize(), GetShadowSize(),186 MREMAP_MAYMOVE | MREMAP_FIXED, (void *)main_shadow);187 CHECK(res != MAP_FAILED);188#elif SANITIZER_NETBSD189 void *res = mremap((void *)shadow_, GetShadowSize(), (void *)main_shadow,190 GetShadowSize(), MAP_FIXED);191 CHECK(res != MAP_FAILED);192#else193 void *res = MmapFixedOrDie(shadow_, GetShadowSize(), "cfi shadow");194 CHECK(res != MAP_FAILED);195 ::memcpy(&shadow_, &main_shadow, GetShadowSize());196#endif197 } else {198 // Initial setup.199 CHECK_EQ(kCfiShadowLimitsStorageSize, GetPageSizeCached());200 CHECK_EQ(0, GetShadow());201 cfi_shadow_limits_storage.limits.start = shadow_;202 MprotectReadOnly((uptr)&cfi_shadow_limits_storage,203 sizeof(cfi_shadow_limits_storage));204 CHECK_EQ(shadow_, GetShadow());205 }206}207#else208#error not implemented209#endif210 211// This is a workaround for a glibc bug:212// https://sourceware.org/bugzilla/show_bug.cgi?id=15199213// Other platforms can, hopefully, just do214// dlopen(RTLD_NOLOAD | RTLD_LAZY)215// dlsym("__cfi_check").216uptr find_cfi_check_in_dso(dl_phdr_info *info) {217 const Elf_Dyn *dynamic = nullptr;218 for (int i = 0; i < info->dlpi_phnum; ++i) {219 if (info->dlpi_phdr[i].p_type == PT_DYNAMIC) {220 dynamic =221 (const Elf_Dyn *)(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr);222 break;223 }224 }225 if (!dynamic) return 0;226 uptr strtab = 0, symtab = 0, strsz = 0;227 for (const Elf_Dyn *p = dynamic; p->d_tag != PT_NULL; ++p) {228 if (p->d_tag == DT_SYMTAB)229 symtab = p->d_un.d_ptr;230 else if (p->d_tag == DT_STRTAB)231 strtab = p->d_un.d_ptr;232 else if (p->d_tag == DT_STRSZ)233 strsz = p->d_un.d_ptr;234 }235 236 if (symtab > strtab) {237 VReport(1, "Can not handle: symtab > strtab (%zx > %zx)\n", symtab, strtab);238 return 0;239 }240 241 // Verify that strtab and symtab are inside of the same LOAD segment.242 // This excludes VDSO, which has (very high) bogus strtab and symtab pointers.243 int phdr_idx;244 for (phdr_idx = 0; phdr_idx < info->dlpi_phnum; phdr_idx++) {245 const Elf_Phdr *phdr = &info->dlpi_phdr[phdr_idx];246 if (phdr->p_type == PT_LOAD) {247 uptr beg = info->dlpi_addr + phdr->p_vaddr;248 uptr end = beg + phdr->p_memsz;249 if (strtab >= beg && strtab + strsz < end && symtab >= beg &&250 symtab < end)251 break;252 }253 }254 if (phdr_idx == info->dlpi_phnum) {255 // Nope, either different segments or just bogus pointers.256 // Can not handle this.257 VReport(1, "Can not handle: symtab %zx, strtab %zx\n", symtab, strtab);258 return 0;259 }260 261 for (const Elf_Sym *p = (const Elf_Sym *)symtab; (Elf_Addr)p < strtab;262 ++p) {263 // There is no reliable way to find the end of the symbol table. In264 // lld-produces files, there are other sections between symtab and strtab.265 // Stop looking when the symbol name is not inside strtab.266 if (p->st_name >= strsz) break;267 char *name = (char*)(strtab + p->st_name);268 if (strcmp(name, "__cfi_check") == 0) {269 assert(p->st_info == ELF32_ST_INFO(STB_GLOBAL, STT_FUNC) ||270 p->st_info == ELF32_ST_INFO(STB_WEAK, STT_FUNC));271 uptr addr = info->dlpi_addr + p->st_value;272 return addr;273 }274 }275 return 0;276}277 278int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *data) {279 uptr cfi_check = find_cfi_check_in_dso(info);280 if (cfi_check)281 VReport(1, "Module '%s' __cfi_check %zx\n", info->dlpi_name, cfi_check);282 283 ShadowBuilder *b = reinterpret_cast<ShadowBuilder *>(data);284 285 for (int i = 0; i < info->dlpi_phnum; i++) {286 const Elf_Phdr *phdr = &info->dlpi_phdr[i];287 if (phdr->p_type == PT_LOAD) {288 // Jump tables are in the executable segment.289 // VTables are in the non-executable one.290 // Need to fill shadow for both.291 // FIXME: reject writable if vtables are in the r/o segment. Depend on292 // PT_RELRO?293 uptr cur_beg = info->dlpi_addr + phdr->p_vaddr;294 uptr cur_end = cur_beg + phdr->p_memsz;295 if (cfi_check) {296 VReport(1, " %zx .. %zx\n", cur_beg, cur_end);297 b->Add(cur_beg, cur_end, cfi_check);298 } else {299 b->AddUnchecked(cur_beg, cur_end);300 }301 }302 }303 return 0;304}305 306// Init or update shadow for the current set of loaded libraries.307void UpdateShadow() {308 ShadowBuilder b;309 b.Start();310 dl_iterate_phdr(dl_iterate_phdr_cb, &b);311 b.Install();312}313 314void InitShadow() {315 CHECK_EQ(0, GetShadow());316 CHECK_EQ(0, GetShadowSize());317 318 uptr vma = GetMaxUserVirtualAddress();319 // Shadow is 2 -> 2**kShadowGranularity.320 SetShadowSize((vma >> (kShadowGranularity - 1)) + 1);321 VReport(1, "CFI: VMA size %zx, shadow size %zx\n", vma, GetShadowSize());322 323 UpdateShadow();324}325 326THREADLOCAL int in_loader;327Mutex shadow_update_lock;328 329void EnterLoader() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {330 if (in_loader == 0) {331 shadow_update_lock.Lock();332 }333 ++in_loader;334}335 336void ExitLoader() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {337 CHECK(in_loader > 0);338 --in_loader;339 UpdateShadow();340 if (in_loader == 0) {341 shadow_update_lock.Unlock();342 }343}344 345ALWAYS_INLINE void CfiSlowPathCommon(u64 CallSiteTypeId, void *Ptr,346 void *DiagData) {347 uptr Addr = (uptr)Ptr;348 VReport(3, "__cfi_slowpath: %llx, %p\n", CallSiteTypeId, Ptr);349 ShadowValue sv = ShadowValue::load(Addr);350 if (sv.is_invalid()) {351 VReport(1, "CFI: invalid memory region for a check target: %p\n", Ptr);352#ifdef CFI_ENABLE_DIAG353 if (DiagData) {354 __ubsan_handle_cfi_check_fail(355 reinterpret_cast<__ubsan::CFICheckFailData *>(DiagData), Addr, false);356 return;357 }358#endif359 Trap();360 }361 if (sv.is_unchecked()) {362 VReport(2, "CFI: unchecked call (shadow=FFFF): %p\n", Ptr);363 return;364 }365 CFICheckFn cfi_check = sv.get_cfi_check();366 VReport(2, "__cfi_check at %p\n", (void *)cfi_check);367 cfi_check(CallSiteTypeId, Ptr, DiagData);368}369 370void InitializeFlags() {371 SetCommonFlagsDefaults();372#ifdef CFI_ENABLE_DIAG373 __ubsan::Flags *uf = __ubsan::flags();374 uf->SetDefaults();375#endif376 377 FlagParser cfi_parser;378 RegisterCommonFlags(&cfi_parser);379 cfi_parser.ParseStringFromEnv("CFI_OPTIONS");380 381#ifdef CFI_ENABLE_DIAG382 FlagParser ubsan_parser;383 __ubsan::RegisterUbsanFlags(&ubsan_parser, uf);384 RegisterCommonFlags(&ubsan_parser);385 386 const char *ubsan_default_options = __ubsan_default_options();387 ubsan_parser.ParseString(ubsan_default_options);388 ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");389#endif390 391 InitializeCommonFlags();392 393 if (Verbosity())394 ReportUnrecognizedFlags();395 396 if (common_flags()->help) {397 cfi_parser.PrintFlagDescriptions();398 }399}400 401} // namespace __cfi402 403using namespace __cfi;404 405extern "C" SANITIZER_INTERFACE_ATTRIBUTE void406__cfi_slowpath(u64 CallSiteTypeId, void *Ptr) {407 CfiSlowPathCommon(CallSiteTypeId, Ptr, nullptr);408}409 410#ifdef CFI_ENABLE_DIAG411extern "C" SANITIZER_INTERFACE_ATTRIBUTE void412__cfi_slowpath_diag(u64 CallSiteTypeId, void *Ptr, void *DiagData) {413 CfiSlowPathCommon(CallSiteTypeId, Ptr, DiagData);414}415#endif416 417static void EnsureInterceptorsInitialized();418 419// Setup shadow for dlopen()ed libraries.420// The actual shadow setup happens after dlopen() returns, which means that421// a library can not be a target of any CFI checks while its constructors are422// running. It's unclear how to fix this without some extra help from libc.423// In glibc, mmap inside dlopen is not interceptable.424// Maybe a seccomp-bpf filter?425// We could insert a high-priority constructor into the library, but that would426// not help with the uninstrumented libraries.427INTERCEPTOR(void*, dlopen, const char *filename, int flag) {428 EnsureInterceptorsInitialized();429 EnterLoader();430 void *handle = REAL(dlopen)(filename, flag);431 ExitLoader();432 return handle;433}434 435INTERCEPTOR(int, dlclose, void *handle) {436 EnsureInterceptorsInitialized();437 EnterLoader();438 int res = REAL(dlclose)(handle);439 ExitLoader();440 return res;441}442 443static Mutex interceptor_init_lock;444static bool interceptors_inited = false;445 446static void EnsureInterceptorsInitialized() {447 Lock lock(&interceptor_init_lock);448 if (interceptors_inited)449 return;450 451 INTERCEPT_FUNCTION(dlopen);452 INTERCEPT_FUNCTION(dlclose);453 454 interceptors_inited = true;455}456 457extern "C" SANITIZER_INTERFACE_ATTRIBUTE458#if !SANITIZER_CAN_USE_PREINIT_ARRAY459// On ELF platforms, the constructor is invoked using .preinit_array (see below)460__attribute__((constructor(0)))461#endif462void __cfi_init() {463 SanitizerToolName = "CFI";464 InitializeFlags();465 InitShadow();466 467#ifdef CFI_ENABLE_DIAG468 __ubsan::InitAsPlugin();469#endif470}471 472#if SANITIZER_CAN_USE_PREINIT_ARRAY473// On ELF platforms, run cfi initialization before any other constructors.474// On other platforms we use the constructor attribute to arrange to run our475// initialization early.476extern "C" {477__attribute__((section(".preinit_array"),478 used)) void (*__cfi_preinit)(void) = __cfi_init;479}480#endif481