322 lines · plain
1//===-- xray_trampoline_x86.s -----------------------------------*- ASM -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file is a part of XRay, a dynamic runtime instrumentation system.10//11// This implements the X86-specific assembler for the trampolines.12//13//===----------------------------------------------------------------------===//14 15#include "../builtins/assembly.h"16#include "../sanitizer_common/sanitizer_asm.h"17 18// XRay trampolines which are not produced by intrinsics are not System V AMD6419// ABI compliant because they are called with a stack that is always misaligned20// by 8 bytes with respect to a 16 bytes alignment. This is because they are21// called immediately after the call to, or immediately before returning from,22// the function being instrumented. This saves space in the patch point, but23// misaligns the stack by 8 bytes.24 25.macro ALIGN_STACK_16B26#if defined(__APPLE__)27 subq $$8, %rsp28#else29 subq $8, %rsp30#endif31 CFI_ADJUST_CFA_OFFSET(8)32.endm33 34.macro RESTORE_STACK_ALIGNMENT35#if defined(__APPLE__)36 addq $$8, %rsp37#else38 addq $8, %rsp39#endif40 CFI_ADJUST_CFA_OFFSET(-8)41.endm42 43// This macro should lower the stack pointer by an odd multiple of 8.44.macro SAVE_REGISTERS45 pushfq46 CFI_ADJUST_CFA_OFFSET(8)47 subq $240, %rsp48 CFI_ADJUST_CFA_OFFSET(240)49 movq %rbp, 232(%rsp)50 movupd %xmm0, 216(%rsp)51 movupd %xmm1, 200(%rsp)52 movupd %xmm2, 184(%rsp)53 movupd %xmm3, 168(%rsp)54 movupd %xmm4, 152(%rsp)55 movupd %xmm5, 136(%rsp)56 movupd %xmm6, 120(%rsp)57 movupd %xmm7, 104(%rsp)58 movq %rdi, 96(%rsp)59 movq %rax, 88(%rsp)60 movq %rdx, 80(%rsp)61 movq %rsi, 72(%rsp)62 movq %rcx, 64(%rsp)63 movq %r8, 56(%rsp)64 movq %r9, 48(%rsp)65 movq %r10, 40(%rsp)66 movq %r11, 32(%rsp)67 movq %r12, 24(%rsp)68 movq %r13, 16(%rsp)69 movq %r14, 8(%rsp)70 movq %r15, 0(%rsp)71.endm72 73.macro RESTORE_REGISTERS74 movq 232(%rsp), %rbp75 movupd 216(%rsp), %xmm076 movupd 200(%rsp), %xmm177 movupd 184(%rsp), %xmm278 movupd 168(%rsp), %xmm379 movupd 152(%rsp), %xmm480 movupd 136(%rsp), %xmm581 movupd 120(%rsp) , %xmm682 movupd 104(%rsp) , %xmm783 movq 96(%rsp), %rdi84 movq 88(%rsp), %rax85 movq 80(%rsp), %rdx86 movq 72(%rsp), %rsi87 movq 64(%rsp), %rcx88 movq 56(%rsp), %r889 movq 48(%rsp), %r990 movq 40(%rsp), %r1091 movq 32(%rsp), %r1192 movq 24(%rsp), %r1293 movq 16(%rsp), %r1394 movq 8(%rsp), %r1495 movq 0(%rsp), %r1596 addq $240, %rsp97 CFI_ADJUST_CFA_OFFSET(-240)98 popfq99 CFI_ADJUST_CFA_OFFSET(-8)100.endm101 102 .text103#if !defined(__APPLE__)104 .section .text105 .file "xray_trampoline_x86.S"106#else107 .section __TEXT,__text108#endif109 110.macro LOAD_HANDLER_ADDR handler111#if !defined(XRAY_PIC)112 movq ASM_SYMBOL(\handler)(%rip), %rax113#else114 movq ASM_SYMBOL(\handler)@GOTPCREL(%rip), %rax115 movq (%rax), %rax116#endif117.endm118 119 120//===----------------------------------------------------------------------===//121 122 .globl ASM_SYMBOL(__xray_FunctionEntry)123 ASM_HIDDEN(__xray_FunctionEntry)124 .align 16, 0x90125 ASM_TYPE_FUNCTION(__xray_FunctionEntry)126# LLVM-MCA-BEGIN __xray_FunctionEntry127ASM_SYMBOL(__xray_FunctionEntry):128 CFI_STARTPROC129 SAVE_REGISTERS130 ALIGN_STACK_16B131 132 // This load has to be atomic, it's concurrent with __xray_patch().133 // On x86/amd64, a simple (type-aligned) MOV instruction is enough.134 LOAD_HANDLER_ADDR _ZN6__xray19XRayPatchedFunctionE135 testq %rax, %rax136 je LOCAL_LABEL(tmp0)137 138 // The patched function prologue puts its xray_instr_map index into %r10d.139 movl %r10d, %edi140 xor %esi,%esi141 callq *%rax142 143LOCAL_LABEL(tmp0):144 RESTORE_STACK_ALIGNMENT145 RESTORE_REGISTERS146 retq147# LLVM-MCA-END148 ASM_SIZE(__xray_FunctionEntry)149 CFI_ENDPROC150 151//===----------------------------------------------------------------------===//152 153 .globl ASM_SYMBOL(__xray_FunctionExit)154 ASM_HIDDEN(__xray_FunctionExit)155 .align 16, 0x90156 ASM_TYPE_FUNCTION(__xray_FunctionExit)157# LLVM-MCA-BEGIN __xray_FunctionExit158ASM_SYMBOL(__xray_FunctionExit):159 CFI_STARTPROC160 ALIGN_STACK_16B161 162 // Save the important registers first. Since we're assuming that this163 // function is only jumped into, we only preserve the registers for164 // returning.165 subq $64, %rsp166 CFI_ADJUST_CFA_OFFSET(64)167 movq %rbp, 48(%rsp)168 movupd %xmm0, 32(%rsp)169 movupd %xmm1, 16(%rsp)170 movq %rax, 8(%rsp)171 movq %rdx, 0(%rsp)172 LOAD_HANDLER_ADDR _ZN6__xray19XRayPatchedFunctionE173 testq %rax,%rax174 je LOCAL_LABEL(tmp2)175 176 movl %r10d, %edi177 movl $1, %esi178 callq *%rax179 180LOCAL_LABEL(tmp2):181 // Restore the important registers.182 movq 48(%rsp), %rbp183 movupd 32(%rsp), %xmm0184 movupd 16(%rsp), %xmm1185 movq 8(%rsp), %rax186 movq 0(%rsp), %rdx187 addq $64, %rsp188 CFI_ADJUST_CFA_OFFSET(-64)189 190 RESTORE_STACK_ALIGNMENT191 retq192# LLVM-MCA-END193 ASM_SIZE(__xray_FunctionExit)194 CFI_ENDPROC195 196//===----------------------------------------------------------------------===//197 198 .globl ASM_SYMBOL(__xray_FunctionTailExit)199 ASM_HIDDEN(__xray_FunctionTailExit)200 .align 16, 0x90201 ASM_TYPE_FUNCTION(__xray_FunctionTailExit)202# LLVM-MCA-BEGIN __xray_FunctionTailExit203ASM_SYMBOL(__xray_FunctionTailExit):204 CFI_STARTPROC205 SAVE_REGISTERS206 ALIGN_STACK_16B207 208 LOAD_HANDLER_ADDR _ZN6__xray19XRayPatchedFunctionE209 testq %rax,%rax210 je LOCAL_LABEL(tmp4)211 212 movl %r10d, %edi213 movl $2, %esi214 callq *%rax215 216LOCAL_LABEL(tmp4):217 RESTORE_STACK_ALIGNMENT218 RESTORE_REGISTERS219 retq220# LLVM-MCA-END221 ASM_SIZE(__xray_FunctionTailExit)222 CFI_ENDPROC223 224//===----------------------------------------------------------------------===//225 226 .globl ASM_SYMBOL(__xray_ArgLoggerEntry)227 ASM_HIDDEN(__xray_ArgLoggerEntry)228 .align 16, 0x90229 ASM_TYPE_FUNCTION(__xray_ArgLoggerEntry)230# LLVM-MCA-BEGIN __xray_ArgLoggerEntry231ASM_SYMBOL(__xray_ArgLoggerEntry):232 CFI_STARTPROC233 SAVE_REGISTERS234 ALIGN_STACK_16B235 236 // Again, these function pointer loads must be atomic; MOV is fine.237 LOAD_HANDLER_ADDR _ZN6__xray13XRayArgLoggerE238 testq %rax, %rax239 jne LOCAL_LABEL(arg1entryLog)240 241 // If [arg1 logging handler] not set, defer to no-arg logging.242 LOAD_HANDLER_ADDR _ZN6__xray19XRayPatchedFunctionE243 testq %rax, %rax244 je LOCAL_LABEL(arg1entryFail)245 246LOCAL_LABEL(arg1entryLog):247 248 // First argument will become the third249 movq %rdi, %rdx250 251 // XRayEntryType::LOG_ARGS_ENTRY into the second252 mov $0x3, %esi253 254 // 32-bit function ID becomes the first255 movl %r10d, %edi256 257 callq *%rax258 259LOCAL_LABEL(arg1entryFail):260 RESTORE_STACK_ALIGNMENT261 RESTORE_REGISTERS262 retq263# LLVM-MCA-END264 ASM_SIZE(__xray_ArgLoggerEntry)265 CFI_ENDPROC266 267//===----------------------------------------------------------------------===//268 269// __xray_*Event have default visibility so that they can be referenced by user270// DSOs that do not link against the runtime.271 .global ASM_SYMBOL(__xray_CustomEvent)272 .align 16, 0x90273 ASM_TYPE_FUNCTION(__xray_CustomEvent)274# LLVM-MCA-BEGIN __xray_CustomEvent275ASM_SYMBOL(__xray_CustomEvent):276 CFI_STARTPROC277 SAVE_REGISTERS278 279 // We take two arguments to this trampoline, which should be in rdi and rsi280 // already.281 LOAD_HANDLER_ADDR _ZN6__xray22XRayPatchedCustomEventE282 testq %rax,%rax283 je LOCAL_LABEL(customEventCleanup)284 285 callq *%rax286 287LOCAL_LABEL(customEventCleanup):288 RESTORE_REGISTERS289 retq290# LLVM-MCA-END291 ASM_SIZE(__xray_CustomEvent)292 CFI_ENDPROC293 294//===----------------------------------------------------------------------===//295 296 .global ASM_SYMBOL(__xray_TypedEvent)297 .align 16, 0x90298 ASM_TYPE_FUNCTION(__xray_TypedEvent)299# LLVM-MCA-BEGIN __xray_TypedEvent300ASM_SYMBOL(__xray_TypedEvent):301 CFI_STARTPROC302 SAVE_REGISTERS303 304 // We pass three arguments to this trampoline, which should be in rdi, rsi305 // and rdx without our intervention.306 LOAD_HANDLER_ADDR _ZN6__xray21XRayPatchedTypedEventE307 testq %rax,%rax308 je LOCAL_LABEL(typedEventCleanup)309 310 callq *%rax311 312LOCAL_LABEL(typedEventCleanup):313 RESTORE_REGISTERS314 retq315# LLVM-MCA-END316 ASM_SIZE(__xray_TypedEvent)317 CFI_ENDPROC318 319//===----------------------------------------------------------------------===//320 321NO_EXEC_STACK_DIRECTIVE322