397 lines · plain
1## This test checks that BOLT correctly inlines memcpy calls on AArch64.2 3# REQUIRES: system-linux, aarch64-registered-target4 5# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown %s -o %t.o6# RUN: %clang %cflags -no-pie %t.o -o %t.exe -Wl,-q7# RUN: llvm-bolt %t.exe --inline-memcpy -o %t.bolt 2>&1 | FileCheck %s --check-prefix=CHECK-INLINE8# RUN: llvm-objdump -d %t.bolt | FileCheck %s --check-prefix=CHECK-ASM9 10# Verify BOLT reports that it inlined memcpy calls (11 successful inlines out of 17 total calls)11# CHECK-INLINE: BOLT-INFO: inlined 11 memcpy() calls12 13# Each function should use optimal size-specific instructions and NO memcpy calls14 15# 1-byte copy should use single byte load/store (ldrb/strb)16# CHECK-ASM-LABEL: <test_1_byte_direct>:17# CHECK-ASM: ldrb{{.*}}w9, [x1]18# CHECK-ASM-NEXT: strb{{.*}}w9, [x0]19# CHECK-ASM-NOT: bl{{.*}}<memcpy20 21# 2-byte copy should use single 16-bit load/store (ldrh/strh)22# CHECK-ASM-LABEL: <test_2_byte_direct>:23# CHECK-ASM: ldrh{{.*}}w9, [x1]24# CHECK-ASM-NEXT: strh{{.*}}w9, [x0]25# CHECK-ASM-NOT: bl{{.*}}<memcpy26 27# 4-byte copy should use single 32-bit load/store (w register)28# CHECK-ASM-LABEL: <test_4_byte_direct>:29# CHECK-ASM: ldr{{.*}}w9, [x1]30# CHECK-ASM-NEXT: str{{.*}}w9, [x0]31# CHECK-ASM-NOT: bl{{.*}}<memcpy32 33# 8-byte copy should use single 64-bit load/store (x register)34# CHECK-ASM-LABEL: <test_8_byte_direct>:35# CHECK-ASM: ldr{{.*}}x9, [x1]36# CHECK-ASM-NEXT: str{{.*}}x9, [x0]37# CHECK-ASM-NOT: bl{{.*}}<memcpy38 39# 16-byte copy should use single 128-bit SIMD load/store (q register)40# CHECK-ASM-LABEL: <test_16_byte_direct>:41# CHECK-ASM: ldr{{.*}}q16, [x1]42# CHECK-ASM-NEXT: str{{.*}}q16, [x0]43# CHECK-ASM-NOT: bl{{.*}}<memcpy44 45# 32-byte copy should use two 128-bit SIMD operations46# CHECK-ASM-LABEL: <test_32_byte_direct>:47# CHECK-ASM: ldr{{.*}}q16, [x1]48# CHECK-ASM-NEXT: str{{.*}}q16, [x0]49# CHECK-ASM-NEXT: ldr{{.*}}q17, [x1, #0x10]50# CHECK-ASM-NEXT: str{{.*}}q17, [x0, #0x10]51# CHECK-ASM-NOT: bl{{.*}}<memcpy52 53# 37-byte copy should use greedy decomposition: (2*16) + (1*4) + (1*1)54# CHECK-ASM-LABEL: <test_37_byte_arbitrary>:55# CHECK-ASM: ldr{{.*}}q16, [x1]56# CHECK-ASM-NEXT: str{{.*}}q16, [x0]57# CHECK-ASM-NEXT: ldr{{.*}}q16, [x1, #0x10]58# CHECK-ASM-NEXT: str{{.*}}q16, [x0, #0x10]59# CHECK-ASM-NEXT: ldr{{.*}}w9, [x1, #0x20]60# CHECK-ASM-NEXT: str{{.*}}w9, [x0, #0x20]61# CHECK-ASM-NEXT: ldrb{{.*}}w9, [x1, #0x24]62# CHECK-ASM-NEXT: strb{{.*}}w9, [x0, #0x24]63# CHECK-ASM-NOT: bl{{.*}}<memcpy64 65# 0-byte copy should be inlined with no load/store instructions (nothing to copy)66# CHECK-ASM-LABEL: <test_0_byte>:67# CHECK-ASM-NOT: ldr68# CHECK-ASM-NOT: str69# CHECK-ASM-NOT: bl{{.*}}<memcpy70 71# Negative size should NOT be inlined (invalid size parameter)72# CHECK-ASM-LABEL: <test_negative_size>:73# CHECK-ASM: bl{{.*}}<memcpy74 75# 128-byte copy should NOT be inlined (too large, original call preserved)76# CHECK-ASM-LABEL: <test_128_byte_too_large>:77# CHECK-ASM: bl{{.*}}<memcpy78 79# ADD immediate with non-zero source should NOT be inlined (can't track mov+add chain)80# CHECK-ASM-LABEL: <test_4_byte_add_immediate>:81# CHECK-ASM: bl{{.*}}<memcpy82 83# Register move should NOT be inlined (size unknown at compile time)84# CHECK-ASM-LABEL: <test_register_move_unknown>:85# CHECK-ASM: bl{{.*}}<memcpy86 87# CHECK-ASM-LABEL: <test_x2_rewrite_unknown>:88# CHECK-ASM: bl{{.*}}<memcpy89 90# Live-in parameter should NOT be inlined (size unknown at compile time)91# CHECK-ASM-LABEL: <test_live_in_unknown>:92# CHECK-ASM: bl{{.*}}<memcpy93 94# _memcpy8 should be inlined with end-pointer return (dest+size)95# CHECK-ASM-LABEL: <test_memcpy8_4_byte>:96# CHECK-ASM: ldr{{.*}}w9, [x1]97# CHECK-ASM-NEXT: str{{.*}}w9, [x0]98# CHECK-ASM-NEXT: add{{.*}}x0, x0, #0x499# CHECK-ASM-NOT: bl{{.*}}<_memcpy8100 101# Complex function with caller-saved X9 should inline 8-byte memcpy using X9 as temp register102# CHECK-ASM-LABEL: <complex_operation>:103# CHECK-ASM: ldr{{.*}}x9, [x1]104# CHECK-ASM-NEXT: str{{.*}}x9, [x0]105# CHECK-ASM-NOT: bl{{.*}}<memcpy106 107# Complex function with caller-saved Q16/Q17 should inline 64-byte memcpy using Q16 as temp register108# CHECK-ASM-LABEL: <complex_fp_operation>:109# CHECK-ASM: ldr{{.*}}q16, [x1]110# CHECK-ASM-NEXT: str{{.*}}q16, [x0]111# CHECK-ASM-NEXT: ldr{{.*}}q16, [x1, #0x10]112# CHECK-ASM-NEXT: str{{.*}}q16, [x0, #0x10]113# CHECK-ASM-NEXT: ldr{{.*}}q16, [x1, #0x20]114# CHECK-ASM-NEXT: str{{.*}}q16, [x0, #0x20]115# CHECK-ASM-NEXT: ldr{{.*}}q16, [x1, #0x30]116# CHECK-ASM-NEXT: str{{.*}}q16, [x0, #0x30]117# CHECK-ASM-NOT: bl{{.*}}<memcpy118 119 .text120 .globl test_1_byte_direct121 .type test_1_byte_direct,@function122test_1_byte_direct:123 stp x29, x30, [sp, #-32]!124 mov x29, sp125 add x1, sp, #16126 add x0, sp, #8127 mov x2, #1128 bl memcpy129 ldp x29, x30, [sp], #32130 ret131 .size test_1_byte_direct, .-test_1_byte_direct132 133 .globl test_2_byte_direct134 .type test_2_byte_direct,@function135test_2_byte_direct:136 stp x29, x30, [sp, #-32]!137 mov x29, sp138 add x1, sp, #16139 add x0, sp, #8140 mov x2, #2141 bl memcpy142 ldp x29, x30, [sp], #32143 ret144 .size test_2_byte_direct, .-test_2_byte_direct145 146 .globl test_4_byte_direct147 .type test_4_byte_direct,@function148test_4_byte_direct:149 stp x29, x30, [sp, #-32]!150 mov x29, sp151 add x1, sp, #16152 add x0, sp, #8153 mov x2, #4154 bl memcpy155 ldp x29, x30, [sp], #32156 ret157 .size test_4_byte_direct, .-test_4_byte_direct158 159 .globl test_8_byte_direct160 .type test_8_byte_direct,@function161test_8_byte_direct:162 stp x29, x30, [sp, #-32]!163 mov x29, sp164 add x1, sp, #16165 add x0, sp, #8166 mov x2, #8167 bl memcpy168 ldp x29, x30, [sp], #32169 ret170 .size test_8_byte_direct, .-test_8_byte_direct171 172 .globl test_16_byte_direct173 .type test_16_byte_direct,@function174test_16_byte_direct:175 stp x29, x30, [sp, #-48]!176 mov x29, sp177 add x1, sp, #16178 add x0, sp, #32179 mov x2, #16180 bl memcpy181 ldp x29, x30, [sp], #48182 ret183 .size test_16_byte_direct, .-test_16_byte_direct184 185 .globl test_32_byte_direct186 .type test_32_byte_direct,@function187test_32_byte_direct:188 stp x29, x30, [sp, #-80]!189 mov x29, sp190 add x1, sp, #16191 add x0, sp, #48192 mov x2, #32193 bl memcpy194 ldp x29, x30, [sp], #80195 ret196 .size test_32_byte_direct, .-test_32_byte_direct197 198 .globl test_37_byte_arbitrary199 .type test_37_byte_arbitrary,@function200test_37_byte_arbitrary:201 stp x29, x30, [sp, #-96]!202 mov x29, sp203 add x1, sp, #16204 add x0, sp, #56205 mov x2, #37206 bl memcpy207 ldp x29, x30, [sp], #96208 ret209 .size test_37_byte_arbitrary, .-test_37_byte_arbitrary210 211 .globl test_0_byte212 .type test_0_byte,@function213test_0_byte:214 stp x29, x30, [sp, #-32]!215 mov x29, sp216 add x1, sp, #16217 add x0, sp, #8218 mov x2, #0219 bl memcpy220 ldp x29, x30, [sp], #32221 ret222 .size test_0_byte, .-test_0_byte223 224 .globl test_negative_size225 .type test_negative_size,@function226test_negative_size:227 # Negative size should not be inlined228 stp x29, x30, [sp, #-32]!229 mov x29, sp230 add x1, sp, #16231 add x0, sp, #8232 mov x2, #-1233 bl memcpy234 ldp x29, x30, [sp], #32235 ret236 .size test_negative_size, .-test_negative_size237 238 .globl test_128_byte_too_large239 .type test_128_byte_too_large,@function240test_128_byte_too_large:241 stp x29, x30, [sp, #-288]!242 mov x29, sp243 add x1, sp, #16244 add x0, sp, #152245 mov x2, #128246 bl memcpy247 ldp x29, x30, [sp], #288248 ret249 .size test_128_byte_too_large, .-test_128_byte_too_large250 251 .globl test_4_byte_add_immediate252 .type test_4_byte_add_immediate,@function253test_4_byte_add_immediate:254 stp x29, x30, [sp, #-32]!255 mov x29, sp256 add x1, sp, #16257 add x0, sp, #8258 mov x3, #0259 add x2, x3, #4260 bl memcpy261 ldp x29, x30, [sp], #32262 ret263 .size test_4_byte_add_immediate, .-test_4_byte_add_immediate264 265 .globl test_register_move_unknown266 .type test_register_move_unknown,@function267test_register_move_unknown:268 stp x29, x30, [sp, #-32]!269 mov x29, sp270 add x1, sp, #16271 add x0, sp, #8272 mov x6, #4273 mov x2, x6274 bl memcpy275 ldp x29, x30, [sp], #32276 ret277 .size test_register_move_unknown, .-test_register_move_unknown278 279 .globl test_x2_rewrite_unknown280 .type test_x2_rewrite_unknown,@function281test_x2_rewrite_unknown:282 mov x2, #8283 ldr x2, [sp, #24]284 bl memcpy285 ret286 .size test_x2_rewrite_unknown, .-test_x2_rewrite_unknown287 288 .globl test_live_in_unknown289 .type test_live_in_unknown,@function290test_live_in_unknown:291 # x2 comes in as parameter, no instruction sets it (should NOT inline)292 stp x29, x30, [sp, #-32]!293 mov x29, sp294 add x1, sp, #16295 add x0, sp, #8296 # x2 is live-in, no size-setting instruction297 bl memcpy298 ldp x29, x30, [sp], #32299 ret300 .size test_live_in_unknown, .-test_live_in_unknown301 302 .globl test_memcpy8_4_byte303 .type test_memcpy8_4_byte,@function304test_memcpy8_4_byte:305 stp x29, x30, [sp, #-32]!306 mov x29, sp307 add x1, sp, #16308 add x0, sp, #8309 mov x2, #4310 bl _memcpy8311 ldp x29, x30, [sp], #32312 ret313 .size test_memcpy8_4_byte, .-test_memcpy8_4_byte314 315 # Simple _memcpy8 implementation that calls memcpy and returns dest+size316 .globl _memcpy8317 .type _memcpy8,@function318_memcpy8:319 stp x29, x30, [sp, #-16]!320 mov x29, sp321 mov x3, x0322 bl memcpy323 add x0, x3, x2324 ldp x29, x30, [sp], #16325 ret326 .size _memcpy8, .-_memcpy8327 328 .globl complex_operation329 .type complex_operation,@function330complex_operation:331 stp x29, x30, [sp, #-32]!332 str x19, [sp, #16]333 mov x29, sp334 ldp x9, x10, [x0]335 ldp x11, x12, [x0, #16]336 mov x19, x1337 mov x8, x0338 add x0, x1, #32339 madd x9, x9, x2, x3340 and x10, x10, x4341 asr x12, x12, #2342 mov w2, #8343 orr x11, x12, x11, lsl #3344 eor x12, x9, x10345 mul x10, x11, x10346 eor x12, x12, x11347 add x13, x12, x9348 add x9, x11, x9, asr #4349 stp x13, x10, [x1]350 mov w10, w12351 stp x9, x10, [x1, #16]352 add x1, x8, #32353 bl memcpy354 ldr x0, [x19, #16]355 ldr x19, [sp, #16]356 ldp x29, x30, [sp], #32357 b use358 .size complex_operation, .-complex_operation359 360 .globl use361 .type use,@function362use:363 ret364 .size use, .-use365 366# Same as above but using FP caller-saved registers (Q16/17)367 .globl complex_fp_operation368 .type complex_fp_operation,@function369complex_fp_operation:370 stp x29, x30, [sp, #-48]!371 stp q8, q9, [sp, #16]372 mov x29, sp373 ldr q16, [x0]374 ldr q17, [x0, #16]375 mov x8, x0376 add x0, x1, #32377 fadd v16.4s, v16.4s, v17.4s378 fmul v17.4s, v16.4s, v17.4s379 fsub v16.2d, v16.2d, v17.2d380 mov w2, #64381 fmax v17.4s, v16.4s, v17.4s382 fmin v16.2d, v16.2d, v17.2d383 str q16, [x1]384 str q17, [x1, #16]385 add x1, x8, #32386 bl memcpy387 ldp q8, q9, [sp, #16]388 ldp x29, x30, [sp], #48389 b use_fp390 .size complex_fp_operation, .-complex_fp_operation391 392 .globl use_fp393 .type use_fp,@function394use_fp:395 ret396 .size use_fp, .-use_fp397