994 lines · c
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */2/*3 * rseq-x86-bits.h4 *5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>6 */7 8#include "rseq-bits-template.h"9 10#ifdef __x86_64__11 12#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \13 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))14 15static inline __attribute__((always_inline))16int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)17{18 RSEQ_INJECT_C(9)19 20 __asm__ __volatile__ goto (21 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */22 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])23#ifdef RSEQ_COMPARE_TWICE24 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])25 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])26#endif27 /* Start rseq by storing table entry pointer into rseq_cs. */28 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))29 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)30 RSEQ_INJECT_ASM(3)31 "cmpq %[v], %[expect]\n\t"32 "jnz %l[cmpfail]\n\t"33 RSEQ_INJECT_ASM(4)34#ifdef RSEQ_COMPARE_TWICE35 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])36 "cmpq %[v], %[expect]\n\t"37 "jnz %l[error2]\n\t"38#endif39 /* final store */40 "movq %[newv], %[v]\n\t"41 "2:\n\t"42 RSEQ_INJECT_ASM(5)43 RSEQ_ASM_DEFINE_ABORT(4, "", abort)44 : /* gcc asm goto does not allow outputs */45 : [cpu_id] "r" (cpu),46 [rseq_offset] "r" (rseq_offset),47 [v] "m" (*v),48 [expect] "r" (expect),49 [newv] "r" (newv)50 : "memory", "cc", "rax"51 RSEQ_INJECT_CLOBBER52 : abort, cmpfail53#ifdef RSEQ_COMPARE_TWICE54 , error1, error255#endif56 );57 rseq_after_asm_goto();58 return 0;59abort:60 rseq_after_asm_goto();61 RSEQ_INJECT_FAILED62 return -1;63cmpfail:64 rseq_after_asm_goto();65 return 1;66#ifdef RSEQ_COMPARE_TWICE67error1:68 rseq_after_asm_goto();69 rseq_bug("cpu_id comparison failed");70error2:71 rseq_after_asm_goto();72 rseq_bug("expected value comparison failed");73#endif74}75 76/*77 * Compare @v against @expectnot. When it does _not_ match, load @v78 * into @load, and store the content of *@v + voffp into @v.79 */80static inline __attribute__((always_inline))81int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,82 long voffp, intptr_t *load, int cpu)83{84 RSEQ_INJECT_C(9)85 86 __asm__ __volatile__ goto (87 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */88 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])89#ifdef RSEQ_COMPARE_TWICE90 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])91 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])92#endif93 /* Start rseq by storing table entry pointer into rseq_cs. */94 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))95 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)96 RSEQ_INJECT_ASM(3)97 "movq %[v], %%rbx\n\t"98 "cmpq %%rbx, %[expectnot]\n\t"99 "je %l[cmpfail]\n\t"100 RSEQ_INJECT_ASM(4)101#ifdef RSEQ_COMPARE_TWICE102 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])103 "movq %[v], %%rbx\n\t"104 "cmpq %%rbx, %[expectnot]\n\t"105 "je %l[error2]\n\t"106#endif107 "movq %%rbx, %[load]\n\t"108 "addq %[voffp], %%rbx\n\t"109 "movq (%%rbx), %%rbx\n\t"110 /* final store */111 "movq %%rbx, %[v]\n\t"112 "2:\n\t"113 RSEQ_INJECT_ASM(5)114 RSEQ_ASM_DEFINE_ABORT(4, "", abort)115 : /* gcc asm goto does not allow outputs */116 : [cpu_id] "r" (cpu),117 [rseq_offset] "r" (rseq_offset),118 /* final store input */119 [v] "m" (*v),120 [expectnot] "r" (expectnot),121 [voffp] "er" (voffp),122 [load] "m" (*load)123 : "memory", "cc", "rax", "rbx"124 RSEQ_INJECT_CLOBBER125 : abort, cmpfail126#ifdef RSEQ_COMPARE_TWICE127 , error1, error2128#endif129 );130 rseq_after_asm_goto();131 return 0;132abort:133 rseq_after_asm_goto();134 RSEQ_INJECT_FAILED135 return -1;136cmpfail:137 rseq_after_asm_goto();138 return 1;139#ifdef RSEQ_COMPARE_TWICE140error1:141 rseq_after_asm_goto();142 rseq_bug("cpu_id comparison failed");143error2:144 rseq_after_asm_goto();145 rseq_bug("expected value comparison failed");146#endif147}148 149static inline __attribute__((always_inline))150int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)151{152 RSEQ_INJECT_C(9)153 154 __asm__ __volatile__ goto (155 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */156#ifdef RSEQ_COMPARE_TWICE157 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])158#endif159 /* Start rseq by storing table entry pointer into rseq_cs. */160 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))161 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)162 RSEQ_INJECT_ASM(3)163#ifdef RSEQ_COMPARE_TWICE164 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])165#endif166 /* final store */167 "addq %[count], %[v]\n\t"168 "2:\n\t"169 RSEQ_INJECT_ASM(4)170 RSEQ_ASM_DEFINE_ABORT(4, "", abort)171 : /* gcc asm goto does not allow outputs */172 : [cpu_id] "r" (cpu),173 [rseq_offset] "r" (rseq_offset),174 /* final store input */175 [v] "m" (*v),176 [count] "er" (count)177 : "memory", "cc", "rax"178 RSEQ_INJECT_CLOBBER179 : abort180#ifdef RSEQ_COMPARE_TWICE181 , error1182#endif183 );184 rseq_after_asm_goto();185 return 0;186abort:187 rseq_after_asm_goto();188 RSEQ_INJECT_FAILED189 return -1;190#ifdef RSEQ_COMPARE_TWICE191error1:192 rseq_after_asm_goto();193 rseq_bug("cpu_id comparison failed");194#endif195}196 197#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV198 199/*200 * pval = *(ptr+off)201 * *pval += inc;202 */203static inline __attribute__((always_inline))204int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, long off, intptr_t inc, int cpu)205{206 RSEQ_INJECT_C(9)207 208 __asm__ __volatile__ goto (209 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */210#ifdef RSEQ_COMPARE_TWICE211 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])212#endif213 /* Start rseq by storing table entry pointer into rseq_cs. */214 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))215 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)216 RSEQ_INJECT_ASM(3)217#ifdef RSEQ_COMPARE_TWICE218 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])219#endif220 /* get p+v */221 "movq %[ptr], %%rbx\n\t"222 "addq %[off], %%rbx\n\t"223 /* get pv */224 "movq (%%rbx), %%rcx\n\t"225 /* *pv += inc */226 "addq %[inc], (%%rcx)\n\t"227 "2:\n\t"228 RSEQ_INJECT_ASM(4)229 RSEQ_ASM_DEFINE_ABORT(4, "", abort)230 : /* gcc asm goto does not allow outputs */231 : [cpu_id] "r" (cpu),232 [rseq_offset] "r" (rseq_offset),233 /* final store input */234 [ptr] "m" (*ptr),235 [off] "er" (off),236 [inc] "er" (inc)237 : "memory", "cc", "rax", "rbx", "rcx"238 RSEQ_INJECT_CLOBBER239 : abort240#ifdef RSEQ_COMPARE_TWICE241 , error1242#endif243 );244 return 0;245abort:246 RSEQ_INJECT_FAILED247 return -1;248#ifdef RSEQ_COMPARE_TWICE249error1:250 rseq_bug("cpu_id comparison failed");251#endif252}253 254static inline __attribute__((always_inline))255int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,256 intptr_t *v2, intptr_t expect2,257 intptr_t newv, int cpu)258{259 RSEQ_INJECT_C(9)260 261 __asm__ __volatile__ goto (262 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */263 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])264#ifdef RSEQ_COMPARE_TWICE265 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])266 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])267 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])268#endif269 /* Start rseq by storing table entry pointer into rseq_cs. */270 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))271 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)272 RSEQ_INJECT_ASM(3)273 "cmpq %[v], %[expect]\n\t"274 "jnz %l[cmpfail]\n\t"275 RSEQ_INJECT_ASM(4)276 "cmpq %[v2], %[expect2]\n\t"277 "jnz %l[cmpfail]\n\t"278 RSEQ_INJECT_ASM(5)279#ifdef RSEQ_COMPARE_TWICE280 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])281 "cmpq %[v], %[expect]\n\t"282 "jnz %l[error2]\n\t"283 "cmpq %[v2], %[expect2]\n\t"284 "jnz %l[error3]\n\t"285#endif286 /* final store */287 "movq %[newv], %[v]\n\t"288 "2:\n\t"289 RSEQ_INJECT_ASM(6)290 RSEQ_ASM_DEFINE_ABORT(4, "", abort)291 : /* gcc asm goto does not allow outputs */292 : [cpu_id] "r" (cpu),293 [rseq_offset] "r" (rseq_offset),294 /* cmp2 input */295 [v2] "m" (*v2),296 [expect2] "r" (expect2),297 /* final store input */298 [v] "m" (*v),299 [expect] "r" (expect),300 [newv] "r" (newv)301 : "memory", "cc", "rax"302 RSEQ_INJECT_CLOBBER303 : abort, cmpfail304#ifdef RSEQ_COMPARE_TWICE305 , error1, error2, error3306#endif307 );308 rseq_after_asm_goto();309 return 0;310abort:311 rseq_after_asm_goto();312 RSEQ_INJECT_FAILED313 return -1;314cmpfail:315 rseq_after_asm_goto();316 return 1;317#ifdef RSEQ_COMPARE_TWICE318error1:319 rseq_after_asm_goto();320 rseq_bug("cpu_id comparison failed");321error2:322 rseq_after_asm_goto();323 rseq_bug("1st expected value comparison failed");324error3:325 rseq_after_asm_goto();326 rseq_bug("2nd expected value comparison failed");327#endif328}329 330#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&331 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */332 333#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \334 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))335 336static inline __attribute__((always_inline))337int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,338 intptr_t *v2, intptr_t newv2,339 intptr_t newv, int cpu)340{341 RSEQ_INJECT_C(9)342 343 __asm__ __volatile__ goto (344 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */345 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])346#ifdef RSEQ_COMPARE_TWICE347 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])348 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])349#endif350 /* Start rseq by storing table entry pointer into rseq_cs. */351 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))352 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)353 RSEQ_INJECT_ASM(3)354 "cmpq %[v], %[expect]\n\t"355 "jnz %l[cmpfail]\n\t"356 RSEQ_INJECT_ASM(4)357#ifdef RSEQ_COMPARE_TWICE358 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])359 "cmpq %[v], %[expect]\n\t"360 "jnz %l[error2]\n\t"361#endif362 /* try store */363 "movq %[newv2], %[v2]\n\t"364 RSEQ_INJECT_ASM(5)365 /* final store */366 "movq %[newv], %[v]\n\t"367 "2:\n\t"368 RSEQ_INJECT_ASM(6)369 RSEQ_ASM_DEFINE_ABORT(4, "", abort)370 : /* gcc asm goto does not allow outputs */371 : [cpu_id] "r" (cpu),372 [rseq_offset] "r" (rseq_offset),373 /* try store input */374 [v2] "m" (*v2),375 [newv2] "r" (newv2),376 /* final store input */377 [v] "m" (*v),378 [expect] "r" (expect),379 [newv] "r" (newv)380 : "memory", "cc", "rax"381 RSEQ_INJECT_CLOBBER382 : abort, cmpfail383#ifdef RSEQ_COMPARE_TWICE384 , error1, error2385#endif386 );387 rseq_after_asm_goto();388 return 0;389abort:390 rseq_after_asm_goto();391 RSEQ_INJECT_FAILED392 return -1;393cmpfail:394 rseq_after_asm_goto();395 return 1;396#ifdef RSEQ_COMPARE_TWICE397error1:398 rseq_after_asm_goto();399 rseq_bug("cpu_id comparison failed");400error2:401 rseq_after_asm_goto();402 rseq_bug("expected value comparison failed");403#endif404}405 406static inline __attribute__((always_inline))407int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,408 void *dst, void *src, size_t len,409 intptr_t newv, int cpu)410{411 uint64_t rseq_scratch[3];412 413 RSEQ_INJECT_C(9)414 415 __asm__ __volatile__ goto (416 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */417 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])418#ifdef RSEQ_COMPARE_TWICE419 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])420 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])421#endif422 "movq %[src], %[rseq_scratch0]\n\t"423 "movq %[dst], %[rseq_scratch1]\n\t"424 "movq %[len], %[rseq_scratch2]\n\t"425 /* Start rseq by storing table entry pointer into rseq_cs. */426 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))427 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)428 RSEQ_INJECT_ASM(3)429 "cmpq %[v], %[expect]\n\t"430 "jnz 5f\n\t"431 RSEQ_INJECT_ASM(4)432#ifdef RSEQ_COMPARE_TWICE433 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f)434 "cmpq %[v], %[expect]\n\t"435 "jnz 7f\n\t"436#endif437 /* try memcpy */438 "test %[len], %[len]\n\t" \439 "jz 333f\n\t" \440 "222:\n\t" \441 "movb (%[src]), %%al\n\t" \442 "movb %%al, (%[dst])\n\t" \443 "inc %[src]\n\t" \444 "inc %[dst]\n\t" \445 "dec %[len]\n\t" \446 "jnz 222b\n\t" \447 "333:\n\t" \448 RSEQ_INJECT_ASM(5)449 /* final store */450 "movq %[newv], %[v]\n\t"451 "2:\n\t"452 RSEQ_INJECT_ASM(6)453 /* teardown */454 "movq %[rseq_scratch2], %[len]\n\t"455 "movq %[rseq_scratch1], %[dst]\n\t"456 "movq %[rseq_scratch0], %[src]\n\t"457 RSEQ_ASM_DEFINE_ABORT(4,458 "movq %[rseq_scratch2], %[len]\n\t"459 "movq %[rseq_scratch1], %[dst]\n\t"460 "movq %[rseq_scratch0], %[src]\n\t",461 abort)462 RSEQ_ASM_DEFINE_CMPFAIL(5,463 "movq %[rseq_scratch2], %[len]\n\t"464 "movq %[rseq_scratch1], %[dst]\n\t"465 "movq %[rseq_scratch0], %[src]\n\t",466 cmpfail)467#ifdef RSEQ_COMPARE_TWICE468 RSEQ_ASM_DEFINE_CMPFAIL(6,469 "movq %[rseq_scratch2], %[len]\n\t"470 "movq %[rseq_scratch1], %[dst]\n\t"471 "movq %[rseq_scratch0], %[src]\n\t",472 error1)473 RSEQ_ASM_DEFINE_CMPFAIL(7,474 "movq %[rseq_scratch2], %[len]\n\t"475 "movq %[rseq_scratch1], %[dst]\n\t"476 "movq %[rseq_scratch0], %[src]\n\t",477 error2)478#endif479 : /* gcc asm goto does not allow outputs */480 : [cpu_id] "r" (cpu),481 [rseq_offset] "r" (rseq_offset),482 /* final store input */483 [v] "m" (*v),484 [expect] "r" (expect),485 [newv] "r" (newv),486 /* try memcpy input */487 [dst] "r" (dst),488 [src] "r" (src),489 [len] "r" (len),490 [rseq_scratch0] "m" (rseq_scratch[0]),491 [rseq_scratch1] "m" (rseq_scratch[1]),492 [rseq_scratch2] "m" (rseq_scratch[2])493 : "memory", "cc", "rax"494 RSEQ_INJECT_CLOBBER495 : abort, cmpfail496#ifdef RSEQ_COMPARE_TWICE497 , error1, error2498#endif499 );500 rseq_after_asm_goto();501 return 0;502abort:503 rseq_after_asm_goto();504 RSEQ_INJECT_FAILED505 return -1;506cmpfail:507 rseq_after_asm_goto();508 return 1;509#ifdef RSEQ_COMPARE_TWICE510error1:511 rseq_after_asm_goto();512 rseq_bug("cpu_id comparison failed");513error2:514 rseq_after_asm_goto();515 rseq_bug("expected value comparison failed");516#endif517}518 519#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&520 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */521 522#elif defined(__i386__)523 524#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \525 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))526 527static inline __attribute__((always_inline))528int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)529{530 RSEQ_INJECT_C(9)531 532 __asm__ __volatile__ goto (533 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */534 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])535#ifdef RSEQ_COMPARE_TWICE536 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])537 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])538#endif539 /* Start rseq by storing table entry pointer into rseq_cs. */540 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))541 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)542 RSEQ_INJECT_ASM(3)543 "cmpl %[v], %[expect]\n\t"544 "jnz %l[cmpfail]\n\t"545 RSEQ_INJECT_ASM(4)546#ifdef RSEQ_COMPARE_TWICE547 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])548 "cmpl %[v], %[expect]\n\t"549 "jnz %l[error2]\n\t"550#endif551 /* final store */552 "movl %[newv], %[v]\n\t"553 "2:\n\t"554 RSEQ_INJECT_ASM(5)555 RSEQ_ASM_DEFINE_ABORT(4, "", abort)556 : /* gcc asm goto does not allow outputs */557 : [cpu_id] "r" (cpu),558 [rseq_offset] "r" (rseq_offset),559 [v] "m" (*v),560 [expect] "r" (expect),561 [newv] "r" (newv)562 : "memory", "cc", "eax"563 RSEQ_INJECT_CLOBBER564 : abort, cmpfail565#ifdef RSEQ_COMPARE_TWICE566 , error1, error2567#endif568 );569 rseq_after_asm_goto();570 return 0;571abort:572 rseq_after_asm_goto();573 RSEQ_INJECT_FAILED574 return -1;575cmpfail:576 rseq_after_asm_goto();577 return 1;578#ifdef RSEQ_COMPARE_TWICE579error1:580 rseq_after_asm_goto();581 rseq_bug("cpu_id comparison failed");582error2:583 rseq_after_asm_goto();584 rseq_bug("expected value comparison failed");585#endif586}587 588/*589 * Compare @v against @expectnot. When it does _not_ match, load @v590 * into @load, and store the content of *@v + voffp into @v.591 */592static inline __attribute__((always_inline))593int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,594 long voffp, intptr_t *load, int cpu)595{596 RSEQ_INJECT_C(9)597 598 __asm__ __volatile__ goto (599 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */600 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])601#ifdef RSEQ_COMPARE_TWICE602 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])603 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])604#endif605 /* Start rseq by storing table entry pointer into rseq_cs. */606 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))607 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)608 RSEQ_INJECT_ASM(3)609 "movl %[v], %%ebx\n\t"610 "cmpl %%ebx, %[expectnot]\n\t"611 "je %l[cmpfail]\n\t"612 RSEQ_INJECT_ASM(4)613#ifdef RSEQ_COMPARE_TWICE614 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])615 "movl %[v], %%ebx\n\t"616 "cmpl %%ebx, %[expectnot]\n\t"617 "je %l[error2]\n\t"618#endif619 "movl %%ebx, %[load]\n\t"620 "addl %[voffp], %%ebx\n\t"621 "movl (%%ebx), %%ebx\n\t"622 /* final store */623 "movl %%ebx, %[v]\n\t"624 "2:\n\t"625 RSEQ_INJECT_ASM(5)626 RSEQ_ASM_DEFINE_ABORT(4, "", abort)627 : /* gcc asm goto does not allow outputs */628 : [cpu_id] "r" (cpu),629 [rseq_offset] "r" (rseq_offset),630 /* final store input */631 [v] "m" (*v),632 [expectnot] "r" (expectnot),633 [voffp] "ir" (voffp),634 [load] "m" (*load)635 : "memory", "cc", "eax", "ebx"636 RSEQ_INJECT_CLOBBER637 : abort, cmpfail638#ifdef RSEQ_COMPARE_TWICE639 , error1, error2640#endif641 );642 rseq_after_asm_goto();643 return 0;644abort:645 rseq_after_asm_goto();646 RSEQ_INJECT_FAILED647 return -1;648cmpfail:649 rseq_after_asm_goto();650 return 1;651#ifdef RSEQ_COMPARE_TWICE652error1:653 rseq_after_asm_goto();654 rseq_bug("cpu_id comparison failed");655error2:656 rseq_after_asm_goto();657 rseq_bug("expected value comparison failed");658#endif659}660 661static inline __attribute__((always_inline))662int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)663{664 RSEQ_INJECT_C(9)665 666 __asm__ __volatile__ goto (667 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */668#ifdef RSEQ_COMPARE_TWICE669 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])670#endif671 /* Start rseq by storing table entry pointer into rseq_cs. */672 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))673 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)674 RSEQ_INJECT_ASM(3)675#ifdef RSEQ_COMPARE_TWICE676 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])677#endif678 /* final store */679 "addl %[count], %[v]\n\t"680 "2:\n\t"681 RSEQ_INJECT_ASM(4)682 RSEQ_ASM_DEFINE_ABORT(4, "", abort)683 : /* gcc asm goto does not allow outputs */684 : [cpu_id] "r" (cpu),685 [rseq_offset] "r" (rseq_offset),686 /* final store input */687 [v] "m" (*v),688 [count] "ir" (count)689 : "memory", "cc", "eax"690 RSEQ_INJECT_CLOBBER691 : abort692#ifdef RSEQ_COMPARE_TWICE693 , error1694#endif695 );696 rseq_after_asm_goto();697 return 0;698abort:699 rseq_after_asm_goto();700 RSEQ_INJECT_FAILED701 return -1;702#ifdef RSEQ_COMPARE_TWICE703error1:704 rseq_after_asm_goto();705 rseq_bug("cpu_id comparison failed");706#endif707}708 709static inline __attribute__((always_inline))710int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,711 intptr_t *v2, intptr_t expect2,712 intptr_t newv, int cpu)713{714 RSEQ_INJECT_C(9)715 716 __asm__ __volatile__ goto (717 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */718 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])719#ifdef RSEQ_COMPARE_TWICE720 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])721 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])722 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])723#endif724 /* Start rseq by storing table entry pointer into rseq_cs. */725 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))726 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)727 RSEQ_INJECT_ASM(3)728 "cmpl %[v], %[expect]\n\t"729 "jnz %l[cmpfail]\n\t"730 RSEQ_INJECT_ASM(4)731 "cmpl %[expect2], %[v2]\n\t"732 "jnz %l[cmpfail]\n\t"733 RSEQ_INJECT_ASM(5)734#ifdef RSEQ_COMPARE_TWICE735 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])736 "cmpl %[v], %[expect]\n\t"737 "jnz %l[error2]\n\t"738 "cmpl %[expect2], %[v2]\n\t"739 "jnz %l[error3]\n\t"740#endif741 "movl %[newv], %%eax\n\t"742 /* final store */743 "movl %%eax, %[v]\n\t"744 "2:\n\t"745 RSEQ_INJECT_ASM(6)746 RSEQ_ASM_DEFINE_ABORT(4, "", abort)747 : /* gcc asm goto does not allow outputs */748 : [cpu_id] "r" (cpu),749 [rseq_offset] "r" (rseq_offset),750 /* cmp2 input */751 [v2] "m" (*v2),752 [expect2] "r" (expect2),753 /* final store input */754 [v] "m" (*v),755 [expect] "r" (expect),756 [newv] "m" (newv)757 : "memory", "cc", "eax"758 RSEQ_INJECT_CLOBBER759 : abort, cmpfail760#ifdef RSEQ_COMPARE_TWICE761 , error1, error2, error3762#endif763 );764 rseq_after_asm_goto();765 return 0;766abort:767 rseq_after_asm_goto();768 RSEQ_INJECT_FAILED769 return -1;770cmpfail:771 rseq_after_asm_goto();772 return 1;773#ifdef RSEQ_COMPARE_TWICE774error1:775 rseq_after_asm_goto();776 rseq_bug("cpu_id comparison failed");777error2:778 rseq_after_asm_goto();779 rseq_bug("1st expected value comparison failed");780error3:781 rseq_after_asm_goto();782 rseq_bug("2nd expected value comparison failed");783#endif784}785 786#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&787 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */788 789#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \790 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))791 792static inline __attribute__((always_inline))793int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,794 intptr_t *v2, intptr_t newv2,795 intptr_t newv, int cpu)796{797 RSEQ_INJECT_C(9)798 799 __asm__ __volatile__ goto (800 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */801 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])802#ifdef RSEQ_COMPARE_TWICE803 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])804 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])805#endif806 /* Start rseq by storing table entry pointer into rseq_cs. */807 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))808 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)809 RSEQ_INJECT_ASM(3)810 "movl %[expect], %%eax\n\t"811 "cmpl %[v], %%eax\n\t"812 "jnz %l[cmpfail]\n\t"813 RSEQ_INJECT_ASM(4)814#ifdef RSEQ_COMPARE_TWICE815 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])816 "movl %[expect], %%eax\n\t"817 "cmpl %[v], %%eax\n\t"818 "jnz %l[error2]\n\t"819#endif820 /* try store */821 "movl %[newv2], %[v2]\n\t"822 RSEQ_INJECT_ASM(5)823#ifdef RSEQ_TEMPLATE_MO_RELEASE824 "lock; addl $0,-128(%%esp)\n\t"825#endif826 /* final store */827 "movl %[newv], %[v]\n\t"828 "2:\n\t"829 RSEQ_INJECT_ASM(6)830 RSEQ_ASM_DEFINE_ABORT(4, "", abort)831 : /* gcc asm goto does not allow outputs */832 : [cpu_id] "r" (cpu),833 [rseq_offset] "r" (rseq_offset),834 /* try store input */835 [v2] "m" (*v2),836 [newv2] "r" (newv2),837 /* final store input */838 [v] "m" (*v),839 [expect] "m" (expect),840 [newv] "r" (newv)841 : "memory", "cc", "eax"842 RSEQ_INJECT_CLOBBER843 : abort, cmpfail844#ifdef RSEQ_COMPARE_TWICE845 , error1, error2846#endif847 );848 rseq_after_asm_goto();849 return 0;850abort:851 rseq_after_asm_goto();852 RSEQ_INJECT_FAILED853 return -1;854cmpfail:855 rseq_after_asm_goto();856 return 1;857#ifdef RSEQ_COMPARE_TWICE858error1:859 rseq_after_asm_goto();860 rseq_bug("cpu_id comparison failed");861error2:862 rseq_after_asm_goto();863 rseq_bug("expected value comparison failed");864#endif865 866}867 868/* TODO: implement a faster memcpy. */869static inline __attribute__((always_inline))870int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,871 void *dst, void *src, size_t len,872 intptr_t newv, int cpu)873{874 uint32_t rseq_scratch[3];875 876 RSEQ_INJECT_C(9)877 878 __asm__ __volatile__ goto (879 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */880 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])881#ifdef RSEQ_COMPARE_TWICE882 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])883 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])884#endif885 "movl %[src], %[rseq_scratch0]\n\t"886 "movl %[dst], %[rseq_scratch1]\n\t"887 "movl %[len], %[rseq_scratch2]\n\t"888 /* Start rseq by storing table entry pointer into rseq_cs. */889 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))890 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)891 RSEQ_INJECT_ASM(3)892 "movl %[expect], %%eax\n\t"893 "cmpl %%eax, %[v]\n\t"894 "jnz 5f\n\t"895 RSEQ_INJECT_ASM(4)896#ifdef RSEQ_COMPARE_TWICE897 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f)898 "movl %[expect], %%eax\n\t"899 "cmpl %%eax, %[v]\n\t"900 "jnz 7f\n\t"901#endif902 /* try memcpy */903 "test %[len], %[len]\n\t" \904 "jz 333f\n\t" \905 "222:\n\t" \906 "movb (%[src]), %%al\n\t" \907 "movb %%al, (%[dst])\n\t" \908 "inc %[src]\n\t" \909 "inc %[dst]\n\t" \910 "dec %[len]\n\t" \911 "jnz 222b\n\t" \912 "333:\n\t" \913 RSEQ_INJECT_ASM(5)914#ifdef RSEQ_TEMPLATE_MO_RELEASE915 "lock; addl $0,-128(%%esp)\n\t"916#endif917 "movl %[newv], %%eax\n\t"918 /* final store */919 "movl %%eax, %[v]\n\t"920 "2:\n\t"921 RSEQ_INJECT_ASM(6)922 /* teardown */923 "movl %[rseq_scratch2], %[len]\n\t"924 "movl %[rseq_scratch1], %[dst]\n\t"925 "movl %[rseq_scratch0], %[src]\n\t"926 RSEQ_ASM_DEFINE_ABORT(4,927 "movl %[rseq_scratch2], %[len]\n\t"928 "movl %[rseq_scratch1], %[dst]\n\t"929 "movl %[rseq_scratch0], %[src]\n\t",930 abort)931 RSEQ_ASM_DEFINE_CMPFAIL(5,932 "movl %[rseq_scratch2], %[len]\n\t"933 "movl %[rseq_scratch1], %[dst]\n\t"934 "movl %[rseq_scratch0], %[src]\n\t",935 cmpfail)936#ifdef RSEQ_COMPARE_TWICE937 RSEQ_ASM_DEFINE_CMPFAIL(6,938 "movl %[rseq_scratch2], %[len]\n\t"939 "movl %[rseq_scratch1], %[dst]\n\t"940 "movl %[rseq_scratch0], %[src]\n\t",941 error1)942 RSEQ_ASM_DEFINE_CMPFAIL(7,943 "movl %[rseq_scratch2], %[len]\n\t"944 "movl %[rseq_scratch1], %[dst]\n\t"945 "movl %[rseq_scratch0], %[src]\n\t",946 error2)947#endif948 : /* gcc asm goto does not allow outputs */949 : [cpu_id] "r" (cpu),950 [rseq_offset] "r" (rseq_offset),951 /* final store input */952 [v] "m" (*v),953 [expect] "m" (expect),954 [newv] "m" (newv),955 /* try memcpy input */956 [dst] "r" (dst),957 [src] "r" (src),958 [len] "r" (len),959 [rseq_scratch0] "m" (rseq_scratch[0]),960 [rseq_scratch1] "m" (rseq_scratch[1]),961 [rseq_scratch2] "m" (rseq_scratch[2])962 : "memory", "cc", "eax"963 RSEQ_INJECT_CLOBBER964 : abort, cmpfail965#ifdef RSEQ_COMPARE_TWICE966 , error1, error2967#endif968 );969 rseq_after_asm_goto();970 return 0;971abort:972 rseq_after_asm_goto();973 RSEQ_INJECT_FAILED974 return -1;975cmpfail:976 rseq_after_asm_goto();977 return 1;978#ifdef RSEQ_COMPARE_TWICE979error1:980 rseq_after_asm_goto();981 rseq_bug("cpu_id comparison failed");982error2:983 rseq_after_asm_goto();984 rseq_bug("expected value comparison failed");985#endif986}987 988#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&989 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */990 991#endif992 993#include "rseq-bits-reset.h"994