475 lines · c
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */2 3#include "rseq-bits-template.h"4 5#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \6 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))7 8static inline __attribute__((always_inline))9int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)10{11 RSEQ_INJECT_C(9)12 13 __asm__ __volatile__ goto (14 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */15 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])16#ifdef RSEQ_COMPARE_TWICE17 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])18 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])19#endif20 /* Start rseq by storing table entry pointer into rseq_cs. */21 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)22 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)23 RSEQ_INJECT_ASM(3)24 LONG_CMP " %[expect], %[v]\n\t"25 "jnz %l[cmpfail]\n\t"26 RSEQ_INJECT_ASM(4)27#ifdef RSEQ_COMPARE_TWICE28 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])29 LONG_CMP " %[expect], %[v]\n\t"30 "jnz %l[error2]\n\t"31#endif32 /* final store */33 LONG_S " %[newv], %[v]\n\t"34 "2:\n\t"35 RSEQ_INJECT_ASM(5)36 RSEQ_ASM_DEFINE_ABORT(4, "", abort)37 : /* gcc asm goto does not allow outputs */38 : [cpu_id] "r" (cpu),39 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),40 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),41 [v] "m" (*v),42 [expect] "r" (expect),43 [newv] "r" (newv)44 RSEQ_INJECT_INPUT45 : "memory", "cc", "r0"46 RSEQ_INJECT_CLOBBER47 : abort, cmpfail48#ifdef RSEQ_COMPARE_TWICE49 , error1, error250#endif51 );52 rseq_after_asm_goto();53 return 0;54abort:55 rseq_after_asm_goto();56 RSEQ_INJECT_FAILED57 return -1;58cmpfail:59 rseq_after_asm_goto();60 return 1;61#ifdef RSEQ_COMPARE_TWICE62error1:63 rseq_after_asm_goto();64 rseq_bug("cpu_id comparison failed");65error2:66 rseq_after_asm_goto();67 rseq_bug("expected value comparison failed");68#endif69}70 71/*72 * Compare @v against @expectnot. When it does _not_ match, load @v73 * into @load, and store the content of *@v + voffp into @v.74 */75static inline __attribute__((always_inline))76int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,77 long voffp, intptr_t *load, int cpu)78{79 RSEQ_INJECT_C(9)80 81 __asm__ __volatile__ goto (82 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */83 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])84#ifdef RSEQ_COMPARE_TWICE85 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])86 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])87#endif88 /* Start rseq by storing table entry pointer into rseq_cs. */89 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)90 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)91 RSEQ_INJECT_ASM(3)92 LONG_L " %%r1, %[v]\n\t"93 LONG_CMP_R " %%r1, %[expectnot]\n\t"94 "je %l[cmpfail]\n\t"95 RSEQ_INJECT_ASM(4)96#ifdef RSEQ_COMPARE_TWICE97 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])98 LONG_L " %%r1, %[v]\n\t"99 LONG_CMP_R " %%r1, %[expectnot]\n\t"100 "je %l[error2]\n\t"101#endif102 LONG_S " %%r1, %[load]\n\t"103 LONG_ADD_R " %%r1, %[voffp]\n\t"104 LONG_L " %%r1, 0(%%r1)\n\t"105 /* final store */106 LONG_S " %%r1, %[v]\n\t"107 "2:\n\t"108 RSEQ_INJECT_ASM(5)109 RSEQ_ASM_DEFINE_ABORT(4, "", abort)110 : /* gcc asm goto does not allow outputs */111 : [cpu_id] "r" (cpu),112 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),113 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),114 /* final store input */115 [v] "m" (*v),116 [expectnot] "r" (expectnot),117 [voffp] "r" (voffp),118 [load] "m" (*load)119 RSEQ_INJECT_INPUT120 : "memory", "cc", "r0", "r1"121 RSEQ_INJECT_CLOBBER122 : abort, cmpfail123#ifdef RSEQ_COMPARE_TWICE124 , error1, error2125#endif126 );127 rseq_after_asm_goto();128 return 0;129abort:130 rseq_after_asm_goto();131 RSEQ_INJECT_FAILED132 return -1;133cmpfail:134 rseq_after_asm_goto();135 return 1;136#ifdef RSEQ_COMPARE_TWICE137error1:138 rseq_after_asm_goto();139 rseq_bug("cpu_id comparison failed");140error2:141 rseq_after_asm_goto();142 rseq_bug("expected value comparison failed");143#endif144}145 146static inline __attribute__((always_inline))147int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)148{149 RSEQ_INJECT_C(9)150 151 __asm__ __volatile__ goto (152 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */153#ifdef RSEQ_COMPARE_TWICE154 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])155#endif156 /* Start rseq by storing table entry pointer into rseq_cs. */157 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)158 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)159 RSEQ_INJECT_ASM(3)160#ifdef RSEQ_COMPARE_TWICE161 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])162#endif163 LONG_L " %%r0, %[v]\n\t"164 LONG_ADD_R " %%r0, %[count]\n\t"165 /* final store */166 LONG_S " %%r0, %[v]\n\t"167 "2:\n\t"168 RSEQ_INJECT_ASM(4)169 RSEQ_ASM_DEFINE_ABORT(4, "", abort)170 : /* gcc asm goto does not allow outputs */171 : [cpu_id] "r" (cpu),172 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),173 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),174 /* final store input */175 [v] "m" (*v),176 [count] "r" (count)177 RSEQ_INJECT_INPUT178 : "memory", "cc", "r0"179 RSEQ_INJECT_CLOBBER180 : abort181#ifdef RSEQ_COMPARE_TWICE182 , error1183#endif184 );185 rseq_after_asm_goto();186 return 0;187abort:188 rseq_after_asm_goto();189 RSEQ_INJECT_FAILED190 return -1;191#ifdef RSEQ_COMPARE_TWICE192error1:193 rseq_after_asm_goto();194 rseq_bug("cpu_id comparison failed");195#endif196}197 198static inline __attribute__((always_inline))199int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,200 intptr_t *v2, intptr_t expect2,201 intptr_t newv, int cpu)202{203 RSEQ_INJECT_C(9)204 205 __asm__ __volatile__ goto (206 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */207 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])208#ifdef RSEQ_COMPARE_TWICE209 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])210 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])211 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])212#endif213 /* Start rseq by storing table entry pointer into rseq_cs. */214 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)215 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)216 RSEQ_INJECT_ASM(3)217 LONG_CMP " %[expect], %[v]\n\t"218 "jnz %l[cmpfail]\n\t"219 RSEQ_INJECT_ASM(4)220 LONG_CMP " %[expect2], %[v2]\n\t"221 "jnz %l[cmpfail]\n\t"222 RSEQ_INJECT_ASM(5)223#ifdef RSEQ_COMPARE_TWICE224 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])225 LONG_CMP " %[expect], %[v]\n\t"226 "jnz %l[error2]\n\t"227 LONG_CMP " %[expect2], %[v2]\n\t"228 "jnz %l[error3]\n\t"229#endif230 /* final store */231 LONG_S " %[newv], %[v]\n\t"232 "2:\n\t"233 RSEQ_INJECT_ASM(6)234 RSEQ_ASM_DEFINE_ABORT(4, "", abort)235 : /* gcc asm goto does not allow outputs */236 : [cpu_id] "r" (cpu),237 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),238 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),239 /* cmp2 input */240 [v2] "m" (*v2),241 [expect2] "r" (expect2),242 /* final store input */243 [v] "m" (*v),244 [expect] "r" (expect),245 [newv] "r" (newv)246 RSEQ_INJECT_INPUT247 : "memory", "cc", "r0"248 RSEQ_INJECT_CLOBBER249 : abort, cmpfail250#ifdef RSEQ_COMPARE_TWICE251 , error1, error2, error3252#endif253 );254 rseq_after_asm_goto();255 return 0;256abort:257 rseq_after_asm_goto();258 RSEQ_INJECT_FAILED259 return -1;260cmpfail:261 rseq_after_asm_goto();262 return 1;263#ifdef RSEQ_COMPARE_TWICE264error1:265 rseq_after_asm_goto();266 rseq_bug("cpu_id comparison failed");267error2:268 rseq_after_asm_goto();269 rseq_bug("1st expected value comparison failed");270error3:271 rseq_after_asm_goto();272 rseq_bug("2nd expected value comparison failed");273#endif274}275 276#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&277 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */278 279#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \280 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))281 282/* s390 is TSO. */283static inline __attribute__((always_inline))284int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,285 intptr_t *v2, intptr_t newv2,286 intptr_t newv, int cpu)287{288 RSEQ_INJECT_C(9)289 290 __asm__ __volatile__ goto (291 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */292 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])293#ifdef RSEQ_COMPARE_TWICE294 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])295 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])296#endif297 /* Start rseq by storing table entry pointer into rseq_cs. */298 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)299 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)300 RSEQ_INJECT_ASM(3)301 LONG_CMP " %[expect], %[v]\n\t"302 "jnz %l[cmpfail]\n\t"303 RSEQ_INJECT_ASM(4)304#ifdef RSEQ_COMPARE_TWICE305 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])306 LONG_CMP " %[expect], %[v]\n\t"307 "jnz %l[error2]\n\t"308#endif309 /* try store */310 LONG_S " %[newv2], %[v2]\n\t"311 RSEQ_INJECT_ASM(5)312 /* final store */313 LONG_S " %[newv], %[v]\n\t"314 "2:\n\t"315 RSEQ_INJECT_ASM(6)316 RSEQ_ASM_DEFINE_ABORT(4, "", abort)317 : /* gcc asm goto does not allow outputs */318 : [cpu_id] "r" (cpu),319 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),320 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),321 /* try store input */322 [v2] "m" (*v2),323 [newv2] "r" (newv2),324 /* final store input */325 [v] "m" (*v),326 [expect] "r" (expect),327 [newv] "r" (newv)328 RSEQ_INJECT_INPUT329 : "memory", "cc", "r0"330 RSEQ_INJECT_CLOBBER331 : abort, cmpfail332#ifdef RSEQ_COMPARE_TWICE333 , error1, error2334#endif335 );336 rseq_after_asm_goto();337 return 0;338abort:339 rseq_after_asm_goto();340 RSEQ_INJECT_FAILED341 return -1;342cmpfail:343 rseq_after_asm_goto();344 return 1;345#ifdef RSEQ_COMPARE_TWICE346error1:347 rseq_after_asm_goto();348 rseq_bug("cpu_id comparison failed");349error2:350 rseq_after_asm_goto();351 rseq_bug("expected value comparison failed");352#endif353}354 355/* s390 is TSO. */356static inline __attribute__((always_inline))357int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,358 void *dst, void *src, size_t len,359 intptr_t newv, int cpu)360{361 uint64_t rseq_scratch[3];362 363 RSEQ_INJECT_C(9)364 365 __asm__ __volatile__ goto (366 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */367 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])368#ifdef RSEQ_COMPARE_TWICE369 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])370 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])371#endif372 LONG_S " %[src], %[rseq_scratch0]\n\t"373 LONG_S " %[dst], %[rseq_scratch1]\n\t"374 LONG_S " %[len], %[rseq_scratch2]\n\t"375 /* Start rseq by storing table entry pointer into rseq_cs. */376 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)377 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)378 RSEQ_INJECT_ASM(3)379 LONG_CMP " %[expect], %[v]\n\t"380 "jnz 5f\n\t"381 RSEQ_INJECT_ASM(4)382#ifdef RSEQ_COMPARE_TWICE383 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)384 LONG_CMP " %[expect], %[v]\n\t"385 "jnz 7f\n\t"386#endif387 /* try memcpy */388 LONG_LT_R " %[len], %[len]\n\t"389 "jz 333f\n\t"390 "222:\n\t"391 "ic %%r0,0(%[src])\n\t"392 "stc %%r0,0(%[dst])\n\t"393 LONG_ADDI " %[src], 1\n\t"394 LONG_ADDI " %[dst], 1\n\t"395 LONG_ADDI " %[len], -1\n\t"396 "jnz 222b\n\t"397 "333:\n\t"398 RSEQ_INJECT_ASM(5)399 /* final store */400 LONG_S " %[newv], %[v]\n\t"401 "2:\n\t"402 RSEQ_INJECT_ASM(6)403 /* teardown */404 LONG_L " %[len], %[rseq_scratch2]\n\t"405 LONG_L " %[dst], %[rseq_scratch1]\n\t"406 LONG_L " %[src], %[rseq_scratch0]\n\t"407 RSEQ_ASM_DEFINE_ABORT(4,408 LONG_L " %[len], %[rseq_scratch2]\n\t"409 LONG_L " %[dst], %[rseq_scratch1]\n\t"410 LONG_L " %[src], %[rseq_scratch0]\n\t",411 abort)412 RSEQ_ASM_DEFINE_CMPFAIL(5,413 LONG_L " %[len], %[rseq_scratch2]\n\t"414 LONG_L " %[dst], %[rseq_scratch1]\n\t"415 LONG_L " %[src], %[rseq_scratch0]\n\t",416 cmpfail)417#ifdef RSEQ_COMPARE_TWICE418 RSEQ_ASM_DEFINE_CMPFAIL(6,419 LONG_L " %[len], %[rseq_scratch2]\n\t"420 LONG_L " %[dst], %[rseq_scratch1]\n\t"421 LONG_L " %[src], %[rseq_scratch0]\n\t",422 error1)423 RSEQ_ASM_DEFINE_CMPFAIL(7,424 LONG_L " %[len], %[rseq_scratch2]\n\t"425 LONG_L " %[dst], %[rseq_scratch1]\n\t"426 LONG_L " %[src], %[rseq_scratch0]\n\t",427 error2)428#endif429 : /* gcc asm goto does not allow outputs */430 : [cpu_id] "r" (cpu),431 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),432 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),433 /* final store input */434 [v] "m" (*v),435 [expect] "r" (expect),436 [newv] "r" (newv),437 /* try memcpy input */438 [dst] "r" (dst),439 [src] "r" (src),440 [len] "r" (len),441 [rseq_scratch0] "m" (rseq_scratch[0]),442 [rseq_scratch1] "m" (rseq_scratch[1]),443 [rseq_scratch2] "m" (rseq_scratch[2])444 RSEQ_INJECT_INPUT445 : "memory", "cc", "r0"446 RSEQ_INJECT_CLOBBER447 : abort, cmpfail448#ifdef RSEQ_COMPARE_TWICE449 , error1, error2450#endif451 );452 rseq_after_asm_goto();453 return 0;454abort:455 rseq_after_asm_goto();456 RSEQ_INJECT_FAILED457 return -1;458cmpfail:459 rseq_after_asm_goto();460 return 1;461#ifdef RSEQ_COMPARE_TWICE462error1:463 rseq_after_asm_goto();464 rseq_bug("cpu_id comparison failed");465error2:466 rseq_after_asm_goto();467 rseq_bug("expected value comparison failed");468#endif469}470 471#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&472 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */473 474#include "rseq-bits-reset.h"475