506 lines · c
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */2/*3 * rseq-arm-bits.h4 *5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>6 */7 8#include "rseq-bits-template.h"9 10#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \11 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))12 13static inline __attribute__((always_inline))14int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)15{16 RSEQ_INJECT_C(9)17 18 __asm__ __volatile__ goto (19 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */20 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])21#ifdef RSEQ_COMPARE_TWICE22 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])23 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])24#endif25 /* Start rseq by storing table entry pointer into rseq_cs. */26 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)27 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)28 RSEQ_INJECT_ASM(3)29 "ldr r0, %[v]\n\t"30 "cmp %[expect], r0\n\t"31 "bne %l[cmpfail]\n\t"32 RSEQ_INJECT_ASM(4)33#ifdef RSEQ_COMPARE_TWICE34 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])35 "ldr r0, %[v]\n\t"36 "cmp %[expect], r0\n\t"37 "bne %l[error2]\n\t"38#endif39 /* final store */40 "str %[newv], %[v]\n\t"41 "2:\n\t"42 RSEQ_INJECT_ASM(5)43 "b 5f\n\t"44 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)45 "5:\n\t"46 : /* gcc asm goto does not allow outputs */47 : [cpu_id] "r" (cpu),48 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),49 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),50 [v] "m" (*v),51 [expect] "r" (expect),52 [newv] "r" (newv)53 RSEQ_INJECT_INPUT54 : "r0", "memory", "cc"55 RSEQ_INJECT_CLOBBER56 : abort, cmpfail57#ifdef RSEQ_COMPARE_TWICE58 , error1, error259#endif60 );61 rseq_after_asm_goto();62 return 0;63abort:64 rseq_after_asm_goto();65 RSEQ_INJECT_FAILED66 return -1;67cmpfail:68 rseq_after_asm_goto();69 return 1;70#ifdef RSEQ_COMPARE_TWICE71error1:72 rseq_after_asm_goto();73 rseq_bug("cpu_id comparison failed");74error2:75 rseq_after_asm_goto();76 rseq_bug("expected value comparison failed");77#endif78}79 80static inline __attribute__((always_inline))81int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,82 long voffp, intptr_t *load, int cpu)83{84 RSEQ_INJECT_C(9)85 86 __asm__ __volatile__ goto (87 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */88 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])89#ifdef RSEQ_COMPARE_TWICE90 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])91 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])92#endif93 /* Start rseq by storing table entry pointer into rseq_cs. */94 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)95 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)96 RSEQ_INJECT_ASM(3)97 "ldr r0, %[v]\n\t"98 "cmp %[expectnot], r0\n\t"99 "beq %l[cmpfail]\n\t"100 RSEQ_INJECT_ASM(4)101#ifdef RSEQ_COMPARE_TWICE102 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])103 "ldr r0, %[v]\n\t"104 "cmp %[expectnot], r0\n\t"105 "beq %l[error2]\n\t"106#endif107 "str r0, %[load]\n\t"108 "add r0, %[voffp]\n\t"109 "ldr r0, [r0]\n\t"110 /* final store */111 "str r0, %[v]\n\t"112 "2:\n\t"113 RSEQ_INJECT_ASM(5)114 "b 5f\n\t"115 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)116 "5:\n\t"117 : /* gcc asm goto does not allow outputs */118 : [cpu_id] "r" (cpu),119 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),120 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),121 /* final store input */122 [v] "m" (*v),123 [expectnot] "r" (expectnot),124 [voffp] "Ir" (voffp),125 [load] "m" (*load)126 RSEQ_INJECT_INPUT127 : "r0", "memory", "cc"128 RSEQ_INJECT_CLOBBER129 : abort, cmpfail130#ifdef RSEQ_COMPARE_TWICE131 , error1, error2132#endif133 );134 rseq_after_asm_goto();135 return 0;136abort:137 rseq_after_asm_goto();138 RSEQ_INJECT_FAILED139 return -1;140cmpfail:141 rseq_after_asm_goto();142 return 1;143#ifdef RSEQ_COMPARE_TWICE144error1:145 rseq_after_asm_goto();146 rseq_bug("cpu_id comparison failed");147error2:148 rseq_after_asm_goto();149 rseq_bug("expected value comparison failed");150#endif151}152 153static inline __attribute__((always_inline))154int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)155{156 RSEQ_INJECT_C(9)157 158 __asm__ __volatile__ goto (159 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */160#ifdef RSEQ_COMPARE_TWICE161 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])162#endif163 /* Start rseq by storing table entry pointer into rseq_cs. */164 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)165 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)166 RSEQ_INJECT_ASM(3)167#ifdef RSEQ_COMPARE_TWICE168 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])169#endif170 "ldr r0, %[v]\n\t"171 "add r0, %[count]\n\t"172 /* final store */173 "str r0, %[v]\n\t"174 "2:\n\t"175 RSEQ_INJECT_ASM(4)176 "b 5f\n\t"177 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)178 "5:\n\t"179 : /* gcc asm goto does not allow outputs */180 : [cpu_id] "r" (cpu),181 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),182 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),183 [v] "m" (*v),184 [count] "Ir" (count)185 RSEQ_INJECT_INPUT186 : "r0", "memory", "cc"187 RSEQ_INJECT_CLOBBER188 : abort189#ifdef RSEQ_COMPARE_TWICE190 , error1191#endif192 );193 rseq_after_asm_goto();194 return 0;195abort:196 rseq_after_asm_goto();197 RSEQ_INJECT_FAILED198 return -1;199#ifdef RSEQ_COMPARE_TWICE200error1:201 rseq_after_asm_goto();202 rseq_bug("cpu_id comparison failed");203#endif204}205 206static inline __attribute__((always_inline))207int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,208 intptr_t *v2, intptr_t expect2,209 intptr_t newv, int cpu)210{211 RSEQ_INJECT_C(9)212 213 __asm__ __volatile__ goto (214 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])216#ifdef RSEQ_COMPARE_TWICE217 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])218 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])219 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])220#endif221 /* Start rseq by storing table entry pointer into rseq_cs. */222 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)223 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)224 RSEQ_INJECT_ASM(3)225 "ldr r0, %[v]\n\t"226 "cmp %[expect], r0\n\t"227 "bne %l[cmpfail]\n\t"228 RSEQ_INJECT_ASM(4)229 "ldr r0, %[v2]\n\t"230 "cmp %[expect2], r0\n\t"231 "bne %l[cmpfail]\n\t"232 RSEQ_INJECT_ASM(5)233#ifdef RSEQ_COMPARE_TWICE234 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])235 "ldr r0, %[v]\n\t"236 "cmp %[expect], r0\n\t"237 "bne %l[error2]\n\t"238 "ldr r0, %[v2]\n\t"239 "cmp %[expect2], r0\n\t"240 "bne %l[error3]\n\t"241#endif242 /* final store */243 "str %[newv], %[v]\n\t"244 "2:\n\t"245 RSEQ_INJECT_ASM(6)246 "b 5f\n\t"247 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)248 "5:\n\t"249 : /* gcc asm goto does not allow outputs */250 : [cpu_id] "r" (cpu),251 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),252 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),253 /* cmp2 input */254 [v2] "m" (*v2),255 [expect2] "r" (expect2),256 /* final store input */257 [v] "m" (*v),258 [expect] "r" (expect),259 [newv] "r" (newv)260 RSEQ_INJECT_INPUT261 : "r0", "memory", "cc"262 RSEQ_INJECT_CLOBBER263 : abort, cmpfail264#ifdef RSEQ_COMPARE_TWICE265 , error1, error2, error3266#endif267 );268 rseq_after_asm_goto();269 return 0;270abort:271 rseq_after_asm_goto();272 RSEQ_INJECT_FAILED273 return -1;274cmpfail:275 rseq_after_asm_goto();276 return 1;277#ifdef RSEQ_COMPARE_TWICE278error1:279 rseq_after_asm_goto();280 rseq_bug("cpu_id comparison failed");281error2:282 rseq_after_asm_goto();283 rseq_bug("1st expected value comparison failed");284error3:285 rseq_after_asm_goto();286 rseq_bug("2nd expected value comparison failed");287#endif288}289 290#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&291 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */292 293#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \294 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))295 296static inline __attribute__((always_inline))297int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,298 intptr_t *v2, intptr_t newv2,299 intptr_t newv, int cpu)300{301 RSEQ_INJECT_C(9)302 303 __asm__ __volatile__ goto (304 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */305 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])306#ifdef RSEQ_COMPARE_TWICE307 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])308 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])309#endif310 /* Start rseq by storing table entry pointer into rseq_cs. */311 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)312 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)313 RSEQ_INJECT_ASM(3)314 "ldr r0, %[v]\n\t"315 "cmp %[expect], r0\n\t"316 "bne %l[cmpfail]\n\t"317 RSEQ_INJECT_ASM(4)318#ifdef RSEQ_COMPARE_TWICE319 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])320 "ldr r0, %[v]\n\t"321 "cmp %[expect], r0\n\t"322 "bne %l[error2]\n\t"323#endif324 /* try store */325 "str %[newv2], %[v2]\n\t"326 RSEQ_INJECT_ASM(5)327#ifdef RSEQ_TEMPLATE_MO_RELEASE328 "dmb\n\t" /* full mb provides store-release */329#endif330 /* final store */331 "str %[newv], %[v]\n\t"332 "2:\n\t"333 RSEQ_INJECT_ASM(6)334 "b 5f\n\t"335 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)336 "5:\n\t"337 : /* gcc asm goto does not allow outputs */338 : [cpu_id] "r" (cpu),339 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),340 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),341 /* try store input */342 [v2] "m" (*v2),343 [newv2] "r" (newv2),344 /* final store input */345 [v] "m" (*v),346 [expect] "r" (expect),347 [newv] "r" (newv)348 RSEQ_INJECT_INPUT349 : "r0", "memory", "cc"350 RSEQ_INJECT_CLOBBER351 : abort, cmpfail352#ifdef RSEQ_COMPARE_TWICE353 , error1, error2354#endif355 );356 rseq_after_asm_goto();357 return 0;358abort:359 rseq_after_asm_goto();360 RSEQ_INJECT_FAILED361 return -1;362cmpfail:363 rseq_after_asm_goto();364 return 1;365#ifdef RSEQ_COMPARE_TWICE366error1:367 rseq_after_asm_goto();368 rseq_bug("cpu_id comparison failed");369error2:370 rseq_after_asm_goto();371 rseq_bug("expected value comparison failed");372#endif373}374 375 376static inline __attribute__((always_inline))377int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,378 void *dst, void *src, size_t len,379 intptr_t newv, int cpu)380{381 uint32_t rseq_scratch[3];382 383 RSEQ_INJECT_C(9)384 385 __asm__ __volatile__ goto (386 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */387 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])388#ifdef RSEQ_COMPARE_TWICE389 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])390 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])391#endif392 "str %[src], %[rseq_scratch0]\n\t"393 "str %[dst], %[rseq_scratch1]\n\t"394 "str %[len], %[rseq_scratch2]\n\t"395 /* Start rseq by storing table entry pointer into rseq_cs. */396 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)397 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)398 RSEQ_INJECT_ASM(3)399 "ldr r0, %[v]\n\t"400 "cmp %[expect], r0\n\t"401 "bne 5f\n\t"402 RSEQ_INJECT_ASM(4)403#ifdef RSEQ_COMPARE_TWICE404 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)405 "ldr r0, %[v]\n\t"406 "cmp %[expect], r0\n\t"407 "bne 7f\n\t"408#endif409 /* try memcpy */410 "cmp %[len], #0\n\t" \411 "beq 333f\n\t" \412 "222:\n\t" \413 "ldrb %%r0, [%[src]]\n\t" \414 "strb %%r0, [%[dst]]\n\t" \415 "adds %[src], #1\n\t" \416 "adds %[dst], #1\n\t" \417 "subs %[len], #1\n\t" \418 "bne 222b\n\t" \419 "333:\n\t" \420 RSEQ_INJECT_ASM(5)421#ifdef RSEQ_TEMPLATE_MO_RELEASE422 "dmb\n\t" /* full mb provides store-release */423#endif424 /* final store */425 "str %[newv], %[v]\n\t"426 "2:\n\t"427 RSEQ_INJECT_ASM(6)428 /* teardown */429 "ldr %[len], %[rseq_scratch2]\n\t"430 "ldr %[dst], %[rseq_scratch1]\n\t"431 "ldr %[src], %[rseq_scratch0]\n\t"432 "b 8f\n\t"433 RSEQ_ASM_DEFINE_ABORT(3, 4,434 /* teardown */435 "ldr %[len], %[rseq_scratch2]\n\t"436 "ldr %[dst], %[rseq_scratch1]\n\t"437 "ldr %[src], %[rseq_scratch0]\n\t",438 abort, 1b, 2b, 4f)439 RSEQ_ASM_DEFINE_CMPFAIL(5,440 /* teardown */441 "ldr %[len], %[rseq_scratch2]\n\t"442 "ldr %[dst], %[rseq_scratch1]\n\t"443 "ldr %[src], %[rseq_scratch0]\n\t",444 cmpfail)445#ifdef RSEQ_COMPARE_TWICE446 RSEQ_ASM_DEFINE_CMPFAIL(6,447 /* teardown */448 "ldr %[len], %[rseq_scratch2]\n\t"449 "ldr %[dst], %[rseq_scratch1]\n\t"450 "ldr %[src], %[rseq_scratch0]\n\t",451 error1)452 RSEQ_ASM_DEFINE_CMPFAIL(7,453 /* teardown */454 "ldr %[len], %[rseq_scratch2]\n\t"455 "ldr %[dst], %[rseq_scratch1]\n\t"456 "ldr %[src], %[rseq_scratch0]\n\t",457 error2)458#endif459 "8:\n\t"460 : /* gcc asm goto does not allow outputs */461 : [cpu_id] "r" (cpu),462 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),463 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),464 /* final store input */465 [v] "m" (*v),466 [expect] "r" (expect),467 [newv] "r" (newv),468 /* try memcpy input */469 [dst] "r" (dst),470 [src] "r" (src),471 [len] "r" (len),472 [rseq_scratch0] "m" (rseq_scratch[0]),473 [rseq_scratch1] "m" (rseq_scratch[1]),474 [rseq_scratch2] "m" (rseq_scratch[2])475 RSEQ_INJECT_INPUT476 : "r0", "memory", "cc"477 RSEQ_INJECT_CLOBBER478 : abort, cmpfail479#ifdef RSEQ_COMPARE_TWICE480 , error1, error2481#endif482 );483 rseq_after_asm_goto();484 return 0;485abort:486 rseq_after_asm_goto();487 RSEQ_INJECT_FAILED488 return -1;489cmpfail:490 rseq_after_asm_goto();491 return 1;492#ifdef RSEQ_COMPARE_TWICE493error1:494 rseq_after_asm_goto();495 rseq_bug("cpu_id comparison failed");496error2:497 rseq_after_asm_goto();498 rseq_bug("expected value comparison failed");499#endif500}501 502#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&503 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */504 505#include "rseq-bits-reset.h"506