455 lines · c
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */2/*3 * rseq-ppc-bits.h4 *5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>7 */8 9#include "rseq-bits-template.h"10 11#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \12 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))13 14static inline __attribute__((always_inline))15int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)16{17 RSEQ_INJECT_C(9)18 19 __asm__ __volatile__ goto (20 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */21 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])22#ifdef RSEQ_COMPARE_TWICE23 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])24 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])25#endif26 /* Start rseq by storing table entry pointer into rseq_cs. */27 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)28 /* cmp cpuid */29 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)30 RSEQ_INJECT_ASM(3)31 /* cmp @v equal to @expect */32 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])33 RSEQ_INJECT_ASM(4)34#ifdef RSEQ_COMPARE_TWICE35 /* cmp cpuid */36 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])37 /* cmp @v equal to @expect */38 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])39#endif40 /* final store */41 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)42 RSEQ_INJECT_ASM(5)43 RSEQ_ASM_DEFINE_ABORT(4, abort)44 : /* gcc asm goto does not allow outputs */45 : [cpu_id] "r" (cpu),46 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),47 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),48 [v] "m" (*v),49 [expect] "r" (expect),50 [newv] "r" (newv)51 RSEQ_INJECT_INPUT52 : "memory", "cc", "r17"53 RSEQ_INJECT_CLOBBER54 : abort, cmpfail55#ifdef RSEQ_COMPARE_TWICE56 , error1, error257#endif58 );59 rseq_after_asm_goto();60 return 0;61abort:62 rseq_after_asm_goto();63 RSEQ_INJECT_FAILED64 return -1;65cmpfail:66 rseq_after_asm_goto();67 return 1;68#ifdef RSEQ_COMPARE_TWICE69error1:70 rseq_after_asm_goto();71 rseq_bug("cpu_id comparison failed");72error2:73 rseq_after_asm_goto();74 rseq_bug("expected value comparison failed");75#endif76}77 78static inline __attribute__((always_inline))79int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,80 long voffp, intptr_t *load, int cpu)81{82 RSEQ_INJECT_C(9)83 84 __asm__ __volatile__ goto (85 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */86 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])87#ifdef RSEQ_COMPARE_TWICE88 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])89 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])90#endif91 /* Start rseq by storing table entry pointer into rseq_cs. */92 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)93 /* cmp cpuid */94 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)95 RSEQ_INJECT_ASM(3)96 /* cmp @v not equal to @expectnot */97 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])98 RSEQ_INJECT_ASM(4)99#ifdef RSEQ_COMPARE_TWICE100 /* cmp cpuid */101 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])102 /* cmp @v not equal to @expectnot */103 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])104#endif105 /* load the value of @v */106 RSEQ_ASM_OP_R_LOAD(v)107 /* store it in @load */108 RSEQ_ASM_OP_R_STORE(load)109 /* dereference voffp(v) */110 RSEQ_ASM_OP_R_LOADX(voffp)111 /* final store the value at voffp(v) */112 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)113 RSEQ_INJECT_ASM(5)114 RSEQ_ASM_DEFINE_ABORT(4, abort)115 : /* gcc asm goto does not allow outputs */116 : [cpu_id] "r" (cpu),117 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),118 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),119 /* final store input */120 [v] "m" (*v),121 [expectnot] "r" (expectnot),122 [voffp] "b" (voffp),123 [load] "m" (*load)124 RSEQ_INJECT_INPUT125 : "memory", "cc", "r17"126 RSEQ_INJECT_CLOBBER127 : abort, cmpfail128#ifdef RSEQ_COMPARE_TWICE129 , error1, error2130#endif131 );132 rseq_after_asm_goto();133 return 0;134abort:135 rseq_after_asm_goto();136 RSEQ_INJECT_FAILED137 return -1;138cmpfail:139 rseq_after_asm_goto();140 return 1;141#ifdef RSEQ_COMPARE_TWICE142error1:143 rseq_after_asm_goto();144 rseq_bug("cpu_id comparison failed");145error2:146 rseq_after_asm_goto();147 rseq_bug("expected value comparison failed");148#endif149}150 151static inline __attribute__((always_inline))152int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)153{154 RSEQ_INJECT_C(9)155 156 __asm__ __volatile__ goto (157 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */158#ifdef RSEQ_COMPARE_TWICE159 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])160#endif161 /* Start rseq by storing table entry pointer into rseq_cs. */162 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)163 /* cmp cpuid */164 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)165 RSEQ_INJECT_ASM(3)166#ifdef RSEQ_COMPARE_TWICE167 /* cmp cpuid */168 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])169#endif170 /* load the value of @v */171 RSEQ_ASM_OP_R_LOAD(v)172 /* add @count to it */173 RSEQ_ASM_OP_R_ADD(count)174 /* final store */175 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)176 RSEQ_INJECT_ASM(4)177 RSEQ_ASM_DEFINE_ABORT(4, abort)178 : /* gcc asm goto does not allow outputs */179 : [cpu_id] "r" (cpu),180 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),181 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),182 /* final store input */183 [v] "m" (*v),184 [count] "r" (count)185 RSEQ_INJECT_INPUT186 : "memory", "cc", "r17"187 RSEQ_INJECT_CLOBBER188 : abort189#ifdef RSEQ_COMPARE_TWICE190 , error1191#endif192 );193 rseq_after_asm_goto();194 return 0;195abort:196 rseq_after_asm_goto();197 RSEQ_INJECT_FAILED198 return -1;199#ifdef RSEQ_COMPARE_TWICE200error1:201 rseq_after_asm_goto();202 rseq_bug("cpu_id comparison failed");203#endif204}205 206static inline __attribute__((always_inline))207int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,208 intptr_t *v2, intptr_t expect2,209 intptr_t newv, int cpu)210{211 RSEQ_INJECT_C(9)212 213 __asm__ __volatile__ goto (214 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])216#ifdef RSEQ_COMPARE_TWICE217 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])218 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])219 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])220#endif221 /* Start rseq by storing table entry pointer into rseq_cs. */222 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)223 /* cmp cpuid */224 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)225 RSEQ_INJECT_ASM(3)226 /* cmp @v equal to @expect */227 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])228 RSEQ_INJECT_ASM(4)229 /* cmp @v2 equal to @expct2 */230 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])231 RSEQ_INJECT_ASM(5)232#ifdef RSEQ_COMPARE_TWICE233 /* cmp cpuid */234 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])235 /* cmp @v equal to @expect */236 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])237 /* cmp @v2 equal to @expct2 */238 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])239#endif240 /* final store */241 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)242 RSEQ_INJECT_ASM(6)243 RSEQ_ASM_DEFINE_ABORT(4, abort)244 : /* gcc asm goto does not allow outputs */245 : [cpu_id] "r" (cpu),246 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),247 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),248 /* cmp2 input */249 [v2] "m" (*v2),250 [expect2] "r" (expect2),251 /* final store input */252 [v] "m" (*v),253 [expect] "r" (expect),254 [newv] "r" (newv)255 RSEQ_INJECT_INPUT256 : "memory", "cc", "r17"257 RSEQ_INJECT_CLOBBER258 : abort, cmpfail259#ifdef RSEQ_COMPARE_TWICE260 , error1, error2, error3261#endif262 );263 rseq_after_asm_goto();264 return 0;265abort:266 rseq_after_asm_goto();267 RSEQ_INJECT_FAILED268 return -1;269cmpfail:270 rseq_after_asm_goto();271 return 1;272#ifdef RSEQ_COMPARE_TWICE273error1:274 rseq_after_asm_goto();275 rseq_bug("cpu_id comparison failed");276error2:277 rseq_after_asm_goto();278 rseq_bug("1st expected value comparison failed");279error3:280 rseq_after_asm_goto();281 rseq_bug("2nd expected value comparison failed");282#endif283}284 285#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&286 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */287 288#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \289 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))290 291static inline __attribute__((always_inline))292int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,293 intptr_t *v2, intptr_t newv2,294 intptr_t newv, int cpu)295{296 RSEQ_INJECT_C(9)297 298 __asm__ __volatile__ goto (299 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */300 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])301#ifdef RSEQ_COMPARE_TWICE302 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])303 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])304#endif305 /* Start rseq by storing table entry pointer into rseq_cs. */306 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)307 /* cmp cpuid */308 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)309 RSEQ_INJECT_ASM(3)310 /* cmp @v equal to @expect */311 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])312 RSEQ_INJECT_ASM(4)313#ifdef RSEQ_COMPARE_TWICE314 /* cmp cpuid */315 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])316 /* cmp @v equal to @expect */317 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])318#endif319 /* try store */320 RSEQ_ASM_OP_STORE(newv2, v2)321 RSEQ_INJECT_ASM(5)322#ifdef RSEQ_TEMPLATE_MO_RELEASE323 /* for 'release' */324 "lwsync\n\t"325#endif326 /* final store */327 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)328 RSEQ_INJECT_ASM(6)329 RSEQ_ASM_DEFINE_ABORT(4, abort)330 : /* gcc asm goto does not allow outputs */331 : [cpu_id] "r" (cpu),332 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),333 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),334 /* try store input */335 [v2] "m" (*v2),336 [newv2] "r" (newv2),337 /* final store input */338 [v] "m" (*v),339 [expect] "r" (expect),340 [newv] "r" (newv)341 RSEQ_INJECT_INPUT342 : "memory", "cc", "r17"343 RSEQ_INJECT_CLOBBER344 : abort, cmpfail345#ifdef RSEQ_COMPARE_TWICE346 , error1, error2347#endif348 );349 rseq_after_asm_goto();350 return 0;351abort:352 rseq_after_asm_goto();353 RSEQ_INJECT_FAILED354 return -1;355cmpfail:356 rseq_after_asm_goto();357 return 1;358#ifdef RSEQ_COMPARE_TWICE359error1:360 rseq_after_asm_goto();361 rseq_bug("cpu_id comparison failed");362error2:363 rseq_after_asm_goto();364 rseq_bug("expected value comparison failed");365#endif366}367 368static inline __attribute__((always_inline))369int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,370 void *dst, void *src, size_t len,371 intptr_t newv, int cpu)372{373 RSEQ_INJECT_C(9)374 375 __asm__ __volatile__ goto (376 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */377 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])378#ifdef RSEQ_COMPARE_TWICE379 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])380 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])381#endif382 /* setup for mempcy */383 "mr %%r19, %[len]\n\t"384 "mr %%r20, %[src]\n\t"385 "mr %%r21, %[dst]\n\t"386 /* Start rseq by storing table entry pointer into rseq_cs. */387 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)388 /* cmp cpuid */389 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)390 RSEQ_INJECT_ASM(3)391 /* cmp @v equal to @expect */392 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])393 RSEQ_INJECT_ASM(4)394#ifdef RSEQ_COMPARE_TWICE395 /* cmp cpuid */396 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])397 /* cmp @v equal to @expect */398 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])399#endif400 /* try memcpy */401 RSEQ_ASM_OP_R_MEMCPY()402 RSEQ_INJECT_ASM(5)403#ifdef RSEQ_TEMPLATE_MO_RELEASE404 /* for 'release' */405 "lwsync\n\t"406#endif407 /* final store */408 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)409 RSEQ_INJECT_ASM(6)410 /* teardown */411 RSEQ_ASM_DEFINE_ABORT(4, abort)412 : /* gcc asm goto does not allow outputs */413 : [cpu_id] "r" (cpu),414 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),415 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),416 /* final store input */417 [v] "m" (*v),418 [expect] "r" (expect),419 [newv] "r" (newv),420 /* try memcpy input */421 [dst] "r" (dst),422 [src] "r" (src),423 [len] "r" (len)424 RSEQ_INJECT_INPUT425 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"426 RSEQ_INJECT_CLOBBER427 : abort, cmpfail428#ifdef RSEQ_COMPARE_TWICE429 , error1, error2430#endif431 );432 rseq_after_asm_goto();433 return 0;434abort:435 rseq_after_asm_goto();436 RSEQ_INJECT_FAILED437 return -1;438cmpfail:439 rseq_after_asm_goto();440 return 1;441#ifdef RSEQ_COMPARE_TWICE442error1:443 rseq_after_asm_goto();444 rseq_bug("cpu_id comparison failed");445error2:446 rseq_after_asm_goto();447 rseq_bug("expected value comparison failed");448#endif449}450 451#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&452 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */453 454#include "rseq-bits-reset.h"455