411 lines · c
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */2 3#include "rseq-bits-template.h"4 5#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \6 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))7 8static inline __always_inline9int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)10{11 RSEQ_INJECT_C(9)12 13 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)14 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")15#ifdef RSEQ_COMPARE_TWICE16 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")17 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")18#endif19 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)20 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)21 RSEQ_INJECT_ASM(3)22 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")23 RSEQ_INJECT_ASM(4)24#ifdef RSEQ_COMPARE_TWICE25 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")26 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")27#endif28 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)29 RSEQ_INJECT_ASM(5)30 RSEQ_ASM_DEFINE_ABORT(4, abort)31 : /* gcc asm goto does not allow outputs */32 : [cpu_id] "r" (cpu),33 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),34 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),35 [v] "m" (*v),36 [expect] "r" (expect),37 [newv] "r" (newv)38 RSEQ_INJECT_INPUT39 : "memory", RSEQ_ASM_TMP_REG_140 RSEQ_INJECT_CLOBBER41 : abort, cmpfail42#ifdef RSEQ_COMPARE_TWICE43 , error1, error244#endif45 );46 47 return 0;48abort:49 RSEQ_INJECT_FAILED50 return -1;51cmpfail:52 return 1;53#ifdef RSEQ_COMPARE_TWICE54error1:55 rseq_bug("cpu_id comparison failed");56error2:57 rseq_bug("expected value comparison failed");58#endif59}60 61static inline __always_inline62int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,63 off_t voffp, intptr_t *load, int cpu)64{65 RSEQ_INJECT_C(9)66 67 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)68 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")69#ifdef RSEQ_COMPARE_TWICE70 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")71 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")72#endif73 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)74 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)75 RSEQ_INJECT_ASM(3)76 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]")77 RSEQ_INJECT_ASM(4)78#ifdef RSEQ_COMPARE_TWICE79 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")80 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]")81#endif82 RSEQ_ASM_OP_R_LOAD(v)83 RSEQ_ASM_OP_R_STORE(load)84 RSEQ_ASM_OP_R_LOAD_OFF(voffp)85 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)86 RSEQ_INJECT_ASM(5)87 RSEQ_ASM_DEFINE_ABORT(4, abort)88 : /* gcc asm goto does not allow outputs */89 : [cpu_id] "r" (cpu),90 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),91 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),92 [v] "m" (*v),93 [expectnot] "r" (expectnot),94 [load] "m" (*load),95 [voffp] "r" (voffp)96 RSEQ_INJECT_INPUT97 : "memory", RSEQ_ASM_TMP_REG_198 RSEQ_INJECT_CLOBBER99 : abort, cmpfail100#ifdef RSEQ_COMPARE_TWICE101 , error1, error2102#endif103 );104 return 0;105abort:106 RSEQ_INJECT_FAILED107 return -1;108cmpfail:109 return 1;110#ifdef RSEQ_COMPARE_TWICE111error1:112 rseq_bug("cpu_id comparison failed");113error2:114 rseq_bug("expected value comparison failed");115#endif116}117 118static inline __always_inline119int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)120{121 RSEQ_INJECT_C(9)122 123 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)124#ifdef RSEQ_COMPARE_TWICE125 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")126#endif127 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)128 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)129 RSEQ_INJECT_ASM(3)130#ifdef RSEQ_COMPARE_TWICE131 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")132#endif133 RSEQ_ASM_OP_R_LOAD(v)134 RSEQ_ASM_OP_R_ADD(count)135 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)136 RSEQ_INJECT_ASM(4)137 RSEQ_ASM_DEFINE_ABORT(4, abort)138 : /* gcc asm goto does not allow outputs */139 : [cpu_id] "r" (cpu),140 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),141 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),142 [v] "m" (*v),143 [count] "r" (count)144 RSEQ_INJECT_INPUT145 : "memory", RSEQ_ASM_TMP_REG_1146 RSEQ_INJECT_CLOBBER147 : abort148#ifdef RSEQ_COMPARE_TWICE149 , error1150#endif151 );152 return 0;153abort:154 RSEQ_INJECT_FAILED155 return -1;156#ifdef RSEQ_COMPARE_TWICE157error1:158 rseq_bug("cpu_id comparison failed");159#endif160}161 162static inline __always_inline163int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,164 intptr_t *v2, intptr_t expect2,165 intptr_t newv, int cpu)166{167 RSEQ_INJECT_C(9)168 169 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)170 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")171#ifdef RSEQ_COMPARE_TWICE172 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")173 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")174 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")175#endif176 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)177 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)178 RSEQ_INJECT_ASM(3)179 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")180 RSEQ_INJECT_ASM(4)181 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]")182 RSEQ_INJECT_ASM(5)183#ifdef RSEQ_COMPARE_TWICE184 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")185 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")186 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]")187#endif188 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)189 RSEQ_INJECT_ASM(6)190 RSEQ_ASM_DEFINE_ABORT(4, abort)191 : /* gcc asm goto does not allow outputs */192 : [cpu_id] "r" (cpu),193 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),194 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),195 [v] "m" (*v),196 [expect] "r" (expect),197 [v2] "m" (*v2),198 [expect2] "r" (expect2),199 [newv] "r" (newv)200 RSEQ_INJECT_INPUT201 : "memory", RSEQ_ASM_TMP_REG_1202 RSEQ_INJECT_CLOBBER203 : abort, cmpfail204#ifdef RSEQ_COMPARE_TWICE205 , error1, error2, error3206#endif207 );208 209 return 0;210abort:211 RSEQ_INJECT_FAILED212 return -1;213cmpfail:214 return 1;215#ifdef RSEQ_COMPARE_TWICE216error1:217 rseq_bug("cpu_id comparison failed");218error2:219 rseq_bug("expected value comparison failed");220error3:221 rseq_bug("2nd expected value comparison failed");222#endif223}224 225#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV226 227/*228 * pval = *(ptr+off)229 * *pval += inc;230 */231static inline __always_inline232int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc, int cpu)233{234 RSEQ_INJECT_C(9)235 236 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)237#ifdef RSEQ_COMPARE_TWICE238 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")239#endif240 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)241 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)242 RSEQ_INJECT_ASM(3)243#ifdef RSEQ_COMPARE_TWICE244 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")245#endif246 RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3)247 RSEQ_INJECT_ASM(4)248 RSEQ_ASM_DEFINE_ABORT(4, abort)249 : /* gcc asm goto does not allow outputs */250 : [cpu_id] "r" (cpu),251 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),252 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),253 [ptr] "r" (ptr),254 [off] "er" (off),255 [inc] "er" (inc)256 RSEQ_INJECT_INPUT257 : "memory", RSEQ_ASM_TMP_REG_1258 RSEQ_INJECT_CLOBBER259 : abort260#ifdef RSEQ_COMPARE_TWICE261 , error1262#endif263 );264 return 0;265abort:266 RSEQ_INJECT_FAILED267 return -1;268#ifdef RSEQ_COMPARE_TWICE269error1:270 rseq_bug("cpu_id comparison failed");271#endif272}273 274#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&275 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */276 277#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \278 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))279 280static inline __always_inline281int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,282 intptr_t *v2, intptr_t newv2,283 intptr_t newv, int cpu)284{285 RSEQ_INJECT_C(9)286 287 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)288 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")289#ifdef RSEQ_COMPARE_TWICE290 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")291 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")292#endif293 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)294 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)295 RSEQ_INJECT_ASM(3)296 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")297 RSEQ_INJECT_ASM(4)298#ifdef RSEQ_COMPARE_TWICE299 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")300 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")301#endif302 RSEQ_ASM_OP_STORE(newv2, v2)303 RSEQ_INJECT_ASM(5)304#ifdef RSEQ_TEMPLATE_MO_RELEASE305 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)306#else307 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)308#endif309 RSEQ_INJECT_ASM(6)310 RSEQ_ASM_DEFINE_ABORT(4, abort)311 : /* gcc asm goto does not allow outputs */312 : [cpu_id] "r" (cpu),313 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),314 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),315 [expect] "r" (expect),316 [v] "m" (*v),317 [newv] "r" (newv),318 [v2] "m" (*v2),319 [newv2] "r" (newv2)320 RSEQ_INJECT_INPUT321 : "memory", RSEQ_ASM_TMP_REG_1322 RSEQ_INJECT_CLOBBER323 : abort, cmpfail324#ifdef RSEQ_COMPARE_TWICE325 , error1, error2326#endif327 );328 329 return 0;330abort:331 RSEQ_INJECT_FAILED332 return -1;333cmpfail:334 return 1;335#ifdef RSEQ_COMPARE_TWICE336error1:337 rseq_bug("cpu_id comparison failed");338error2:339 rseq_bug("expected value comparison failed");340#endif341}342 343static inline __always_inline344int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,345 void *dst, void *src, size_t len,346 intptr_t newv, int cpu)347{348 RSEQ_INJECT_C(9)349 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)350 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")351#ifdef RSEQ_COMPARE_TWICE352 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")353 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")354#endif355 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)356 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)357 RSEQ_INJECT_ASM(3)358 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")359 RSEQ_INJECT_ASM(4)360#ifdef RSEQ_COMPARE_TWICE361 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")362 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")363#endif364 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)365 RSEQ_INJECT_ASM(5)366#ifdef RSEQ_TEMPLATE_MO_RELEASE367 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)368#else369 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)370#endif371 RSEQ_INJECT_ASM(6)372 RSEQ_ASM_DEFINE_ABORT(4, abort)373 : /* gcc asm goto does not allow outputs */374 : [cpu_id] "r" (cpu),375 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),376 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),377 [expect] "r" (expect),378 [v] "m" (*v),379 [newv] "r" (newv),380 [dst] "r" (dst),381 [src] "r" (src),382 [len] "r" (len)383 RSEQ_INJECT_INPUT384 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,385 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4386 RSEQ_INJECT_CLOBBER387 : abort, cmpfail388#ifdef RSEQ_COMPARE_TWICE389 , error1, error2390#endif391 );392 393 return 0;394abort:395 RSEQ_INJECT_FAILED396 return -1;397cmpfail:398 return 1;399#ifdef RSEQ_COMPARE_TWICE400error1:401 rseq_bug("cpu_id comparison failed");402error2:403 rseq_bug("expected value comparison failed");404#endif405}406 407#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&408 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */409 410#include "rseq-bits-reset.h"411