463 lines · c
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */2/*3 * Author: Paul Burton <paul.burton@mips.com>4 * (C) Copyright 2018 MIPS Tech LLC5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>6 */7 8#include "rseq-bits-template.h"9 10#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \11 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))12 13static inline __attribute__((always_inline))14int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)15{16 RSEQ_INJECT_C(9)17 18 __asm__ __volatile__ goto (19 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */20 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])21#ifdef RSEQ_COMPARE_TWICE22 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])23 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])24#endif25 /* Start rseq by storing table entry pointer into rseq_cs. */26 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)27 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)28 RSEQ_INJECT_ASM(3)29 LONG_L " $4, %[v]\n\t"30 "bne $4, %[expect], %l[cmpfail]\n\t"31 RSEQ_INJECT_ASM(4)32#ifdef RSEQ_COMPARE_TWICE33 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])34 LONG_L " $4, %[v]\n\t"35 "bne $4, %[expect], %l[error2]\n\t"36#endif37 /* final store */38 LONG_S " %[newv], %[v]\n\t"39 "2:\n\t"40 RSEQ_INJECT_ASM(5)41 "b 5f\n\t"42 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)43 "5:\n\t"44 : /* gcc asm goto does not allow outputs */45 : [cpu_id] "r" (cpu),46 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),47 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),48 [v] "m" (*v),49 [expect] "r" (expect),50 [newv] "r" (newv)51 RSEQ_INJECT_INPUT52 : "$4", "memory"53 RSEQ_INJECT_CLOBBER54 : abort, cmpfail55#ifdef RSEQ_COMPARE_TWICE56 , error1, error257#endif58 );59 return 0;60abort:61 RSEQ_INJECT_FAILED62 return -1;63cmpfail:64 return 1;65#ifdef RSEQ_COMPARE_TWICE66error1:67 rseq_bug("cpu_id comparison failed");68error2:69 rseq_bug("expected value comparison failed");70#endif71}72 73static inline __attribute__((always_inline))74int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,75 long voffp, intptr_t *load, int cpu)76{77 RSEQ_INJECT_C(9)78 79 __asm__ __volatile__ goto (80 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */81 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])82#ifdef RSEQ_COMPARE_TWICE83 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])84 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])85#endif86 /* Start rseq by storing table entry pointer into rseq_cs. */87 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)88 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)89 RSEQ_INJECT_ASM(3)90 LONG_L " $4, %[v]\n\t"91 "beq $4, %[expectnot], %l[cmpfail]\n\t"92 RSEQ_INJECT_ASM(4)93#ifdef RSEQ_COMPARE_TWICE94 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])95 LONG_L " $4, %[v]\n\t"96 "beq $4, %[expectnot], %l[error2]\n\t"97#endif98 LONG_S " $4, %[load]\n\t"99 LONG_ADDI " $4, %[voffp]\n\t"100 LONG_L " $4, 0($4)\n\t"101 /* final store */102 LONG_S " $4, %[v]\n\t"103 "2:\n\t"104 RSEQ_INJECT_ASM(5)105 "b 5f\n\t"106 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)107 "5:\n\t"108 : /* gcc asm goto does not allow outputs */109 : [cpu_id] "r" (cpu),110 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),111 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),112 /* final store input */113 [v] "m" (*v),114 [expectnot] "r" (expectnot),115 [voffp] "Ir" (voffp),116 [load] "m" (*load)117 RSEQ_INJECT_INPUT118 : "$4", "memory"119 RSEQ_INJECT_CLOBBER120 : abort, cmpfail121#ifdef RSEQ_COMPARE_TWICE122 , error1, error2123#endif124 );125 return 0;126abort:127 RSEQ_INJECT_FAILED128 return -1;129cmpfail:130 return 1;131#ifdef RSEQ_COMPARE_TWICE132error1:133 rseq_bug("cpu_id comparison failed");134error2:135 rseq_bug("expected value comparison failed");136#endif137}138 139static inline __attribute__((always_inline))140int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)141{142 RSEQ_INJECT_C(9)143 144 __asm__ __volatile__ goto (145 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */146#ifdef RSEQ_COMPARE_TWICE147 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])148#endif149 /* Start rseq by storing table entry pointer into rseq_cs. */150 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)151 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)152 RSEQ_INJECT_ASM(3)153#ifdef RSEQ_COMPARE_TWICE154 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])155#endif156 LONG_L " $4, %[v]\n\t"157 LONG_ADDI " $4, %[count]\n\t"158 /* final store */159 LONG_S " $4, %[v]\n\t"160 "2:\n\t"161 RSEQ_INJECT_ASM(4)162 "b 5f\n\t"163 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)164 "5:\n\t"165 : /* gcc asm goto does not allow outputs */166 : [cpu_id] "r" (cpu),167 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),168 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),169 [v] "m" (*v),170 [count] "Ir" (count)171 RSEQ_INJECT_INPUT172 : "$4", "memory"173 RSEQ_INJECT_CLOBBER174 : abort175#ifdef RSEQ_COMPARE_TWICE176 , error1177#endif178 );179 return 0;180abort:181 RSEQ_INJECT_FAILED182 return -1;183#ifdef RSEQ_COMPARE_TWICE184error1:185 rseq_bug("cpu_id comparison failed");186#endif187}188 189static inline __attribute__((always_inline))190int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,191 intptr_t *v2, intptr_t expect2,192 intptr_t newv, int cpu)193{194 RSEQ_INJECT_C(9)195 196 __asm__ __volatile__ goto (197 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */198 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])199#ifdef RSEQ_COMPARE_TWICE200 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])201 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])202 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])203#endif204 /* Start rseq by storing table entry pointer into rseq_cs. */205 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)206 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)207 RSEQ_INJECT_ASM(3)208 LONG_L " $4, %[v]\n\t"209 "bne $4, %[expect], %l[cmpfail]\n\t"210 RSEQ_INJECT_ASM(4)211 LONG_L " $4, %[v2]\n\t"212 "bne $4, %[expect2], %l[cmpfail]\n\t"213 RSEQ_INJECT_ASM(5)214#ifdef RSEQ_COMPARE_TWICE215 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])216 LONG_L " $4, %[v]\n\t"217 "bne $4, %[expect], %l[error2]\n\t"218 LONG_L " $4, %[v2]\n\t"219 "bne $4, %[expect2], %l[error3]\n\t"220#endif221 /* final store */222 LONG_S " %[newv], %[v]\n\t"223 "2:\n\t"224 RSEQ_INJECT_ASM(6)225 "b 5f\n\t"226 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)227 "5:\n\t"228 : /* gcc asm goto does not allow outputs */229 : [cpu_id] "r" (cpu),230 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),231 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),232 /* cmp2 input */233 [v2] "m" (*v2),234 [expect2] "r" (expect2),235 /* final store input */236 [v] "m" (*v),237 [expect] "r" (expect),238 [newv] "r" (newv)239 RSEQ_INJECT_INPUT240 : "$4", "memory"241 RSEQ_INJECT_CLOBBER242 : abort, cmpfail243#ifdef RSEQ_COMPARE_TWICE244 , error1, error2, error3245#endif246 );247 return 0;248abort:249 RSEQ_INJECT_FAILED250 return -1;251cmpfail:252 return 1;253#ifdef RSEQ_COMPARE_TWICE254error1:255 rseq_bug("cpu_id comparison failed");256error2:257 rseq_bug("1st expected value comparison failed");258error3:259 rseq_bug("2nd expected value comparison failed");260#endif261}262 263#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&264 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */265 266#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \267 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))268 269static inline __attribute__((always_inline))270int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,271 intptr_t *v2, intptr_t newv2,272 intptr_t newv, int cpu)273{274 RSEQ_INJECT_C(9)275 276 __asm__ __volatile__ goto (277 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */278 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])279#ifdef RSEQ_COMPARE_TWICE280 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])281 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])282#endif283 /* Start rseq by storing table entry pointer into rseq_cs. */284 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)285 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)286 RSEQ_INJECT_ASM(3)287 LONG_L " $4, %[v]\n\t"288 "bne $4, %[expect], %l[cmpfail]\n\t"289 RSEQ_INJECT_ASM(4)290#ifdef RSEQ_COMPARE_TWICE291 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])292 LONG_L " $4, %[v]\n\t"293 "bne $4, %[expect], %l[error2]\n\t"294#endif295 /* try store */296 LONG_S " %[newv2], %[v2]\n\t"297 RSEQ_INJECT_ASM(5)298#ifdef RSEQ_TEMPLATE_MO_RELEASE299 "sync\n\t" /* full sync provides store-release */300#endif301 /* final store */302 LONG_S " %[newv], %[v]\n\t"303 "2:\n\t"304 RSEQ_INJECT_ASM(6)305 "b 5f\n\t"306 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)307 "5:\n\t"308 : /* gcc asm goto does not allow outputs */309 : [cpu_id] "r" (cpu),310 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),311 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),312 /* try store input */313 [v2] "m" (*v2),314 [newv2] "r" (newv2),315 /* final store input */316 [v] "m" (*v),317 [expect] "r" (expect),318 [newv] "r" (newv)319 RSEQ_INJECT_INPUT320 : "$4", "memory"321 RSEQ_INJECT_CLOBBER322 : abort, cmpfail323#ifdef RSEQ_COMPARE_TWICE324 , error1, error2325#endif326 );327 return 0;328abort:329 RSEQ_INJECT_FAILED330 return -1;331cmpfail:332 return 1;333#ifdef RSEQ_COMPARE_TWICE334error1:335 rseq_bug("cpu_id comparison failed");336error2:337 rseq_bug("expected value comparison failed");338#endif339}340 341static inline __attribute__((always_inline))342int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,343 void *dst, void *src, size_t len,344 intptr_t newv, int cpu)345{346 uintptr_t rseq_scratch[3];347 348 RSEQ_INJECT_C(9)349 350 __asm__ __volatile__ goto (351 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */352 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])353#ifdef RSEQ_COMPARE_TWICE354 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])355 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])356#endif357 LONG_S " %[src], %[rseq_scratch0]\n\t"358 LONG_S " %[dst], %[rseq_scratch1]\n\t"359 LONG_S " %[len], %[rseq_scratch2]\n\t"360 /* Start rseq by storing table entry pointer into rseq_cs. */361 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)362 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)363 RSEQ_INJECT_ASM(3)364 LONG_L " $4, %[v]\n\t"365 "bne $4, %[expect], 5f\n\t"366 RSEQ_INJECT_ASM(4)367#ifdef RSEQ_COMPARE_TWICE368 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)369 LONG_L " $4, %[v]\n\t"370 "bne $4, %[expect], 7f\n\t"371#endif372 /* try memcpy */373 "beqz %[len], 333f\n\t" \374 "222:\n\t" \375 "lb $4, 0(%[src])\n\t" \376 "sb $4, 0(%[dst])\n\t" \377 LONG_ADDI " %[src], 1\n\t" \378 LONG_ADDI " %[dst], 1\n\t" \379 LONG_ADDI " %[len], -1\n\t" \380 "bnez %[len], 222b\n\t" \381 "333:\n\t" \382 RSEQ_INJECT_ASM(5)383#ifdef RSEQ_TEMPLATE_MO_RELEASE384 "sync\n\t" /* full sync provides store-release */385#endif386 /* final store */387 LONG_S " %[newv], %[v]\n\t"388 "2:\n\t"389 RSEQ_INJECT_ASM(6)390 /* teardown */391 LONG_L " %[len], %[rseq_scratch2]\n\t"392 LONG_L " %[dst], %[rseq_scratch1]\n\t"393 LONG_L " %[src], %[rseq_scratch0]\n\t"394 "b 8f\n\t"395 RSEQ_ASM_DEFINE_ABORT(3, 4,396 /* teardown */397 LONG_L " %[len], %[rseq_scratch2]\n\t"398 LONG_L " %[dst], %[rseq_scratch1]\n\t"399 LONG_L " %[src], %[rseq_scratch0]\n\t",400 abort, 1b, 2b, 4f)401 RSEQ_ASM_DEFINE_CMPFAIL(5,402 /* teardown */403 LONG_L " %[len], %[rseq_scratch2]\n\t"404 LONG_L " %[dst], %[rseq_scratch1]\n\t"405 LONG_L " %[src], %[rseq_scratch0]\n\t",406 cmpfail)407#ifdef RSEQ_COMPARE_TWICE408 RSEQ_ASM_DEFINE_CMPFAIL(6,409 /* teardown */410 LONG_L " %[len], %[rseq_scratch2]\n\t"411 LONG_L " %[dst], %[rseq_scratch1]\n\t"412 LONG_L " %[src], %[rseq_scratch0]\n\t",413 error1)414 RSEQ_ASM_DEFINE_CMPFAIL(7,415 /* teardown */416 LONG_L " %[len], %[rseq_scratch2]\n\t"417 LONG_L " %[dst], %[rseq_scratch1]\n\t"418 LONG_L " %[src], %[rseq_scratch0]\n\t",419 error2)420#endif421 "8:\n\t"422 : /* gcc asm goto does not allow outputs */423 : [cpu_id] "r" (cpu),424 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),425 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),426 /* final store input */427 [v] "m" (*v),428 [expect] "r" (expect),429 [newv] "r" (newv),430 /* try memcpy input */431 [dst] "r" (dst),432 [src] "r" (src),433 [len] "r" (len),434 [rseq_scratch0] "m" (rseq_scratch[0]),435 [rseq_scratch1] "m" (rseq_scratch[1]),436 [rseq_scratch2] "m" (rseq_scratch[2])437 RSEQ_INJECT_INPUT438 : "$4", "memory"439 RSEQ_INJECT_CLOBBER440 : abort, cmpfail441#ifdef RSEQ_COMPARE_TWICE442 , error1, error2443#endif444 );445 return 0;446abort:447 RSEQ_INJECT_FAILED448 return -1;449cmpfail:450 return 1;451#ifdef RSEQ_COMPARE_TWICE452error1:453 rseq_bug("cpu_id comparison failed");454error2:455 rseq_bug("expected value comparison failed");456#endif457}458 459#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&460 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */461 462#include "rseq-bits-reset.h"463