393 lines · c
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */2/*3 * rseq-arm64-bits.h4 *5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>7 */8 9#include "rseq-bits-template.h"10 11#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \12 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))13 14static inline __attribute__((always_inline))15int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)16{17 RSEQ_INJECT_C(9)18 19 __asm__ __volatile__ goto (20 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)21 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])22#ifdef RSEQ_COMPARE_TWICE23 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])24 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])25#endif26 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)27 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)28 RSEQ_INJECT_ASM(3)29 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])30 RSEQ_INJECT_ASM(4)31#ifdef RSEQ_COMPARE_TWICE32 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])33 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])34#endif35 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)36 RSEQ_INJECT_ASM(5)37 RSEQ_ASM_DEFINE_ABORT(4, abort)38 : /* gcc asm goto does not allow outputs */39 : [cpu_id] "r" (cpu),40 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),41 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),42 [v] "Qo" (*v),43 [expect] "r" (expect),44 [newv] "r" (newv)45 RSEQ_INJECT_INPUT46 : "memory", RSEQ_ASM_TMP_REG47 : abort, cmpfail48#ifdef RSEQ_COMPARE_TWICE49 , error1, error250#endif51 );52 rseq_after_asm_goto();53 return 0;54abort:55 rseq_after_asm_goto();56 RSEQ_INJECT_FAILED57 return -1;58cmpfail:59 rseq_after_asm_goto();60 return 1;61#ifdef RSEQ_COMPARE_TWICE62error1:63 rseq_after_asm_goto();64 rseq_bug("cpu_id comparison failed");65error2:66 rseq_after_asm_goto();67 rseq_bug("expected value comparison failed");68#endif69}70 71static inline __attribute__((always_inline))72int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,73 long voffp, intptr_t *load, int cpu)74{75 RSEQ_INJECT_C(9)76 77 __asm__ __volatile__ goto (78 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)79 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])80#ifdef RSEQ_COMPARE_TWICE81 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])82 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])83#endif84 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)85 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)86 RSEQ_INJECT_ASM(3)87 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])88 RSEQ_INJECT_ASM(4)89#ifdef RSEQ_COMPARE_TWICE90 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])91 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])92#endif93 RSEQ_ASM_OP_R_LOAD(v)94 RSEQ_ASM_OP_R_STORE(load)95 RSEQ_ASM_OP_R_LOAD_OFF(voffp)96 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)97 RSEQ_INJECT_ASM(5)98 RSEQ_ASM_DEFINE_ABORT(4, abort)99 : /* gcc asm goto does not allow outputs */100 : [cpu_id] "r" (cpu),101 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),102 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),103 [v] "Qo" (*v),104 [expectnot] "r" (expectnot),105 [load] "Qo" (*load),106 [voffp] "r" (voffp)107 RSEQ_INJECT_INPUT108 : "memory", RSEQ_ASM_TMP_REG109 : abort, cmpfail110#ifdef RSEQ_COMPARE_TWICE111 , error1, error2112#endif113 );114 rseq_after_asm_goto();115 return 0;116abort:117 rseq_after_asm_goto();118 RSEQ_INJECT_FAILED119 return -1;120cmpfail:121 rseq_after_asm_goto();122 return 1;123#ifdef RSEQ_COMPARE_TWICE124error1:125 rseq_after_asm_goto();126 rseq_bug("cpu_id comparison failed");127error2:128 rseq_after_asm_goto();129 rseq_bug("expected value comparison failed");130#endif131}132 133static inline __attribute__((always_inline))134int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)135{136 RSEQ_INJECT_C(9)137 138 __asm__ __volatile__ goto (139 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)140#ifdef RSEQ_COMPARE_TWICE141 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])142#endif143 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)144 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)145 RSEQ_INJECT_ASM(3)146#ifdef RSEQ_COMPARE_TWICE147 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])148#endif149 RSEQ_ASM_OP_R_LOAD(v)150 RSEQ_ASM_OP_R_ADD(count)151 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)152 RSEQ_INJECT_ASM(4)153 RSEQ_ASM_DEFINE_ABORT(4, abort)154 : /* gcc asm goto does not allow outputs */155 : [cpu_id] "r" (cpu),156 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),157 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),158 [v] "Qo" (*v),159 [count] "r" (count)160 RSEQ_INJECT_INPUT161 : "memory", RSEQ_ASM_TMP_REG162 : abort163#ifdef RSEQ_COMPARE_TWICE164 , error1165#endif166 );167 rseq_after_asm_goto();168 return 0;169abort:170 rseq_after_asm_goto();171 RSEQ_INJECT_FAILED172 return -1;173#ifdef RSEQ_COMPARE_TWICE174error1:175 rseq_after_asm_goto();176 rseq_bug("cpu_id comparison failed");177#endif178}179 180static inline __attribute__((always_inline))181int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,182 intptr_t *v2, intptr_t expect2,183 intptr_t newv, int cpu)184{185 RSEQ_INJECT_C(9)186 187 __asm__ __volatile__ goto (188 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)189 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])190#ifdef RSEQ_COMPARE_TWICE191 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])192 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])193 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])194#endif195 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)196 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)197 RSEQ_INJECT_ASM(3)198 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])199 RSEQ_INJECT_ASM(4)200 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])201 RSEQ_INJECT_ASM(5)202#ifdef RSEQ_COMPARE_TWICE203 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])204 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])205 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])206#endif207 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)208 RSEQ_INJECT_ASM(6)209 RSEQ_ASM_DEFINE_ABORT(4, abort)210 : /* gcc asm goto does not allow outputs */211 : [cpu_id] "r" (cpu),212 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),213 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),214 [v] "Qo" (*v),215 [expect] "r" (expect),216 [v2] "Qo" (*v2),217 [expect2] "r" (expect2),218 [newv] "r" (newv)219 RSEQ_INJECT_INPUT220 : "memory", RSEQ_ASM_TMP_REG221 : abort, cmpfail222#ifdef RSEQ_COMPARE_TWICE223 , error1, error2, error3224#endif225 );226 rseq_after_asm_goto();227 return 0;228abort:229 rseq_after_asm_goto();230 RSEQ_INJECT_FAILED231 return -1;232cmpfail:233 rseq_after_asm_goto();234 return 1;235#ifdef RSEQ_COMPARE_TWICE236error1:237 rseq_after_asm_goto();238 rseq_bug("cpu_id comparison failed");239error2:240 rseq_after_asm_goto();241 rseq_bug("expected value comparison failed");242error3:243 rseq_after_asm_goto();244 rseq_bug("2nd expected value comparison failed");245#endif246}247 248#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&249 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */250 251#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \252 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))253 254static inline __attribute__((always_inline))255int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,256 intptr_t *v2, intptr_t newv2,257 intptr_t newv, int cpu)258{259 RSEQ_INJECT_C(9)260 261 __asm__ __volatile__ goto (262 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)263 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])264#ifdef RSEQ_COMPARE_TWICE265 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])266 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])267#endif268 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)269 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)270 RSEQ_INJECT_ASM(3)271 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])272 RSEQ_INJECT_ASM(4)273#ifdef RSEQ_COMPARE_TWICE274 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])275 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])276#endif277 RSEQ_ASM_OP_STORE(newv2, v2)278 RSEQ_INJECT_ASM(5)279#ifdef RSEQ_TEMPLATE_MO_RELEASE280 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)281#else282 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)283#endif284 RSEQ_INJECT_ASM(6)285 RSEQ_ASM_DEFINE_ABORT(4, abort)286 : /* gcc asm goto does not allow outputs */287 : [cpu_id] "r" (cpu),288 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),289 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),290 [expect] "r" (expect),291 [v] "Qo" (*v),292 [newv] "r" (newv),293 [v2] "Qo" (*v2),294 [newv2] "r" (newv2)295 RSEQ_INJECT_INPUT296 : "memory", RSEQ_ASM_TMP_REG297 : abort, cmpfail298#ifdef RSEQ_COMPARE_TWICE299 , error1, error2300#endif301 );302 rseq_after_asm_goto();303 return 0;304abort:305 rseq_after_asm_goto();306 RSEQ_INJECT_FAILED307 return -1;308cmpfail:309 rseq_after_asm_goto();310 return 1;311#ifdef RSEQ_COMPARE_TWICE312error1:313 rseq_after_asm_goto();314 rseq_bug("cpu_id comparison failed");315error2:316 rseq_after_asm_goto();317 rseq_bug("expected value comparison failed");318#endif319}320 321static inline __attribute__((always_inline))322int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,323 void *dst, void *src, size_t len,324 intptr_t newv, int cpu)325{326 RSEQ_INJECT_C(9)327 328 __asm__ __volatile__ goto (329 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)330 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])331#ifdef RSEQ_COMPARE_TWICE332 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])333 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])334#endif335 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)336 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)337 RSEQ_INJECT_ASM(3)338 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])339 RSEQ_INJECT_ASM(4)340#ifdef RSEQ_COMPARE_TWICE341 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])342 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])343#endif344 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)345 RSEQ_INJECT_ASM(5)346#ifdef RSEQ_TEMPLATE_MO_RELEASE347 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)348#else349 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)350#endif351 RSEQ_INJECT_ASM(6)352 RSEQ_ASM_DEFINE_ABORT(4, abort)353 : /* gcc asm goto does not allow outputs */354 : [cpu_id] "r" (cpu),355 [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),356 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),357 [expect] "r" (expect),358 [v] "Qo" (*v),359 [newv] "r" (newv),360 [dst] "r" (dst),361 [src] "r" (src),362 [len] "r" (len)363 RSEQ_INJECT_INPUT364 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2365 : abort, cmpfail366#ifdef RSEQ_COMPARE_TWICE367 , error1, error2368#endif369 );370 rseq_after_asm_goto();371 return 0;372abort:373 rseq_after_asm_goto();374 RSEQ_INJECT_FAILED375 return -1;376cmpfail:377 rseq_after_asm_goto();378 return 1;379#ifdef RSEQ_COMPARE_TWICE380error1:381 rseq_after_asm_goto();382 rseq_bug("cpu_id comparison failed");383error2:384 rseq_after_asm_goto();385 rseq_bug("expected value comparison failed");386#endif387}388 389#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&390 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */391 392#include "rseq-bits-reset.h"393