239 lines · c
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */2/*3 * rseq-ppc.h4 *5 * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>7 */8 9/*10 * RSEQ_SIG is used with the following trap instruction:11 *12 * powerpc-be: 0f e5 00 0b twui r5,1113 * powerpc64-le: 0b 00 e5 0f twui r5,1114 * powerpc64-be: 0f e5 00 0b twui r5,1115 */16 17#define RSEQ_SIG 0x0fe5000b18 19#define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc")20#define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc")21#define rseq_smp_rmb() rseq_smp_lwsync()22#define rseq_smp_wmb() rseq_smp_lwsync()23 24#define rseq_smp_load_acquire(p) \25__extension__ ({ \26 rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \27 rseq_smp_lwsync(); \28 ____p1; \29})30 31#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync()32 33#define rseq_smp_store_release(p, v) \34do { \35 rseq_smp_lwsync(); \36 RSEQ_WRITE_ONCE(*(p), v); \37} while (0)38 39/*40 * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to41 * better handle single-stepping through the restartable critical sections.42 */43 44#ifdef __PPC64__45 46#define RSEQ_STORE_LONG(arg) "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */47#define RSEQ_STORE_INT(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */48#define RSEQ_LOAD_LONG(arg) "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */49#define RSEQ_LOAD_INT(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */50#define RSEQ_LOADX_LONG "ldx " /* From base register ("b" constraint) */51#define RSEQ_CMP_LONG "cmpd "52#define RSEQ_CMP_LONG_INT "cmpdi "53 54#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \55 start_ip, post_commit_offset, abort_ip) \56 ".pushsection __rseq_cs, \"aw\"\n\t" \57 ".balign 32\n\t" \58 __rseq_str(label) ":\n\t" \59 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \60 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \61 ".popsection\n\t" \62 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \63 ".quad " __rseq_str(label) "b\n\t" \64 ".popsection\n\t"65 66#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \67 RSEQ_INJECT_ASM(1) \68 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \69 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \70 "rldicr %%r17, %%r17, 32, 31\n\t" \71 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \72 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \73 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \74 __rseq_str(label) ":\n\t"75 76/*77 * Exit points of a rseq critical section consist of all instructions outside78 * of the critical section where a critical section can either branch to or79 * reach through the normal course of its execution. The abort IP and the80 * post-commit IP are already part of the __rseq_cs section and should not be81 * explicitly defined as additional exit points. Knowing all exit points is82 * useful to assist debuggers stepping over the critical section.83 */84#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \85 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \86 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \87 ".popsection\n\t"88 89#else /* #ifdef __PPC64__ */90 91#define RSEQ_STORE_LONG(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */92#define RSEQ_STORE_INT(arg) RSEQ_STORE_LONG(arg) /* To memory ("m" constraint) */93#define RSEQ_LOAD_LONG(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */94#define RSEQ_LOAD_INT(arg) RSEQ_LOAD_LONG(arg) /* From memory ("m" constraint) */95#define RSEQ_LOADX_LONG "lwzx " /* From base register ("b" constraint) */96#define RSEQ_CMP_LONG "cmpw "97#define RSEQ_CMP_LONG_INT "cmpwi "98 99#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \100 start_ip, post_commit_offset, abort_ip) \101 ".pushsection __rseq_cs, \"aw\"\n\t" \102 ".balign 32\n\t" \103 __rseq_str(label) ":\n\t" \104 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \105 /* 32-bit only supported on BE */ \106 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \107 ".popsection\n\t" \108 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \109 ".long 0x0, " __rseq_str(label) "b\n\t" \110 ".popsection\n\t"111 112/*113 * Exit points of a rseq critical section consist of all instructions outside114 * of the critical section where a critical section can either branch to or115 * reach through the normal course of its execution. The abort IP and the116 * post-commit IP are already part of the __rseq_cs section and should not be117 * explicitly defined as additional exit points. Knowing all exit points is118 * useful to assist debuggers stepping over the critical section.119 */120#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \121 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \122 /* 32-bit only supported on BE */ \123 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \124 ".popsection\n\t"125 126#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \127 RSEQ_INJECT_ASM(1) \128 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \129 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \130 RSEQ_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \131 __rseq_str(label) ":\n\t"132 133#endif /* #ifdef __PPC64__ */134 135#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \136 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \137 (post_commit_ip - start_ip), abort_ip)138 139#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \140 RSEQ_INJECT_ASM(2) \141 RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \142 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \143 "bne- cr7, " __rseq_str(label) "\n\t"144 145#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \146 ".pushsection __rseq_failure, \"ax\"\n\t" \147 ".long " __rseq_str(RSEQ_SIG) "\n\t" \148 __rseq_str(label) ":\n\t" \149 "b %l[" __rseq_str(abort_label) "]\n\t" \150 ".popsection\n\t"151 152/*153 * RSEQ_ASM_OPs: asm operations for rseq154 * RSEQ_ASM_OP_R_*: has hard-code registers in it155 * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)156 */157#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \158 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \159 RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \160 "bne- cr7, " __rseq_str(label) "\n\t"161 162#define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \163 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \164 RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \165 "beq- cr7, " __rseq_str(label) "\n\t"166 167#define RSEQ_ASM_OP_STORE(value, var) \168 RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"169 170/* Load @var to r17 */171#define RSEQ_ASM_OP_R_LOAD(var) \172 RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"173 174/* Store r17 to @var */175#define RSEQ_ASM_OP_R_STORE(var) \176 RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"177 178/* Add @count to r17 */179#define RSEQ_ASM_OP_R_ADD(count) \180 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"181 182/* Load (r17 + voffp) to r17 */183#define RSEQ_ASM_OP_R_LOADX(voffp) \184 RSEQ_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"185 186/* TODO: implement a faster memcpy. */187#define RSEQ_ASM_OP_R_MEMCPY() \188 RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \189 "beq 333f\n\t" \190 "addi %%r20, %%r20, -1\n\t" \191 "addi %%r21, %%r21, -1\n\t" \192 "222:\n\t" \193 "lbzu %%r18, 1(%%r20)\n\t" \194 "stbu %%r18, 1(%%r21)\n\t" \195 "addi %%r19, %%r19, -1\n\t" \196 RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \197 "bne 222b\n\t" \198 "333:\n\t" \199 200#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \201 RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \202 __rseq_str(post_commit_label) ":\n\t"203 204#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \205 RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \206 __rseq_str(post_commit_label) ":\n\t"207 208/* Per-cpu-id indexing. */209 210#define RSEQ_TEMPLATE_CPU_ID211#define RSEQ_TEMPLATE_MO_RELAXED212#include "rseq-ppc-bits.h"213#undef RSEQ_TEMPLATE_MO_RELAXED214 215#define RSEQ_TEMPLATE_MO_RELEASE216#include "rseq-ppc-bits.h"217#undef RSEQ_TEMPLATE_MO_RELEASE218#undef RSEQ_TEMPLATE_CPU_ID219 220/* Per-mm-cid indexing. */221 222#define RSEQ_TEMPLATE_MM_CID223#define RSEQ_TEMPLATE_MO_RELAXED224#include "rseq-ppc-bits.h"225#undef RSEQ_TEMPLATE_MO_RELAXED226 227#define RSEQ_TEMPLATE_MO_RELEASE228#include "rseq-ppc-bits.h"229#undef RSEQ_TEMPLATE_MO_RELEASE230#undef RSEQ_TEMPLATE_MM_CID231 232/* APIs which are not based on cpu ids. */233 234#define RSEQ_TEMPLATE_CPU_ID_NONE235#define RSEQ_TEMPLATE_MO_RELAXED236#include "rseq-ppc-bits.h"237#undef RSEQ_TEMPLATE_MO_RELAXED238#undef RSEQ_TEMPLATE_CPU_ID_NONE239