1733 lines · cpp
1//===-- NativeRegisterContextLinux_arm64.cpp ------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#if defined(__arm64__) || defined(__aarch64__)10 11#include "NativeRegisterContextLinux_arm64.h"12#include "NativeRegisterContextLinux_arm.h"13#include "NativeRegisterContextLinux_arm64dbreg.h"14 15#include "lldb/Host/HostInfo.h"16#include "lldb/Host/common/NativeProcessProtocol.h"17#include "lldb/Host/linux/Ptrace.h"18#include "lldb/Utility/DataBufferHeap.h"19#include "lldb/Utility/Log.h"20#include "lldb/Utility/RegisterValue.h"21#include "lldb/Utility/Status.h"22 23#include "Plugins/Process/Linux/NativeProcessLinux.h"24#include "Plugins/Process/Linux/Procfs.h"25#include "Plugins/Process/POSIX/ProcessPOSIXLog.h"26#include "Plugins/Process/Utility/MemoryTagManagerAArch64MTE.h"27#include "Plugins/Process/Utility/RegisterFlagsDetector_arm64.h"28#include "Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h"29 30// System includes - They have to be included after framework includes because31// they define some macros which collide with variable names in other modules32#include <sys/uio.h>33// NT_PRSTATUS and NT_FPREGSET definition34#include <elf.h>35#include <mutex>36#include <optional>37 38#ifndef NT_ARM_SVE39#define NT_ARM_SVE 0x405 /* ARM Scalable Vector Extension */40#endif41 42#ifndef NT_ARM_SSVE43#define NT_ARM_SSVE \44 0x40b /* ARM Scalable Matrix Extension, Streaming SVE mode */45#endif46 47#ifndef NT_ARM_ZA48#define NT_ARM_ZA 0x40c /* ARM Scalable Matrix Extension, Array Storage */49#endif50 51#ifndef NT_ARM_ZT52#define NT_ARM_ZT \53 0x40d /* ARM Scalable Matrix Extension 2, lookup table register */54#endif55 56#ifndef NT_ARM_PAC_MASK57#define NT_ARM_PAC_MASK 0x406 /* Pointer authentication code masks */58#endif59 60#ifndef NT_ARM_TAGGED_ADDR_CTRL61#define NT_ARM_TAGGED_ADDR_CTRL 0x409 /* Tagged address control register */62#endif63 64#ifndef NT_ARM_FPMR65#define NT_ARM_FPMR 0x40e /* Floating point mode register */66#endif67 68#ifndef NT_ARM_GCS69#define NT_ARM_GCS 0x410 /* Guarded Control Stack control registers */70#endif71 72#define HWCAP_PACA (1 << 30)73 74#define HWCAP_GCS (1UL << 32)75 76#define HWCAP2_MTE (1 << 18)77 78#define HWCAP2_FPMR (1UL << 48)79 80using namespace lldb;81using namespace lldb_private;82using namespace lldb_private::process_linux;83 84// A NativeRegisterContext is constructed per thread, but all threads' registers85// will contain the same fields. Therefore this mutex prevents each instance86// competing with the other, and subsequent instances from having to detect the87// fields all over again.88static std::mutex g_register_flags_detector_mutex;89static Arm64RegisterFlagsDetector g_register_flags_detector;90 91std::unique_ptr<NativeRegisterContextLinux>92NativeRegisterContextLinux::CreateHostNativeRegisterContextLinux(93 const ArchSpec &target_arch, NativeThreadLinux &native_thread) {94 switch (target_arch.GetMachine()) {95 case llvm::Triple::arm:96 return std::make_unique<NativeRegisterContextLinux_arm>(target_arch,97 native_thread);98 case llvm::Triple::aarch64: {99 // Configure register sets supported by this AArch64 target.100 // Read SVE header to check for SVE support.101 struct sve::user_sve_header sve_header;102 struct iovec ioVec;103 ioVec.iov_base = &sve_header;104 ioVec.iov_len = sizeof(sve_header);105 unsigned int regset = NT_ARM_SVE;106 107 Flags opt_regsets;108 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,109 native_thread.GetID(), ®set,110 &ioVec, sizeof(sve_header))111 .Success()) {112 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSVE);113 114 // We may also have the Scalable Matrix Extension (SME) which adds a115 // streaming SVE mode.116 ioVec.iov_len = sizeof(sve_header);117 regset = NT_ARM_SSVE;118 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,119 native_thread.GetID(), ®set,120 &ioVec, sizeof(sve_header))121 .Success())122 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskSSVE);123 }124 125 sve::user_za_header za_header;126 ioVec.iov_base = &za_header;127 ioVec.iov_len = sizeof(za_header);128 regset = NT_ARM_ZA;129 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,130 native_thread.GetID(), ®set,131 &ioVec, sizeof(za_header))132 .Success())133 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskZA);134 135 // SME's ZT0 is a 512 bit register.136 std::array<uint8_t, 64> zt_reg;137 ioVec.iov_base = zt_reg.data();138 ioVec.iov_len = zt_reg.size();139 regset = NT_ARM_ZT;140 if (NativeProcessLinux::PtraceWrapper(PTRACE_GETREGSET,141 native_thread.GetID(), ®set,142 &ioVec, zt_reg.size())143 .Success())144 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskZT);145 146 NativeProcessLinux &process = native_thread.GetProcess();147 148 std::optional<uint64_t> auxv_at_hwcap =149 process.GetAuxValue(AuxVector::AUXV_AT_HWCAP);150 if (auxv_at_hwcap && (*auxv_at_hwcap & HWCAP_PACA))151 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskPAuth);152 153 std::optional<uint64_t> auxv_at_hwcap2 =154 process.GetAuxValue(AuxVector::AUXV_AT_HWCAP2);155 if (auxv_at_hwcap2) {156 if (*auxv_at_hwcap2 & HWCAP2_MTE)157 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskMTE);158 if (*auxv_at_hwcap2 & HWCAP2_FPMR)159 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskFPMR);160 if (*auxv_at_hwcap & HWCAP_GCS)161 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskGCS);162 }163 164 opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskTLS);165 166 std::optional<uint64_t> auxv_at_hwcap3 =167 process.GetAuxValue(AuxVector::AUXV_AT_HWCAP3);168 std::lock_guard<std::mutex> lock(g_register_flags_detector_mutex);169 if (!g_register_flags_detector.HasDetected())170 g_register_flags_detector.DetectFields(auxv_at_hwcap.value_or(0),171 auxv_at_hwcap2.value_or(0),172 auxv_at_hwcap3.value_or(0));173 174 auto register_info_up =175 std::make_unique<RegisterInfoPOSIX_arm64>(target_arch, opt_regsets);176 return std::make_unique<NativeRegisterContextLinux_arm64>(177 target_arch, native_thread, std::move(register_info_up));178 }179 default:180 llvm_unreachable("have no register context for architecture");181 }182}183 184llvm::Expected<ArchSpec>185NativeRegisterContextLinux::DetermineArchitecture(lldb::tid_t tid) {186 return DetermineArchitectureViaGPR(187 tid, RegisterInfoPOSIX_arm64::GetGPRSizeStatic());188}189 190NativeRegisterContextLinux_arm64::NativeRegisterContextLinux_arm64(191 const ArchSpec &target_arch, NativeThreadProtocol &native_thread,192 std::unique_ptr<RegisterInfoPOSIX_arm64> register_info_up)193 : NativeRegisterContextRegisterInfo(native_thread,194 register_info_up.release()),195 NativeRegisterContextLinux(native_thread) {196 g_register_flags_detector.UpdateRegisterInfo(197 GetRegisterInfoInterface().GetRegisterInfo(),198 GetRegisterInfoInterface().GetRegisterCount());199 200 ::memset(&m_fpr, 0, sizeof(m_fpr));201 ::memset(&m_gpr_arm64, 0, sizeof(m_gpr_arm64));202 ::memset(&m_hwp_regs, 0, sizeof(m_hwp_regs));203 ::memset(&m_hbp_regs, 0, sizeof(m_hbp_regs));204 ::memset(&m_sve_header, 0, sizeof(m_sve_header));205 ::memset(&m_pac_mask, 0, sizeof(m_pac_mask));206 ::memset(&m_tls_regs, 0, sizeof(m_tls_regs));207 ::memset(&m_sme_pseudo_regs, 0, sizeof(m_sme_pseudo_regs));208 ::memset(&m_gcs_regs, 0, sizeof(m_gcs_regs));209 std::fill(m_zt_reg.begin(), m_zt_reg.end(), 0);210 211 m_mte_ctrl_reg = 0;212 m_fpmr_reg = 0;213 214 // 16 is just a maximum value, query hardware for actual watchpoint count215 m_max_hwp_supported = 16;216 m_max_hbp_supported = 16;217 218 m_refresh_hwdebug_info = true;219 220 m_gpr_is_valid = false;221 m_fpu_is_valid = false;222 m_sve_buffer_is_valid = false;223 m_sve_header_is_valid = false;224 m_pac_mask_is_valid = false;225 m_mte_ctrl_is_valid = false;226 m_tls_is_valid = false;227 m_zt_buffer_is_valid = false;228 m_fpmr_is_valid = false;229 m_gcs_is_valid = false;230 231 // SME adds the tpidr2 register232 m_tls_size = GetRegisterInfo().IsSSVEPresent() ? sizeof(m_tls_regs)233 : sizeof(m_tls_regs.tpidr_reg);234 235 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent())236 m_sve_state = SVEState::Unknown;237 else238 m_sve_state = SVEState::Disabled;239}240 241RegisterInfoPOSIX_arm64 &242NativeRegisterContextLinux_arm64::GetRegisterInfo() const {243 return static_cast<RegisterInfoPOSIX_arm64 &>(*m_register_info_interface_up);244}245 246uint32_t NativeRegisterContextLinux_arm64::GetRegisterSetCount() const {247 return GetRegisterInfo().GetRegisterSetCount();248}249 250const RegisterSet *251NativeRegisterContextLinux_arm64::GetRegisterSet(uint32_t set_index) const {252 return GetRegisterInfo().GetRegisterSet(set_index);253}254 255uint32_t NativeRegisterContextLinux_arm64::GetUserRegisterCount() const {256 uint32_t count = 0;257 for (uint32_t set_index = 0; set_index < GetRegisterSetCount(); ++set_index)258 count += GetRegisterSet(set_index)->num_registers;259 return count;260}261 262Status263NativeRegisterContextLinux_arm64::ReadRegister(const RegisterInfo *reg_info,264 RegisterValue ®_value) {265 Status error;266 267 if (!reg_info) {268 error = Status::FromErrorString("reg_info NULL");269 return error;270 }271 272 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];273 274 if (reg == LLDB_INVALID_REGNUM)275 return Status::FromErrorStringWithFormat(276 "no lldb regnum for %s",277 reg_info && reg_info->name ? reg_info->name : "<unknown register>");278 279 uint8_t *src;280 uint32_t offset = LLDB_INVALID_INDEX32;281 uint64_t sve_vg;282 std::vector<uint8_t> sve_reg_non_live;283 284 if (IsGPR(reg)) {285 error = ReadGPR();286 if (error.Fail())287 return error;288 289 offset = reg_info->byte_offset;290 assert(offset < GetGPRSize());291 src = (uint8_t *)GetGPRBuffer() + offset;292 293 } else if (IsFPR(reg)) {294 if (m_sve_state == SVEState::Disabled) {295 // SVE is disabled take legacy route for FPU register access296 error = ReadFPR();297 if (error.Fail())298 return error;299 300 offset = CalculateFprOffset(reg_info);301 assert(offset < GetFPRSize());302 src = (uint8_t *)GetFPRBuffer() + offset;303 } else {304 // SVE or SSVE enabled, we will read and cache SVE ptrace data.305 // In SIMD or Full mode, the data comes from the SVE regset. In streaming306 // mode it comes from the streaming SVE regset.307 error = ReadAllSVE();308 if (error.Fail())309 return error;310 311 // FPSR and FPCR will be located right after Z registers in312 // SVEState::FPSIMD while in SVEState::Full or SVEState::Streaming they313 // will be located at the end of register data after an alignment314 // correction based on currently selected vector length.315 uint32_t sve_reg_num = LLDB_INVALID_REGNUM;316 if (reg == GetRegisterInfo().GetRegNumFPSR()) {317 sve_reg_num = reg;318 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)319 offset = sve::PTraceFPSROffset(sve::vq_from_vl(m_sve_header.vl));320 else if (m_sve_state == SVEState::FPSIMD)321 offset = sve::ptrace_fpsimd_offset + (32 * 16);322 } else if (reg == GetRegisterInfo().GetRegNumFPCR()) {323 sve_reg_num = reg;324 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)325 offset = sve::PTraceFPCROffset(sve::vq_from_vl(m_sve_header.vl));326 else if (m_sve_state == SVEState::FPSIMD)327 offset = sve::ptrace_fpsimd_offset + (32 * 16) + 4;328 } else {329 // Extract SVE Z register value register number for this reg_info330 if (reg_info->value_regs &&331 reg_info->value_regs[0] != LLDB_INVALID_REGNUM)332 sve_reg_num = reg_info->value_regs[0];333 offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num));334 }335 336 assert(offset < GetSVEBufferSize());337 src = (uint8_t *)GetSVEBuffer() + offset;338 }339 } else if (IsTLS(reg)) {340 error = ReadTLS();341 if (error.Fail())342 return error;343 344 offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset();345 assert(offset < GetTLSBufferSize());346 src = (uint8_t *)GetTLSBuffer() + offset;347 } else if (IsSVE(reg)) {348 if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown)349 return Status::FromErrorString("SVE disabled or not supported");350 351 if (GetRegisterInfo().IsSVERegVG(reg)) {352 sve_vg = GetSVERegVG();353 src = (uint8_t *)&sve_vg;354 } else {355 // SVE enabled, we will read and cache SVE ptrace data356 error = ReadAllSVE();357 if (error.Fail())358 return error;359 360 if (m_sve_state == SVEState::FPSIMD) {361 // In FPSIMD state SVE payload mirrors legacy fpsimd struct and so362 // just copy 16 bytes of v register to the start of z register. All363 // other SVE register will be set to zero.364 sve_reg_non_live.resize(reg_info->byte_size, 0);365 src = sve_reg_non_live.data();366 367 if (GetRegisterInfo().IsSVEZReg(reg)) {368 offset = CalculateSVEOffset(reg_info);369 assert(offset < GetSVEBufferSize());370 ::memcpy(sve_reg_non_live.data(), (uint8_t *)GetSVEBuffer() + offset,371 16);372 }373 } else {374 offset = CalculateSVEOffset(reg_info);375 assert(offset < GetSVEBufferSize());376 src = (uint8_t *)GetSVEBuffer() + offset;377 }378 }379 } else if (IsPAuth(reg)) {380 error = ReadPAuthMask();381 if (error.Fail())382 return error;383 384 offset = reg_info->byte_offset - GetRegisterInfo().GetPAuthOffset();385 assert(offset < GetPACMaskSize());386 src = (uint8_t *)GetPACMask() + offset;387 } else if (IsMTE(reg)) {388 error = ReadMTEControl();389 if (error.Fail())390 return error;391 392 offset = reg_info->byte_offset - GetRegisterInfo().GetMTEOffset();393 assert(offset < GetMTEControlSize());394 src = (uint8_t *)GetMTEControl() + offset;395 } else if (IsSME(reg)) {396 if (GetRegisterInfo().IsSMERegZA(reg)) {397 error = ReadZAHeader();398 if (error.Fail())399 return error;400 401 // If there is only a header and no registers, ZA is inactive. Read as 0402 // in this case.403 if (m_za_header.size == sizeof(m_za_header)) {404 // This will get reconfigured/reset later, so we are safe to use it.405 // ZA is a square of VL * VL and the ptrace buffer also includes the406 // header itself.407 m_za_ptrace_payload.resize(((m_za_header.vl) * (m_za_header.vl)) +408 GetZAHeaderSize());409 std::fill(m_za_ptrace_payload.begin(), m_za_ptrace_payload.end(), 0);410 } else {411 // ZA is active, read the real register.412 error = ReadZA();413 if (error.Fail())414 return error;415 }416 417 // ZA is part of the SME set but uses a separate member buffer for418 // storage. Therefore its effective byte offset is always 0 even if it419 // isn't 0 within the SME register set.420 src = (uint8_t *)GetZABuffer() + GetZAHeaderSize();421 } else if (GetRegisterInfo().IsSMERegZT(reg)) {422 // Unlike ZA, the kernel will return register data for ZT0 when ZA is not423 // enabled. This data will be all 0s so we don't have to invent anything424 // like we did for ZA.425 error = ReadZT();426 if (error.Fail())427 return error;428 429 src = (uint8_t *)GetZTBuffer();430 } else {431 error = ReadSMESVG();432 if (error.Fail())433 return error;434 435 // This is a psuedo so it never fails.436 ReadSMEControl();437 438 offset = reg_info->byte_offset - GetRegisterInfo().GetSMEOffset();439 assert(offset < GetSMEPseudoBufferSize());440 src = (uint8_t *)GetSMEPseudoBuffer() + offset;441 }442 } else if (IsFPMR(reg)) {443 error = ReadFPMR();444 if (error.Fail())445 return error;446 447 offset = reg_info->byte_offset - GetRegisterInfo().GetFPMROffset();448 assert(offset < GetFPMRBufferSize());449 src = (uint8_t *)GetFPMRBuffer() + offset;450 } else if (IsGCS(reg)) {451 error = ReadGCS();452 if (error.Fail())453 return error;454 455 offset = reg_info->byte_offset - GetRegisterInfo().GetGCSOffset();456 assert(offset < GetGCSBufferSize());457 src = (uint8_t *)GetGCSBuffer() + offset;458 } else459 return Status::FromErrorString(460 "failed - register wasn't recognized to be a GPR or an FPR, "461 "write strategy unknown");462 463 reg_value.SetFromMemoryData(*reg_info, src, reg_info->byte_size,464 eByteOrderLittle, error);465 466 return error;467}468 469Status NativeRegisterContextLinux_arm64::WriteRegister(470 const RegisterInfo *reg_info, const RegisterValue ®_value) {471 Status error;472 473 if (!reg_info)474 return Status::FromErrorString("reg_info NULL");475 476 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];477 478 if (reg == LLDB_INVALID_REGNUM)479 return Status::FromErrorStringWithFormat(480 "no lldb regnum for %s",481 reg_info && reg_info->name ? reg_info->name : "<unknown register>");482 483 uint8_t *dst;484 uint32_t offset = LLDB_INVALID_INDEX32;485 std::vector<uint8_t> sve_reg_non_live;486 487 if (IsGPR(reg)) {488 error = ReadGPR();489 if (error.Fail())490 return error;491 492 assert(reg_info->byte_offset < GetGPRSize());493 dst = (uint8_t *)GetGPRBuffer() + reg_info->byte_offset;494 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);495 496 return WriteGPR();497 } else if (IsFPR(reg)) {498 if (m_sve_state == SVEState::Disabled) {499 // SVE is disabled take legacy route for FPU register access500 error = ReadFPR();501 if (error.Fail())502 return error;503 504 offset = CalculateFprOffset(reg_info);505 assert(offset < GetFPRSize());506 dst = (uint8_t *)GetFPRBuffer() + offset;507 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);508 509 return WriteFPR();510 } else {511 // SVE enabled, we will read and cache SVE ptrace data.512 error = ReadAllSVE();513 if (error.Fail())514 return error;515 516 // FPSR and FPCR will be located right after Z registers in517 // SVEState::FPSIMD while in SVEState::Full or SVEState::Streaming they518 // will be located at the end of register data after an alignment519 // correction based on currently selected vector length.520 uint32_t sve_reg_num = LLDB_INVALID_REGNUM;521 if (reg == GetRegisterInfo().GetRegNumFPSR()) {522 sve_reg_num = reg;523 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)524 offset = sve::PTraceFPSROffset(sve::vq_from_vl(m_sve_header.vl));525 else if (m_sve_state == SVEState::FPSIMD)526 offset = sve::ptrace_fpsimd_offset + (32 * 16);527 } else if (reg == GetRegisterInfo().GetRegNumFPCR()) {528 sve_reg_num = reg;529 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::Streaming)530 offset = sve::PTraceFPCROffset(sve::vq_from_vl(m_sve_header.vl));531 else if (m_sve_state == SVEState::FPSIMD)532 offset = sve::ptrace_fpsimd_offset + (32 * 16) + 4;533 } else {534 // Extract SVE Z register value register number for this reg_info535 if (reg_info->value_regs &&536 reg_info->value_regs[0] != LLDB_INVALID_REGNUM)537 sve_reg_num = reg_info->value_regs[0];538 offset = CalculateSVEOffset(GetRegisterInfoAtIndex(sve_reg_num));539 }540 541 assert(offset < GetSVEBufferSize());542 dst = (uint8_t *)GetSVEBuffer() + offset;543 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);544 return WriteAllSVE();545 }546 } else if (IsSVE(reg)) {547 if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown)548 return Status::FromErrorString("SVE disabled or not supported");549 else {550 // Target has SVE enabled, we will read and cache SVE ptrace data551 error = ReadAllSVE();552 if (error.Fail())553 return error;554 555 if (GetRegisterInfo().IsSVERegVG(reg)) {556 uint64_t vg_value = reg_value.GetAsUInt64();557 558 if (sve::vl_valid(vg_value * 8)) {559 if (m_sve_header_is_valid && vg_value == GetSVERegVG())560 return error;561 562 SetSVERegVG(vg_value);563 564 error = WriteSVEHeader();565 if (error.Success()) {566 // Changing VG during streaming mode also changes the size of ZA.567 if (m_sve_state == SVEState::Streaming)568 m_za_header_is_valid = false;569 ConfigureRegisterContext();570 }571 572 if (m_sve_header_is_valid && vg_value == GetSVERegVG())573 return error;574 }575 576 return Status::FromErrorString("SVE vector length update failed.");577 }578 579 // If target supports SVE but currently in FPSIMD mode.580 if (m_sve_state == SVEState::FPSIMD) {581 // Here we will check if writing this SVE register enables582 // SVEState::Full583 bool set_sve_state_full = false;584 const uint8_t *reg_bytes = (const uint8_t *)reg_value.GetBytes();585 if (GetRegisterInfo().IsSVEZReg(reg)) {586 for (uint32_t i = 16; i < reg_info->byte_size; i++) {587 if (reg_bytes[i]) {588 set_sve_state_full = true;589 break;590 }591 }592 } else if (GetRegisterInfo().IsSVEPReg(reg) ||593 reg == GetRegisterInfo().GetRegNumSVEFFR()) {594 for (uint32_t i = 0; i < reg_info->byte_size; i++) {595 if (reg_bytes[i]) {596 set_sve_state_full = true;597 break;598 }599 }600 }601 602 if (!set_sve_state_full && GetRegisterInfo().IsSVEZReg(reg)) {603 // We are writing a Z register which is zero beyond 16 bytes so copy604 // first 16 bytes only as SVE payload mirrors legacy fpsimd structure605 offset = CalculateSVEOffset(reg_info);606 assert(offset < GetSVEBufferSize());607 dst = (uint8_t *)GetSVEBuffer() + offset;608 ::memcpy(dst, reg_value.GetBytes(), 16);609 610 return WriteAllSVE();611 } else612 return Status::FromErrorString(613 "SVE state change operation not supported");614 } else {615 offset = CalculateSVEOffset(reg_info);616 assert(offset < GetSVEBufferSize());617 dst = (uint8_t *)GetSVEBuffer() + offset;618 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);619 return WriteAllSVE();620 }621 }622 } else if (IsMTE(reg)) {623 error = ReadMTEControl();624 if (error.Fail())625 return error;626 627 offset = reg_info->byte_offset - GetRegisterInfo().GetMTEOffset();628 assert(offset < GetMTEControlSize());629 dst = (uint8_t *)GetMTEControl() + offset;630 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);631 632 return WriteMTEControl();633 } else if (IsTLS(reg)) {634 error = ReadTLS();635 if (error.Fail())636 return error;637 638 offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset();639 assert(offset < GetTLSBufferSize());640 dst = (uint8_t *)GetTLSBuffer() + offset;641 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);642 643 return WriteTLS();644 } else if (IsSME(reg)) {645 if (GetRegisterInfo().IsSMERegZA(reg)) {646 error = ReadZA();647 if (error.Fail())648 return error;649 650 // ZA is part of the SME set but not stored with the other SME registers.651 // So its byte offset is effectively always 0.652 dst = (uint8_t *)GetZABuffer() + GetZAHeaderSize();653 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);654 655 // While this is writing a header that contains a vector length, the only656 // way to change that is via the vg register. So here we assume the length657 // will always be the current length and no reconfigure is needed.658 return WriteZA();659 } else if (GetRegisterInfo().IsSMERegZT(reg)) {660 error = ReadZT();661 if (error.Fail())662 return error;663 664 dst = (uint8_t *)GetZTBuffer();665 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);666 667 return WriteZT();668 } else669 return Status::FromErrorString(670 "Writing to SVG or SVCR is not supported.");671 } else if (IsFPMR(reg)) {672 error = ReadFPMR();673 if (error.Fail())674 return error;675 676 offset = reg_info->byte_offset - GetRegisterInfo().GetFPMROffset();677 assert(offset < GetFPMRBufferSize());678 dst = (uint8_t *)GetFPMRBuffer() + offset;679 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);680 681 return WriteFPMR();682 } else if (IsGCS(reg)) {683 error = ReadGCS();684 if (error.Fail())685 return error;686 687 offset = reg_info->byte_offset - GetRegisterInfo().GetGCSOffset();688 assert(offset < GetGCSBufferSize());689 dst = (uint8_t *)GetGCSBuffer() + offset;690 ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);691 692 return WriteGCS();693 }694 695 return Status::FromErrorString("Failed to write register value");696}697 698enum RegisterSetType : uint32_t {699 GPR,700 SVE, // Used for SVE and SSVE.701 FPR, // When there is no SVE, or SVE in FPSIMD mode.702 // Pointer authentication registers are read only, so not included here.703 MTE,704 TLS,705 SME, // ZA only, because SVCR and SVG are pseudo registers.706 SME2, // ZT only.707 FPMR,708 GCS, // Guarded Control Stack registers.709};710 711static uint8_t *AddRegisterSetType(uint8_t *dst,712 RegisterSetType register_set_type) {713 *(reinterpret_cast<uint32_t *>(dst)) = register_set_type;714 return dst + sizeof(uint32_t);715}716 717static uint8_t *AddSavedRegistersData(uint8_t *dst, void *src, size_t size) {718 ::memcpy(dst, src, size);719 return dst + size;720}721 722static uint8_t *AddSavedRegisters(uint8_t *dst,723 enum RegisterSetType register_set_type,724 void *src, size_t size) {725 dst = AddRegisterSetType(dst, register_set_type);726 return AddSavedRegistersData(dst, src, size);727}728 729Status730NativeRegisterContextLinux_arm64::CacheAllRegisters(uint32_t &cached_size) {731 Status error;732 cached_size = sizeof(RegisterSetType) + GetGPRBufferSize();733 error = ReadGPR();734 if (error.Fail())735 return error;736 737 if (GetRegisterInfo().IsZAPresent()) {738 error = ReadZAHeader();739 if (error.Fail())740 return error;741 // Use header size here because the buffer may contain fake data when ZA is742 // disabled. We do not want to write this fake data (all 0s) because this743 // would tell the kernel that we want ZA to become active. Which is the744 // opposite of what we want in the case where it is currently inactive.745 cached_size += sizeof(RegisterSetType) + m_za_header.size;746 // For the same reason, we need to force it to be re-read so that it will747 // always contain the real header.748 m_za_buffer_is_valid = false;749 error = ReadZA();750 if (error.Fail())751 return error;752 753 // We will only be restoring ZT data if ZA is active. As writing to an754 // inactive ZT enables ZA, which may not be desireable.755 if (756 // If we have ZT0, or in other words, if we have SME2.757 GetRegisterInfo().IsZTPresent() &&758 // And ZA is active, which means that ZT0 is also active.759 m_za_header.size > sizeof(m_za_header)) {760 cached_size += sizeof(RegisterSetType) + GetZTBufferSize();761 // The kernel handles an inactive ZT0 for us, and it will read as 0s if762 // inactive (unlike ZA where we fake that behaviour).763 error = ReadZT();764 if (error.Fail())765 return error;766 }767 }768 769 // If SVE is enabled we need not copy FPR separately.770 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) {771 // Store mode and register data.772 cached_size +=773 sizeof(RegisterSetType) + sizeof(m_sve_state) + GetSVEBufferSize();774 error = ReadAllSVE();775 } else {776 cached_size += sizeof(RegisterSetType) + GetFPRSize();777 error = ReadFPR();778 }779 if (error.Fail())780 return error;781 782 if (GetRegisterInfo().IsMTEPresent()) {783 cached_size += sizeof(RegisterSetType) + GetMTEControlSize();784 error = ReadMTEControl();785 if (error.Fail())786 return error;787 }788 789 if (GetRegisterInfo().IsFPMRPresent()) {790 cached_size += sizeof(RegisterSetType) + GetFPMRBufferSize();791 error = ReadFPMR();792 if (error.Fail())793 return error;794 }795 796 if (GetRegisterInfo().IsGCSPresent()) {797 cached_size += sizeof(RegisterSetType) + GetGCSBufferSize();798 error = ReadGCS();799 if (error.Fail())800 return error;801 }802 803 // tpidr is always present but tpidr2 depends on SME.804 cached_size += sizeof(RegisterSetType) + GetTLSBufferSize();805 error = ReadTLS();806 807 return error;808}809 810Status NativeRegisterContextLinux_arm64::ReadAllRegisterValues(811 lldb::WritableDataBufferSP &data_sp) {812 // AArch64 register data must contain GPRs and either FPR or SVE registers.813 // SVE registers can be non-streaming (aka SVE) or streaming (aka SSVE).814 // Finally an optional MTE register. Pointer Authentication (PAC) registers815 // are read-only and will be skipped.816 817 // In order to create register data checkpoint we first read all register818 // values if not done already and calculate total size of register set data.819 // We store all register values in data_sp by copying full PTrace data that820 // corresponds to register sets enabled by current register context.821 822 uint32_t reg_data_byte_size = 0;823 Status error = CacheAllRegisters(reg_data_byte_size);824 if (error.Fail())825 return error;826 827 data_sp.reset(new DataBufferHeap(reg_data_byte_size, 0));828 uint8_t *dst = data_sp->GetBytes();829 830 dst = AddSavedRegisters(dst, RegisterSetType::GPR, GetGPRBuffer(),831 GetGPRBufferSize());832 833 // Streaming SVE and the ZA register both use the streaming vector length.834 // When you change this, the kernel will invalidate parts of the process835 // state. Therefore we need a specific order of restoration for each mode, if836 // we also have ZA to restore.837 //838 // Streaming mode enabled, ZA enabled:839 // * Write streaming registers. This sets SVCR.SM and clears SVCR.ZA.840 // * Write ZA, this set SVCR.ZA. The register data we provide is written to841 // ZA.842 // * Result is SVCR.SM and SVCR.ZA set, with the expected data in both843 // register sets.844 //845 // Streaming mode disabled, ZA enabled:846 // * Write ZA. This sets SVCR.ZA, and the ZA content. In the majority of cases847 // the streaming vector length is changing, so the thread is converted into848 // an FPSIMD thread if it is not already one. This also clears SVCR.SM.849 // * Write SVE registers, which also clears SVCR.SM but most importantly, puts850 // us into full SVE mode instead of FPSIMD mode (where the registers are851 // actually the 128 bit Neon registers).852 // * Result is we have SVCR.SM = 0, SVCR.ZA = 1 and the expected register853 // state.854 //855 // Restoring in different orders leads to things like the SVE registers being856 // truncated due to the FPSIMD mode and ZA being disabled or filled with 0s857 // (disabled and 0s looks the same from inside lldb since we fake the value858 // when it's disabled).859 //860 // For more information on this, look up the uses of the relevant NT_ARM_861 // constants and the functions vec_set_vector_length, sve_set_common and862 // za_set in the Linux Kernel.863 864 if ((m_sve_state != SVEState::Streaming) && GetRegisterInfo().IsZAPresent()) {865 // Use the header size not the buffer size, as we may be using the buffer866 // for fake data, which we do not want to write out.867 assert(m_za_header.size <= GetZABufferSize());868 dst = AddSavedRegisters(dst, RegisterSetType::SME, GetZABuffer(),869 m_za_header.size);870 }871 872 if (GetRegisterInfo().IsSVEPresent() || GetRegisterInfo().IsSSVEPresent()) {873 dst = AddRegisterSetType(dst, RegisterSetType::SVE);874 *(reinterpret_cast<SVEState *>(dst)) = m_sve_state;875 dst += sizeof(m_sve_state);876 dst = AddSavedRegistersData(dst, GetSVEBuffer(), GetSVEBufferSize());877 } else {878 dst = AddSavedRegisters(dst, RegisterSetType::FPR, GetFPRBuffer(),879 GetFPRSize());880 }881 882 if ((m_sve_state == SVEState::Streaming) && GetRegisterInfo().IsZAPresent()) {883 assert(m_za_header.size <= GetZABufferSize());884 dst = AddSavedRegisters(dst, RegisterSetType::SME, GetZABuffer(),885 m_za_header.size);886 }887 888 // If ZT0 is present and we are going to be restoring an active ZA (which889 // implies an active ZT0), then restore ZT0 after ZA has been set. This890 // prevents us enabling ZA accidentally after the restore of ZA disabled it.891 // If we leave ZA/ZT0 inactive and read ZT0, the kernel returns 0s. Therefore892 // there's nothing for us to restore if ZA was originally inactive.893 if (894 // If we have SME2 and therefore ZT0.895 GetRegisterInfo().IsZTPresent() &&896 // And ZA is enabled.897 m_za_header.size > sizeof(m_za_header))898 dst = AddSavedRegisters(dst, RegisterSetType::SME2, GetZTBuffer(),899 GetZTBufferSize());900 901 if (GetRegisterInfo().IsMTEPresent()) {902 dst = AddSavedRegisters(dst, RegisterSetType::MTE, GetMTEControl(),903 GetMTEControlSize());904 }905 906 if (GetRegisterInfo().IsFPMRPresent()) {907 dst = AddSavedRegisters(dst, RegisterSetType::FPMR, GetFPMRBuffer(),908 GetFPMRBufferSize());909 }910 911 if (GetRegisterInfo().IsGCSPresent()) {912 dst = AddSavedRegisters(dst, RegisterSetType::GCS, GetGCSBuffer(),913 GetGCSBufferSize());914 }915 916 dst = AddSavedRegisters(dst, RegisterSetType::TLS, GetTLSBuffer(),917 GetTLSBufferSize());918 919 return error;920}921 922static Status RestoreRegisters(void *buffer, const uint8_t **src, size_t len,923 bool &is_valid, std::function<Status()> writer) {924 ::memcpy(buffer, *src, len);925 is_valid = true;926 *src += len;927 return writer();928}929 930Status NativeRegisterContextLinux_arm64::WriteAllRegisterValues(931 const lldb::DataBufferSP &data_sp) {932 // AArch64 register data must contain GPRs, either FPR or SVE registers933 // (which can be streaming or non-streaming) and optional MTE register.934 // Pointer Authentication (PAC) registers are read-only and will be skipped.935 936 // We store all register values in data_sp by copying full PTrace data that937 // corresponds to register sets enabled by current register context. In order938 // to restore from register data checkpoint we will first restore GPRs, based939 // on size of remaining register data either SVE or FPRs should be restored940 // next. SVE is not enabled if we have register data size less than or equal941 // to size of GPR + FPR + MTE.942 943 Status error;944 if (!data_sp) {945 error = Status::FromErrorStringWithFormat(946 "NativeRegisterContextLinux_arm64::%s invalid data_sp provided",947 __FUNCTION__);948 return error;949 }950 951 const uint8_t *src = data_sp->GetBytes();952 if (src == nullptr) {953 error = Status::FromErrorStringWithFormat(954 "NativeRegisterContextLinux_arm64::%s "955 "DataBuffer::GetBytes() returned a null "956 "pointer",957 __FUNCTION__);958 return error;959 }960 961 uint64_t reg_data_min_size =962 GetGPRBufferSize() + GetFPRSize() + 2 * (sizeof(RegisterSetType));963 if (data_sp->GetByteSize() < reg_data_min_size) {964 error = Status::FromErrorStringWithFormat(965 "NativeRegisterContextLinux_arm64::%s data_sp contained insufficient "966 "register data bytes, expected at least %" PRIu64 ", actual %" PRIu64,967 __FUNCTION__, reg_data_min_size, data_sp->GetByteSize());968 return error;969 }970 971 const uint8_t *end = src + data_sp->GetByteSize();972 while (src < end) {973 const RegisterSetType kind =974 *reinterpret_cast<const RegisterSetType *>(src);975 src += sizeof(RegisterSetType);976 977 switch (kind) {978 case RegisterSetType::GPR:979 error = RestoreRegisters(980 GetGPRBuffer(), &src, GetGPRBufferSize(), m_gpr_is_valid,981 std::bind(&NativeRegisterContextLinux_arm64::WriteGPR, this));982 break;983 case RegisterSetType::SVE:984 // Restore to the correct mode, streaming or not.985 m_sve_state = static_cast<SVEState>(*src);986 src += sizeof(m_sve_state);987 988 // First write SVE header. We do not use RestoreRegisters because we do989 // not want src to be modified yet.990 ::memcpy(GetSVEHeader(), src, GetSVEHeaderSize());991 if (!sve::vl_valid(m_sve_header.vl)) {992 m_sve_header_is_valid = false;993 error = Status::FromErrorStringWithFormat(994 "NativeRegisterContextLinux_arm64::%s "995 "Invalid SVE header in data_sp",996 __FUNCTION__);997 return error;998 }999 m_sve_header_is_valid = true;1000 error = WriteSVEHeader();1001 if (error.Fail())1002 return error;1003 1004 // SVE header has been written configure SVE vector length if needed.1005 // This could change ZA data too, but that will be restored again later1006 // anyway.1007 ConfigureRegisterContext();1008 1009 // Write header and register data, incrementing src this time.1010 error = RestoreRegisters(1011 GetSVEBuffer(), &src, GetSVEBufferSize(), m_sve_buffer_is_valid,1012 std::bind(&NativeRegisterContextLinux_arm64::WriteAllSVE, this));1013 break;1014 case RegisterSetType::FPR:1015 error = RestoreRegisters(1016 GetFPRBuffer(), &src, GetFPRSize(), m_fpu_is_valid,1017 std::bind(&NativeRegisterContextLinux_arm64::WriteFPR, this));1018 break;1019 case RegisterSetType::MTE:1020 error = RestoreRegisters(1021 GetMTEControl(), &src, GetMTEControlSize(), m_mte_ctrl_is_valid,1022 std::bind(&NativeRegisterContextLinux_arm64::WriteMTEControl, this));1023 break;1024 case RegisterSetType::TLS:1025 error = RestoreRegisters(1026 GetTLSBuffer(), &src, GetTLSBufferSize(), m_tls_is_valid,1027 std::bind(&NativeRegisterContextLinux_arm64::WriteTLS, this));1028 break;1029 case RegisterSetType::SME:1030 // To enable or disable ZA you write the regset with or without register1031 // data. The kernel detects this by looking at the ioVec's length, not the1032 // ZA header size you pass in. Therefore we must write header and register1033 // data (if present) in one go every time. Read the header only first just1034 // to get the size.1035 ::memcpy(GetZAHeader(), src, GetZAHeaderSize());1036 // Read the header and register data. Can't use the buffer size here, it1037 // may be incorrect due to being filled with dummy data previously. Resize1038 // this so WriteZA uses the correct size.1039 m_za_ptrace_payload.resize(m_za_header.size);1040 ::memcpy(GetZABuffer(), src, GetZABufferSize());1041 m_za_buffer_is_valid = true;1042 1043 error = WriteZA();1044 if (error.Fail())1045 return error;1046 1047 // Update size of ZA, which resizes the ptrace payload potentially1048 // trashing our copy of the data we just wrote.1049 ConfigureRegisterContext();1050 1051 // ZA buffer now has proper size, read back the data we wrote above, from1052 // ptrace.1053 error = ReadZA();1054 src += GetZABufferSize();1055 break;1056 case RegisterSetType::SME2:1057 // Doing this would activate an inactive ZA, however we will only get here1058 // if the state we are restoring had an active ZA. Restoring ZT0 will1059 // always come after restoring ZA.1060 error = RestoreRegisters(1061 GetZTBuffer(), &src, GetZTBufferSize(), m_zt_buffer_is_valid,1062 std::bind(&NativeRegisterContextLinux_arm64::WriteZT, this));1063 break;1064 case RegisterSetType::FPMR:1065 error = RestoreRegisters(1066 GetFPMRBuffer(), &src, GetFPMRBufferSize(), m_fpmr_is_valid,1067 std::bind(&NativeRegisterContextLinux_arm64::WriteFPMR, this));1068 break;1069 case RegisterSetType::GCS:1070 // It is not permitted to enable GCS via ptrace. We can disable it, but1071 // to keep things simple we will not revert any change to the1072 // PR_SHADOW_STACK_ENABLE bit. Instead patch in the current enable bit1073 // into the registers we are about to restore.1074 m_gcs_is_valid = false;1075 error = ReadGCS();1076 if (error.Fail())1077 return error;1078 1079 uint64_t enable_bit = m_gcs_regs.features_enabled & 1UL;1080 gcs_regs new_gcs_regs = *reinterpret_cast<const gcs_regs *>(src);1081 new_gcs_regs.features_enabled =1082 (new_gcs_regs.features_enabled & ~1UL) | enable_bit;1083 1084 const uint8_t *new_gcs_src =1085 reinterpret_cast<const uint8_t *>(&new_gcs_regs);1086 error = RestoreRegisters(1087 GetGCSBuffer(), &new_gcs_src, GetGCSBufferSize(), m_gcs_is_valid,1088 std::bind(&NativeRegisterContextLinux_arm64::WriteGCS, this));1089 src += GetGCSBufferSize();1090 1091 break;1092 }1093 1094 if (error.Fail())1095 return error;1096 }1097 1098 return error;1099}1100 1101bool NativeRegisterContextLinux_arm64::IsGPR(unsigned reg) const {1102 if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) ==1103 RegisterInfoPOSIX_arm64::GPRegSet)1104 return true;1105 return false;1106}1107 1108bool NativeRegisterContextLinux_arm64::IsFPR(unsigned reg) const {1109 if (GetRegisterInfo().GetRegisterSetFromRegisterIndex(reg) ==1110 RegisterInfoPOSIX_arm64::FPRegSet)1111 return true;1112 return false;1113}1114 1115bool NativeRegisterContextLinux_arm64::IsSVE(unsigned reg) const {1116 return GetRegisterInfo().IsSVEReg(reg);1117}1118 1119bool NativeRegisterContextLinux_arm64::IsSME(unsigned reg) const {1120 return GetRegisterInfo().IsSMEReg(reg);1121}1122 1123bool NativeRegisterContextLinux_arm64::IsPAuth(unsigned reg) const {1124 return GetRegisterInfo().IsPAuthReg(reg);1125}1126 1127bool NativeRegisterContextLinux_arm64::IsMTE(unsigned reg) const {1128 return GetRegisterInfo().IsMTEReg(reg);1129}1130 1131bool NativeRegisterContextLinux_arm64::IsTLS(unsigned reg) const {1132 return GetRegisterInfo().IsTLSReg(reg);1133}1134 1135bool NativeRegisterContextLinux_arm64::IsFPMR(unsigned reg) const {1136 return GetRegisterInfo().IsFPMRReg(reg);1137}1138 1139bool NativeRegisterContextLinux_arm64::IsGCS(unsigned reg) const {1140 return GetRegisterInfo().IsGCSReg(reg);1141}1142 1143llvm::Error NativeRegisterContextLinux_arm64::ReadHardwareDebugInfo() {1144 if (!m_refresh_hwdebug_info) {1145 return llvm::Error::success();1146 }1147 1148 ::pid_t tid = m_thread.GetID();1149 1150 Status error = arm64::ReadHardwareDebugInfo(tid, m_max_hwp_supported,1151 m_max_hbp_supported);1152 if (error.Fail())1153 return error.ToError();1154 1155 m_refresh_hwdebug_info = false;1156 1157 return llvm::Error::success();1158}1159 1160llvm::Error1161NativeRegisterContextLinux_arm64::WriteHardwareDebugRegs(DREGType hwbType) {1162 uint32_t max_supported =1163 (hwbType == eDREGTypeWATCH) ? m_max_hwp_supported : m_max_hbp_supported;1164 auto ®s = (hwbType == eDREGTypeWATCH) ? m_hwp_regs : m_hbp_regs;1165 return arm64::WriteHardwareDebugRegs(hwbType, m_thread.GetID(), max_supported,1166 regs)1167 .ToError();1168}1169 1170Status NativeRegisterContextLinux_arm64::ReadGPR() {1171 Status error;1172 1173 if (m_gpr_is_valid)1174 return error;1175 1176 struct iovec ioVec;1177 ioVec.iov_base = GetGPRBuffer();1178 ioVec.iov_len = GetGPRBufferSize();1179 1180 error = ReadRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);1181 1182 if (error.Success())1183 m_gpr_is_valid = true;1184 1185 return error;1186}1187 1188Status NativeRegisterContextLinux_arm64::WriteGPR() {1189 Status error = ReadGPR();1190 if (error.Fail())1191 return error;1192 1193 struct iovec ioVec;1194 ioVec.iov_base = GetGPRBuffer();1195 ioVec.iov_len = GetGPRBufferSize();1196 1197 m_gpr_is_valid = false;1198 1199 return WriteRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);1200}1201 1202Status NativeRegisterContextLinux_arm64::ReadFPR() {1203 Status error;1204 1205 if (m_fpu_is_valid)1206 return error;1207 1208 struct iovec ioVec;1209 ioVec.iov_base = GetFPRBuffer();1210 ioVec.iov_len = GetFPRSize();1211 1212 error = ReadRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET);1213 1214 if (error.Success())1215 m_fpu_is_valid = true;1216 1217 return error;1218}1219 1220Status NativeRegisterContextLinux_arm64::WriteFPR() {1221 Status error = ReadFPR();1222 if (error.Fail())1223 return error;1224 1225 struct iovec ioVec;1226 ioVec.iov_base = GetFPRBuffer();1227 ioVec.iov_len = GetFPRSize();1228 1229 m_fpu_is_valid = false;1230 1231 return WriteRegisterSet(&ioVec, GetFPRSize(), NT_FPREGSET);1232}1233 1234void NativeRegisterContextLinux_arm64::InvalidateAllRegisters() {1235 m_gpr_is_valid = false;1236 m_fpu_is_valid = false;1237 m_sve_buffer_is_valid = false;1238 m_sve_header_is_valid = false;1239 m_za_buffer_is_valid = false;1240 m_za_header_is_valid = false;1241 m_pac_mask_is_valid = false;1242 m_mte_ctrl_is_valid = false;1243 m_tls_is_valid = false;1244 m_zt_buffer_is_valid = false;1245 m_fpmr_is_valid = false;1246 m_gcs_is_valid = false;1247 1248 // Update SVE and ZA registers in case there is change in configuration.1249 ConfigureRegisterContext();1250}1251 1252unsigned NativeRegisterContextLinux_arm64::GetSVERegSet() {1253 return m_sve_state == SVEState::Streaming ? NT_ARM_SSVE : NT_ARM_SVE;1254}1255 1256Status NativeRegisterContextLinux_arm64::ReadSVEHeader() {1257 Status error;1258 1259 if (m_sve_header_is_valid)1260 return error;1261 1262 struct iovec ioVec;1263 ioVec.iov_base = GetSVEHeader();1264 ioVec.iov_len = GetSVEHeaderSize();1265 1266 error = ReadRegisterSet(&ioVec, GetSVEHeaderSize(), GetSVERegSet());1267 1268 if (error.Success())1269 m_sve_header_is_valid = true;1270 1271 return error;1272}1273 1274Status NativeRegisterContextLinux_arm64::ReadPAuthMask() {1275 Status error;1276 1277 if (m_pac_mask_is_valid)1278 return error;1279 1280 struct iovec ioVec;1281 ioVec.iov_base = GetPACMask();1282 ioVec.iov_len = GetPACMaskSize();1283 1284 error = ReadRegisterSet(&ioVec, GetPACMaskSize(), NT_ARM_PAC_MASK);1285 1286 if (error.Success())1287 m_pac_mask_is_valid = true;1288 1289 return error;1290}1291 1292Status NativeRegisterContextLinux_arm64::WriteSVEHeader() {1293 Status error;1294 1295 error = ReadSVEHeader();1296 if (error.Fail())1297 return error;1298 1299 struct iovec ioVec;1300 ioVec.iov_base = GetSVEHeader();1301 ioVec.iov_len = GetSVEHeaderSize();1302 1303 m_sve_buffer_is_valid = false;1304 m_sve_header_is_valid = false;1305 m_fpu_is_valid = false;1306 1307 return WriteRegisterSet(&ioVec, GetSVEHeaderSize(), GetSVERegSet());1308}1309 1310Status NativeRegisterContextLinux_arm64::ReadAllSVE() {1311 Status error;1312 if (m_sve_buffer_is_valid)1313 return error;1314 1315 struct iovec ioVec;1316 ioVec.iov_base = GetSVEBuffer();1317 ioVec.iov_len = GetSVEBufferSize();1318 1319 error = ReadRegisterSet(&ioVec, GetSVEBufferSize(), GetSVERegSet());1320 1321 if (error.Success())1322 m_sve_buffer_is_valid = true;1323 1324 return error;1325}1326 1327Status NativeRegisterContextLinux_arm64::WriteAllSVE() {1328 Status error;1329 1330 error = ReadAllSVE();1331 if (error.Fail())1332 return error;1333 1334 struct iovec ioVec;1335 1336 ioVec.iov_base = GetSVEBuffer();1337 ioVec.iov_len = GetSVEBufferSize();1338 1339 m_sve_buffer_is_valid = false;1340 m_sve_header_is_valid = false;1341 m_fpu_is_valid = false;1342 1343 return WriteRegisterSet(&ioVec, GetSVEBufferSize(), GetSVERegSet());1344}1345 1346Status NativeRegisterContextLinux_arm64::ReadSMEControl() {1347 // The real register is SVCR and is accessible from EL0. However we don't want1348 // to have to JIT code into the target process so we'll just recreate it using1349 // what we know from ptrace.1350 1351 // Bit 0 indicates whether streaming mode is active.1352 m_sme_pseudo_regs.ctrl_reg = m_sve_state == SVEState::Streaming;1353 1354 // Bit 1 indicates whether the array storage is active.1355 // It is active if we can read the header and the size field tells us that1356 // there is register data following it.1357 Status error = ReadZAHeader();1358 if (error.Success() && (m_za_header.size > sizeof(m_za_header)))1359 m_sme_pseudo_regs.ctrl_reg |= 2;1360 1361 return error;1362}1363 1364Status NativeRegisterContextLinux_arm64::ReadMTEControl() {1365 Status error;1366 1367 if (m_mte_ctrl_is_valid)1368 return error;1369 1370 struct iovec ioVec;1371 ioVec.iov_base = GetMTEControl();1372 ioVec.iov_len = GetMTEControlSize();1373 1374 error = ReadRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);1375 1376 if (error.Success())1377 m_mte_ctrl_is_valid = true;1378 1379 return error;1380}1381 1382Status NativeRegisterContextLinux_arm64::WriteMTEControl() {1383 Status error;1384 1385 error = ReadMTEControl();1386 if (error.Fail())1387 return error;1388 1389 struct iovec ioVec;1390 ioVec.iov_base = GetMTEControl();1391 ioVec.iov_len = GetMTEControlSize();1392 1393 m_mte_ctrl_is_valid = false;1394 1395 return WriteRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);1396}1397 1398Status NativeRegisterContextLinux_arm64::ReadTLS() {1399 Status error;1400 1401 if (m_tls_is_valid)1402 return error;1403 1404 struct iovec ioVec;1405 ioVec.iov_base = GetTLSBuffer();1406 ioVec.iov_len = GetTLSBufferSize();1407 1408 error = ReadRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS);1409 1410 if (error.Success())1411 m_tls_is_valid = true;1412 1413 return error;1414}1415 1416Status NativeRegisterContextLinux_arm64::WriteTLS() {1417 Status error;1418 1419 error = ReadTLS();1420 if (error.Fail())1421 return error;1422 1423 struct iovec ioVec;1424 ioVec.iov_base = GetTLSBuffer();1425 ioVec.iov_len = GetTLSBufferSize();1426 1427 m_tls_is_valid = false;1428 1429 return WriteRegisterSet(&ioVec, GetTLSBufferSize(), NT_ARM_TLS);1430}1431 1432Status NativeRegisterContextLinux_arm64::ReadGCS() {1433 Status error;1434 1435 if (m_gcs_is_valid)1436 return error;1437 1438 struct iovec ioVec;1439 ioVec.iov_base = GetGCSBuffer();1440 ioVec.iov_len = GetGCSBufferSize();1441 1442 error = ReadRegisterSet(&ioVec, GetGCSBufferSize(), NT_ARM_GCS);1443 1444 if (error.Success())1445 m_gcs_is_valid = true;1446 1447 return error;1448}1449 1450Status NativeRegisterContextLinux_arm64::WriteGCS() {1451 Status error;1452 1453 error = ReadGCS();1454 if (error.Fail())1455 return error;1456 1457 struct iovec ioVec;1458 ioVec.iov_base = GetGCSBuffer();1459 ioVec.iov_len = GetGCSBufferSize();1460 1461 m_gcs_is_valid = false;1462 1463 return WriteRegisterSet(&ioVec, GetGCSBufferSize(), NT_ARM_GCS);1464}1465 1466Status NativeRegisterContextLinux_arm64::ReadZAHeader() {1467 Status error;1468 1469 if (m_za_header_is_valid)1470 return error;1471 1472 struct iovec ioVec;1473 ioVec.iov_base = GetZAHeader();1474 ioVec.iov_len = GetZAHeaderSize();1475 1476 error = ReadRegisterSet(&ioVec, GetZAHeaderSize(), NT_ARM_ZA);1477 1478 if (error.Success())1479 m_za_header_is_valid = true;1480 1481 return error;1482}1483 1484Status NativeRegisterContextLinux_arm64::ReadZA() {1485 Status error;1486 1487 if (m_za_buffer_is_valid)1488 return error;1489 1490 struct iovec ioVec;1491 ioVec.iov_base = GetZABuffer();1492 ioVec.iov_len = GetZABufferSize();1493 1494 error = ReadRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA);1495 1496 if (error.Success())1497 m_za_buffer_is_valid = true;1498 1499 return error;1500}1501 1502Status NativeRegisterContextLinux_arm64::WriteZA() {1503 // Note that because the ZA ptrace payload contains the header also, this1504 // method will write both. This is done because writing only the header1505 // will disable ZA, even if .size in the header is correct for an enabled ZA.1506 Status error;1507 1508 error = ReadZA();1509 if (error.Fail())1510 return error;1511 1512 struct iovec ioVec;1513 ioVec.iov_base = GetZABuffer();1514 ioVec.iov_len = GetZABufferSize();1515 1516 m_za_buffer_is_valid = false;1517 m_za_header_is_valid = false;1518 // Writing to ZA may enable ZA, which means ZT0 may change too.1519 m_zt_buffer_is_valid = false;1520 1521 return WriteRegisterSet(&ioVec, GetZABufferSize(), NT_ARM_ZA);1522}1523 1524Status NativeRegisterContextLinux_arm64::ReadZT() {1525 Status error;1526 1527 if (m_zt_buffer_is_valid)1528 return error;1529 1530 struct iovec ioVec;1531 ioVec.iov_base = GetZTBuffer();1532 ioVec.iov_len = GetZTBufferSize();1533 1534 error = ReadRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT);1535 m_zt_buffer_is_valid = error.Success();1536 1537 return error;1538}1539 1540Status NativeRegisterContextLinux_arm64::WriteZT() {1541 Status error;1542 1543 error = ReadZT();1544 if (error.Fail())1545 return error;1546 1547 struct iovec ioVec;1548 ioVec.iov_base = GetZTBuffer();1549 ioVec.iov_len = GetZTBufferSize();1550 1551 m_zt_buffer_is_valid = false;1552 // Writing to an inactive ZT0 will enable ZA as well, which invalidates our1553 // current copy of it.1554 m_za_buffer_is_valid = false;1555 m_za_header_is_valid = false;1556 1557 return WriteRegisterSet(&ioVec, GetZTBufferSize(), NT_ARM_ZT);1558}1559 1560Status NativeRegisterContextLinux_arm64::ReadFPMR() {1561 Status error;1562 1563 if (m_fpmr_is_valid)1564 return error;1565 1566 struct iovec ioVec;1567 ioVec.iov_base = GetFPMRBuffer();1568 ioVec.iov_len = GetFPMRBufferSize();1569 1570 error = ReadRegisterSet(&ioVec, GetFPMRBufferSize(), NT_ARM_FPMR);1571 1572 if (error.Success())1573 m_fpmr_is_valid = true;1574 1575 return error;1576}1577 1578Status NativeRegisterContextLinux_arm64::WriteFPMR() {1579 Status error;1580 1581 error = ReadFPMR();1582 if (error.Fail())1583 return error;1584 1585 struct iovec ioVec;1586 ioVec.iov_base = GetFPMRBuffer();1587 ioVec.iov_len = GetFPMRBufferSize();1588 1589 m_fpmr_is_valid = false;1590 1591 return WriteRegisterSet(&ioVec, GetFPMRBufferSize(), NT_ARM_FPMR);1592}1593 1594void NativeRegisterContextLinux_arm64::ConfigureRegisterContext() {1595 // ConfigureRegisterContext gets called from InvalidateAllRegisters1596 // on every stop and configures SVE vector length and whether we are in1597 // streaming SVE mode.1598 // If m_sve_state is set to SVEState::Disabled on first stop, code below will1599 // be deemed non operational for the lifetime of current process.1600 if (!m_sve_header_is_valid && m_sve_state != SVEState::Disabled) {1601 // If we have SVE we may also have the SVE streaming mode that SME added.1602 // We can read the header of either mode, but only the active mode will1603 // have valid register data.1604 1605 // Check whether SME is present and the streaming SVE mode is active.1606 m_sve_header_is_valid = false;1607 m_sve_buffer_is_valid = false;1608 m_sve_state = SVEState::Streaming;1609 Status error = ReadSVEHeader();1610 1611 // Streaming mode is active if the header has the SVE active flag set.1612 if (!(error.Success() && ((m_sve_header.flags & sve::ptrace_regs_mask) ==1613 sve::ptrace_regs_sve))) {1614 // Non-streaming might be active instead.1615 m_sve_header_is_valid = false;1616 m_sve_buffer_is_valid = false;1617 m_sve_state = SVEState::Full;1618 error = ReadSVEHeader();1619 if (error.Success()) {1620 // If SVE is enabled thread can switch between SVEState::FPSIMD and1621 // SVEState::Full on every stop.1622 if ((m_sve_header.flags & sve::ptrace_regs_mask) ==1623 sve::ptrace_regs_fpsimd)1624 m_sve_state = SVEState::FPSIMD;1625 // Else we are in SVEState::Full.1626 } else {1627 m_sve_state = SVEState::Disabled;1628 }1629 }1630 1631 if (m_sve_state == SVEState::Full || m_sve_state == SVEState::FPSIMD ||1632 m_sve_state == SVEState::Streaming) {1633 // On every stop we configure SVE vector length by calling1634 // ConfigureVectorLengthSVE regardless of current SVEState of this thread.1635 uint32_t vq = RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64SVE;1636 if (sve::vl_valid(m_sve_header.vl))1637 vq = sve::vq_from_vl(m_sve_header.vl);1638 1639 GetRegisterInfo().ConfigureVectorLengthSVE(vq);1640 m_sve_ptrace_payload.resize(sve::PTraceSize(vq, sve::ptrace_regs_sve));1641 }1642 }1643 1644 if (!m_za_header_is_valid) {1645 Status error = ReadZAHeader();1646 if (error.Success()) {1647 uint32_t vq = RegisterInfoPOSIX_arm64::eVectorQuadwordAArch64SVE;1648 if (sve::vl_valid(m_za_header.vl))1649 vq = sve::vq_from_vl(m_za_header.vl);1650 1651 GetRegisterInfo().ConfigureVectorLengthZA(vq);1652 m_za_ptrace_payload.resize(m_za_header.size);1653 m_za_buffer_is_valid = false;1654 }1655 }1656}1657 1658uint32_t NativeRegisterContextLinux_arm64::CalculateFprOffset(1659 const RegisterInfo *reg_info) const {1660 return reg_info->byte_offset - GetGPRSize();1661}1662 1663uint32_t NativeRegisterContextLinux_arm64::CalculateSVEOffset(1664 const RegisterInfo *reg_info) const {1665 // Start of Z0 data is after GPRs plus 8 bytes of vg register1666 uint32_t sve_reg_offset = LLDB_INVALID_INDEX32;1667 if (m_sve_state == SVEState::FPSIMD) {1668 const uint32_t reg = reg_info->kinds[lldb::eRegisterKindLLDB];1669 sve_reg_offset = sve::ptrace_fpsimd_offset +1670 (reg - GetRegisterInfo().GetRegNumSVEZ0()) * 16;1671 // Between non-streaming and streaming mode, the layout is identical.1672 } else if (m_sve_state == SVEState::Full ||1673 m_sve_state == SVEState::Streaming) {1674 uint32_t sve_z0_offset = GetGPRSize() + 16;1675 sve_reg_offset =1676 sve::SigRegsOffset() + reg_info->byte_offset - sve_z0_offset;1677 }1678 return sve_reg_offset;1679}1680 1681Status NativeRegisterContextLinux_arm64::ReadSMESVG() {1682 // This register is the streaming vector length, so we will get it from1683 // NT_ARM_ZA regardless of the current streaming mode.1684 Status error = ReadZAHeader();1685 if (error.Success())1686 m_sme_pseudo_regs.svg_reg = m_za_header.vl / 8;1687 1688 return error;1689}1690 1691std::vector<uint32_t> NativeRegisterContextLinux_arm64::GetExpeditedRegisters(1692 ExpeditedRegs expType) const {1693 std::vector<uint32_t> expedited_reg_nums =1694 NativeRegisterContext::GetExpeditedRegisters(expType);1695 // SVE, non-streaming vector length.1696 if (m_sve_state == SVEState::FPSIMD || m_sve_state == SVEState::Full)1697 expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSVEVG());1698 // SME, streaming vector length. This is used by the ZA register which is1699 // present even when streaming mode is not enabled.1700 if (GetRegisterInfo().IsSSVEPresent())1701 expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSMESVG());1702 1703 return expedited_reg_nums;1704}1705 1706llvm::Expected<NativeRegisterContextLinux::MemoryTaggingDetails>1707NativeRegisterContextLinux_arm64::GetMemoryTaggingDetails(int32_t type) {1708 if (type == MemoryTagManagerAArch64MTE::eMTE_allocation) {1709 return MemoryTaggingDetails{std::make_unique<MemoryTagManagerAArch64MTE>(),1710 PTRACE_PEEKMTETAGS, PTRACE_POKEMTETAGS};1711 }1712 1713 return llvm::createStringError(llvm::inconvertibleErrorCode(),1714 "Unknown AArch64 memory tag type %d", type);1715}1716 1717lldb::addr_t NativeRegisterContextLinux_arm64::FixWatchpointHitAddress(1718 lldb::addr_t hit_addr) {1719 // Linux configures user-space virtual addresses with top byte ignored.1720 // We set default value of mask such that top byte is masked out.1721 lldb::addr_t mask = ~((1ULL << 56) - 1);1722 1723 // Try to read pointer authentication data_mask register and calculate a1724 // consolidated data address mask after ignoring the top byte.1725 if (ReadPAuthMask().Success())1726 mask |= m_pac_mask.data_mask;1727 1728 return hit_addr & ~mask;1729 ;1730}1731 1732#endif // defined (__arm64__) || defined (__aarch64__)1733