585 lines · plain
1// SPDX-License-Identifier: GPL-2.0-only2// Copyright (C) 2015-2019 ARM Limited.3// Original author: Dave Martin <Dave.Martin@arm.com>4//5// Simple Scalable Vector Extension context switch test6// Repeatedly writes unique test patterns into each SVE register7// and reads them back to verify integrity.8//9// for x in `seq 1 NR_CPUS`; do sve-test & pids=$pids\ $! ; done10// (leave it running for as long as you want...)11// kill $pids12 13#include <asm/unistd.h>14#include "assembler.h"15#include "asm-offsets.h"16#include "sme-inst.h"17 18#define NZR 3219#define NPR 1620#define MAXVL_B (2048 / 8)21 22.arch_extension sve23 24.macro _sve_ldr_v zt, xn25 ldr z\zt, [x\xn]26.endm27 28.macro _sve_str_v zt, xn29 str z\zt, [x\xn]30.endm31 32.macro _sve_ldr_p pt, xn33 ldr p\pt, [x\xn]34.endm35 36.macro _sve_str_p pt, xn37 str p\pt, [x\xn]38.endm39 40// Generate accessor functions to read/write programmatically selected41// SVE registers.42// x0 is the register index to access43// x1 is the memory address to read from (getz,setp) or store to (setz,setp)44// All clobber x0-x245define_accessor setz, NZR, _sve_ldr_v46define_accessor getz, NZR, _sve_str_v47define_accessor setp, NPR, _sve_ldr_p48define_accessor getp, NPR, _sve_str_p49 50// Declare some storate space to shadow the SVE register contents:51.pushsection .text52.data53.align 454zref:55 .space MAXVL_B * NZR56pref:57 .space MAXVL_B / 8 * NPR58ffrref:59 .space MAXVL_B / 860scratch:61 .space MAXVL_B62.popsection63 64// Generate a test pattern for storage in SVE registers65// x0: pid (16 bits)66// x1: register number (6 bits)67// x2: generation (4 bits)68 69// These values are used to constuct a 32-bit pattern that is repeated in the70// scratch buffer as many times as will fit:71// bits 31:28 generation number (increments once per test_loop)72// bits 27:22 32-bit lane index73// bits 21:16 register number74// bits 15: 0 pid75 76function pattern77 orr w1, w0, w1, lsl #1678 orr w2, w1, w2, lsl #2879 80 ldr x0, =scratch81 mov w1, #MAXVL_B / 482 830: str w2, [x0], #484 add w2, w2, #(1 << 22)85 subs w1, w1, #186 bne 0b87 88 ret89endfunction90 91// Get the address of shadow data for SVE Z-register Z<xn>92.macro _adrz xd, xn, nrtmp93 ldr \xd, =zref94 rdvl x\nrtmp, #195 madd \xd, x\nrtmp, \xn, \xd96.endm97 98// Get the address of shadow data for SVE P-register P<xn - NZR>99.macro _adrp xd, xn, nrtmp100 ldr \xd, =pref101 rdvl x\nrtmp, #1102 lsr x\nrtmp, x\nrtmp, #3103 sub \xn, \xn, #NZR104 madd \xd, x\nrtmp, \xn, \xd105.endm106 107// Set up test pattern in a SVE Z-register108// x0: pid109// x1: register number110// x2: generation111function setup_zreg112 mov x4, x30113 114 mov x6, x1115 bl pattern116 _adrz x0, x6, 2117 mov x5, x0118 ldr x1, =scratch119 bl memcpy120 121 mov x0, x6122 mov x1, x5123 bl setz124 125 ret x4126endfunction127 128// Set up test pattern in a SVE P-register129// x0: pid130// x1: register number131// x2: generation132function setup_preg133 mov x4, x30134 135 mov x6, x1136 bl pattern137 _adrp x0, x6, 2138 mov x5, x0139 ldr x1, =scratch140 bl memcpy141 142 mov x0, x6143 mov x1, x5144 bl setp145 146 ret x4147endfunction148 149// Set up test pattern in the FFR150// x0: pid151// x2: generation152//153// We need to generate a canonical FFR value, which consists of a number of154// low "1" bits, followed by a number of zeros. This gives us 17 unique values155// per 16 bits of FFR, so we create a 4 bit signature out of the PID and156// generation, and use that as the initial number of ones in the pattern.157// We fill the upper lanes of FFR with zeros.158// Beware: corrupts P0.159function setup_ffr160#ifndef SSVE161 mov x4, x30162 163 and w0, w0, #0x3164 bfi w0, w2, #2, #2165 mov w1, #1166 lsl w1, w1, w0167 sub w1, w1, #1168 169 ldr x0, =ffrref170 strh w1, [x0], 2171 rdvl x1, #1172 lsr x1, x1, #3173 sub x1, x1, #2174 bl memclr175 176 mov x0, #0177 ldr x1, =ffrref178 bl setp179 180 wrffr p0.b181 182 ret x4183#else184 ret185#endif186endfunction187 188// Trivial memory compare: compare x2 bytes starting at address x0 with189// bytes starting at address x1.190// Returns only if all bytes match; otherwise, the program is aborted.191// Clobbers x0-x5.192function memcmp193 cbz x2, 2f194 195 stp x0, x1, [sp, #-0x20]!196 str x2, [sp, #0x10]197 198 mov x5, #01990: ldrb w3, [x0, x5]200 ldrb w4, [x1, x5]201 add x5, x5, #1202 cmp w3, w4203 b.ne 1f204 subs x2, x2, #1205 b.ne 0b206 2071: ldr x2, [sp, #0x10]208 ldp x0, x1, [sp], #0x20209 b.ne barf210 2112: ret212endfunction213 214// Verify that a SVE Z-register matches its shadow in memory, else abort215// x0: reg number216// Clobbers x0-x7.217function check_zreg218 mov x3, x30219 220 _adrz x5, x0, 6221 mov x4, x0222 ldr x7, =scratch223 224 mov x0, x7225 mov x1, x6226 bl memfill_ae227 228 mov x0, x4229 mov x1, x7230 bl getz231 232 mov x0, x5233 mov x1, x7234 mov x2, x6235 mov x30, x3236 b memcmp237endfunction238 239// Verify that a SVE P-register matches its shadow in memory, else abort240// x0: reg number241// Clobbers x0-x7.242function check_preg243 mov x3, x30244 245 _adrp x5, x0, 6246 mov x4, x0247 ldr x7, =scratch248 249 mov x0, x7250 mov x1, x6251 bl memfill_ae252 253 mov x0, x4254 mov x1, x7255 bl getp256 257 mov x0, x5258 mov x1, x7259 mov x2, x6260 mov x30, x3261 b memcmp262endfunction263 264// Verify that the FFR matches its shadow in memory, else abort265// Beware -- corrupts P0.266// Clobbers x0-x5.267function check_ffr268#ifndef SSVE269 mov x3, x30270 271 ldr x4, =scratch272 rdvl x5, #1273 lsr x5, x5, #3274 275 mov x0, x4276 mov x1, x5277 bl memfill_ae278 279 rdffr p0.b280 mov x0, #0281 mov x1, x4282 bl getp283 284 ldr x0, =ffrref285 mov x1, x4286 mov x2, x5287 mov x30, x3288 b memcmp289#else290 ret291#endif292endfunction293 294// Any SVE register modified here can cause corruption in the main295// thread -- but *only* the registers modified here.296function irritator_handler297 // Increment the irritation signal count (x23):298 ldr x0, [x2, #ucontext_regs + 8 * 23]299 add x0, x0, #1300 str x0, [x2, #ucontext_regs + 8 * 23]301 302 // Corrupt some random Z-regs303 adr x0, .text + (irritator_handler - .text) / 16 * 16304 movi v0.8b, #1305 movi v9.16b, #2306 movi v31.8b, #3307#ifndef SSVE308 // And P0309 rdffr p0.b310 // And FFR311 wrffr p15.b312#endif313 314 ret315endfunction316 317function tickle_handler318 // Increment the signal count (x23):319 ldr x0, [x2, #ucontext_regs + 8 * 23]320 add x0, x0, #1321 str x0, [x2, #ucontext_regs + 8 * 23]322 323 ret324endfunction325 326function terminate_handler327 mov w21, w0328 mov x20, x2329 330 puts "Terminated by signal "331 mov w0, w21332 bl putdec333 puts ", no error, iterations="334 ldr x0, [x20, #ucontext_regs + 8 * 22]335 bl putdec336 puts ", signals="337 ldr x0, [x20, #ucontext_regs + 8 * 23]338 bl putdecn339 340 mov x0, #0341 mov x8, #__NR_exit342 svc #0343endfunction344 345// w0: signal number346// x1: sa_action347// w2: sa_flags348// Clobbers x0-x6,x8349function setsignal350 str x30, [sp, #-((sa_sz + 15) / 16 * 16 + 16)]!351 352 mov w4, w0353 mov x5, x1354 mov w6, w2355 356 add x0, sp, #16357 mov x1, #sa_sz358 bl memclr359 360 mov w0, w4361 add x1, sp, #16362 str w6, [x1, #sa_flags]363 str x5, [x1, #sa_handler]364 mov x2, #0365 mov x3, #sa_mask_sz366 mov x8, #__NR_rt_sigaction367 svc #0368 369 cbz w0, 1f370 371 puts "sigaction failure\n"372 b .Labort373 3741: ldr x30, [sp], #((sa_sz + 15) / 16 * 16 + 16)375 ret376endfunction377 378// Main program entry point379.globl _start380function _start381 mov x23, #0 // Irritation signal count382 383 mov w0, #SIGINT384 adr x1, terminate_handler385 mov w2, #SA_SIGINFO386 bl setsignal387 388 mov w0, #SIGTERM389 adr x1, terminate_handler390 mov w2, #SA_SIGINFO391 bl setsignal392 393 mov w0, #SIGUSR1394 adr x1, irritator_handler395 mov w2, #SA_SIGINFO396 orr w2, w2, #SA_NODEFER397 bl setsignal398 399 mov w0, #SIGUSR2400 adr x1, tickle_handler401 mov w2, #SA_SIGINFO402 orr w2, w2, #SA_NODEFER403 bl setsignal404 405#ifdef SSVE406 puts "Streaming mode "407 smstart_sm408#endif409 410 // Sanity-check and report the vector length411 412 rdvl x19, #8413 cmp x19, #128414 b.lo 1f415 cmp x19, #2048416 b.hi 1f417 tst x19, #(8 - 1)418 b.eq 2f419 4201: puts "Bad vector length: "421 mov x0, x19422 bl putdecn423 b .Labort424 4252: puts "Vector length:\t"426 mov x0, x19427 bl putdec428 puts " bits\n"429 430 // Obtain our PID, to ensure test pattern uniqueness between processes431 432 mov x8, #__NR_getpid433 svc #0434 mov x20, x0435 436 puts "PID:\t"437 mov x0, x20438 bl putdecn439 440#ifdef SSVE441 smstart_sm // syscalls will have exited streaming mode442#endif443 444 mov x22, #0 // generation number, increments per iteration445.Ltest_loop:446 rdvl x0, #8447 cmp x0, x19448 b.ne vl_barf449 450 mov x21, #0 // Set up Z-regs & shadow with test pattern4510: mov x0, x20452 mov x1, x21453 and x2, x22, #0xf454 bl setup_zreg455 add x21, x21, #1456 cmp x21, #NZR457 b.lo 0b458 459 mov x0, x20 // Set up FFR & shadow with test pattern460 mov x1, #NZR + NPR461 and x2, x22, #0xf462 bl setup_ffr463 4640: mov x0, x20 // Set up P-regs & shadow with test pattern465 mov x1, x21466 and x2, x22, #0xf467 bl setup_preg468 add x21, x21, #1469 cmp x21, #NZR + NPR470 b.lo 0b471 472// Can't do this when SVE state is volatile across SVC:473// mov x8, #__NR_sched_yield // Encourage preemption474// svc #0475 476#ifdef SSVE477 mrs x0, S3_3_C4_C2_2 // SVCR should have ZA=0,SM=1478 and x1, x0, #3479 cmp x1, #1480 b.ne svcr_barf481#endif482 483 mov x21, #04840: mov x0, x21485 bl check_zreg486 add x21, x21, #1487 cmp x21, #NZR488 b.lo 0b489 4900: mov x0, x21491 bl check_preg492 add x21, x21, #1493 cmp x21, #NZR + NPR494 b.lo 0b495 496 bl check_ffr497 498 add x22, x22, #1499 b .Ltest_loop500 501.Labort:502 mov x0, #0503 mov x1, #SIGABRT504 mov x8, #__NR_kill505 svc #0506endfunction507 508function barf509// fpsimd.c acitivty log dump hack510// ldr w0, =0xdeadc0de511// mov w8, #__NR_exit512// svc #0513// end hack514 mov x10, x0 // expected data515 mov x11, x1 // actual data516 mov x12, x2 // data size517 518#ifdef SSVE519 mrs x13, S3_3_C4_C2_2520#endif521 522 puts "Mismatch: PID="523 mov x0, x20524 bl putdec525 puts ", iteration="526 mov x0, x22527 bl putdec528 puts ", reg="529 mov x0, x21530 bl putdecn531 puts "\tExpected ["532 mov x0, x10533 mov x1, x12534 bl dumphex535 puts "]\n\tGot ["536 mov x0, x11537 mov x1, x12538 bl dumphex539 puts "]\n"540 541#ifdef SSVE542 puts "\tSVCR: "543 mov x0, x13544 bl putdecn545#endif546 547 mov x8, #__NR_getpid548 svc #0549// fpsimd.c acitivty log dump hack550// ldr w0, =0xdeadc0de551// mov w8, #__NR_exit552// svc #0553// ^ end of hack554 mov x1, #SIGABRT555 mov x8, #__NR_kill556 svc #0557// mov x8, #__NR_exit558// mov x1, #1559// svc #0560endfunction561 562function vl_barf563 mov x10, x0564 565 puts "Bad active VL: "566 mov x0, x10567 bl putdecn568 569 mov x8, #__NR_exit570 mov x1, #1571 svc #0572endfunction573 574function svcr_barf575 mov x10, x0576 577 puts "Bad SVCR: "578 mov x0, x10579 bl putdecn580 581 mov x8, #__NR_exit582 mov x1, #1583 svc #0584endfunction585