1188 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Test for s390x KVM_S390_MEM_OP4 *5 * Copyright (C) 2019, Red Hat, Inc.6 */7#include <stdio.h>8#include <stdlib.h>9#include <string.h>10#include <sys/ioctl.h>11#include <pthread.h>12 13#include <linux/bits.h>14 15#include "test_util.h"16#include "kvm_util.h"17#include "kselftest.h"18#include "ucall_common.h"19#include "processor.h"20 21enum mop_target {22 LOGICAL,23 SIDA,24 ABSOLUTE,25 INVALID,26};27 28enum mop_access_mode {29 READ,30 WRITE,31 CMPXCHG,32};33 34struct mop_desc {35 uintptr_t gaddr;36 uintptr_t gaddr_v;37 uint64_t set_flags;38 unsigned int f_check : 1;39 unsigned int f_inject : 1;40 unsigned int f_key : 1;41 unsigned int _gaddr_v : 1;42 unsigned int _set_flags : 1;43 unsigned int _sida_offset : 1;44 unsigned int _ar : 1;45 uint32_t size;46 enum mop_target target;47 enum mop_access_mode mode;48 void *buf;49 uint32_t sida_offset;50 void *old;51 uint8_t old_value[16];52 bool *cmpxchg_success;53 uint8_t ar;54 uint8_t key;55};56 57const uint8_t NO_KEY = 0xff;58 59static struct kvm_s390_mem_op ksmo_from_desc(struct mop_desc *desc)60{61 struct kvm_s390_mem_op ksmo = {62 .gaddr = (uintptr_t)desc->gaddr,63 .size = desc->size,64 .buf = ((uintptr_t)desc->buf),65 .reserved = "ignored_ignored_ignored_ignored"66 };67 68 switch (desc->target) {69 case LOGICAL:70 if (desc->mode == READ)71 ksmo.op = KVM_S390_MEMOP_LOGICAL_READ;72 if (desc->mode == WRITE)73 ksmo.op = KVM_S390_MEMOP_LOGICAL_WRITE;74 break;75 case SIDA:76 if (desc->mode == READ)77 ksmo.op = KVM_S390_MEMOP_SIDA_READ;78 if (desc->mode == WRITE)79 ksmo.op = KVM_S390_MEMOP_SIDA_WRITE;80 break;81 case ABSOLUTE:82 if (desc->mode == READ)83 ksmo.op = KVM_S390_MEMOP_ABSOLUTE_READ;84 if (desc->mode == WRITE)85 ksmo.op = KVM_S390_MEMOP_ABSOLUTE_WRITE;86 if (desc->mode == CMPXCHG) {87 ksmo.op = KVM_S390_MEMOP_ABSOLUTE_CMPXCHG;88 ksmo.old_addr = (uint64_t)desc->old;89 memcpy(desc->old_value, desc->old, desc->size);90 }91 break;92 case INVALID:93 ksmo.op = -1;94 }95 if (desc->f_check)96 ksmo.flags |= KVM_S390_MEMOP_F_CHECK_ONLY;97 if (desc->f_inject)98 ksmo.flags |= KVM_S390_MEMOP_F_INJECT_EXCEPTION;99 if (desc->_set_flags)100 ksmo.flags = desc->set_flags;101 if (desc->f_key && desc->key != NO_KEY) {102 ksmo.flags |= KVM_S390_MEMOP_F_SKEY_PROTECTION;103 ksmo.key = desc->key;104 }105 if (desc->_ar)106 ksmo.ar = desc->ar;107 else108 ksmo.ar = 0;109 if (desc->_sida_offset)110 ksmo.sida_offset = desc->sida_offset;111 112 return ksmo;113}114 115struct test_info {116 struct kvm_vm *vm;117 struct kvm_vcpu *vcpu;118};119 120#define PRINT_MEMOP false121static void print_memop(struct kvm_vcpu *vcpu, const struct kvm_s390_mem_op *ksmo)122{123 if (!PRINT_MEMOP)124 return;125 126 if (!vcpu)127 printf("vm memop(");128 else129 printf("vcpu memop(");130 switch (ksmo->op) {131 case KVM_S390_MEMOP_LOGICAL_READ:132 printf("LOGICAL, READ, ");133 break;134 case KVM_S390_MEMOP_LOGICAL_WRITE:135 printf("LOGICAL, WRITE, ");136 break;137 case KVM_S390_MEMOP_SIDA_READ:138 printf("SIDA, READ, ");139 break;140 case KVM_S390_MEMOP_SIDA_WRITE:141 printf("SIDA, WRITE, ");142 break;143 case KVM_S390_MEMOP_ABSOLUTE_READ:144 printf("ABSOLUTE, READ, ");145 break;146 case KVM_S390_MEMOP_ABSOLUTE_WRITE:147 printf("ABSOLUTE, WRITE, ");148 break;149 case KVM_S390_MEMOP_ABSOLUTE_CMPXCHG:150 printf("ABSOLUTE, CMPXCHG, ");151 break;152 }153 printf("gaddr=%llu, size=%u, buf=%llu, ar=%u, key=%u, old_addr=%llx",154 ksmo->gaddr, ksmo->size, ksmo->buf, ksmo->ar, ksmo->key,155 ksmo->old_addr);156 if (ksmo->flags & KVM_S390_MEMOP_F_CHECK_ONLY)157 printf(", CHECK_ONLY");158 if (ksmo->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION)159 printf(", INJECT_EXCEPTION");160 if (ksmo->flags & KVM_S390_MEMOP_F_SKEY_PROTECTION)161 printf(", SKEY_PROTECTION");162 puts(")");163}164 165static int err_memop_ioctl(struct test_info info, struct kvm_s390_mem_op *ksmo,166 struct mop_desc *desc)167{168 struct kvm_vcpu *vcpu = info.vcpu;169 170 if (!vcpu)171 return __vm_ioctl(info.vm, KVM_S390_MEM_OP, ksmo);172 else173 return __vcpu_ioctl(vcpu, KVM_S390_MEM_OP, ksmo);174}175 176static void memop_ioctl(struct test_info info, struct kvm_s390_mem_op *ksmo,177 struct mop_desc *desc)178{179 int r;180 181 r = err_memop_ioctl(info, ksmo, desc);182 if (ksmo->op == KVM_S390_MEMOP_ABSOLUTE_CMPXCHG) {183 if (desc->cmpxchg_success) {184 int diff = memcmp(desc->old_value, desc->old, desc->size);185 *desc->cmpxchg_success = !diff;186 }187 }188 TEST_ASSERT(!r, __KVM_IOCTL_ERROR("KVM_S390_MEM_OP", r));189}190 191#define MEMOP(err, info_p, mop_target_p, access_mode_p, buf_p, size_p, ...) \192({ \193 struct test_info __info = (info_p); \194 struct mop_desc __desc = { \195 .target = (mop_target_p), \196 .mode = (access_mode_p), \197 .buf = (buf_p), \198 .size = (size_p), \199 __VA_ARGS__ \200 }; \201 struct kvm_s390_mem_op __ksmo; \202 \203 if (__desc._gaddr_v) { \204 if (__desc.target == ABSOLUTE) \205 __desc.gaddr = addr_gva2gpa(__info.vm, __desc.gaddr_v); \206 else \207 __desc.gaddr = __desc.gaddr_v; \208 } \209 __ksmo = ksmo_from_desc(&__desc); \210 print_memop(__info.vcpu, &__ksmo); \211 err##memop_ioctl(__info, &__ksmo, &__desc); \212})213 214#define MOP(...) MEMOP(, __VA_ARGS__)215#define ERR_MOP(...) MEMOP(err_, __VA_ARGS__)216 217#define GADDR(a) .gaddr = ((uintptr_t)a)218#define GADDR_V(v) ._gaddr_v = 1, .gaddr_v = ((uintptr_t)v)219#define CHECK_ONLY .f_check = 1220#define SET_FLAGS(f) ._set_flags = 1, .set_flags = (f)221#define SIDA_OFFSET(o) ._sida_offset = 1, .sida_offset = (o)222#define AR(a) ._ar = 1, .ar = (a)223#define KEY(a) .f_key = 1, .key = (a)224#define INJECT .f_inject = 1225#define CMPXCHG_OLD(o) .old = (o)226#define CMPXCHG_SUCCESS(s) .cmpxchg_success = (s)227 228#define CHECK_N_DO(f, ...) ({ f(__VA_ARGS__, CHECK_ONLY); f(__VA_ARGS__); })229 230#define CR0_FETCH_PROTECTION_OVERRIDE (1UL << (63 - 38))231#define CR0_STORAGE_PROTECTION_OVERRIDE (1UL << (63 - 39))232 233static uint8_t __aligned(PAGE_SIZE) mem1[65536];234static uint8_t __aligned(PAGE_SIZE) mem2[65536];235 236struct test_default {237 struct kvm_vm *kvm_vm;238 struct test_info vm;239 struct test_info vcpu;240 struct kvm_run *run;241 int size;242};243 244static struct test_default test_default_init(void *guest_code)245{246 struct kvm_vcpu *vcpu;247 struct test_default t;248 249 t.size = min((size_t)kvm_check_cap(KVM_CAP_S390_MEM_OP), sizeof(mem1));250 t.kvm_vm = vm_create_with_one_vcpu(&vcpu, guest_code);251 t.vm = (struct test_info) { t.kvm_vm, NULL };252 t.vcpu = (struct test_info) { t.kvm_vm, vcpu };253 t.run = vcpu->run;254 return t;255}256 257enum stage {258 /* Synced state set by host, e.g. DAT */259 STAGE_INITED,260 /* Guest did nothing */261 STAGE_IDLED,262 /* Guest set storage keys (specifics up to test case) */263 STAGE_SKEYS_SET,264 /* Guest copied memory (locations up to test case) */265 STAGE_COPIED,266 /* End of guest code reached */267 STAGE_DONE,268};269 270#define HOST_SYNC(info_p, stage) \271({ \272 struct test_info __info = (info_p); \273 struct kvm_vcpu *__vcpu = __info.vcpu; \274 struct ucall uc; \275 int __stage = (stage); \276 \277 vcpu_run(__vcpu); \278 get_ucall(__vcpu, &uc); \279 if (uc.cmd == UCALL_ABORT) { \280 REPORT_GUEST_ASSERT(uc); \281 } \282 TEST_ASSERT_EQ(uc.cmd, UCALL_SYNC); \283 TEST_ASSERT_EQ(uc.args[1], __stage); \284}) \285 286static void prepare_mem12(void)287{288 int i;289 290 for (i = 0; i < sizeof(mem1); i++)291 mem1[i] = rand();292 memset(mem2, 0xaa, sizeof(mem2));293}294 295#define ASSERT_MEM_EQ(p1, p2, size) \296 TEST_ASSERT(!memcmp(p1, p2, size), "Memory contents do not match!")297 298static void default_write_read(struct test_info copy_cpu, struct test_info mop_cpu,299 enum mop_target mop_target, uint32_t size, uint8_t key)300{301 prepare_mem12();302 CHECK_N_DO(MOP, mop_cpu, mop_target, WRITE, mem1, size,303 GADDR_V(mem1), KEY(key));304 HOST_SYNC(copy_cpu, STAGE_COPIED);305 CHECK_N_DO(MOP, mop_cpu, mop_target, READ, mem2, size,306 GADDR_V(mem2), KEY(key));307 ASSERT_MEM_EQ(mem1, mem2, size);308}309 310static void default_read(struct test_info copy_cpu, struct test_info mop_cpu,311 enum mop_target mop_target, uint32_t size, uint8_t key)312{313 prepare_mem12();314 CHECK_N_DO(MOP, mop_cpu, mop_target, WRITE, mem1, size, GADDR_V(mem1));315 HOST_SYNC(copy_cpu, STAGE_COPIED);316 CHECK_N_DO(MOP, mop_cpu, mop_target, READ, mem2, size,317 GADDR_V(mem2), KEY(key));318 ASSERT_MEM_EQ(mem1, mem2, size);319}320 321static void default_cmpxchg(struct test_default *test, uint8_t key)322{323 for (int size = 1; size <= 16; size *= 2) {324 for (int offset = 0; offset < 16; offset += size) {325 uint8_t __aligned(16) new[16] = {};326 uint8_t __aligned(16) old[16];327 bool succ;328 329 prepare_mem12();330 default_write_read(test->vcpu, test->vcpu, LOGICAL, 16, NO_KEY);331 332 memcpy(&old, mem1, 16);333 MOP(test->vm, ABSOLUTE, CMPXCHG, new + offset,334 size, GADDR_V(mem1 + offset),335 CMPXCHG_OLD(old + offset),336 CMPXCHG_SUCCESS(&succ), KEY(key));337 HOST_SYNC(test->vcpu, STAGE_COPIED);338 MOP(test->vm, ABSOLUTE, READ, mem2, 16, GADDR_V(mem2));339 TEST_ASSERT(succ, "exchange of values should succeed");340 memcpy(mem1 + offset, new + offset, size);341 ASSERT_MEM_EQ(mem1, mem2, 16);342 343 memcpy(&old, mem1, 16);344 new[offset]++;345 old[offset]++;346 MOP(test->vm, ABSOLUTE, CMPXCHG, new + offset,347 size, GADDR_V(mem1 + offset),348 CMPXCHG_OLD(old + offset),349 CMPXCHG_SUCCESS(&succ), KEY(key));350 HOST_SYNC(test->vcpu, STAGE_COPIED);351 MOP(test->vm, ABSOLUTE, READ, mem2, 16, GADDR_V(mem2));352 TEST_ASSERT(!succ, "exchange of values should not succeed");353 ASSERT_MEM_EQ(mem1, mem2, 16);354 ASSERT_MEM_EQ(&old, mem1, 16);355 }356 }357}358 359static void guest_copy(void)360{361 GUEST_SYNC(STAGE_INITED);362 memcpy(&mem2, &mem1, sizeof(mem2));363 GUEST_SYNC(STAGE_COPIED);364}365 366static void test_copy(void)367{368 struct test_default t = test_default_init(guest_copy);369 370 HOST_SYNC(t.vcpu, STAGE_INITED);371 372 default_write_read(t.vcpu, t.vcpu, LOGICAL, t.size, NO_KEY);373 374 kvm_vm_free(t.kvm_vm);375}376 377static void test_copy_access_register(void)378{379 struct test_default t = test_default_init(guest_copy);380 381 HOST_SYNC(t.vcpu, STAGE_INITED);382 383 prepare_mem12();384 t.run->psw_mask &= ~(3UL << (63 - 17));385 t.run->psw_mask |= 1UL << (63 - 17); /* Enable AR mode */386 387 /*388 * Primary address space gets used if an access register389 * contains zero. The host makes use of AR[1] so is a good390 * candidate to ensure the guest AR (of zero) is used.391 */392 CHECK_N_DO(MOP, t.vcpu, LOGICAL, WRITE, mem1, t.size,393 GADDR_V(mem1), AR(1));394 HOST_SYNC(t.vcpu, STAGE_COPIED);395 396 CHECK_N_DO(MOP, t.vcpu, LOGICAL, READ, mem2, t.size,397 GADDR_V(mem2), AR(1));398 ASSERT_MEM_EQ(mem1, mem2, t.size);399 400 kvm_vm_free(t.kvm_vm);401}402 403static void set_storage_key_range(void *addr, size_t len, uint8_t key)404{405 uintptr_t _addr, abs, i;406 int not_mapped = 0;407 408 _addr = (uintptr_t)addr;409 for (i = _addr & PAGE_MASK; i < _addr + len; i += PAGE_SIZE) {410 abs = i;411 asm volatile (412 "lra %[abs], 0(0,%[abs])\n"413 " jz 0f\n"414 " llill %[not_mapped],1\n"415 " j 1f\n"416 "0: sske %[key], %[abs]\n"417 "1:"418 : [abs] "+&a" (abs), [not_mapped] "+r" (not_mapped)419 : [key] "r" (key)420 : "cc"421 );422 GUEST_ASSERT_EQ(not_mapped, 0);423 }424}425 426static void guest_copy_key(void)427{428 set_storage_key_range(mem1, sizeof(mem1), 0x90);429 set_storage_key_range(mem2, sizeof(mem2), 0x90);430 GUEST_SYNC(STAGE_SKEYS_SET);431 432 for (;;) {433 memcpy(&mem2, &mem1, sizeof(mem2));434 GUEST_SYNC(STAGE_COPIED);435 }436}437 438static void test_copy_key(void)439{440 struct test_default t = test_default_init(guest_copy_key);441 442 HOST_SYNC(t.vcpu, STAGE_SKEYS_SET);443 444 /* vm, no key */445 default_write_read(t.vcpu, t.vm, ABSOLUTE, t.size, NO_KEY);446 447 /* vm/vcpu, machting key or key 0 */448 default_write_read(t.vcpu, t.vcpu, LOGICAL, t.size, 0);449 default_write_read(t.vcpu, t.vcpu, LOGICAL, t.size, 9);450 default_write_read(t.vcpu, t.vm, ABSOLUTE, t.size, 0);451 default_write_read(t.vcpu, t.vm, ABSOLUTE, t.size, 9);452 /*453 * There used to be different code paths for key handling depending on454 * if the region crossed a page boundary.455 * There currently are not, but the more tests the merrier.456 */457 default_write_read(t.vcpu, t.vcpu, LOGICAL, 1, 0);458 default_write_read(t.vcpu, t.vcpu, LOGICAL, 1, 9);459 default_write_read(t.vcpu, t.vm, ABSOLUTE, 1, 0);460 default_write_read(t.vcpu, t.vm, ABSOLUTE, 1, 9);461 462 /* vm/vcpu, mismatching keys on read, but no fetch protection */463 default_read(t.vcpu, t.vcpu, LOGICAL, t.size, 2);464 default_read(t.vcpu, t.vm, ABSOLUTE, t.size, 2);465 466 kvm_vm_free(t.kvm_vm);467}468 469static void test_cmpxchg_key(void)470{471 struct test_default t = test_default_init(guest_copy_key);472 473 HOST_SYNC(t.vcpu, STAGE_SKEYS_SET);474 475 default_cmpxchg(&t, NO_KEY);476 default_cmpxchg(&t, 0);477 default_cmpxchg(&t, 9);478 479 kvm_vm_free(t.kvm_vm);480}481 482static __uint128_t cut_to_size(int size, __uint128_t val)483{484 switch (size) {485 case 1:486 return (uint8_t)val;487 case 2:488 return (uint16_t)val;489 case 4:490 return (uint32_t)val;491 case 8:492 return (uint64_t)val;493 case 16:494 return val;495 }496 GUEST_FAIL("Invalid size = %u", size);497 return 0;498}499 500static bool popcount_eq(__uint128_t a, __uint128_t b)501{502 unsigned int count_a, count_b;503 504 count_a = __builtin_popcountl((uint64_t)(a >> 64)) +505 __builtin_popcountl((uint64_t)a);506 count_b = __builtin_popcountl((uint64_t)(b >> 64)) +507 __builtin_popcountl((uint64_t)b);508 return count_a == count_b;509}510 511static __uint128_t rotate(int size, __uint128_t val, int amount)512{513 unsigned int bits = size * 8;514 515 amount = (amount + bits) % bits;516 val = cut_to_size(size, val);517 if (!amount)518 return val;519 return (val << (bits - amount)) | (val >> amount);520}521 522const unsigned int max_block = 16;523 524static void choose_block(bool guest, int i, int *size, int *offset)525{526 unsigned int rand;527 528 rand = i;529 if (guest) {530 rand = rand * 19 + 11;531 *size = 1 << ((rand % 3) + 2);532 rand = rand * 19 + 11;533 *offset = (rand % max_block) & ~(*size - 1);534 } else {535 rand = rand * 17 + 5;536 *size = 1 << (rand % 5);537 rand = rand * 17 + 5;538 *offset = (rand % max_block) & ~(*size - 1);539 }540}541 542static __uint128_t permutate_bits(bool guest, int i, int size, __uint128_t old)543{544 unsigned int rand;545 int amount;546 bool swap;547 548 rand = i;549 rand = rand * 3 + 1;550 if (guest)551 rand = rand * 3 + 1;552 swap = rand % 2 == 0;553 if (swap) {554 int i, j;555 __uint128_t new;556 uint8_t byte0, byte1;557 558 rand = rand * 3 + 1;559 i = rand % size;560 rand = rand * 3 + 1;561 j = rand % size;562 if (i == j)563 return old;564 new = rotate(16, old, i * 8);565 byte0 = new & 0xff;566 new &= ~0xff;567 new = rotate(16, new, -i * 8);568 new = rotate(16, new, j * 8);569 byte1 = new & 0xff;570 new = (new & ~0xff) | byte0;571 new = rotate(16, new, -j * 8);572 new = rotate(16, new, i * 8);573 new = new | byte1;574 new = rotate(16, new, -i * 8);575 return new;576 }577 rand = rand * 3 + 1;578 amount = rand % (size * 8);579 return rotate(size, old, amount);580}581 582static bool _cmpxchg(int size, void *target, __uint128_t *old_addr, __uint128_t new)583{584 bool ret;585 586 switch (size) {587 case 4: {588 uint32_t old = *old_addr;589 590 asm volatile ("cs %[old],%[new],%[address]"591 : [old] "+d" (old),592 [address] "+Q" (*(uint32_t *)(target))593 : [new] "d" ((uint32_t)new)594 : "cc"595 );596 ret = old == (uint32_t)*old_addr;597 *old_addr = old;598 return ret;599 }600 case 8: {601 uint64_t old = *old_addr;602 603 asm volatile ("csg %[old],%[new],%[address]"604 : [old] "+d" (old),605 [address] "+Q" (*(uint64_t *)(target))606 : [new] "d" ((uint64_t)new)607 : "cc"608 );609 ret = old == (uint64_t)*old_addr;610 *old_addr = old;611 return ret;612 }613 case 16: {614 __uint128_t old = *old_addr;615 616 asm volatile ("cdsg %[old],%[new],%[address]"617 : [old] "+d" (old),618 [address] "+Q" (*(__uint128_t *)(target))619 : [new] "d" (new)620 : "cc"621 );622 ret = old == *old_addr;623 *old_addr = old;624 return ret;625 }626 }627 GUEST_FAIL("Invalid size = %u", size);628 return 0;629}630 631const unsigned int cmpxchg_iter_outer = 100, cmpxchg_iter_inner = 10000;632 633static void guest_cmpxchg_key(void)634{635 int size, offset;636 __uint128_t old, new;637 638 set_storage_key_range(mem1, max_block, 0x10);639 set_storage_key_range(mem2, max_block, 0x10);640 GUEST_SYNC(STAGE_SKEYS_SET);641 642 for (int i = 0; i < cmpxchg_iter_outer; i++) {643 do {644 old = 1;645 } while (!_cmpxchg(16, mem1, &old, 0));646 for (int j = 0; j < cmpxchg_iter_inner; j++) {647 choose_block(true, i + j, &size, &offset);648 do {649 new = permutate_bits(true, i + j, size, old);650 } while (!_cmpxchg(size, mem2 + offset, &old, new));651 }652 }653 654 GUEST_SYNC(STAGE_DONE);655}656 657static void *run_guest(void *data)658{659 struct test_info *info = data;660 661 HOST_SYNC(*info, STAGE_DONE);662 return NULL;663}664 665static char *quad_to_char(__uint128_t *quad, int size)666{667 return ((char *)quad) + (sizeof(*quad) - size);668}669 670static void test_cmpxchg_key_concurrent(void)671{672 struct test_default t = test_default_init(guest_cmpxchg_key);673 int size, offset;674 __uint128_t old, new;675 bool success;676 pthread_t thread;677 678 HOST_SYNC(t.vcpu, STAGE_SKEYS_SET);679 prepare_mem12();680 MOP(t.vcpu, LOGICAL, WRITE, mem1, max_block, GADDR_V(mem2));681 pthread_create(&thread, NULL, run_guest, &t.vcpu);682 683 for (int i = 0; i < cmpxchg_iter_outer; i++) {684 do {685 old = 0;686 new = 1;687 MOP(t.vm, ABSOLUTE, CMPXCHG, &new,688 sizeof(new), GADDR_V(mem1),689 CMPXCHG_OLD(&old),690 CMPXCHG_SUCCESS(&success), KEY(1));691 } while (!success);692 for (int j = 0; j < cmpxchg_iter_inner; j++) {693 choose_block(false, i + j, &size, &offset);694 do {695 new = permutate_bits(false, i + j, size, old);696 MOP(t.vm, ABSOLUTE, CMPXCHG, quad_to_char(&new, size),697 size, GADDR_V(mem2 + offset),698 CMPXCHG_OLD(quad_to_char(&old, size)),699 CMPXCHG_SUCCESS(&success), KEY(1));700 } while (!success);701 }702 }703 704 pthread_join(thread, NULL);705 706 MOP(t.vcpu, LOGICAL, READ, mem2, max_block, GADDR_V(mem2));707 TEST_ASSERT(popcount_eq(*(__uint128_t *)mem1, *(__uint128_t *)mem2),708 "Must retain number of set bits");709 710 kvm_vm_free(t.kvm_vm);711}712 713static void guest_copy_key_fetch_prot(void)714{715 /*716 * For some reason combining the first sync with override enablement717 * results in an exception when calling HOST_SYNC.718 */719 GUEST_SYNC(STAGE_INITED);720 /* Storage protection override applies to both store and fetch. */721 set_storage_key_range(mem1, sizeof(mem1), 0x98);722 set_storage_key_range(mem2, sizeof(mem2), 0x98);723 GUEST_SYNC(STAGE_SKEYS_SET);724 725 for (;;) {726 memcpy(&mem2, &mem1, sizeof(mem2));727 GUEST_SYNC(STAGE_COPIED);728 }729}730 731static void test_copy_key_storage_prot_override(void)732{733 struct test_default t = test_default_init(guest_copy_key_fetch_prot);734 735 HOST_SYNC(t.vcpu, STAGE_INITED);736 t.run->s.regs.crs[0] |= CR0_STORAGE_PROTECTION_OVERRIDE;737 t.run->kvm_dirty_regs = KVM_SYNC_CRS;738 HOST_SYNC(t.vcpu, STAGE_SKEYS_SET);739 740 /* vcpu, mismatching keys, storage protection override in effect */741 default_write_read(t.vcpu, t.vcpu, LOGICAL, t.size, 2);742 743 kvm_vm_free(t.kvm_vm);744}745 746static void test_copy_key_fetch_prot(void)747{748 struct test_default t = test_default_init(guest_copy_key_fetch_prot);749 750 HOST_SYNC(t.vcpu, STAGE_INITED);751 HOST_SYNC(t.vcpu, STAGE_SKEYS_SET);752 753 /* vm/vcpu, matching key, fetch protection in effect */754 default_read(t.vcpu, t.vcpu, LOGICAL, t.size, 9);755 default_read(t.vcpu, t.vm, ABSOLUTE, t.size, 9);756 757 kvm_vm_free(t.kvm_vm);758}759 760#define ERR_PROT_MOP(...) \761({ \762 int rv; \763 \764 rv = ERR_MOP(__VA_ARGS__); \765 TEST_ASSERT(rv == 4, "Should result in protection exception"); \766})767 768static void guest_error_key(void)769{770 GUEST_SYNC(STAGE_INITED);771 set_storage_key_range(mem1, PAGE_SIZE, 0x18);772 set_storage_key_range(mem1 + PAGE_SIZE, sizeof(mem1) - PAGE_SIZE, 0x98);773 GUEST_SYNC(STAGE_SKEYS_SET);774 GUEST_SYNC(STAGE_IDLED);775}776 777static void test_errors_key(void)778{779 struct test_default t = test_default_init(guest_error_key);780 781 HOST_SYNC(t.vcpu, STAGE_INITED);782 HOST_SYNC(t.vcpu, STAGE_SKEYS_SET);783 784 /* vm/vcpu, mismatching keys, fetch protection in effect */785 CHECK_N_DO(ERR_PROT_MOP, t.vcpu, LOGICAL, WRITE, mem1, t.size, GADDR_V(mem1), KEY(2));786 CHECK_N_DO(ERR_PROT_MOP, t.vcpu, LOGICAL, READ, mem2, t.size, GADDR_V(mem1), KEY(2));787 CHECK_N_DO(ERR_PROT_MOP, t.vm, ABSOLUTE, WRITE, mem1, t.size, GADDR_V(mem1), KEY(2));788 CHECK_N_DO(ERR_PROT_MOP, t.vm, ABSOLUTE, READ, mem2, t.size, GADDR_V(mem1), KEY(2));789 790 kvm_vm_free(t.kvm_vm);791}792 793static void test_errors_cmpxchg_key(void)794{795 struct test_default t = test_default_init(guest_copy_key_fetch_prot);796 int i;797 798 HOST_SYNC(t.vcpu, STAGE_INITED);799 HOST_SYNC(t.vcpu, STAGE_SKEYS_SET);800 801 for (i = 1; i <= 16; i *= 2) {802 __uint128_t old = 0;803 804 ERR_PROT_MOP(t.vm, ABSOLUTE, CMPXCHG, mem2, i, GADDR_V(mem2),805 CMPXCHG_OLD(&old), KEY(2));806 }807 808 kvm_vm_free(t.kvm_vm);809}810 811static void test_termination(void)812{813 struct test_default t = test_default_init(guest_error_key);814 uint64_t prefix;815 uint64_t teid;816 uint64_t teid_mask = BIT(63 - 56) | BIT(63 - 60) | BIT(63 - 61);817 uint64_t psw[2];818 819 HOST_SYNC(t.vcpu, STAGE_INITED);820 HOST_SYNC(t.vcpu, STAGE_SKEYS_SET);821 822 /* vcpu, mismatching keys after first page */823 ERR_PROT_MOP(t.vcpu, LOGICAL, WRITE, mem1, t.size, GADDR_V(mem1), KEY(1), INJECT);824 /*825 * The memop injected a program exception and the test needs to check the826 * Translation-Exception Identification (TEID). It is necessary to run827 * the guest in order to be able to read the TEID from guest memory.828 * Set the guest program new PSW, so the guest state is not clobbered.829 */830 prefix = t.run->s.regs.prefix;831 psw[0] = t.run->psw_mask;832 psw[1] = t.run->psw_addr;833 MOP(t.vm, ABSOLUTE, WRITE, psw, sizeof(psw), GADDR(prefix + 464));834 HOST_SYNC(t.vcpu, STAGE_IDLED);835 MOP(t.vm, ABSOLUTE, READ, &teid, sizeof(teid), GADDR(prefix + 168));836 /* Bits 56, 60, 61 form a code, 0 being the only one allowing for termination */837 TEST_ASSERT_EQ(teid & teid_mask, 0);838 839 kvm_vm_free(t.kvm_vm);840}841 842static void test_errors_key_storage_prot_override(void)843{844 struct test_default t = test_default_init(guest_copy_key_fetch_prot);845 846 HOST_SYNC(t.vcpu, STAGE_INITED);847 t.run->s.regs.crs[0] |= CR0_STORAGE_PROTECTION_OVERRIDE;848 t.run->kvm_dirty_regs = KVM_SYNC_CRS;849 HOST_SYNC(t.vcpu, STAGE_SKEYS_SET);850 851 /* vm, mismatching keys, storage protection override not applicable to vm */852 CHECK_N_DO(ERR_PROT_MOP, t.vm, ABSOLUTE, WRITE, mem1, t.size, GADDR_V(mem1), KEY(2));853 CHECK_N_DO(ERR_PROT_MOP, t.vm, ABSOLUTE, READ, mem2, t.size, GADDR_V(mem2), KEY(2));854 855 kvm_vm_free(t.kvm_vm);856}857 858const uint64_t last_page_addr = -PAGE_SIZE;859 860static void guest_copy_key_fetch_prot_override(void)861{862 int i;863 char *page_0 = 0;864 865 GUEST_SYNC(STAGE_INITED);866 set_storage_key_range(0, PAGE_SIZE, 0x18);867 set_storage_key_range((void *)last_page_addr, PAGE_SIZE, 0x0);868 asm volatile ("sske %[key],%[addr]\n" :: [addr] "r"(0L), [key] "r"(0x18) : "cc");869 GUEST_SYNC(STAGE_SKEYS_SET);870 871 for (;;) {872 for (i = 0; i < PAGE_SIZE; i++)873 page_0[i] = mem1[i];874 GUEST_SYNC(STAGE_COPIED);875 }876}877 878static void test_copy_key_fetch_prot_override(void)879{880 struct test_default t = test_default_init(guest_copy_key_fetch_prot_override);881 vm_vaddr_t guest_0_page, guest_last_page;882 883 guest_0_page = vm_vaddr_alloc(t.kvm_vm, PAGE_SIZE, 0);884 guest_last_page = vm_vaddr_alloc(t.kvm_vm, PAGE_SIZE, last_page_addr);885 if (guest_0_page != 0 || guest_last_page != last_page_addr) {886 print_skip("did not allocate guest pages at required positions");887 goto out;888 }889 890 HOST_SYNC(t.vcpu, STAGE_INITED);891 t.run->s.regs.crs[0] |= CR0_FETCH_PROTECTION_OVERRIDE;892 t.run->kvm_dirty_regs = KVM_SYNC_CRS;893 HOST_SYNC(t.vcpu, STAGE_SKEYS_SET);894 895 /* vcpu, mismatching keys on fetch, fetch protection override applies */896 prepare_mem12();897 MOP(t.vcpu, LOGICAL, WRITE, mem1, PAGE_SIZE, GADDR_V(mem1));898 HOST_SYNC(t.vcpu, STAGE_COPIED);899 CHECK_N_DO(MOP, t.vcpu, LOGICAL, READ, mem2, 2048, GADDR_V(guest_0_page), KEY(2));900 ASSERT_MEM_EQ(mem1, mem2, 2048);901 902 /*903 * vcpu, mismatching keys on fetch, fetch protection override applies,904 * wraparound905 */906 prepare_mem12();907 MOP(t.vcpu, LOGICAL, WRITE, mem1, 2 * PAGE_SIZE, GADDR_V(guest_last_page));908 HOST_SYNC(t.vcpu, STAGE_COPIED);909 CHECK_N_DO(MOP, t.vcpu, LOGICAL, READ, mem2, PAGE_SIZE + 2048,910 GADDR_V(guest_last_page), KEY(2));911 ASSERT_MEM_EQ(mem1, mem2, 2048);912 913out:914 kvm_vm_free(t.kvm_vm);915}916 917static void test_errors_key_fetch_prot_override_not_enabled(void)918{919 struct test_default t = test_default_init(guest_copy_key_fetch_prot_override);920 vm_vaddr_t guest_0_page, guest_last_page;921 922 guest_0_page = vm_vaddr_alloc(t.kvm_vm, PAGE_SIZE, 0);923 guest_last_page = vm_vaddr_alloc(t.kvm_vm, PAGE_SIZE, last_page_addr);924 if (guest_0_page != 0 || guest_last_page != last_page_addr) {925 print_skip("did not allocate guest pages at required positions");926 goto out;927 }928 HOST_SYNC(t.vcpu, STAGE_INITED);929 HOST_SYNC(t.vcpu, STAGE_SKEYS_SET);930 931 /* vcpu, mismatching keys on fetch, fetch protection override not enabled */932 CHECK_N_DO(ERR_PROT_MOP, t.vcpu, LOGICAL, READ, mem2, 2048, GADDR_V(0), KEY(2));933 934out:935 kvm_vm_free(t.kvm_vm);936}937 938static void test_errors_key_fetch_prot_override_enabled(void)939{940 struct test_default t = test_default_init(guest_copy_key_fetch_prot_override);941 vm_vaddr_t guest_0_page, guest_last_page;942 943 guest_0_page = vm_vaddr_alloc(t.kvm_vm, PAGE_SIZE, 0);944 guest_last_page = vm_vaddr_alloc(t.kvm_vm, PAGE_SIZE, last_page_addr);945 if (guest_0_page != 0 || guest_last_page != last_page_addr) {946 print_skip("did not allocate guest pages at required positions");947 goto out;948 }949 HOST_SYNC(t.vcpu, STAGE_INITED);950 t.run->s.regs.crs[0] |= CR0_FETCH_PROTECTION_OVERRIDE;951 t.run->kvm_dirty_regs = KVM_SYNC_CRS;952 HOST_SYNC(t.vcpu, STAGE_SKEYS_SET);953 954 /*955 * vcpu, mismatching keys on fetch,956 * fetch protection override does not apply because memory range exceeded957 */958 CHECK_N_DO(ERR_PROT_MOP, t.vcpu, LOGICAL, READ, mem2, 2048 + 1, GADDR_V(0), KEY(2));959 CHECK_N_DO(ERR_PROT_MOP, t.vcpu, LOGICAL, READ, mem2, PAGE_SIZE + 2048 + 1,960 GADDR_V(guest_last_page), KEY(2));961 /* vm, fetch protected override does not apply */962 CHECK_N_DO(ERR_PROT_MOP, t.vm, ABSOLUTE, READ, mem2, 2048, GADDR(0), KEY(2));963 CHECK_N_DO(ERR_PROT_MOP, t.vm, ABSOLUTE, READ, mem2, 2048, GADDR_V(guest_0_page), KEY(2));964 965out:966 kvm_vm_free(t.kvm_vm);967}968 969static void guest_idle(void)970{971 GUEST_SYNC(STAGE_INITED); /* for consistency's sake */972 for (;;)973 GUEST_SYNC(STAGE_IDLED);974}975 976static void _test_errors_common(struct test_info info, enum mop_target target, int size)977{978 int rv;979 980 /* Bad size: */981 rv = ERR_MOP(info, target, WRITE, mem1, -1, GADDR_V(mem1));982 TEST_ASSERT(rv == -1 && errno == E2BIG, "ioctl allows insane sizes");983 984 /* Zero size: */985 rv = ERR_MOP(info, target, WRITE, mem1, 0, GADDR_V(mem1));986 TEST_ASSERT(rv == -1 && (errno == EINVAL || errno == ENOMEM),987 "ioctl allows 0 as size");988 989 /* Bad flags: */990 rv = ERR_MOP(info, target, WRITE, mem1, size, GADDR_V(mem1), SET_FLAGS(-1));991 TEST_ASSERT(rv == -1 && errno == EINVAL, "ioctl allows all flags");992 993 /* Bad guest address: */994 rv = ERR_MOP(info, target, WRITE, mem1, size, GADDR((void *)~0xfffUL), CHECK_ONLY);995 TEST_ASSERT(rv > 0, "ioctl does not report bad guest memory address with CHECK_ONLY");996 rv = ERR_MOP(info, target, WRITE, mem1, size, GADDR((void *)~0xfffUL));997 TEST_ASSERT(rv > 0, "ioctl does not report bad guest memory address on write");998 999 /* Bad host address: */1000 rv = ERR_MOP(info, target, WRITE, 0, size, GADDR_V(mem1));1001 TEST_ASSERT(rv == -1 && errno == EFAULT,1002 "ioctl does not report bad host memory address");1003 1004 /* Bad key: */1005 rv = ERR_MOP(info, target, WRITE, mem1, size, GADDR_V(mem1), KEY(17));1006 TEST_ASSERT(rv == -1 && errno == EINVAL, "ioctl allows invalid key");1007}1008 1009static void test_errors(void)1010{1011 struct test_default t = test_default_init(guest_idle);1012 int rv;1013 1014 HOST_SYNC(t.vcpu, STAGE_INITED);1015 1016 _test_errors_common(t.vcpu, LOGICAL, t.size);1017 _test_errors_common(t.vm, ABSOLUTE, t.size);1018 1019 /* Bad operation: */1020 rv = ERR_MOP(t.vcpu, INVALID, WRITE, mem1, t.size, GADDR_V(mem1));1021 TEST_ASSERT(rv == -1 && errno == EINVAL, "ioctl allows bad operations");1022 /* virtual addresses are not translated when passing INVALID */1023 rv = ERR_MOP(t.vm, INVALID, WRITE, mem1, PAGE_SIZE, GADDR(0));1024 TEST_ASSERT(rv == -1 && errno == EINVAL, "ioctl allows bad operations");1025 1026 /* Bad access register: */1027 t.run->psw_mask &= ~(3UL << (63 - 17));1028 t.run->psw_mask |= 1UL << (63 - 17); /* Enable AR mode */1029 HOST_SYNC(t.vcpu, STAGE_IDLED); /* To sync new state to SIE block */1030 rv = ERR_MOP(t.vcpu, LOGICAL, WRITE, mem1, t.size, GADDR_V(mem1), AR(17));1031 TEST_ASSERT(rv == -1 && errno == EINVAL, "ioctl allows ARs > 15");1032 t.run->psw_mask &= ~(3UL << (63 - 17)); /* Disable AR mode */1033 HOST_SYNC(t.vcpu, STAGE_IDLED); /* Run to sync new state */1034 1035 /* Check that the SIDA calls are rejected for non-protected guests */1036 rv = ERR_MOP(t.vcpu, SIDA, READ, mem1, 8, GADDR(0), SIDA_OFFSET(0x1c0));1037 TEST_ASSERT(rv == -1 && errno == EINVAL,1038 "ioctl does not reject SIDA_READ in non-protected mode");1039 rv = ERR_MOP(t.vcpu, SIDA, WRITE, mem1, 8, GADDR(0), SIDA_OFFSET(0x1c0));1040 TEST_ASSERT(rv == -1 && errno == EINVAL,1041 "ioctl does not reject SIDA_WRITE in non-protected mode");1042 1043 kvm_vm_free(t.kvm_vm);1044}1045 1046static void test_errors_cmpxchg(void)1047{1048 struct test_default t = test_default_init(guest_idle);1049 __uint128_t old;1050 int rv, i, power = 1;1051 1052 HOST_SYNC(t.vcpu, STAGE_INITED);1053 1054 for (i = 0; i < 32; i++) {1055 if (i == power) {1056 power *= 2;1057 continue;1058 }1059 rv = ERR_MOP(t.vm, ABSOLUTE, CMPXCHG, mem1, i, GADDR_V(mem1),1060 CMPXCHG_OLD(&old));1061 TEST_ASSERT(rv == -1 && errno == EINVAL,1062 "ioctl allows bad size for cmpxchg");1063 }1064 for (i = 1; i <= 16; i *= 2) {1065 rv = ERR_MOP(t.vm, ABSOLUTE, CMPXCHG, mem1, i, GADDR((void *)~0xfffUL),1066 CMPXCHG_OLD(&old));1067 TEST_ASSERT(rv > 0, "ioctl allows bad guest address for cmpxchg");1068 }1069 for (i = 2; i <= 16; i *= 2) {1070 rv = ERR_MOP(t.vm, ABSOLUTE, CMPXCHG, mem1, i, GADDR_V(mem1 + 1),1071 CMPXCHG_OLD(&old));1072 TEST_ASSERT(rv == -1 && errno == EINVAL,1073 "ioctl allows bad alignment for cmpxchg");1074 }1075 1076 kvm_vm_free(t.kvm_vm);1077}1078 1079int main(int argc, char *argv[])1080{1081 int extension_cap, idx;1082 1083 TEST_REQUIRE(kvm_has_cap(KVM_CAP_S390_MEM_OP));1084 extension_cap = kvm_check_cap(KVM_CAP_S390_MEM_OP_EXTENSION);1085 1086 struct testdef {1087 const char *name;1088 void (*test)(void);1089 bool requirements_met;1090 } testlist[] = {1091 {1092 .name = "simple copy",1093 .test = test_copy,1094 .requirements_met = true,1095 },1096 {1097 .name = "generic error checks",1098 .test = test_errors,1099 .requirements_met = true,1100 },1101 {1102 .name = "copy with storage keys",1103 .test = test_copy_key,1104 .requirements_met = extension_cap > 0,1105 },1106 {1107 .name = "cmpxchg with storage keys",1108 .test = test_cmpxchg_key,1109 .requirements_met = extension_cap & 0x2,1110 },1111 {1112 .name = "concurrently cmpxchg with storage keys",1113 .test = test_cmpxchg_key_concurrent,1114 .requirements_met = extension_cap & 0x2,1115 },1116 {1117 .name = "copy with key storage protection override",1118 .test = test_copy_key_storage_prot_override,1119 .requirements_met = extension_cap > 0,1120 },1121 {1122 .name = "copy with key fetch protection",1123 .test = test_copy_key_fetch_prot,1124 .requirements_met = extension_cap > 0,1125 },1126 {1127 .name = "copy with key fetch protection override",1128 .test = test_copy_key_fetch_prot_override,1129 .requirements_met = extension_cap > 0,1130 },1131 {1132 .name = "copy with access register mode",1133 .test = test_copy_access_register,1134 .requirements_met = true,1135 },1136 {1137 .name = "error checks with key",1138 .test = test_errors_key,1139 .requirements_met = extension_cap > 0,1140 },1141 {1142 .name = "error checks for cmpxchg with key",1143 .test = test_errors_cmpxchg_key,1144 .requirements_met = extension_cap & 0x2,1145 },1146 {1147 .name = "error checks for cmpxchg",1148 .test = test_errors_cmpxchg,1149 .requirements_met = extension_cap & 0x2,1150 },1151 {1152 .name = "termination",1153 .test = test_termination,1154 .requirements_met = extension_cap > 0,1155 },1156 {1157 .name = "error checks with key storage protection override",1158 .test = test_errors_key_storage_prot_override,1159 .requirements_met = extension_cap > 0,1160 },1161 {1162 .name = "error checks without key fetch prot override",1163 .test = test_errors_key_fetch_prot_override_not_enabled,1164 .requirements_met = extension_cap > 0,1165 },1166 {1167 .name = "error checks with key fetch prot override",1168 .test = test_errors_key_fetch_prot_override_enabled,1169 .requirements_met = extension_cap > 0,1170 },1171 };1172 1173 ksft_print_header();1174 ksft_set_plan(ARRAY_SIZE(testlist));1175 1176 for (idx = 0; idx < ARRAY_SIZE(testlist); idx++) {1177 if (testlist[idx].requirements_met) {1178 testlist[idx].test();1179 ksft_test_result_pass("%s\n", testlist[idx].name);1180 } else {1181 ksft_test_result_skip("%s - requirements not met (kernel has extension cap %#x)\n",1182 testlist[idx].name, extension_cap);1183 }1184 }1185 1186 ksft_finished(); /* Print results and exit() accordingly */1187}1188