2060 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * HMM stands for Heterogeneous Memory Management, it is a helper layer inside4 * the linux kernel to help device drivers mirror a process address space in5 * the device. This allows the device to use the same address space which6 * makes communication and data exchange a lot easier.7 *8 * This framework's sole purpose is to exercise various code paths inside9 * the kernel to make sure that HMM performs as expected and to flush out any10 * bugs.11 */12 13#include "../kselftest_harness.h"14 15#include <errno.h>16#include <fcntl.h>17#include <stdio.h>18#include <stdlib.h>19#include <stdint.h>20#include <unistd.h>21#include <strings.h>22#include <time.h>23#include <pthread.h>24#include <sys/types.h>25#include <sys/stat.h>26#include <sys/mman.h>27#include <sys/ioctl.h>28 29 30/*31 * This is a private UAPI to the kernel test module so it isn't exported32 * in the usual include/uapi/... directory.33 */34#include <lib/test_hmm_uapi.h>35#include <mm/gup_test.h>36 37struct hmm_buffer {38 void *ptr;39 void *mirror;40 unsigned long size;41 int fd;42 uint64_t cpages;43 uint64_t faults;44};45 46enum {47 HMM_PRIVATE_DEVICE_ONE,48 HMM_PRIVATE_DEVICE_TWO,49 HMM_COHERENCE_DEVICE_ONE,50 HMM_COHERENCE_DEVICE_TWO,51};52 53#define TWOMEG (1 << 21)54#define HMM_BUFFER_SIZE (1024 << 12)55#define HMM_PATH_MAX 6456#define NTIMES 1057 58#define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1)))59/* Just the flags we need, copied from mm.h: */60 61#ifndef FOLL_WRITE62#define FOLL_WRITE 0x01 /* check pte is writable */63#endif64 65#ifndef FOLL_LONGTERM66#define FOLL_LONGTERM 0x100 /* mapping lifetime is indefinite */67#endif68FIXTURE(hmm)69{70 int fd;71 unsigned int page_size;72 unsigned int page_shift;73};74 75FIXTURE_VARIANT(hmm)76{77 int device_number;78};79 80FIXTURE_VARIANT_ADD(hmm, hmm_device_private)81{82 .device_number = HMM_PRIVATE_DEVICE_ONE,83};84 85FIXTURE_VARIANT_ADD(hmm, hmm_device_coherent)86{87 .device_number = HMM_COHERENCE_DEVICE_ONE,88};89 90FIXTURE(hmm2)91{92 int fd0;93 int fd1;94 unsigned int page_size;95 unsigned int page_shift;96};97 98FIXTURE_VARIANT(hmm2)99{100 int device_number0;101 int device_number1;102};103 104FIXTURE_VARIANT_ADD(hmm2, hmm2_device_private)105{106 .device_number0 = HMM_PRIVATE_DEVICE_ONE,107 .device_number1 = HMM_PRIVATE_DEVICE_TWO,108};109 110FIXTURE_VARIANT_ADD(hmm2, hmm2_device_coherent)111{112 .device_number0 = HMM_COHERENCE_DEVICE_ONE,113 .device_number1 = HMM_COHERENCE_DEVICE_TWO,114};115 116static int hmm_open(int unit)117{118 char pathname[HMM_PATH_MAX];119 int fd;120 121 snprintf(pathname, sizeof(pathname), "/dev/hmm_dmirror%d", unit);122 fd = open(pathname, O_RDWR, 0);123 if (fd < 0)124 fprintf(stderr, "could not open hmm dmirror driver (%s)\n",125 pathname);126 return fd;127}128 129static bool hmm_is_coherent_type(int dev_num)130{131 return (dev_num >= HMM_COHERENCE_DEVICE_ONE);132}133 134FIXTURE_SETUP(hmm)135{136 self->page_size = sysconf(_SC_PAGE_SIZE);137 self->page_shift = ffs(self->page_size) - 1;138 139 self->fd = hmm_open(variant->device_number);140 if (self->fd < 0 && hmm_is_coherent_type(variant->device_number))141 SKIP(return, "DEVICE_COHERENT not available");142 ASSERT_GE(self->fd, 0);143}144 145FIXTURE_SETUP(hmm2)146{147 self->page_size = sysconf(_SC_PAGE_SIZE);148 self->page_shift = ffs(self->page_size) - 1;149 150 self->fd0 = hmm_open(variant->device_number0);151 if (self->fd0 < 0 && hmm_is_coherent_type(variant->device_number0))152 SKIP(return, "DEVICE_COHERENT not available");153 ASSERT_GE(self->fd0, 0);154 self->fd1 = hmm_open(variant->device_number1);155 ASSERT_GE(self->fd1, 0);156}157 158FIXTURE_TEARDOWN(hmm)159{160 int ret = close(self->fd);161 162 ASSERT_EQ(ret, 0);163 self->fd = -1;164}165 166FIXTURE_TEARDOWN(hmm2)167{168 int ret = close(self->fd0);169 170 ASSERT_EQ(ret, 0);171 self->fd0 = -1;172 173 ret = close(self->fd1);174 ASSERT_EQ(ret, 0);175 self->fd1 = -1;176}177 178static int hmm_dmirror_cmd(int fd,179 unsigned long request,180 struct hmm_buffer *buffer,181 unsigned long npages)182{183 struct hmm_dmirror_cmd cmd;184 int ret;185 186 /* Simulate a device reading system memory. */187 cmd.addr = (__u64)buffer->ptr;188 cmd.ptr = (__u64)buffer->mirror;189 cmd.npages = npages;190 191 for (;;) {192 ret = ioctl(fd, request, &cmd);193 if (ret == 0)194 break;195 if (errno == EINTR)196 continue;197 return -errno;198 }199 buffer->cpages = cmd.cpages;200 buffer->faults = cmd.faults;201 202 return 0;203}204 205static void hmm_buffer_free(struct hmm_buffer *buffer)206{207 if (buffer == NULL)208 return;209 210 if (buffer->ptr)211 munmap(buffer->ptr, buffer->size);212 free(buffer->mirror);213 free(buffer);214}215 216/*217 * Create a temporary file that will be deleted on close.218 */219static int hmm_create_file(unsigned long size)220{221 char path[HMM_PATH_MAX];222 int fd;223 224 strcpy(path, "/tmp");225 fd = open(path, O_TMPFILE | O_EXCL | O_RDWR, 0600);226 if (fd >= 0) {227 int r;228 229 do {230 r = ftruncate(fd, size);231 } while (r == -1 && errno == EINTR);232 if (!r)233 return fd;234 close(fd);235 }236 return -1;237}238 239/*240 * Return a random unsigned number.241 */242static unsigned int hmm_random(void)243{244 static int fd = -1;245 unsigned int r;246 247 if (fd < 0) {248 fd = open("/dev/urandom", O_RDONLY);249 if (fd < 0) {250 fprintf(stderr, "%s:%d failed to open /dev/urandom\n",251 __FILE__, __LINE__);252 return ~0U;253 }254 }255 read(fd, &r, sizeof(r));256 return r;257}258 259static void hmm_nanosleep(unsigned int n)260{261 struct timespec t;262 263 t.tv_sec = 0;264 t.tv_nsec = n;265 nanosleep(&t, NULL);266}267 268static int hmm_migrate_sys_to_dev(int fd,269 struct hmm_buffer *buffer,270 unsigned long npages)271{272 return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_DEV, buffer, npages);273}274 275static int hmm_migrate_dev_to_sys(int fd,276 struct hmm_buffer *buffer,277 unsigned long npages)278{279 return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_SYS, buffer, npages);280}281 282/*283 * Simple NULL test of device open/close.284 */285TEST_F(hmm, open_close)286{287}288 289/*290 * Read private anonymous memory.291 */292TEST_F(hmm, anon_read)293{294 struct hmm_buffer *buffer;295 unsigned long npages;296 unsigned long size;297 unsigned long i;298 int *ptr;299 int ret;300 int val;301 302 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;303 ASSERT_NE(npages, 0);304 size = npages << self->page_shift;305 306 buffer = malloc(sizeof(*buffer));307 ASSERT_NE(buffer, NULL);308 309 buffer->fd = -1;310 buffer->size = size;311 buffer->mirror = malloc(size);312 ASSERT_NE(buffer->mirror, NULL);313 314 buffer->ptr = mmap(NULL, size,315 PROT_READ | PROT_WRITE,316 MAP_PRIVATE | MAP_ANONYMOUS,317 buffer->fd, 0);318 ASSERT_NE(buffer->ptr, MAP_FAILED);319 320 /*321 * Initialize buffer in system memory but leave the first two pages322 * zero (pte_none and pfn_zero).323 */324 i = 2 * self->page_size / sizeof(*ptr);325 for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)326 ptr[i] = i;327 328 /* Set buffer permission to read-only. */329 ret = mprotect(buffer->ptr, size, PROT_READ);330 ASSERT_EQ(ret, 0);331 332 /* Populate the CPU page table with a special zero page. */333 val = *(int *)(buffer->ptr + self->page_size);334 ASSERT_EQ(val, 0);335 336 /* Simulate a device reading system memory. */337 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);338 ASSERT_EQ(ret, 0);339 ASSERT_EQ(buffer->cpages, npages);340 ASSERT_EQ(buffer->faults, 1);341 342 /* Check what the device read. */343 ptr = buffer->mirror;344 for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i)345 ASSERT_EQ(ptr[i], 0);346 for (; i < size / sizeof(*ptr); ++i)347 ASSERT_EQ(ptr[i], i);348 349 hmm_buffer_free(buffer);350}351 352/*353 * Read private anonymous memory which has been protected with354 * mprotect() PROT_NONE.355 */356TEST_F(hmm, anon_read_prot)357{358 struct hmm_buffer *buffer;359 unsigned long npages;360 unsigned long size;361 unsigned long i;362 int *ptr;363 int ret;364 365 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;366 ASSERT_NE(npages, 0);367 size = npages << self->page_shift;368 369 buffer = malloc(sizeof(*buffer));370 ASSERT_NE(buffer, NULL);371 372 buffer->fd = -1;373 buffer->size = size;374 buffer->mirror = malloc(size);375 ASSERT_NE(buffer->mirror, NULL);376 377 buffer->ptr = mmap(NULL, size,378 PROT_READ | PROT_WRITE,379 MAP_PRIVATE | MAP_ANONYMOUS,380 buffer->fd, 0);381 ASSERT_NE(buffer->ptr, MAP_FAILED);382 383 /* Initialize buffer in system memory. */384 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)385 ptr[i] = i;386 387 /* Initialize mirror buffer so we can verify it isn't written. */388 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)389 ptr[i] = -i;390 391 /* Protect buffer from reading. */392 ret = mprotect(buffer->ptr, size, PROT_NONE);393 ASSERT_EQ(ret, 0);394 395 /* Simulate a device reading system memory. */396 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);397 ASSERT_EQ(ret, -EFAULT);398 399 /* Allow CPU to read the buffer so we can check it. */400 ret = mprotect(buffer->ptr, size, PROT_READ);401 ASSERT_EQ(ret, 0);402 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)403 ASSERT_EQ(ptr[i], i);404 405 /* Check what the device read. */406 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)407 ASSERT_EQ(ptr[i], -i);408 409 hmm_buffer_free(buffer);410}411 412/*413 * Write private anonymous memory.414 */415TEST_F(hmm, anon_write)416{417 struct hmm_buffer *buffer;418 unsigned long npages;419 unsigned long size;420 unsigned long i;421 int *ptr;422 int ret;423 424 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;425 ASSERT_NE(npages, 0);426 size = npages << self->page_shift;427 428 buffer = malloc(sizeof(*buffer));429 ASSERT_NE(buffer, NULL);430 431 buffer->fd = -1;432 buffer->size = size;433 buffer->mirror = malloc(size);434 ASSERT_NE(buffer->mirror, NULL);435 436 buffer->ptr = mmap(NULL, size,437 PROT_READ | PROT_WRITE,438 MAP_PRIVATE | MAP_ANONYMOUS,439 buffer->fd, 0);440 ASSERT_NE(buffer->ptr, MAP_FAILED);441 442 /* Initialize data that the device will write to buffer->ptr. */443 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)444 ptr[i] = i;445 446 /* Simulate a device writing system memory. */447 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);448 ASSERT_EQ(ret, 0);449 ASSERT_EQ(buffer->cpages, npages);450 ASSERT_EQ(buffer->faults, 1);451 452 /* Check what the device wrote. */453 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)454 ASSERT_EQ(ptr[i], i);455 456 hmm_buffer_free(buffer);457}458 459/*460 * Write private anonymous memory which has been protected with461 * mprotect() PROT_READ.462 */463TEST_F(hmm, anon_write_prot)464{465 struct hmm_buffer *buffer;466 unsigned long npages;467 unsigned long size;468 unsigned long i;469 int *ptr;470 int ret;471 472 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;473 ASSERT_NE(npages, 0);474 size = npages << self->page_shift;475 476 buffer = malloc(sizeof(*buffer));477 ASSERT_NE(buffer, NULL);478 479 buffer->fd = -1;480 buffer->size = size;481 buffer->mirror = malloc(size);482 ASSERT_NE(buffer->mirror, NULL);483 484 buffer->ptr = mmap(NULL, size,485 PROT_READ,486 MAP_PRIVATE | MAP_ANONYMOUS,487 buffer->fd, 0);488 ASSERT_NE(buffer->ptr, MAP_FAILED);489 490 /* Simulate a device reading a zero page of memory. */491 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1);492 ASSERT_EQ(ret, 0);493 ASSERT_EQ(buffer->cpages, 1);494 ASSERT_EQ(buffer->faults, 1);495 496 /* Initialize data that the device will write to buffer->ptr. */497 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)498 ptr[i] = i;499 500 /* Simulate a device writing system memory. */501 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);502 ASSERT_EQ(ret, -EPERM);503 504 /* Check what the device wrote. */505 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)506 ASSERT_EQ(ptr[i], 0);507 508 /* Now allow writing and see that the zero page is replaced. */509 ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ);510 ASSERT_EQ(ret, 0);511 512 /* Simulate a device writing system memory. */513 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);514 ASSERT_EQ(ret, 0);515 ASSERT_EQ(buffer->cpages, npages);516 ASSERT_EQ(buffer->faults, 1);517 518 /* Check what the device wrote. */519 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)520 ASSERT_EQ(ptr[i], i);521 522 hmm_buffer_free(buffer);523}524 525/*526 * Check that a device writing an anonymous private mapping527 * will copy-on-write if a child process inherits the mapping.528 */529TEST_F(hmm, anon_write_child)530{531 struct hmm_buffer *buffer;532 unsigned long npages;533 unsigned long size;534 unsigned long i;535 int *ptr;536 pid_t pid;537 int child_fd;538 int ret;539 540 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;541 ASSERT_NE(npages, 0);542 size = npages << self->page_shift;543 544 buffer = malloc(sizeof(*buffer));545 ASSERT_NE(buffer, NULL);546 547 buffer->fd = -1;548 buffer->size = size;549 buffer->mirror = malloc(size);550 ASSERT_NE(buffer->mirror, NULL);551 552 buffer->ptr = mmap(NULL, size,553 PROT_READ | PROT_WRITE,554 MAP_PRIVATE | MAP_ANONYMOUS,555 buffer->fd, 0);556 ASSERT_NE(buffer->ptr, MAP_FAILED);557 558 /* Initialize buffer->ptr so we can tell if it is written. */559 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)560 ptr[i] = i;561 562 /* Initialize data that the device will write to buffer->ptr. */563 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)564 ptr[i] = -i;565 566 pid = fork();567 if (pid == -1)568 ASSERT_EQ(pid, 0);569 if (pid != 0) {570 waitpid(pid, &ret, 0);571 ASSERT_EQ(WIFEXITED(ret), 1);572 573 /* Check that the parent's buffer did not change. */574 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)575 ASSERT_EQ(ptr[i], i);576 return;577 }578 579 /* Check that we see the parent's values. */580 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)581 ASSERT_EQ(ptr[i], i);582 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)583 ASSERT_EQ(ptr[i], -i);584 585 /* The child process needs its own mirror to its own mm. */586 child_fd = hmm_open(0);587 ASSERT_GE(child_fd, 0);588 589 /* Simulate a device writing system memory. */590 ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages);591 ASSERT_EQ(ret, 0);592 ASSERT_EQ(buffer->cpages, npages);593 ASSERT_EQ(buffer->faults, 1);594 595 /* Check what the device wrote. */596 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)597 ASSERT_EQ(ptr[i], -i);598 599 close(child_fd);600 exit(0);601}602 603/*604 * Check that a device writing an anonymous shared mapping605 * will not copy-on-write if a child process inherits the mapping.606 */607TEST_F(hmm, anon_write_child_shared)608{609 struct hmm_buffer *buffer;610 unsigned long npages;611 unsigned long size;612 unsigned long i;613 int *ptr;614 pid_t pid;615 int child_fd;616 int ret;617 618 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;619 ASSERT_NE(npages, 0);620 size = npages << self->page_shift;621 622 buffer = malloc(sizeof(*buffer));623 ASSERT_NE(buffer, NULL);624 625 buffer->fd = -1;626 buffer->size = size;627 buffer->mirror = malloc(size);628 ASSERT_NE(buffer->mirror, NULL);629 630 buffer->ptr = mmap(NULL, size,631 PROT_READ | PROT_WRITE,632 MAP_SHARED | MAP_ANONYMOUS,633 buffer->fd, 0);634 ASSERT_NE(buffer->ptr, MAP_FAILED);635 636 /* Initialize buffer->ptr so we can tell if it is written. */637 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)638 ptr[i] = i;639 640 /* Initialize data that the device will write to buffer->ptr. */641 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)642 ptr[i] = -i;643 644 pid = fork();645 if (pid == -1)646 ASSERT_EQ(pid, 0);647 if (pid != 0) {648 waitpid(pid, &ret, 0);649 ASSERT_EQ(WIFEXITED(ret), 1);650 651 /* Check that the parent's buffer did change. */652 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)653 ASSERT_EQ(ptr[i], -i);654 return;655 }656 657 /* Check that we see the parent's values. */658 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)659 ASSERT_EQ(ptr[i], i);660 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)661 ASSERT_EQ(ptr[i], -i);662 663 /* The child process needs its own mirror to its own mm. */664 child_fd = hmm_open(0);665 ASSERT_GE(child_fd, 0);666 667 /* Simulate a device writing system memory. */668 ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages);669 ASSERT_EQ(ret, 0);670 ASSERT_EQ(buffer->cpages, npages);671 ASSERT_EQ(buffer->faults, 1);672 673 /* Check what the device wrote. */674 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)675 ASSERT_EQ(ptr[i], -i);676 677 close(child_fd);678 exit(0);679}680 681/*682 * Write private anonymous huge page.683 */684TEST_F(hmm, anon_write_huge)685{686 struct hmm_buffer *buffer;687 unsigned long npages;688 unsigned long size;689 unsigned long i;690 void *old_ptr;691 void *map;692 int *ptr;693 int ret;694 695 size = 2 * TWOMEG;696 697 buffer = malloc(sizeof(*buffer));698 ASSERT_NE(buffer, NULL);699 700 buffer->fd = -1;701 buffer->size = size;702 buffer->mirror = malloc(size);703 ASSERT_NE(buffer->mirror, NULL);704 705 buffer->ptr = mmap(NULL, size,706 PROT_READ | PROT_WRITE,707 MAP_PRIVATE | MAP_ANONYMOUS,708 buffer->fd, 0);709 ASSERT_NE(buffer->ptr, MAP_FAILED);710 711 size = TWOMEG;712 npages = size >> self->page_shift;713 map = (void *)ALIGN((uintptr_t)buffer->ptr, size);714 ret = madvise(map, size, MADV_HUGEPAGE);715 ASSERT_EQ(ret, 0);716 old_ptr = buffer->ptr;717 buffer->ptr = map;718 719 /* Initialize data that the device will write to buffer->ptr. */720 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)721 ptr[i] = i;722 723 /* Simulate a device writing system memory. */724 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);725 ASSERT_EQ(ret, 0);726 ASSERT_EQ(buffer->cpages, npages);727 ASSERT_EQ(buffer->faults, 1);728 729 /* Check what the device wrote. */730 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)731 ASSERT_EQ(ptr[i], i);732 733 buffer->ptr = old_ptr;734 hmm_buffer_free(buffer);735}736 737/*738 * Read numeric data from raw and tagged kernel status files. Used to read739 * /proc and /sys data (without a tag) and from /proc/meminfo (with a tag).740 */741static long file_read_ulong(char *file, const char *tag)742{743 int fd;744 char buf[2048];745 int len;746 char *p, *q;747 long val;748 749 fd = open(file, O_RDONLY);750 if (fd < 0) {751 /* Error opening the file */752 return -1;753 }754 755 len = read(fd, buf, sizeof(buf));756 close(fd);757 if (len < 0) {758 /* Error in reading the file */759 return -1;760 }761 if (len == sizeof(buf)) {762 /* Error file is too large */763 return -1;764 }765 buf[len] = '\0';766 767 /* Search for a tag if provided */768 if (tag) {769 p = strstr(buf, tag);770 if (!p)771 return -1; /* looks like the line we want isn't there */772 p += strlen(tag);773 } else774 p = buf;775 776 val = strtol(p, &q, 0);777 if (*q != ' ') {778 /* Error parsing the file */779 return -1;780 }781 782 return val;783}784 785/*786 * Write huge TLBFS page.787 */788TEST_F(hmm, anon_write_hugetlbfs)789{790 struct hmm_buffer *buffer;791 unsigned long npages;792 unsigned long size;793 unsigned long default_hsize;794 unsigned long i;795 int *ptr;796 int ret;797 798 default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:");799 if (default_hsize < 0 || default_hsize*1024 < default_hsize)800 SKIP(return, "Huge page size could not be determined");801 default_hsize = default_hsize*1024; /* KB to B */802 803 size = ALIGN(TWOMEG, default_hsize);804 npages = size >> self->page_shift;805 806 buffer = malloc(sizeof(*buffer));807 ASSERT_NE(buffer, NULL);808 809 buffer->ptr = mmap(NULL, size,810 PROT_READ | PROT_WRITE,811 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,812 -1, 0);813 if (buffer->ptr == MAP_FAILED) {814 free(buffer);815 SKIP(return, "Huge page could not be allocated");816 }817 818 buffer->fd = -1;819 buffer->size = size;820 buffer->mirror = malloc(size);821 ASSERT_NE(buffer->mirror, NULL);822 823 /* Initialize data that the device will write to buffer->ptr. */824 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)825 ptr[i] = i;826 827 /* Simulate a device writing system memory. */828 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);829 ASSERT_EQ(ret, 0);830 ASSERT_EQ(buffer->cpages, npages);831 ASSERT_EQ(buffer->faults, 1);832 833 /* Check what the device wrote. */834 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)835 ASSERT_EQ(ptr[i], i);836 837 munmap(buffer->ptr, buffer->size);838 buffer->ptr = NULL;839 hmm_buffer_free(buffer);840}841 842/*843 * Read mmap'ed file memory.844 */845TEST_F(hmm, file_read)846{847 struct hmm_buffer *buffer;848 unsigned long npages;849 unsigned long size;850 unsigned long i;851 int *ptr;852 int ret;853 int fd;854 ssize_t len;855 856 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;857 ASSERT_NE(npages, 0);858 size = npages << self->page_shift;859 860 fd = hmm_create_file(size);861 ASSERT_GE(fd, 0);862 863 buffer = malloc(sizeof(*buffer));864 ASSERT_NE(buffer, NULL);865 866 buffer->fd = fd;867 buffer->size = size;868 buffer->mirror = malloc(size);869 ASSERT_NE(buffer->mirror, NULL);870 871 /* Write initial contents of the file. */872 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)873 ptr[i] = i;874 len = pwrite(fd, buffer->mirror, size, 0);875 ASSERT_EQ(len, size);876 memset(buffer->mirror, 0, size);877 878 buffer->ptr = mmap(NULL, size,879 PROT_READ,880 MAP_SHARED,881 buffer->fd, 0);882 ASSERT_NE(buffer->ptr, MAP_FAILED);883 884 /* Simulate a device reading system memory. */885 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);886 ASSERT_EQ(ret, 0);887 ASSERT_EQ(buffer->cpages, npages);888 ASSERT_EQ(buffer->faults, 1);889 890 /* Check what the device read. */891 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)892 ASSERT_EQ(ptr[i], i);893 894 hmm_buffer_free(buffer);895}896 897/*898 * Write mmap'ed file memory.899 */900TEST_F(hmm, file_write)901{902 struct hmm_buffer *buffer;903 unsigned long npages;904 unsigned long size;905 unsigned long i;906 int *ptr;907 int ret;908 int fd;909 ssize_t len;910 911 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;912 ASSERT_NE(npages, 0);913 size = npages << self->page_shift;914 915 fd = hmm_create_file(size);916 ASSERT_GE(fd, 0);917 918 buffer = malloc(sizeof(*buffer));919 ASSERT_NE(buffer, NULL);920 921 buffer->fd = fd;922 buffer->size = size;923 buffer->mirror = malloc(size);924 ASSERT_NE(buffer->mirror, NULL);925 926 buffer->ptr = mmap(NULL, size,927 PROT_READ | PROT_WRITE,928 MAP_SHARED,929 buffer->fd, 0);930 ASSERT_NE(buffer->ptr, MAP_FAILED);931 932 /* Initialize data that the device will write to buffer->ptr. */933 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)934 ptr[i] = i;935 936 /* Simulate a device writing system memory. */937 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);938 ASSERT_EQ(ret, 0);939 ASSERT_EQ(buffer->cpages, npages);940 ASSERT_EQ(buffer->faults, 1);941 942 /* Check what the device wrote. */943 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)944 ASSERT_EQ(ptr[i], i);945 946 /* Check that the device also wrote the file. */947 len = pread(fd, buffer->mirror, size, 0);948 ASSERT_EQ(len, size);949 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)950 ASSERT_EQ(ptr[i], i);951 952 hmm_buffer_free(buffer);953}954 955/*956 * Migrate anonymous memory to device private memory.957 */958TEST_F(hmm, migrate)959{960 struct hmm_buffer *buffer;961 unsigned long npages;962 unsigned long size;963 unsigned long i;964 int *ptr;965 int ret;966 967 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;968 ASSERT_NE(npages, 0);969 size = npages << self->page_shift;970 971 buffer = malloc(sizeof(*buffer));972 ASSERT_NE(buffer, NULL);973 974 buffer->fd = -1;975 buffer->size = size;976 buffer->mirror = malloc(size);977 ASSERT_NE(buffer->mirror, NULL);978 979 buffer->ptr = mmap(NULL, size,980 PROT_READ | PROT_WRITE,981 MAP_PRIVATE | MAP_ANONYMOUS,982 buffer->fd, 0);983 ASSERT_NE(buffer->ptr, MAP_FAILED);984 985 /* Initialize buffer in system memory. */986 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)987 ptr[i] = i;988 989 /* Migrate memory to device. */990 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);991 ASSERT_EQ(ret, 0);992 ASSERT_EQ(buffer->cpages, npages);993 994 /* Check what the device read. */995 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)996 ASSERT_EQ(ptr[i], i);997 998 hmm_buffer_free(buffer);999}1000 1001/*1002 * Migrate anonymous memory to device private memory and fault some of it back1003 * to system memory, then try migrating the resulting mix of system and device1004 * private memory to the device.1005 */1006TEST_F(hmm, migrate_fault)1007{1008 struct hmm_buffer *buffer;1009 unsigned long npages;1010 unsigned long size;1011 unsigned long i;1012 int *ptr;1013 int ret;1014 1015 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;1016 ASSERT_NE(npages, 0);1017 size = npages << self->page_shift;1018 1019 buffer = malloc(sizeof(*buffer));1020 ASSERT_NE(buffer, NULL);1021 1022 buffer->fd = -1;1023 buffer->size = size;1024 buffer->mirror = malloc(size);1025 ASSERT_NE(buffer->mirror, NULL);1026 1027 buffer->ptr = mmap(NULL, size,1028 PROT_READ | PROT_WRITE,1029 MAP_PRIVATE | MAP_ANONYMOUS,1030 buffer->fd, 0);1031 ASSERT_NE(buffer->ptr, MAP_FAILED);1032 1033 /* Initialize buffer in system memory. */1034 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1035 ptr[i] = i;1036 1037 /* Migrate memory to device. */1038 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);1039 ASSERT_EQ(ret, 0);1040 ASSERT_EQ(buffer->cpages, npages);1041 1042 /* Check what the device read. */1043 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)1044 ASSERT_EQ(ptr[i], i);1045 1046 /* Fault half the pages back to system memory and check them. */1047 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)1048 ASSERT_EQ(ptr[i], i);1049 1050 /* Migrate memory to the device again. */1051 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);1052 ASSERT_EQ(ret, 0);1053 ASSERT_EQ(buffer->cpages, npages);1054 1055 /* Check what the device read. */1056 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)1057 ASSERT_EQ(ptr[i], i);1058 1059 hmm_buffer_free(buffer);1060}1061 1062TEST_F(hmm, migrate_release)1063{1064 struct hmm_buffer *buffer;1065 unsigned long npages;1066 unsigned long size;1067 unsigned long i;1068 int *ptr;1069 int ret;1070 1071 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;1072 ASSERT_NE(npages, 0);1073 size = npages << self->page_shift;1074 1075 buffer = malloc(sizeof(*buffer));1076 ASSERT_NE(buffer, NULL);1077 1078 buffer->fd = -1;1079 buffer->size = size;1080 buffer->mirror = malloc(size);1081 ASSERT_NE(buffer->mirror, NULL);1082 1083 buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,1084 MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);1085 ASSERT_NE(buffer->ptr, MAP_FAILED);1086 1087 /* Initialize buffer in system memory. */1088 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1089 ptr[i] = i;1090 1091 /* Migrate memory to device. */1092 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);1093 ASSERT_EQ(ret, 0);1094 ASSERT_EQ(buffer->cpages, npages);1095 1096 /* Check what the device read. */1097 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)1098 ASSERT_EQ(ptr[i], i);1099 1100 /* Release device memory. */1101 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_RELEASE, buffer, npages);1102 ASSERT_EQ(ret, 0);1103 1104 /* Fault pages back to system memory and check them. */1105 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)1106 ASSERT_EQ(ptr[i], i);1107 1108 hmm_buffer_free(buffer);1109}1110 1111/*1112 * Migrate anonymous shared memory to device private memory.1113 */1114TEST_F(hmm, migrate_shared)1115{1116 struct hmm_buffer *buffer;1117 unsigned long npages;1118 unsigned long size;1119 int ret;1120 1121 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;1122 ASSERT_NE(npages, 0);1123 size = npages << self->page_shift;1124 1125 buffer = malloc(sizeof(*buffer));1126 ASSERT_NE(buffer, NULL);1127 1128 buffer->fd = -1;1129 buffer->size = size;1130 buffer->mirror = malloc(size);1131 ASSERT_NE(buffer->mirror, NULL);1132 1133 buffer->ptr = mmap(NULL, size,1134 PROT_READ | PROT_WRITE,1135 MAP_SHARED | MAP_ANONYMOUS,1136 buffer->fd, 0);1137 ASSERT_NE(buffer->ptr, MAP_FAILED);1138 1139 /* Migrate memory to device. */1140 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);1141 ASSERT_EQ(ret, -ENOENT);1142 1143 hmm_buffer_free(buffer);1144}1145 1146/*1147 * Try to migrate various memory types to device private memory.1148 */1149TEST_F(hmm2, migrate_mixed)1150{1151 struct hmm_buffer *buffer;1152 unsigned long npages;1153 unsigned long size;1154 int *ptr;1155 unsigned char *p;1156 int ret;1157 int val;1158 1159 npages = 6;1160 size = npages << self->page_shift;1161 1162 buffer = malloc(sizeof(*buffer));1163 ASSERT_NE(buffer, NULL);1164 1165 buffer->fd = -1;1166 buffer->size = size;1167 buffer->mirror = malloc(size);1168 ASSERT_NE(buffer->mirror, NULL);1169 1170 /* Reserve a range of addresses. */1171 buffer->ptr = mmap(NULL, size,1172 PROT_NONE,1173 MAP_PRIVATE | MAP_ANONYMOUS,1174 buffer->fd, 0);1175 ASSERT_NE(buffer->ptr, MAP_FAILED);1176 p = buffer->ptr;1177 1178 /* Migrating a protected area should be an error. */1179 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages);1180 ASSERT_EQ(ret, -EINVAL);1181 1182 /* Punch a hole after the first page address. */1183 ret = munmap(buffer->ptr + self->page_size, self->page_size);1184 ASSERT_EQ(ret, 0);1185 1186 /* We expect an error if the vma doesn't cover the range. */1187 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 3);1188 ASSERT_EQ(ret, -EINVAL);1189 1190 /* Page 2 will be a read-only zero page. */1191 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size,1192 PROT_READ);1193 ASSERT_EQ(ret, 0);1194 ptr = (int *)(buffer->ptr + 2 * self->page_size);1195 val = *ptr + 3;1196 ASSERT_EQ(val, 3);1197 1198 /* Page 3 will be read-only. */1199 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,1200 PROT_READ | PROT_WRITE);1201 ASSERT_EQ(ret, 0);1202 ptr = (int *)(buffer->ptr + 3 * self->page_size);1203 *ptr = val;1204 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,1205 PROT_READ);1206 ASSERT_EQ(ret, 0);1207 1208 /* Page 4-5 will be read-write. */1209 ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size,1210 PROT_READ | PROT_WRITE);1211 ASSERT_EQ(ret, 0);1212 ptr = (int *)(buffer->ptr + 4 * self->page_size);1213 *ptr = val;1214 ptr = (int *)(buffer->ptr + 5 * self->page_size);1215 *ptr = val;1216 1217 /* Now try to migrate pages 2-5 to device 1. */1218 buffer->ptr = p + 2 * self->page_size;1219 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 4);1220 ASSERT_EQ(ret, 0);1221 ASSERT_EQ(buffer->cpages, 4);1222 1223 /* Page 5 won't be migrated to device 0 because it's on device 1. */1224 buffer->ptr = p + 5 * self->page_size;1225 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1);1226 ASSERT_EQ(ret, -ENOENT);1227 buffer->ptr = p;1228 1229 buffer->ptr = p;1230 hmm_buffer_free(buffer);1231}1232 1233/*1234 * Migrate anonymous memory to device memory and back to system memory1235 * multiple times. In case of private zone configuration, this is done1236 * through fault pages accessed by CPU. In case of coherent zone configuration,1237 * the pages from the device should be explicitly migrated back to system memory.1238 * The reason is Coherent device zone has coherent access by CPU, therefore1239 * it will not generate any page fault.1240 */1241TEST_F(hmm, migrate_multiple)1242{1243 struct hmm_buffer *buffer;1244 unsigned long npages;1245 unsigned long size;1246 unsigned long i;1247 unsigned long c;1248 int *ptr;1249 int ret;1250 1251 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;1252 ASSERT_NE(npages, 0);1253 size = npages << self->page_shift;1254 1255 for (c = 0; c < NTIMES; c++) {1256 buffer = malloc(sizeof(*buffer));1257 ASSERT_NE(buffer, NULL);1258 1259 buffer->fd = -1;1260 buffer->size = size;1261 buffer->mirror = malloc(size);1262 ASSERT_NE(buffer->mirror, NULL);1263 1264 buffer->ptr = mmap(NULL, size,1265 PROT_READ | PROT_WRITE,1266 MAP_PRIVATE | MAP_ANONYMOUS,1267 buffer->fd, 0);1268 ASSERT_NE(buffer->ptr, MAP_FAILED);1269 1270 /* Initialize buffer in system memory. */1271 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1272 ptr[i] = i;1273 1274 /* Migrate memory to device. */1275 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);1276 ASSERT_EQ(ret, 0);1277 ASSERT_EQ(buffer->cpages, npages);1278 1279 /* Check what the device read. */1280 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)1281 ASSERT_EQ(ptr[i], i);1282 1283 /* Migrate back to system memory and check them. */1284 if (hmm_is_coherent_type(variant->device_number)) {1285 ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages);1286 ASSERT_EQ(ret, 0);1287 ASSERT_EQ(buffer->cpages, npages);1288 }1289 1290 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1291 ASSERT_EQ(ptr[i], i);1292 1293 hmm_buffer_free(buffer);1294 }1295}1296 1297/*1298 * Read anonymous memory multiple times.1299 */1300TEST_F(hmm, anon_read_multiple)1301{1302 struct hmm_buffer *buffer;1303 unsigned long npages;1304 unsigned long size;1305 unsigned long i;1306 unsigned long c;1307 int *ptr;1308 int ret;1309 1310 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;1311 ASSERT_NE(npages, 0);1312 size = npages << self->page_shift;1313 1314 for (c = 0; c < NTIMES; c++) {1315 buffer = malloc(sizeof(*buffer));1316 ASSERT_NE(buffer, NULL);1317 1318 buffer->fd = -1;1319 buffer->size = size;1320 buffer->mirror = malloc(size);1321 ASSERT_NE(buffer->mirror, NULL);1322 1323 buffer->ptr = mmap(NULL, size,1324 PROT_READ | PROT_WRITE,1325 MAP_PRIVATE | MAP_ANONYMOUS,1326 buffer->fd, 0);1327 ASSERT_NE(buffer->ptr, MAP_FAILED);1328 1329 /* Initialize buffer in system memory. */1330 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1331 ptr[i] = i + c;1332 1333 /* Simulate a device reading system memory. */1334 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer,1335 npages);1336 ASSERT_EQ(ret, 0);1337 ASSERT_EQ(buffer->cpages, npages);1338 ASSERT_EQ(buffer->faults, 1);1339 1340 /* Check what the device read. */1341 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)1342 ASSERT_EQ(ptr[i], i + c);1343 1344 hmm_buffer_free(buffer);1345 }1346}1347 1348void *unmap_buffer(void *p)1349{1350 struct hmm_buffer *buffer = p;1351 1352 /* Delay for a bit and then unmap buffer while it is being read. */1353 hmm_nanosleep(hmm_random() % 32000);1354 munmap(buffer->ptr + buffer->size / 2, buffer->size / 2);1355 buffer->ptr = NULL;1356 1357 return NULL;1358}1359 1360/*1361 * Try reading anonymous memory while it is being unmapped.1362 */1363TEST_F(hmm, anon_teardown)1364{1365 unsigned long npages;1366 unsigned long size;1367 unsigned long c;1368 void *ret;1369 1370 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;1371 ASSERT_NE(npages, 0);1372 size = npages << self->page_shift;1373 1374 for (c = 0; c < NTIMES; ++c) {1375 pthread_t thread;1376 struct hmm_buffer *buffer;1377 unsigned long i;1378 int *ptr;1379 int rc;1380 1381 buffer = malloc(sizeof(*buffer));1382 ASSERT_NE(buffer, NULL);1383 1384 buffer->fd = -1;1385 buffer->size = size;1386 buffer->mirror = malloc(size);1387 ASSERT_NE(buffer->mirror, NULL);1388 1389 buffer->ptr = mmap(NULL, size,1390 PROT_READ | PROT_WRITE,1391 MAP_PRIVATE | MAP_ANONYMOUS,1392 buffer->fd, 0);1393 ASSERT_NE(buffer->ptr, MAP_FAILED);1394 1395 /* Initialize buffer in system memory. */1396 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1397 ptr[i] = i + c;1398 1399 rc = pthread_create(&thread, NULL, unmap_buffer, buffer);1400 ASSERT_EQ(rc, 0);1401 1402 /* Simulate a device reading system memory. */1403 rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer,1404 npages);1405 if (rc == 0) {1406 ASSERT_EQ(buffer->cpages, npages);1407 ASSERT_EQ(buffer->faults, 1);1408 1409 /* Check what the device read. */1410 for (i = 0, ptr = buffer->mirror;1411 i < size / sizeof(*ptr);1412 ++i)1413 ASSERT_EQ(ptr[i], i + c);1414 }1415 1416 pthread_join(thread, &ret);1417 hmm_buffer_free(buffer);1418 }1419}1420 1421/*1422 * Test memory snapshot without faulting in pages accessed by the device.1423 */1424TEST_F(hmm, mixedmap)1425{1426 struct hmm_buffer *buffer;1427 unsigned long npages;1428 unsigned long size;1429 unsigned char *m;1430 int ret;1431 1432 npages = 1;1433 size = npages << self->page_shift;1434 1435 buffer = malloc(sizeof(*buffer));1436 ASSERT_NE(buffer, NULL);1437 1438 buffer->fd = -1;1439 buffer->size = size;1440 buffer->mirror = malloc(npages);1441 ASSERT_NE(buffer->mirror, NULL);1442 1443 1444 /* Reserve a range of addresses. */1445 buffer->ptr = mmap(NULL, size,1446 PROT_READ | PROT_WRITE,1447 MAP_PRIVATE,1448 self->fd, 0);1449 ASSERT_NE(buffer->ptr, MAP_FAILED);1450 1451 /* Simulate a device snapshotting CPU pagetables. */1452 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);1453 ASSERT_EQ(ret, 0);1454 ASSERT_EQ(buffer->cpages, npages);1455 1456 /* Check what the device saw. */1457 m = buffer->mirror;1458 ASSERT_EQ(m[0], HMM_DMIRROR_PROT_READ);1459 1460 hmm_buffer_free(buffer);1461}1462 1463/*1464 * Test memory snapshot without faulting in pages accessed by the device.1465 */1466TEST_F(hmm2, snapshot)1467{1468 struct hmm_buffer *buffer;1469 unsigned long npages;1470 unsigned long size;1471 int *ptr;1472 unsigned char *p;1473 unsigned char *m;1474 int ret;1475 int val;1476 1477 npages = 7;1478 size = npages << self->page_shift;1479 1480 buffer = malloc(sizeof(*buffer));1481 ASSERT_NE(buffer, NULL);1482 1483 buffer->fd = -1;1484 buffer->size = size;1485 buffer->mirror = malloc(npages);1486 ASSERT_NE(buffer->mirror, NULL);1487 1488 /* Reserve a range of addresses. */1489 buffer->ptr = mmap(NULL, size,1490 PROT_NONE,1491 MAP_PRIVATE | MAP_ANONYMOUS,1492 buffer->fd, 0);1493 ASSERT_NE(buffer->ptr, MAP_FAILED);1494 p = buffer->ptr;1495 1496 /* Punch a hole after the first page address. */1497 ret = munmap(buffer->ptr + self->page_size, self->page_size);1498 ASSERT_EQ(ret, 0);1499 1500 /* Page 2 will be read-only zero page. */1501 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size,1502 PROT_READ);1503 ASSERT_EQ(ret, 0);1504 ptr = (int *)(buffer->ptr + 2 * self->page_size);1505 val = *ptr + 3;1506 ASSERT_EQ(val, 3);1507 1508 /* Page 3 will be read-only. */1509 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,1510 PROT_READ | PROT_WRITE);1511 ASSERT_EQ(ret, 0);1512 ptr = (int *)(buffer->ptr + 3 * self->page_size);1513 *ptr = val;1514 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,1515 PROT_READ);1516 ASSERT_EQ(ret, 0);1517 1518 /* Page 4-6 will be read-write. */1519 ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size,1520 PROT_READ | PROT_WRITE);1521 ASSERT_EQ(ret, 0);1522 ptr = (int *)(buffer->ptr + 4 * self->page_size);1523 *ptr = val;1524 1525 /* Page 5 will be migrated to device 0. */1526 buffer->ptr = p + 5 * self->page_size;1527 ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1);1528 ASSERT_EQ(ret, 0);1529 ASSERT_EQ(buffer->cpages, 1);1530 1531 /* Page 6 will be migrated to device 1. */1532 buffer->ptr = p + 6 * self->page_size;1533 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 1);1534 ASSERT_EQ(ret, 0);1535 ASSERT_EQ(buffer->cpages, 1);1536 1537 /* Simulate a device snapshotting CPU pagetables. */1538 buffer->ptr = p;1539 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages);1540 ASSERT_EQ(ret, 0);1541 ASSERT_EQ(buffer->cpages, npages);1542 1543 /* Check what the device saw. */1544 m = buffer->mirror;1545 ASSERT_EQ(m[0], HMM_DMIRROR_PROT_ERROR);1546 ASSERT_EQ(m[1], HMM_DMIRROR_PROT_ERROR);1547 ASSERT_EQ(m[2], HMM_DMIRROR_PROT_ZERO | HMM_DMIRROR_PROT_READ);1548 ASSERT_EQ(m[3], HMM_DMIRROR_PROT_READ);1549 ASSERT_EQ(m[4], HMM_DMIRROR_PROT_WRITE);1550 if (!hmm_is_coherent_type(variant->device_number0)) {1551 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL |1552 HMM_DMIRROR_PROT_WRITE);1553 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_NONE);1554 } else {1555 ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL |1556 HMM_DMIRROR_PROT_WRITE);1557 ASSERT_EQ(m[6], HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE |1558 HMM_DMIRROR_PROT_WRITE);1559 }1560 1561 hmm_buffer_free(buffer);1562}1563 1564/*1565 * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that1566 * should be mapped by a large page table entry.1567 */1568TEST_F(hmm, compound)1569{1570 struct hmm_buffer *buffer;1571 unsigned long npages;1572 unsigned long size;1573 unsigned long default_hsize;1574 int *ptr;1575 unsigned char *m;1576 int ret;1577 unsigned long i;1578 1579 /* Skip test if we can't allocate a hugetlbfs page. */1580 1581 default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:");1582 if (default_hsize < 0 || default_hsize*1024 < default_hsize)1583 SKIP(return, "Huge page size could not be determined");1584 default_hsize = default_hsize*1024; /* KB to B */1585 1586 size = ALIGN(TWOMEG, default_hsize);1587 npages = size >> self->page_shift;1588 1589 buffer = malloc(sizeof(*buffer));1590 ASSERT_NE(buffer, NULL);1591 1592 buffer->ptr = mmap(NULL, size,1593 PROT_READ | PROT_WRITE,1594 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,1595 -1, 0);1596 if (buffer->ptr == MAP_FAILED) {1597 free(buffer);1598 return;1599 }1600 1601 buffer->size = size;1602 buffer->mirror = malloc(npages);1603 ASSERT_NE(buffer->mirror, NULL);1604 1605 /* Initialize the pages the device will snapshot in buffer->ptr. */1606 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1607 ptr[i] = i;1608 1609 /* Simulate a device snapshotting CPU pagetables. */1610 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);1611 ASSERT_EQ(ret, 0);1612 ASSERT_EQ(buffer->cpages, npages);1613 1614 /* Check what the device saw. */1615 m = buffer->mirror;1616 for (i = 0; i < npages; ++i)1617 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE |1618 HMM_DMIRROR_PROT_PMD);1619 1620 /* Make the region read-only. */1621 ret = mprotect(buffer->ptr, size, PROT_READ);1622 ASSERT_EQ(ret, 0);1623 1624 /* Simulate a device snapshotting CPU pagetables. */1625 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);1626 ASSERT_EQ(ret, 0);1627 ASSERT_EQ(buffer->cpages, npages);1628 1629 /* Check what the device saw. */1630 m = buffer->mirror;1631 for (i = 0; i < npages; ++i)1632 ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ |1633 HMM_DMIRROR_PROT_PMD);1634 1635 munmap(buffer->ptr, buffer->size);1636 buffer->ptr = NULL;1637 hmm_buffer_free(buffer);1638}1639 1640/*1641 * Test two devices reading the same memory (double mapped).1642 */1643TEST_F(hmm2, double_map)1644{1645 struct hmm_buffer *buffer;1646 unsigned long npages;1647 unsigned long size;1648 unsigned long i;1649 int *ptr;1650 int ret;1651 1652 npages = 6;1653 size = npages << self->page_shift;1654 1655 buffer = malloc(sizeof(*buffer));1656 ASSERT_NE(buffer, NULL);1657 1658 buffer->fd = -1;1659 buffer->size = size;1660 buffer->mirror = malloc(size);1661 ASSERT_NE(buffer->mirror, NULL);1662 1663 /* Reserve a range of addresses. */1664 buffer->ptr = mmap(NULL, size,1665 PROT_READ | PROT_WRITE,1666 MAP_PRIVATE | MAP_ANONYMOUS,1667 buffer->fd, 0);1668 ASSERT_NE(buffer->ptr, MAP_FAILED);1669 1670 /* Initialize buffer in system memory. */1671 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1672 ptr[i] = i;1673 1674 /* Make region read-only. */1675 ret = mprotect(buffer->ptr, size, PROT_READ);1676 ASSERT_EQ(ret, 0);1677 1678 /* Simulate device 0 reading system memory. */1679 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages);1680 ASSERT_EQ(ret, 0);1681 ASSERT_EQ(buffer->cpages, npages);1682 ASSERT_EQ(buffer->faults, 1);1683 1684 /* Check what the device read. */1685 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)1686 ASSERT_EQ(ptr[i], i);1687 1688 /* Simulate device 1 reading system memory. */1689 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages);1690 ASSERT_EQ(ret, 0);1691 ASSERT_EQ(buffer->cpages, npages);1692 ASSERT_EQ(buffer->faults, 1);1693 1694 /* Check what the device read. */1695 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)1696 ASSERT_EQ(ptr[i], i);1697 1698 /* Migrate pages to device 1 and try to read from device 0. */1699 ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages);1700 ASSERT_EQ(ret, 0);1701 ASSERT_EQ(buffer->cpages, npages);1702 1703 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages);1704 ASSERT_EQ(ret, 0);1705 ASSERT_EQ(buffer->cpages, npages);1706 ASSERT_EQ(buffer->faults, 1);1707 1708 /* Check what device 0 read. */1709 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)1710 ASSERT_EQ(ptr[i], i);1711 1712 hmm_buffer_free(buffer);1713}1714 1715/*1716 * Basic check of exclusive faulting.1717 */1718TEST_F(hmm, exclusive)1719{1720 struct hmm_buffer *buffer;1721 unsigned long npages;1722 unsigned long size;1723 unsigned long i;1724 int *ptr;1725 int ret;1726 1727 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;1728 ASSERT_NE(npages, 0);1729 size = npages << self->page_shift;1730 1731 buffer = malloc(sizeof(*buffer));1732 ASSERT_NE(buffer, NULL);1733 1734 buffer->fd = -1;1735 buffer->size = size;1736 buffer->mirror = malloc(size);1737 ASSERT_NE(buffer->mirror, NULL);1738 1739 buffer->ptr = mmap(NULL, size,1740 PROT_READ | PROT_WRITE,1741 MAP_PRIVATE | MAP_ANONYMOUS,1742 buffer->fd, 0);1743 ASSERT_NE(buffer->ptr, MAP_FAILED);1744 1745 /* Initialize buffer in system memory. */1746 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1747 ptr[i] = i;1748 1749 /* Map memory exclusively for device access. */1750 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);1751 ASSERT_EQ(ret, 0);1752 ASSERT_EQ(buffer->cpages, npages);1753 1754 /* Check what the device read. */1755 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)1756 ASSERT_EQ(ptr[i], i);1757 1758 /* Fault pages back to system memory and check them. */1759 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1760 ASSERT_EQ(ptr[i]++, i);1761 1762 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1763 ASSERT_EQ(ptr[i], i+1);1764 1765 /* Check atomic access revoked */1766 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_CHECK_EXCLUSIVE, buffer, npages);1767 ASSERT_EQ(ret, 0);1768 1769 hmm_buffer_free(buffer);1770}1771 1772TEST_F(hmm, exclusive_mprotect)1773{1774 struct hmm_buffer *buffer;1775 unsigned long npages;1776 unsigned long size;1777 unsigned long i;1778 int *ptr;1779 int ret;1780 1781 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;1782 ASSERT_NE(npages, 0);1783 size = npages << self->page_shift;1784 1785 buffer = malloc(sizeof(*buffer));1786 ASSERT_NE(buffer, NULL);1787 1788 buffer->fd = -1;1789 buffer->size = size;1790 buffer->mirror = malloc(size);1791 ASSERT_NE(buffer->mirror, NULL);1792 1793 buffer->ptr = mmap(NULL, size,1794 PROT_READ | PROT_WRITE,1795 MAP_PRIVATE | MAP_ANONYMOUS,1796 buffer->fd, 0);1797 ASSERT_NE(buffer->ptr, MAP_FAILED);1798 1799 /* Initialize buffer in system memory. */1800 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1801 ptr[i] = i;1802 1803 /* Map memory exclusively for device access. */1804 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);1805 ASSERT_EQ(ret, 0);1806 ASSERT_EQ(buffer->cpages, npages);1807 1808 /* Check what the device read. */1809 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)1810 ASSERT_EQ(ptr[i], i);1811 1812 ret = mprotect(buffer->ptr, size, PROT_READ);1813 ASSERT_EQ(ret, 0);1814 1815 /* Simulate a device writing system memory. */1816 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);1817 ASSERT_EQ(ret, -EPERM);1818 1819 hmm_buffer_free(buffer);1820}1821 1822/*1823 * Check copy-on-write works.1824 */1825TEST_F(hmm, exclusive_cow)1826{1827 struct hmm_buffer *buffer;1828 unsigned long npages;1829 unsigned long size;1830 unsigned long i;1831 int *ptr;1832 int ret;1833 1834 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;1835 ASSERT_NE(npages, 0);1836 size = npages << self->page_shift;1837 1838 buffer = malloc(sizeof(*buffer));1839 ASSERT_NE(buffer, NULL);1840 1841 buffer->fd = -1;1842 buffer->size = size;1843 buffer->mirror = malloc(size);1844 ASSERT_NE(buffer->mirror, NULL);1845 1846 buffer->ptr = mmap(NULL, size,1847 PROT_READ | PROT_WRITE,1848 MAP_PRIVATE | MAP_ANONYMOUS,1849 buffer->fd, 0);1850 ASSERT_NE(buffer->ptr, MAP_FAILED);1851 1852 /* Initialize buffer in system memory. */1853 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1854 ptr[i] = i;1855 1856 /* Map memory exclusively for device access. */1857 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);1858 ASSERT_EQ(ret, 0);1859 ASSERT_EQ(buffer->cpages, npages);1860 1861 fork();1862 1863 /* Fault pages back to system memory and check them. */1864 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1865 ASSERT_EQ(ptr[i]++, i);1866 1867 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1868 ASSERT_EQ(ptr[i], i+1);1869 1870 hmm_buffer_free(buffer);1871}1872 1873static int gup_test_exec(int gup_fd, unsigned long addr, int cmd,1874 int npages, int size, int flags)1875{1876 struct gup_test gup = {1877 .nr_pages_per_call = npages,1878 .addr = addr,1879 .gup_flags = FOLL_WRITE | flags,1880 .size = size,1881 };1882 1883 if (ioctl(gup_fd, cmd, &gup)) {1884 perror("ioctl on error\n");1885 return errno;1886 }1887 1888 return 0;1889}1890 1891/*1892 * Test get user device pages through gup_test. Setting PIN_LONGTERM flag.1893 * This should trigger a migration back to system memory for both, private1894 * and coherent type pages.1895 * This test makes use of gup_test module. Make sure GUP_TEST_CONFIG is added1896 * to your configuration before you run it.1897 */1898TEST_F(hmm, hmm_gup_test)1899{1900 struct hmm_buffer *buffer;1901 int gup_fd;1902 unsigned long npages;1903 unsigned long size;1904 unsigned long i;1905 int *ptr;1906 int ret;1907 unsigned char *m;1908 1909 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);1910 if (gup_fd == -1)1911 SKIP(return, "Skipping test, could not find gup_test driver");1912 1913 npages = 4;1914 size = npages << self->page_shift;1915 1916 buffer = malloc(sizeof(*buffer));1917 ASSERT_NE(buffer, NULL);1918 1919 buffer->fd = -1;1920 buffer->size = size;1921 buffer->mirror = malloc(size);1922 ASSERT_NE(buffer->mirror, NULL);1923 1924 buffer->ptr = mmap(NULL, size,1925 PROT_READ | PROT_WRITE,1926 MAP_PRIVATE | MAP_ANONYMOUS,1927 buffer->fd, 0);1928 ASSERT_NE(buffer->ptr, MAP_FAILED);1929 1930 /* Initialize buffer in system memory. */1931 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1932 ptr[i] = i;1933 1934 /* Migrate memory to device. */1935 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);1936 ASSERT_EQ(ret, 0);1937 ASSERT_EQ(buffer->cpages, npages);1938 /* Check what the device read. */1939 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)1940 ASSERT_EQ(ptr[i], i);1941 1942 ASSERT_EQ(gup_test_exec(gup_fd,1943 (unsigned long)buffer->ptr,1944 GUP_BASIC_TEST, 1, self->page_size, 0), 0);1945 ASSERT_EQ(gup_test_exec(gup_fd,1946 (unsigned long)buffer->ptr + 1 * self->page_size,1947 GUP_FAST_BENCHMARK, 1, self->page_size, 0), 0);1948 ASSERT_EQ(gup_test_exec(gup_fd,1949 (unsigned long)buffer->ptr + 2 * self->page_size,1950 PIN_FAST_BENCHMARK, 1, self->page_size, FOLL_LONGTERM), 0);1951 ASSERT_EQ(gup_test_exec(gup_fd,1952 (unsigned long)buffer->ptr + 3 * self->page_size,1953 PIN_LONGTERM_BENCHMARK, 1, self->page_size, 0), 0);1954 1955 /* Take snapshot to CPU pagetables */1956 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);1957 ASSERT_EQ(ret, 0);1958 ASSERT_EQ(buffer->cpages, npages);1959 m = buffer->mirror;1960 if (hmm_is_coherent_type(variant->device_number)) {1961 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[0]);1962 ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[1]);1963 } else {1964 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[0]);1965 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[1]);1966 }1967 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[2]);1968 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[3]);1969 /*1970 * Check again the content on the pages. Make sure there's no1971 * corrupted data.1972 */1973 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)1974 ASSERT_EQ(ptr[i], i);1975 1976 close(gup_fd);1977 hmm_buffer_free(buffer);1978}1979 1980/*1981 * Test copy-on-write in device pages.1982 * In case of writing to COW private page(s), a page fault will migrate pages1983 * back to system memory first. Then, these pages will be duplicated. In case1984 * of COW device coherent type, pages are duplicated directly from device1985 * memory.1986 */1987TEST_F(hmm, hmm_cow_in_device)1988{1989 struct hmm_buffer *buffer;1990 unsigned long npages;1991 unsigned long size;1992 unsigned long i;1993 int *ptr;1994 int ret;1995 unsigned char *m;1996 pid_t pid;1997 int status;1998 1999 npages = 4;2000 size = npages << self->page_shift;2001 2002 buffer = malloc(sizeof(*buffer));2003 ASSERT_NE(buffer, NULL);2004 2005 buffer->fd = -1;2006 buffer->size = size;2007 buffer->mirror = malloc(size);2008 ASSERT_NE(buffer->mirror, NULL);2009 2010 buffer->ptr = mmap(NULL, size,2011 PROT_READ | PROT_WRITE,2012 MAP_PRIVATE | MAP_ANONYMOUS,2013 buffer->fd, 0);2014 ASSERT_NE(buffer->ptr, MAP_FAILED);2015 2016 /* Initialize buffer in system memory. */2017 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)2018 ptr[i] = i;2019 2020 /* Migrate memory to device. */2021 2022 ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);2023 ASSERT_EQ(ret, 0);2024 ASSERT_EQ(buffer->cpages, npages);2025 2026 pid = fork();2027 if (pid == -1)2028 ASSERT_EQ(pid, 0);2029 if (!pid) {2030 /* Child process waitd for SIGTERM from the parent. */2031 while (1) {2032 }2033 perror("Should not reach this\n");2034 exit(0);2035 }2036 /* Parent process writes to COW pages(s) and gets a2037 * new copy in system. In case of device private pages,2038 * this write causes a migration to system mem first.2039 */2040 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)2041 ptr[i] = i;2042 2043 /* Terminate child and wait */2044 EXPECT_EQ(0, kill(pid, SIGTERM));2045 EXPECT_EQ(pid, waitpid(pid, &status, 0));2046 EXPECT_NE(0, WIFSIGNALED(status));2047 EXPECT_EQ(SIGTERM, WTERMSIG(status));2048 2049 /* Take snapshot to CPU pagetables */2050 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);2051 ASSERT_EQ(ret, 0);2052 ASSERT_EQ(buffer->cpages, npages);2053 m = buffer->mirror;2054 for (i = 0; i < npages; i++)2055 ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[i]);2056 2057 hmm_buffer_free(buffer);2058}2059TEST_HARNESS_MAIN2060