734 lines · c
1// SPDX-License-Identifier: GPL-2.02 3/*4 * Copyright 2019, Nick Piggin, Gautham R. Shenoy, Aneesh Kumar K.V, IBM Corp.5 */6 7/*8 *9 * Test tlbie/mtpidr race. We have 4 threads doing flush/load/compare/store10 * sequence in a loop. The same threads also rung a context switch task11 * that does sched_yield() in loop.12 *13 * The snapshot thread mark the mmap area PROT_READ in between, make a copy14 * and copy it back to the original area. This helps us to detect if any15 * store continued to happen after we marked the memory PROT_READ.16 */17 18#define _GNU_SOURCE19#include <stdio.h>20#include <sys/mman.h>21#include <sys/types.h>22#include <sys/wait.h>23#include <sys/ipc.h>24#include <sys/shm.h>25#include <sys/stat.h>26#include <sys/time.h>27#include <linux/futex.h>28#include <unistd.h>29#include <asm/unistd.h>30#include <string.h>31#include <stdlib.h>32#include <fcntl.h>33#include <sched.h>34#include <time.h>35#include <stdarg.h>36#include <pthread.h>37#include <signal.h>38#include <sys/prctl.h>39 40static inline void dcbf(volatile unsigned int *addr)41{42 __asm__ __volatile__ ("dcbf %y0; sync" : : "Z"(*(unsigned char *)addr) : "memory");43}44 45static void err_msg(char *msg)46{47 48 time_t now;49 time(&now);50 printf("=================================\n");51 printf(" Error: %s\n", msg);52 printf(" %s", ctime(&now));53 printf("=================================\n");54 exit(1);55}56 57static char *map1;58static char *map2;59static pid_t rim_process_pid;60 61/*62 * A "rim-sequence" is defined to be the sequence of the following63 * operations performed on a memory word:64 * 1) FLUSH the contents of that word.65 * 2) LOAD the contents of that word.66 * 3) COMPARE the contents of that word with the content that was67 * previously stored at that word68 * 4) STORE new content into that word.69 *70 * The threads in this test that perform the rim-sequence are termed71 * as rim_threads.72 */73 74/*75 * A "corruption" is defined to be the failed COMPARE operation in a76 * rim-sequence.77 *78 * A rim_thread that detects a corruption informs about it to all the79 * other rim_threads, and the mem_snapshot thread.80 */81static volatile unsigned int corruption_found;82 83/*84 * This defines the maximum number of rim_threads in this test.85 *86 * The THREAD_ID_BITS denote the number of bits required87 * to represent the thread_ids [0..MAX_THREADS - 1].88 * We are being a bit paranoid here and set it to 8 bits,89 * though 6 bits suffice.90 *91 */92#define MAX_THREADS 6493#define THREAD_ID_BITS 894#define THREAD_ID_MASK ((1 << THREAD_ID_BITS) - 1)95static unsigned int rim_thread_ids[MAX_THREADS];96static pthread_t rim_threads[MAX_THREADS];97 98 99/*100 * Each rim_thread works on an exclusive "chunk" of size101 * RIM_CHUNK_SIZE.102 *103 * The ith rim_thread works on the ith chunk.104 *105 * The ith chunk begins at106 * map1 + (i * RIM_CHUNK_SIZE)107 */108#define RIM_CHUNK_SIZE 1024109#define BITS_PER_BYTE 8110#define WORD_SIZE (sizeof(unsigned int))111#define WORD_BITS (WORD_SIZE * BITS_PER_BYTE)112#define WORDS_PER_CHUNK (RIM_CHUNK_SIZE/WORD_SIZE)113 114static inline char *compute_chunk_start_addr(unsigned int thread_id)115{116 char *chunk_start;117 118 chunk_start = (char *)((unsigned long)map1 +119 (thread_id * RIM_CHUNK_SIZE));120 121 return chunk_start;122}123 124/*125 * The "word-offset" of a word-aligned address inside a chunk, is126 * defined to be the number of words that precede the address in that127 * chunk.128 *129 * WORD_OFFSET_BITS denote the number of bits required to represent130 * the word-offsets of all the word-aligned addresses of a chunk.131 */132#define WORD_OFFSET_BITS (__builtin_ctz(WORDS_PER_CHUNK))133#define WORD_OFFSET_MASK ((1 << WORD_OFFSET_BITS) - 1)134 135static inline unsigned int compute_word_offset(char *start, unsigned int *addr)136{137 unsigned int delta_bytes, ret;138 delta_bytes = (unsigned long)addr - (unsigned long)start;139 140 ret = delta_bytes/WORD_SIZE;141 142 return ret;143}144 145/*146 * A "sweep" is defined to be the sequential execution of the147 * rim-sequence by a rim_thread on its chunk one word at a time,148 * starting from the first word of its chunk and ending with the last149 * word of its chunk.150 *151 * Each sweep of a rim_thread is uniquely identified by a sweep_id.152 * SWEEP_ID_BITS denote the number of bits required to represent153 * the sweep_ids of rim_threads.154 *155 * As to why SWEEP_ID_BITS are computed as a function of THREAD_ID_BITS,156 * WORD_OFFSET_BITS, and WORD_BITS, see the "store-pattern" below.157 */158#define SWEEP_ID_BITS (WORD_BITS - (THREAD_ID_BITS + WORD_OFFSET_BITS))159#define SWEEP_ID_MASK ((1 << SWEEP_ID_BITS) - 1)160 161/*162 * A "store-pattern" is the word-pattern that is stored into a word163 * location in the 4)STORE step of the rim-sequence.164 *165 * In the store-pattern, we shall encode:166 *167 * - The thread-id of the rim_thread performing the store168 * (The most significant THREAD_ID_BITS)169 *170 * - The word-offset of the address into which the store is being171 * performed (The next WORD_OFFSET_BITS)172 *173 * - The sweep_id of the current sweep in which the store is174 * being performed. (The lower SWEEP_ID_BITS)175 *176 * Store Pattern: 32 bits177 * |------------------|--------------------|---------------------------------|178 * | Thread id | Word offset | sweep_id |179 * |------------------|--------------------|---------------------------------|180 * THREAD_ID_BITS WORD_OFFSET_BITS SWEEP_ID_BITS181 *182 * In the store pattern, the (Thread-id + Word-offset) uniquely identify the183 * address to which the store is being performed i.e,184 * address == map1 +185 * (Thread-id * RIM_CHUNK_SIZE) + (Word-offset * WORD_SIZE)186 *187 * And the sweep_id in the store pattern identifies the time when the188 * store was performed by the rim_thread.189 *190 * We shall use this property in the 3)COMPARE step of the191 * rim-sequence.192 */193#define SWEEP_ID_SHIFT 0194#define WORD_OFFSET_SHIFT (SWEEP_ID_BITS)195#define THREAD_ID_SHIFT (WORD_OFFSET_BITS + SWEEP_ID_BITS)196 197/*198 * Compute the store pattern for a given thread with id @tid, at199 * location @addr in the sweep identified by @sweep_id200 */201static inline unsigned int compute_store_pattern(unsigned int tid,202 unsigned int *addr,203 unsigned int sweep_id)204{205 unsigned int ret = 0;206 char *start = compute_chunk_start_addr(tid);207 unsigned int word_offset = compute_word_offset(start, addr);208 209 ret += (tid & THREAD_ID_MASK) << THREAD_ID_SHIFT;210 ret += (word_offset & WORD_OFFSET_MASK) << WORD_OFFSET_SHIFT;211 ret += (sweep_id & SWEEP_ID_MASK) << SWEEP_ID_SHIFT;212 return ret;213}214 215/* Extract the thread-id from the given store-pattern */216static inline unsigned int extract_tid(unsigned int pattern)217{218 unsigned int ret;219 220 ret = (pattern >> THREAD_ID_SHIFT) & THREAD_ID_MASK;221 return ret;222}223 224/* Extract the word-offset from the given store-pattern */225static inline unsigned int extract_word_offset(unsigned int pattern)226{227 unsigned int ret;228 229 ret = (pattern >> WORD_OFFSET_SHIFT) & WORD_OFFSET_MASK;230 231 return ret;232}233 234/* Extract the sweep-id from the given store-pattern */235static inline unsigned int extract_sweep_id(unsigned int pattern)236 237{238 unsigned int ret;239 240 ret = (pattern >> SWEEP_ID_SHIFT) & SWEEP_ID_MASK;241 242 return ret;243}244 245/************************************************************246 * *247 * Logging the output of the verification *248 * *249 ************************************************************/250#define LOGDIR_NAME_SIZE 100251static char logdir[LOGDIR_NAME_SIZE];252 253static FILE *fp[MAX_THREADS];254static const char logfilename[] ="Thread-%02d-Chunk";255 256static inline void start_verification_log(unsigned int tid,257 unsigned int *addr,258 unsigned int cur_sweep_id,259 unsigned int prev_sweep_id)260{261 FILE *f;262 char logfile[30];263 char path[LOGDIR_NAME_SIZE + 30];264 char separator[2] = "/";265 char *chunk_start = compute_chunk_start_addr(tid);266 unsigned int size = RIM_CHUNK_SIZE;267 268 sprintf(logfile, logfilename, tid);269 strcpy(path, logdir);270 strcat(path, separator);271 strcat(path, logfile);272 f = fopen(path, "w");273 274 if (!f) {275 err_msg("Unable to create logfile\n");276 }277 278 fp[tid] = f;279 280 fprintf(f, "----------------------------------------------------------\n");281 fprintf(f, "PID = %d\n", rim_process_pid);282 fprintf(f, "Thread id = %02d\n", tid);283 fprintf(f, "Chunk Start Addr = 0x%016lx\n", (unsigned long)chunk_start);284 fprintf(f, "Chunk Size = %d\n", size);285 fprintf(f, "Next Store Addr = 0x%016lx\n", (unsigned long)addr);286 fprintf(f, "Current sweep-id = 0x%08x\n", cur_sweep_id);287 fprintf(f, "Previous sweep-id = 0x%08x\n", prev_sweep_id);288 fprintf(f, "----------------------------------------------------------\n");289}290 291static inline void log_anamoly(unsigned int tid, unsigned int *addr,292 unsigned int expected, unsigned int observed)293{294 FILE *f = fp[tid];295 296 fprintf(f, "Thread %02d: Addr 0x%lx: Expected 0x%x, Observed 0x%x\n",297 tid, (unsigned long)addr, expected, observed);298 fprintf(f, "Thread %02d: Expected Thread id = %02d\n", tid, extract_tid(expected));299 fprintf(f, "Thread %02d: Observed Thread id = %02d\n", tid, extract_tid(observed));300 fprintf(f, "Thread %02d: Expected Word offset = %03d\n", tid, extract_word_offset(expected));301 fprintf(f, "Thread %02d: Observed Word offset = %03d\n", tid, extract_word_offset(observed));302 fprintf(f, "Thread %02d: Expected sweep-id = 0x%x\n", tid, extract_sweep_id(expected));303 fprintf(f, "Thread %02d: Observed sweep-id = 0x%x\n", tid, extract_sweep_id(observed));304 fprintf(f, "----------------------------------------------------------\n");305}306 307static inline void end_verification_log(unsigned int tid, unsigned nr_anamolies)308{309 FILE *f = fp[tid];310 char logfile[30];311 char path[LOGDIR_NAME_SIZE + 30];312 char separator[] = "/";313 314 fclose(f);315 316 if (nr_anamolies == 0) {317 remove(path);318 return;319 }320 321 sprintf(logfile, logfilename, tid);322 strcpy(path, logdir);323 strcat(path, separator);324 strcat(path, logfile);325 326 printf("Thread %02d chunk has %d corrupted words. For details check %s\n",327 tid, nr_anamolies, path);328}329 330/*331 * When a COMPARE step of a rim-sequence fails, the rim_thread informs332 * everyone else via the shared_memory pointed to by333 * corruption_found variable. On seeing this, every thread verifies the334 * content of its chunk as follows.335 *336 * Suppose a thread identified with @tid was about to store (but not337 * yet stored) to @next_store_addr in its current sweep identified338 * @cur_sweep_id. Let @prev_sweep_id indicate the previous sweep_id.339 *340 * This implies that for all the addresses @addr < @next_store_addr,341 * Thread @tid has already performed a store as part of its current342 * sweep. Hence we expect the content of such @addr to be:343 * |-------------------------------------------------|344 * | tid | word_offset(addr) | cur_sweep_id |345 * |-------------------------------------------------|346 *347 * Since Thread @tid is yet to perform stores on address348 * @next_store_addr and above, we expect the content of such an349 * address @addr to be:350 * |-------------------------------------------------|351 * | tid | word_offset(addr) | prev_sweep_id |352 * |-------------------------------------------------|353 *354 * The verifier function @verify_chunk does this verification and logs355 * any anamolies that it finds.356 */357static void verify_chunk(unsigned int tid, unsigned int *next_store_addr,358 unsigned int cur_sweep_id,359 unsigned int prev_sweep_id)360{361 unsigned int *iter_ptr;362 unsigned int size = RIM_CHUNK_SIZE;363 unsigned int expected;364 unsigned int observed;365 char *chunk_start = compute_chunk_start_addr(tid);366 367 int nr_anamolies = 0;368 369 start_verification_log(tid, next_store_addr,370 cur_sweep_id, prev_sweep_id);371 372 for (iter_ptr = (unsigned int *)chunk_start;373 (unsigned long)iter_ptr < (unsigned long)chunk_start + size;374 iter_ptr++) {375 unsigned int expected_sweep_id;376 377 if (iter_ptr < next_store_addr) {378 expected_sweep_id = cur_sweep_id;379 } else {380 expected_sweep_id = prev_sweep_id;381 }382 383 expected = compute_store_pattern(tid, iter_ptr, expected_sweep_id);384 385 dcbf((volatile unsigned int*)iter_ptr); //Flush before reading386 observed = *iter_ptr;387 388 if (observed != expected) {389 nr_anamolies++;390 log_anamoly(tid, iter_ptr, expected, observed);391 }392 }393 394 end_verification_log(tid, nr_anamolies);395}396 397static void set_pthread_cpu(pthread_t th, int cpu)398{399 cpu_set_t run_cpu_mask;400 struct sched_param param;401 402 CPU_ZERO(&run_cpu_mask);403 CPU_SET(cpu, &run_cpu_mask);404 pthread_setaffinity_np(th, sizeof(cpu_set_t), &run_cpu_mask);405 406 param.sched_priority = 1;407 if (0 && sched_setscheduler(0, SCHED_FIFO, ¶m) == -1) {408 /* haven't reproduced with this setting, it kills random preemption which may be a factor */409 fprintf(stderr, "could not set SCHED_FIFO, run as root?\n");410 }411}412 413static void set_mycpu(int cpu)414{415 cpu_set_t run_cpu_mask;416 struct sched_param param;417 418 CPU_ZERO(&run_cpu_mask);419 CPU_SET(cpu, &run_cpu_mask);420 sched_setaffinity(0, sizeof(cpu_set_t), &run_cpu_mask);421 422 param.sched_priority = 1;423 if (0 && sched_setscheduler(0, SCHED_FIFO, ¶m) == -1) {424 fprintf(stderr, "could not set SCHED_FIFO, run as root?\n");425 }426}427 428static volatile int segv_wait;429 430static void segv_handler(int signo, siginfo_t *info, void *extra)431{432 while (segv_wait) {433 sched_yield();434 }435 436}437 438static void set_segv_handler(void)439{440 struct sigaction sa;441 442 sa.sa_flags = SA_SIGINFO;443 sa.sa_sigaction = segv_handler;444 445 if (sigaction(SIGSEGV, &sa, NULL) == -1) {446 perror("sigaction");447 exit(EXIT_FAILURE);448 }449}450 451int timeout = 0;452/*453 * This function is executed by every rim_thread.454 *455 * This function performs sweeps over the exclusive chunks of the456 * rim_threads executing the rim-sequence one word at a time.457 */458static void *rim_fn(void *arg)459{460 unsigned int tid = *((unsigned int *)arg);461 462 int size = RIM_CHUNK_SIZE;463 char *chunk_start = compute_chunk_start_addr(tid);464 465 unsigned int prev_sweep_id;466 unsigned int cur_sweep_id = 0;467 468 /* word access */469 unsigned int pattern = cur_sweep_id;470 unsigned int *pattern_ptr = &pattern;471 unsigned int *w_ptr, read_data;472 473 set_segv_handler();474 475 /*476 * Let us initialize the chunk:477 *478 * Each word-aligned address addr in the chunk,479 * is initialized to :480 * |-------------------------------------------------|481 * | tid | word_offset(addr) | 0 |482 * |-------------------------------------------------|483 */484 for (w_ptr = (unsigned int *)chunk_start;485 (unsigned long)w_ptr < (unsigned long)(chunk_start) + size;486 w_ptr++) {487 488 *pattern_ptr = compute_store_pattern(tid, w_ptr, cur_sweep_id);489 *w_ptr = *pattern_ptr;490 }491 492 while (!corruption_found && !timeout) {493 prev_sweep_id = cur_sweep_id;494 cur_sweep_id = cur_sweep_id + 1;495 496 for (w_ptr = (unsigned int *)chunk_start;497 (unsigned long)w_ptr < (unsigned long)(chunk_start) + size;498 w_ptr++) {499 unsigned int old_pattern;500 501 /*502 * Compute the pattern that we would have503 * stored at this location in the previous504 * sweep.505 */506 old_pattern = compute_store_pattern(tid, w_ptr, prev_sweep_id);507 508 /*509 * FLUSH:Ensure that we flush the contents of510 * the cache before loading511 */512 dcbf((volatile unsigned int*)w_ptr); //Flush513 514 /* LOAD: Read the value */515 read_data = *w_ptr; //Load516 517 /*518 * COMPARE: Is it the same as what we had stored519 * in the previous sweep ? It better be!520 */521 if (read_data != old_pattern) {522 /* No it isn't! Tell everyone */523 corruption_found = 1;524 }525 526 /*527 * Before performing a store, let us check if528 * any rim_thread has found a corruption.529 */530 if (corruption_found || timeout) {531 /*532 * Yes. Someone (including us!) has found533 * a corruption :(534 *535 * Let us verify that our chunk is536 * correct.537 */538 /* But first, let us allow the dust to settle down! */539 verify_chunk(tid, w_ptr, cur_sweep_id, prev_sweep_id);540 541 return 0;542 }543 544 /*545 * Compute the new pattern that we are going546 * to write to this location547 */548 *pattern_ptr = compute_store_pattern(tid, w_ptr, cur_sweep_id);549 550 /*551 * STORE: Now let us write this pattern into552 * the location553 */554 *w_ptr = *pattern_ptr;555 }556 }557 558 return NULL;559}560 561 562static unsigned long start_cpu = 0;563static unsigned long nrthreads = 4;564 565static pthread_t mem_snapshot_thread;566 567static void *mem_snapshot_fn(void *arg)568{569 int page_size = getpagesize();570 size_t size = page_size;571 void *tmp = malloc(size);572 573 while (!corruption_found && !timeout) {574 /* Stop memory migration once corruption is found */575 segv_wait = 1;576 577 mprotect(map1, size, PROT_READ);578 579 /*580 * Load from the working alias (map1). Loading from map2581 * also fails.582 */583 memcpy(tmp, map1, size);584 585 /*586 * Stores must go via map2 which has write permissions, but587 * the corrupted data tends to be seen in the snapshot buffer,588 * so corruption does not appear to be introduced at the589 * copy-back via map2 alias here.590 */591 memcpy(map2, tmp, size);592 /*593 * Before releasing other threads, must ensure the copy594 * back to595 */596 asm volatile("sync" ::: "memory");597 mprotect(map1, size, PROT_READ|PROT_WRITE);598 asm volatile("sync" ::: "memory");599 segv_wait = 0;600 601 usleep(1); /* This value makes a big difference */602 }603 604 return 0;605}606 607void alrm_sighandler(int sig)608{609 timeout = 1;610}611 612int main(int argc, char *argv[])613{614 int c;615 int page_size = getpagesize();616 time_t now;617 int i, dir_error;618 pthread_attr_t attr;619 key_t shm_key = (key_t) getpid();620 int shmid, run_time = 20 * 60;621 struct sigaction sa_alrm;622 623 snprintf(logdir, LOGDIR_NAME_SIZE,624 "/tmp/logdir-%u", (unsigned int)getpid());625 while ((c = getopt(argc, argv, "r:hn:l:t:")) != -1) {626 switch(c) {627 case 'r':628 start_cpu = strtoul(optarg, NULL, 10);629 break;630 case 'h':631 printf("%s [-r <start_cpu>] [-n <nrthreads>] [-l <logdir>] [-t <timeout>]\n", argv[0]);632 exit(0);633 break;634 case 'n':635 nrthreads = strtoul(optarg, NULL, 10);636 break;637 case 'l':638 strncpy(logdir, optarg, LOGDIR_NAME_SIZE - 1);639 break;640 case 't':641 run_time = strtoul(optarg, NULL, 10);642 break;643 default:644 printf("invalid option\n");645 exit(0);646 break;647 }648 }649 650 if (nrthreads > MAX_THREADS)651 nrthreads = MAX_THREADS;652 653 shmid = shmget(shm_key, page_size, IPC_CREAT|0666);654 if (shmid < 0) {655 err_msg("Failed shmget\n");656 }657 658 map1 = shmat(shmid, NULL, 0);659 if (map1 == (void *) -1) {660 err_msg("Failed shmat");661 }662 663 map2 = shmat(shmid, NULL, 0);664 if (map2 == (void *) -1) {665 err_msg("Failed shmat");666 }667 668 dir_error = mkdir(logdir, 0755);669 670 if (dir_error) {671 err_msg("Failed mkdir");672 }673 674 printf("start_cpu list:%lu\n", start_cpu);675 printf("number of worker threads:%lu + 1 snapshot thread\n", nrthreads);676 printf("Allocated address:0x%016lx + secondary map:0x%016lx\n", (unsigned long)map1, (unsigned long)map2);677 printf("logdir at : %s\n", logdir);678 printf("Timeout: %d seconds\n", run_time);679 680 time(&now);681 printf("=================================\n");682 printf(" Starting Test\n");683 printf(" %s", ctime(&now));684 printf("=================================\n");685 686 for (i = 0; i < nrthreads; i++) {687 if (1 && !fork()) {688 prctl(PR_SET_PDEATHSIG, SIGKILL);689 set_mycpu(start_cpu + i);690 for (;;)691 sched_yield();692 exit(0);693 }694 }695 696 697 sa_alrm.sa_handler = &alrm_sighandler;698 sigemptyset(&sa_alrm.sa_mask);699 sa_alrm.sa_flags = 0;700 701 if (sigaction(SIGALRM, &sa_alrm, 0) == -1) {702 err_msg("Failed signal handler registration\n");703 }704 705 alarm(run_time);706 707 pthread_attr_init(&attr);708 for (i = 0; i < nrthreads; i++) {709 rim_thread_ids[i] = i;710 pthread_create(&rim_threads[i], &attr, rim_fn, &rim_thread_ids[i]);711 set_pthread_cpu(rim_threads[i], start_cpu + i);712 }713 714 pthread_create(&mem_snapshot_thread, &attr, mem_snapshot_fn, map1);715 set_pthread_cpu(mem_snapshot_thread, start_cpu + i);716 717 718 pthread_join(mem_snapshot_thread, NULL);719 for (i = 0; i < nrthreads; i++) {720 pthread_join(rim_threads[i], NULL);721 }722 723 if (!timeout) {724 time(&now);725 printf("=================================\n");726 printf(" Data Corruption Detected\n");727 printf(" %s", ctime(&now));728 printf(" See logfiles in %s\n", logdir);729 printf("=================================\n");730 return 1;731 }732 return 0;733}734