1377 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * DMA Engine test module4 *5 * Copyright (C) 2007 Atmel Corporation6 * Copyright (C) 2013 Intel Corporation7 */8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt9 10#include <linux/err.h>11#include <linux/delay.h>12#include <linux/dma-mapping.h>13#include <linux/dmaengine.h>14#include <linux/freezer.h>15#include <linux/init.h>16#include <linux/kthread.h>17#include <linux/sched/task.h>18#include <linux/module.h>19#include <linux/moduleparam.h>20#include <linux/random.h>21#include <linux/slab.h>22#include <linux/wait.h>23 24static bool nobounce;25module_param(nobounce, bool, 0644);26MODULE_PARM_DESC(nobounce, "Prevent using swiotlb buffer (default: use swiotlb buffer)");27 28static unsigned int test_buf_size = 16384;29module_param(test_buf_size, uint, 0644);30MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");31 32static char test_device[32];33module_param_string(device, test_device, sizeof(test_device), 0644);34MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");35 36static unsigned int threads_per_chan = 1;37module_param(threads_per_chan, uint, 0644);38MODULE_PARM_DESC(threads_per_chan,39 "Number of threads to start per channel (default: 1)");40 41static unsigned int max_channels;42module_param(max_channels, uint, 0644);43MODULE_PARM_DESC(max_channels,44 "Maximum number of channels to use (default: all)");45 46static unsigned int iterations;47module_param(iterations, uint, 0644);48MODULE_PARM_DESC(iterations,49 "Iterations before stopping test (default: infinite)");50 51static unsigned int dmatest;52module_param(dmatest, uint, 0644);53MODULE_PARM_DESC(dmatest,54 "dmatest 0-memcpy 1-memset (default: 0)");55 56static unsigned int xor_sources = 3;57module_param(xor_sources, uint, 0644);58MODULE_PARM_DESC(xor_sources,59 "Number of xor source buffers (default: 3)");60 61static unsigned int pq_sources = 3;62module_param(pq_sources, uint, 0644);63MODULE_PARM_DESC(pq_sources,64 "Number of p+q source buffers (default: 3)");65 66static int timeout = 3000;67module_param(timeout, int, 0644);68MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "69 "Pass -1 for infinite timeout");70 71static bool noverify;72module_param(noverify, bool, 0644);73MODULE_PARM_DESC(noverify, "Disable data verification (default: verify)");74 75static bool norandom;76module_param(norandom, bool, 0644);77MODULE_PARM_DESC(norandom, "Disable random offset setup (default: random)");78 79static bool verbose;80module_param(verbose, bool, 0644);81MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");82 83static int alignment = -1;84module_param(alignment, int, 0644);85MODULE_PARM_DESC(alignment, "Custom data address alignment taken as 2^(alignment) (default: not used (-1))");86 87static unsigned int transfer_size;88module_param(transfer_size, uint, 0644);89MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default: not used (0))");90 91static bool polled;92module_param(polled, bool, 0644);93MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts");94 95/**96 * struct dmatest_params - test parameters.97 * @nobounce: prevent using swiotlb buffer98 * @buf_size: size of the memcpy test buffer99 * @channel: bus ID of the channel to test100 * @device: bus ID of the DMA Engine to test101 * @threads_per_chan: number of threads to start per channel102 * @max_channels: maximum number of channels to use103 * @iterations: iterations before stopping test104 * @xor_sources: number of xor source buffers105 * @pq_sources: number of p+q source buffers106 * @timeout: transfer timeout in msec, -1 for infinite timeout107 * @noverify: disable data verification108 * @norandom: disable random offset setup109 * @alignment: custom data address alignment taken as 2^alignment110 * @transfer_size: custom transfer size in bytes111 * @polled: use polling for completion instead of interrupts112 */113struct dmatest_params {114 bool nobounce;115 unsigned int buf_size;116 char channel[20];117 char device[32];118 unsigned int threads_per_chan;119 unsigned int max_channels;120 unsigned int iterations;121 unsigned int xor_sources;122 unsigned int pq_sources;123 int timeout;124 bool noverify;125 bool norandom;126 int alignment;127 unsigned int transfer_size;128 bool polled;129};130 131/**132 * struct dmatest_info - test information.133 * @params: test parameters134 * @channels: channels under test135 * @nr_channels: number of channels under test136 * @lock: access protection to the fields of this structure137 * @did_init: module has been initialized completely138 * @last_error: test has faced configuration issues139 */140static struct dmatest_info {141 /* Test parameters */142 struct dmatest_params params;143 144 /* Internal state */145 struct list_head channels;146 unsigned int nr_channels;147 int last_error;148 struct mutex lock;149 bool did_init;150} test_info = {151 .channels = LIST_HEAD_INIT(test_info.channels),152 .lock = __MUTEX_INITIALIZER(test_info.lock),153};154 155static int dmatest_run_set(const char *val, const struct kernel_param *kp);156static int dmatest_run_get(char *val, const struct kernel_param *kp);157static const struct kernel_param_ops run_ops = {158 .set = dmatest_run_set,159 .get = dmatest_run_get,160};161static bool dmatest_run;162module_param_cb(run, &run_ops, &dmatest_run, 0644);163MODULE_PARM_DESC(run, "Run the test (default: false)");164 165static int dmatest_chan_set(const char *val, const struct kernel_param *kp);166static int dmatest_chan_get(char *val, const struct kernel_param *kp);167static const struct kernel_param_ops multi_chan_ops = {168 .set = dmatest_chan_set,169 .get = dmatest_chan_get,170};171 172static char test_channel[20];173static struct kparam_string newchan_kps = {174 .string = test_channel,175 .maxlen = 20,176};177module_param_cb(channel, &multi_chan_ops, &newchan_kps, 0644);178MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");179 180static int dmatest_test_list_get(char *val, const struct kernel_param *kp);181static const struct kernel_param_ops test_list_ops = {182 .get = dmatest_test_list_get,183};184module_param_cb(test_list, &test_list_ops, NULL, 0444);185MODULE_PARM_DESC(test_list, "Print current test list");186 187/* Maximum amount of mismatched bytes in buffer to print */188#define MAX_ERROR_COUNT 32189 190/*191 * Initialization patterns. All bytes in the source buffer has bit 7192 * set, all bytes in the destination buffer has bit 7 cleared.193 *194 * Bit 6 is set for all bytes which are to be copied by the DMA195 * engine. Bit 5 is set for all bytes which are to be overwritten by196 * the DMA engine.197 *198 * The remaining bits are the inverse of a counter which increments by199 * one for each byte address.200 */201#define PATTERN_SRC 0x80202#define PATTERN_DST 0x00203#define PATTERN_COPY 0x40204#define PATTERN_OVERWRITE 0x20205#define PATTERN_COUNT_MASK 0x1f206#define PATTERN_MEMSET_IDX 0x01207 208/* Fixed point arithmetic ops */209#define FIXPT_SHIFT 8210#define FIXPNT_MASK 0xFF211#define FIXPT_TO_INT(a) ((a) >> FIXPT_SHIFT)212#define INT_TO_FIXPT(a) ((a) << FIXPT_SHIFT)213#define FIXPT_GET_FRAC(a) ((((a) & FIXPNT_MASK) * 100) >> FIXPT_SHIFT)214 215/* poor man's completion - we want to use wait_event_freezable() on it */216struct dmatest_done {217 bool done;218 wait_queue_head_t *wait;219};220 221struct dmatest_data {222 u8 **raw;223 u8 **aligned;224 gfp_t gfp_flags;225 unsigned int cnt;226 unsigned int off;227};228 229struct dmatest_thread {230 struct list_head node;231 struct dmatest_info *info;232 struct task_struct *task;233 struct dma_chan *chan;234 struct dmatest_data src;235 struct dmatest_data dst;236 enum dma_transaction_type type;237 wait_queue_head_t done_wait;238 struct dmatest_done test_done;239 bool done;240 bool pending;241};242 243struct dmatest_chan {244 struct list_head node;245 struct dma_chan *chan;246 struct list_head threads;247};248 249static DECLARE_WAIT_QUEUE_HEAD(thread_wait);250static bool wait;251 252static bool is_threaded_test_run(struct dmatest_info *info)253{254 struct dmatest_chan *dtc;255 256 list_for_each_entry(dtc, &info->channels, node) {257 struct dmatest_thread *thread;258 259 list_for_each_entry(thread, &dtc->threads, node) {260 if (!thread->done && !thread->pending)261 return true;262 }263 }264 265 return false;266}267 268static bool is_threaded_test_pending(struct dmatest_info *info)269{270 struct dmatest_chan *dtc;271 272 list_for_each_entry(dtc, &info->channels, node) {273 struct dmatest_thread *thread;274 275 list_for_each_entry(thread, &dtc->threads, node) {276 if (thread->pending)277 return true;278 }279 }280 281 return false;282}283 284static int dmatest_wait_get(char *val, const struct kernel_param *kp)285{286 struct dmatest_info *info = &test_info;287 struct dmatest_params *params = &info->params;288 289 if (params->iterations)290 wait_event(thread_wait, !is_threaded_test_run(info));291 wait = true;292 return param_get_bool(val, kp);293}294 295static const struct kernel_param_ops wait_ops = {296 .get = dmatest_wait_get,297 .set = param_set_bool,298};299module_param_cb(wait, &wait_ops, &wait, 0444);300MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");301 302static bool dmatest_match_channel(struct dmatest_params *params,303 struct dma_chan *chan)304{305 if (params->channel[0] == '\0')306 return true;307 return strcmp(dma_chan_name(chan), params->channel) == 0;308}309 310static bool dmatest_match_device(struct dmatest_params *params,311 struct dma_device *device)312{313 if (params->device[0] == '\0')314 return true;315 return strcmp(dev_name(device->dev), params->device) == 0;316}317 318static unsigned long dmatest_random(void)319{320 unsigned long buf;321 322 get_random_bytes(&buf, sizeof(buf));323 return buf;324}325 326static inline u8 gen_inv_idx(u8 index, bool is_memset)327{328 u8 val = is_memset ? PATTERN_MEMSET_IDX : index;329 330 return ~val & PATTERN_COUNT_MASK;331}332 333static inline u8 gen_src_value(u8 index, bool is_memset)334{335 return PATTERN_SRC | gen_inv_idx(index, is_memset);336}337 338static inline u8 gen_dst_value(u8 index, bool is_memset)339{340 return PATTERN_DST | gen_inv_idx(index, is_memset);341}342 343static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,344 unsigned int buf_size, bool is_memset)345{346 unsigned int i;347 u8 *buf;348 349 for (; (buf = *bufs); bufs++) {350 for (i = 0; i < start; i++)351 buf[i] = gen_src_value(i, is_memset);352 for ( ; i < start + len; i++)353 buf[i] = gen_src_value(i, is_memset) | PATTERN_COPY;354 for ( ; i < buf_size; i++)355 buf[i] = gen_src_value(i, is_memset);356 buf++;357 }358}359 360static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,361 unsigned int buf_size, bool is_memset)362{363 unsigned int i;364 u8 *buf;365 366 for (; (buf = *bufs); bufs++) {367 for (i = 0; i < start; i++)368 buf[i] = gen_dst_value(i, is_memset);369 for ( ; i < start + len; i++)370 buf[i] = gen_dst_value(i, is_memset) |371 PATTERN_OVERWRITE;372 for ( ; i < buf_size; i++)373 buf[i] = gen_dst_value(i, is_memset);374 }375}376 377static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,378 unsigned int counter, bool is_srcbuf, bool is_memset)379{380 u8 diff = actual ^ pattern;381 u8 expected = pattern | gen_inv_idx(counter, is_memset);382 const char *thread_name = current->comm;383 384 if (is_srcbuf)385 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",386 thread_name, index, expected, actual);387 else if ((pattern & PATTERN_COPY)388 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))389 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",390 thread_name, index, expected, actual);391 else if (diff & PATTERN_SRC)392 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",393 thread_name, index, expected, actual);394 else395 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",396 thread_name, index, expected, actual);397}398 399static unsigned int dmatest_verify(u8 **bufs, unsigned int start,400 unsigned int end, unsigned int counter, u8 pattern,401 bool is_srcbuf, bool is_memset)402{403 unsigned int i;404 unsigned int error_count = 0;405 u8 actual;406 u8 expected;407 u8 *buf;408 unsigned int counter_orig = counter;409 410 for (; (buf = *bufs); bufs++) {411 counter = counter_orig;412 for (i = start; i < end; i++) {413 actual = buf[i];414 expected = pattern | gen_inv_idx(counter, is_memset);415 if (actual != expected) {416 if (error_count < MAX_ERROR_COUNT)417 dmatest_mismatch(actual, pattern, i,418 counter, is_srcbuf,419 is_memset);420 error_count++;421 }422 counter++;423 }424 }425 426 if (error_count > MAX_ERROR_COUNT)427 pr_warn("%s: %u errors suppressed\n",428 current->comm, error_count - MAX_ERROR_COUNT);429 430 return error_count;431}432 433 434static void dmatest_callback(void *arg)435{436 struct dmatest_done *done = arg;437 struct dmatest_thread *thread =438 container_of(done, struct dmatest_thread, test_done);439 if (!thread->done) {440 done->done = true;441 wake_up_all(done->wait);442 } else {443 /*444 * If thread->done, it means that this callback occurred445 * after the parent thread has cleaned up. This can446 * happen in the case that driver doesn't implement447 * the terminate_all() functionality and a dma operation448 * did not occur within the timeout period449 */450 WARN(1, "dmatest: Kernel memory may be corrupted!!\n");451 }452}453 454static unsigned int min_odd(unsigned int x, unsigned int y)455{456 unsigned int val = min(x, y);457 458 return val % 2 ? val : val - 1;459}460 461static void result(const char *err, unsigned int n, unsigned int src_off,462 unsigned int dst_off, unsigned int len, unsigned long data)463{464 if (IS_ERR_VALUE(data)) {465 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%ld)\n",466 current->comm, n, err, src_off, dst_off, len, data);467 } else {468 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",469 current->comm, n, err, src_off, dst_off, len, data);470 }471}472 473static void dbg_result(const char *err, unsigned int n, unsigned int src_off,474 unsigned int dst_off, unsigned int len,475 unsigned long data)476{477 pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",478 current->comm, n, err, src_off, dst_off, len, data);479}480 481#define verbose_result(err, n, src_off, dst_off, len, data) ({ \482 if (verbose) \483 result(err, n, src_off, dst_off, len, data); \484 else \485 dbg_result(err, n, src_off, dst_off, len, data);\486})487 488static unsigned long long dmatest_persec(s64 runtime, unsigned int val)489{490 unsigned long long per_sec = 1000000;491 492 if (runtime <= 0)493 return 0;494 495 /* drop precision until runtime is 32-bits */496 while (runtime > UINT_MAX) {497 runtime >>= 1;498 per_sec <<= 1;499 }500 501 per_sec *= val;502 per_sec = INT_TO_FIXPT(per_sec);503 do_div(per_sec, (u32)runtime);504 505 return per_sec;506}507 508static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)509{510 return FIXPT_TO_INT(dmatest_persec(runtime, len >> 10));511}512 513static void __dmatest_free_test_data(struct dmatest_data *d, unsigned int cnt)514{515 unsigned int i;516 517 for (i = 0; i < cnt; i++)518 kfree(d->raw[i]);519 520 kfree(d->aligned);521 kfree(d->raw);522}523 524static void dmatest_free_test_data(struct dmatest_data *d)525{526 __dmatest_free_test_data(d, d->cnt);527}528 529static int dmatest_alloc_test_data(struct dmatest_data *d,530 unsigned int buf_size, u8 align)531{532 unsigned int i = 0;533 534 d->raw = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);535 if (!d->raw)536 return -ENOMEM;537 538 d->aligned = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);539 if (!d->aligned)540 goto err;541 542 for (i = 0; i < d->cnt; i++) {543 d->raw[i] = kmalloc(buf_size + align, d->gfp_flags);544 if (!d->raw[i])545 goto err;546 547 /* align to alignment restriction */548 if (align)549 d->aligned[i] = PTR_ALIGN(d->raw[i], align);550 else551 d->aligned[i] = d->raw[i];552 }553 554 return 0;555err:556 __dmatest_free_test_data(d, i);557 return -ENOMEM;558}559 560/*561 * This function repeatedly tests DMA transfers of various lengths and562 * offsets for a given operation type until it is told to exit by563 * kthread_stop(). There may be multiple threads running this function564 * in parallel for a single channel, and there may be multiple channels565 * being tested in parallel.566 *567 * Before each test, the source and destination buffer is initialized568 * with a known pattern. This pattern is different depending on569 * whether it's in an area which is supposed to be copied or570 * overwritten, and different in the source and destination buffers.571 * So if the DMA engine doesn't copy exactly what we tell it to copy,572 * we'll notice.573 */574static int dmatest_func(void *data)575{576 struct dmatest_thread *thread = data;577 struct dmatest_done *done = &thread->test_done;578 struct dmatest_info *info;579 struct dmatest_params *params;580 struct dma_chan *chan;581 struct dma_device *dev;582 struct device *dma_dev;583 unsigned int error_count;584 unsigned int failed_tests = 0;585 unsigned int total_tests = 0;586 dma_cookie_t cookie;587 enum dma_status status;588 enum dma_ctrl_flags flags;589 u8 *pq_coefs = NULL;590 int ret;591 unsigned int buf_size;592 struct dmatest_data *src;593 struct dmatest_data *dst;594 int i;595 ktime_t ktime, start, diff;596 ktime_t filltime = 0;597 ktime_t comparetime = 0;598 s64 runtime = 0;599 unsigned long long total_len = 0;600 unsigned long long iops = 0;601 u8 align = 0;602 bool is_memset = false;603 dma_addr_t *srcs;604 dma_addr_t *dma_pq;605 606 set_freezable();607 608 ret = -ENOMEM;609 610 smp_rmb();611 thread->pending = false;612 info = thread->info;613 params = &info->params;614 chan = thread->chan;615 dev = chan->device;616 dma_dev = dmaengine_get_dma_device(chan);617 618 src = &thread->src;619 dst = &thread->dst;620 if (thread->type == DMA_MEMCPY) {621 align = params->alignment < 0 ? dev->copy_align :622 params->alignment;623 src->cnt = dst->cnt = 1;624 } else if (thread->type == DMA_MEMSET) {625 align = params->alignment < 0 ? dev->fill_align :626 params->alignment;627 src->cnt = dst->cnt = 1;628 is_memset = true;629 } else if (thread->type == DMA_XOR) {630 /* force odd to ensure dst = src */631 src->cnt = min_odd(params->xor_sources | 1, dev->max_xor);632 dst->cnt = 1;633 align = params->alignment < 0 ? dev->xor_align :634 params->alignment;635 } else if (thread->type == DMA_PQ) {636 /* force odd to ensure dst = src */637 src->cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));638 dst->cnt = 2;639 align = params->alignment < 0 ? dev->pq_align :640 params->alignment;641 642 pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL);643 if (!pq_coefs)644 goto err_thread_type;645 646 for (i = 0; i < src->cnt; i++)647 pq_coefs[i] = 1;648 } else649 goto err_thread_type;650 651 /* Check if buffer count fits into map count variable (u8) */652 if ((src->cnt + dst->cnt) >= 255) {653 pr_err("too many buffers (%d of 255 supported)\n",654 src->cnt + dst->cnt);655 goto err_free_coefs;656 }657 658 buf_size = params->buf_size;659 if (1 << align > buf_size) {660 pr_err("%u-byte buffer too small for %d-byte alignment\n",661 buf_size, 1 << align);662 goto err_free_coefs;663 }664 665 src->gfp_flags = GFP_KERNEL;666 dst->gfp_flags = GFP_KERNEL;667 if (params->nobounce) {668 src->gfp_flags = GFP_DMA;669 dst->gfp_flags = GFP_DMA;670 }671 672 if (dmatest_alloc_test_data(src, buf_size, align) < 0)673 goto err_free_coefs;674 675 if (dmatest_alloc_test_data(dst, buf_size, align) < 0)676 goto err_src;677 678 set_user_nice(current, 10);679 680 srcs = kcalloc(src->cnt, sizeof(dma_addr_t), GFP_KERNEL);681 if (!srcs)682 goto err_dst;683 684 dma_pq = kcalloc(dst->cnt, sizeof(dma_addr_t), GFP_KERNEL);685 if (!dma_pq)686 goto err_srcs_array;687 688 /*689 * src and dst buffers are freed by ourselves below690 */691 if (params->polled)692 flags = DMA_CTRL_ACK;693 else694 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;695 696 ktime = ktime_get();697 while (!(kthread_should_stop() ||698 (params->iterations && total_tests >= params->iterations))) {699 struct dma_async_tx_descriptor *tx = NULL;700 struct dmaengine_unmap_data *um;701 dma_addr_t *dsts;702 unsigned int len;703 704 total_tests++;705 706 if (params->transfer_size) {707 if (params->transfer_size >= buf_size) {708 pr_err("%u-byte transfer size must be lower than %u-buffer size\n",709 params->transfer_size, buf_size);710 break;711 }712 len = params->transfer_size;713 } else if (params->norandom) {714 len = buf_size;715 } else {716 len = dmatest_random() % buf_size + 1;717 }718 719 /* Do not alter transfer size explicitly defined by user */720 if (!params->transfer_size) {721 len = (len >> align) << align;722 if (!len)723 len = 1 << align;724 }725 total_len += len;726 727 if (params->norandom) {728 src->off = 0;729 dst->off = 0;730 } else {731 src->off = dmatest_random() % (buf_size - len + 1);732 dst->off = dmatest_random() % (buf_size - len + 1);733 734 src->off = (src->off >> align) << align;735 dst->off = (dst->off >> align) << align;736 }737 738 if (!params->noverify) {739 start = ktime_get();740 dmatest_init_srcs(src->aligned, src->off, len,741 buf_size, is_memset);742 dmatest_init_dsts(dst->aligned, dst->off, len,743 buf_size, is_memset);744 745 diff = ktime_sub(ktime_get(), start);746 filltime = ktime_add(filltime, diff);747 }748 749 um = dmaengine_get_unmap_data(dma_dev, src->cnt + dst->cnt,750 GFP_KERNEL);751 if (!um) {752 failed_tests++;753 result("unmap data NULL", total_tests,754 src->off, dst->off, len, ret);755 continue;756 }757 758 um->len = buf_size;759 for (i = 0; i < src->cnt; i++) {760 void *buf = src->aligned[i];761 struct page *pg = virt_to_page(buf);762 unsigned long pg_off = offset_in_page(buf);763 764 um->addr[i] = dma_map_page(dma_dev, pg, pg_off,765 um->len, DMA_TO_DEVICE);766 srcs[i] = um->addr[i] + src->off;767 ret = dma_mapping_error(dma_dev, um->addr[i]);768 if (ret) {769 result("src mapping error", total_tests,770 src->off, dst->off, len, ret);771 goto error_unmap_continue;772 }773 um->to_cnt++;774 }775 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */776 dsts = &um->addr[src->cnt];777 for (i = 0; i < dst->cnt; i++) {778 void *buf = dst->aligned[i];779 struct page *pg = virt_to_page(buf);780 unsigned long pg_off = offset_in_page(buf);781 782 dsts[i] = dma_map_page(dma_dev, pg, pg_off, um->len,783 DMA_BIDIRECTIONAL);784 ret = dma_mapping_error(dma_dev, dsts[i]);785 if (ret) {786 result("dst mapping error", total_tests,787 src->off, dst->off, len, ret);788 goto error_unmap_continue;789 }790 um->bidi_cnt++;791 }792 793 if (thread->type == DMA_MEMCPY)794 tx = dev->device_prep_dma_memcpy(chan,795 dsts[0] + dst->off,796 srcs[0], len, flags);797 else if (thread->type == DMA_MEMSET)798 tx = dev->device_prep_dma_memset(chan,799 dsts[0] + dst->off,800 *(src->aligned[0] + src->off),801 len, flags);802 else if (thread->type == DMA_XOR)803 tx = dev->device_prep_dma_xor(chan,804 dsts[0] + dst->off,805 srcs, src->cnt,806 len, flags);807 else if (thread->type == DMA_PQ) {808 for (i = 0; i < dst->cnt; i++)809 dma_pq[i] = dsts[i] + dst->off;810 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,811 src->cnt, pq_coefs,812 len, flags);813 }814 815 if (!tx) {816 result("prep error", total_tests, src->off,817 dst->off, len, ret);818 msleep(100);819 goto error_unmap_continue;820 }821 822 done->done = false;823 if (!params->polled) {824 tx->callback = dmatest_callback;825 tx->callback_param = done;826 }827 cookie = tx->tx_submit(tx);828 829 if (dma_submit_error(cookie)) {830 result("submit error", total_tests, src->off,831 dst->off, len, ret);832 msleep(100);833 goto error_unmap_continue;834 }835 836 if (params->polled) {837 status = dma_sync_wait(chan, cookie);838 dmaengine_terminate_sync(chan);839 if (status == DMA_COMPLETE)840 done->done = true;841 } else {842 dma_async_issue_pending(chan);843 844 wait_event_freezable_timeout(thread->done_wait,845 done->done,846 msecs_to_jiffies(params->timeout));847 848 status = dma_async_is_tx_complete(chan, cookie, NULL,849 NULL);850 }851 852 if (!done->done) {853 result("test timed out", total_tests, src->off, dst->off,854 len, 0);855 goto error_unmap_continue;856 } else if (status != DMA_COMPLETE &&857 !(dma_has_cap(DMA_COMPLETION_NO_ORDER,858 dev->cap_mask) &&859 status == DMA_OUT_OF_ORDER)) {860 result(status == DMA_ERROR ?861 "completion error status" :862 "completion busy status", total_tests, src->off,863 dst->off, len, ret);864 goto error_unmap_continue;865 }866 867 dmaengine_unmap_put(um);868 869 if (params->noverify) {870 verbose_result("test passed", total_tests, src->off,871 dst->off, len, 0);872 continue;873 }874 875 start = ktime_get();876 pr_debug("%s: verifying source buffer...\n", current->comm);877 error_count = dmatest_verify(src->aligned, 0, src->off,878 0, PATTERN_SRC, true, is_memset);879 error_count += dmatest_verify(src->aligned, src->off,880 src->off + len, src->off,881 PATTERN_SRC | PATTERN_COPY, true, is_memset);882 error_count += dmatest_verify(src->aligned, src->off + len,883 buf_size, src->off + len,884 PATTERN_SRC, true, is_memset);885 886 pr_debug("%s: verifying dest buffer...\n", current->comm);887 error_count += dmatest_verify(dst->aligned, 0, dst->off,888 0, PATTERN_DST, false, is_memset);889 890 error_count += dmatest_verify(dst->aligned, dst->off,891 dst->off + len, src->off,892 PATTERN_SRC | PATTERN_COPY, false, is_memset);893 894 error_count += dmatest_verify(dst->aligned, dst->off + len,895 buf_size, dst->off + len,896 PATTERN_DST, false, is_memset);897 898 diff = ktime_sub(ktime_get(), start);899 comparetime = ktime_add(comparetime, diff);900 901 if (error_count) {902 result("data error", total_tests, src->off, dst->off,903 len, error_count);904 failed_tests++;905 } else {906 verbose_result("test passed", total_tests, src->off,907 dst->off, len, 0);908 }909 910 continue;911 912error_unmap_continue:913 dmaengine_unmap_put(um);914 failed_tests++;915 }916 ktime = ktime_sub(ktime_get(), ktime);917 ktime = ktime_sub(ktime, comparetime);918 ktime = ktime_sub(ktime, filltime);919 runtime = ktime_to_us(ktime);920 921 ret = 0;922 kfree(dma_pq);923err_srcs_array:924 kfree(srcs);925err_dst:926 dmatest_free_test_data(dst);927err_src:928 dmatest_free_test_data(src);929err_free_coefs:930 kfree(pq_coefs);931err_thread_type:932 iops = dmatest_persec(runtime, total_tests);933 pr_info("%s: summary %u tests, %u failures %llu.%02llu iops %llu KB/s (%d)\n",934 current->comm, total_tests, failed_tests,935 FIXPT_TO_INT(iops), FIXPT_GET_FRAC(iops),936 dmatest_KBs(runtime, total_len), ret);937 938 /* terminate all transfers on specified channels */939 if (ret || failed_tests)940 dmaengine_terminate_sync(chan);941 942 thread->done = true;943 wake_up(&thread_wait);944 945 return ret;946}947 948static void dmatest_cleanup_channel(struct dmatest_chan *dtc)949{950 struct dmatest_thread *thread;951 struct dmatest_thread *_thread;952 int ret;953 954 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {955 ret = kthread_stop(thread->task);956 pr_debug("thread %s exited with status %d\n",957 thread->task->comm, ret);958 list_del(&thread->node);959 put_task_struct(thread->task);960 kfree(thread);961 }962 963 /* terminate all transfers on specified channels */964 dmaengine_terminate_sync(dtc->chan);965 966 kfree(dtc);967}968 969static int dmatest_add_threads(struct dmatest_info *info,970 struct dmatest_chan *dtc, enum dma_transaction_type type)971{972 struct dmatest_params *params = &info->params;973 struct dmatest_thread *thread;974 struct dma_chan *chan = dtc->chan;975 char *op;976 unsigned int i;977 978 if (type == DMA_MEMCPY)979 op = "copy";980 else if (type == DMA_MEMSET)981 op = "set";982 else if (type == DMA_XOR)983 op = "xor";984 else if (type == DMA_PQ)985 op = "pq";986 else987 return -EINVAL;988 989 for (i = 0; i < params->threads_per_chan; i++) {990 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);991 if (!thread) {992 pr_warn("No memory for %s-%s%u\n",993 dma_chan_name(chan), op, i);994 break;995 }996 thread->info = info;997 thread->chan = dtc->chan;998 thread->type = type;999 thread->test_done.wait = &thread->done_wait;1000 init_waitqueue_head(&thread->done_wait);1001 smp_wmb();1002 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",1003 dma_chan_name(chan), op, i);1004 if (IS_ERR(thread->task)) {1005 pr_warn("Failed to create thread %s-%s%u\n",1006 dma_chan_name(chan), op, i);1007 kfree(thread);1008 break;1009 }1010 1011 /* srcbuf and dstbuf are allocated by the thread itself */1012 get_task_struct(thread->task);1013 list_add_tail(&thread->node, &dtc->threads);1014 thread->pending = true;1015 }1016 1017 return i;1018}1019 1020static int dmatest_add_channel(struct dmatest_info *info,1021 struct dma_chan *chan)1022{1023 struct dmatest_chan *dtc;1024 struct dma_device *dma_dev = chan->device;1025 unsigned int thread_count = 0;1026 int cnt;1027 1028 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);1029 if (!dtc) {1030 pr_warn("No memory for %s\n", dma_chan_name(chan));1031 return -ENOMEM;1032 }1033 1034 dtc->chan = chan;1035 INIT_LIST_HEAD(&dtc->threads);1036 1037 if (dma_has_cap(DMA_COMPLETION_NO_ORDER, dma_dev->cap_mask) &&1038 info->params.polled) {1039 info->params.polled = false;1040 pr_warn("DMA_COMPLETION_NO_ORDER, polled disabled\n");1041 }1042 1043 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {1044 if (dmatest == 0) {1045 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);1046 thread_count += cnt > 0 ? cnt : 0;1047 }1048 }1049 1050 if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {1051 if (dmatest == 1) {1052 cnt = dmatest_add_threads(info, dtc, DMA_MEMSET);1053 thread_count += cnt > 0 ? cnt : 0;1054 }1055 }1056 1057 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {1058 cnt = dmatest_add_threads(info, dtc, DMA_XOR);1059 thread_count += cnt > 0 ? cnt : 0;1060 }1061 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {1062 cnt = dmatest_add_threads(info, dtc, DMA_PQ);1063 thread_count += cnt > 0 ? cnt : 0;1064 }1065 1066 pr_info("Added %u threads using %s\n",1067 thread_count, dma_chan_name(chan));1068 1069 list_add_tail(&dtc->node, &info->channels);1070 info->nr_channels++;1071 1072 return 0;1073}1074 1075static bool filter(struct dma_chan *chan, void *param)1076{1077 return dmatest_match_channel(param, chan) && dmatest_match_device(param, chan->device);1078}1079 1080static void request_channels(struct dmatest_info *info,1081 enum dma_transaction_type type)1082{1083 dma_cap_mask_t mask;1084 1085 dma_cap_zero(mask);1086 dma_cap_set(type, mask);1087 for (;;) {1088 struct dmatest_params *params = &info->params;1089 struct dma_chan *chan;1090 1091 chan = dma_request_channel(mask, filter, params);1092 if (chan) {1093 if (dmatest_add_channel(info, chan)) {1094 dma_release_channel(chan);1095 break; /* add_channel failed, punt */1096 }1097 } else1098 break; /* no more channels available */1099 if (params->max_channels &&1100 info->nr_channels >= params->max_channels)1101 break; /* we have all we need */1102 }1103}1104 1105static void add_threaded_test(struct dmatest_info *info)1106{1107 struct dmatest_params *params = &info->params;1108 1109 /* Copy test parameters */1110 params->nobounce = nobounce;1111 params->buf_size = test_buf_size;1112 strscpy(params->channel, strim(test_channel), sizeof(params->channel));1113 strscpy(params->device, strim(test_device), sizeof(params->device));1114 params->threads_per_chan = threads_per_chan;1115 params->max_channels = max_channels;1116 params->iterations = iterations;1117 params->xor_sources = xor_sources;1118 params->pq_sources = pq_sources;1119 params->timeout = timeout;1120 params->noverify = noverify;1121 params->norandom = norandom;1122 params->alignment = alignment;1123 params->transfer_size = transfer_size;1124 params->polled = polled;1125 1126 request_channels(info, DMA_MEMCPY);1127 request_channels(info, DMA_MEMSET);1128 request_channels(info, DMA_XOR);1129 request_channels(info, DMA_PQ);1130}1131 1132static void run_pending_tests(struct dmatest_info *info)1133{1134 struct dmatest_chan *dtc;1135 unsigned int thread_count = 0;1136 1137 list_for_each_entry(dtc, &info->channels, node) {1138 struct dmatest_thread *thread;1139 1140 thread_count = 0;1141 list_for_each_entry(thread, &dtc->threads, node) {1142 wake_up_process(thread->task);1143 thread_count++;1144 }1145 pr_info("Started %u threads using %s\n",1146 thread_count, dma_chan_name(dtc->chan));1147 }1148}1149 1150static void stop_threaded_test(struct dmatest_info *info)1151{1152 struct dmatest_chan *dtc, *_dtc;1153 struct dma_chan *chan;1154 1155 list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {1156 list_del(&dtc->node);1157 chan = dtc->chan;1158 dmatest_cleanup_channel(dtc);1159 pr_debug("dropped channel %s\n", dma_chan_name(chan));1160 dma_release_channel(chan);1161 }1162 1163 info->nr_channels = 0;1164}1165 1166static void start_threaded_tests(struct dmatest_info *info)1167{1168 /* we might be called early to set run=, defer running until all1169 * parameters have been evaluated1170 */1171 if (!info->did_init)1172 return;1173 1174 run_pending_tests(info);1175}1176 1177static int dmatest_run_get(char *val, const struct kernel_param *kp)1178{1179 struct dmatest_info *info = &test_info;1180 1181 mutex_lock(&info->lock);1182 if (is_threaded_test_run(info)) {1183 dmatest_run = true;1184 } else {1185 if (!is_threaded_test_pending(info))1186 stop_threaded_test(info);1187 dmatest_run = false;1188 }1189 mutex_unlock(&info->lock);1190 1191 return param_get_bool(val, kp);1192}1193 1194static int dmatest_run_set(const char *val, const struct kernel_param *kp)1195{1196 struct dmatest_info *info = &test_info;1197 int ret;1198 1199 mutex_lock(&info->lock);1200 ret = param_set_bool(val, kp);1201 if (ret) {1202 mutex_unlock(&info->lock);1203 return ret;1204 } else if (dmatest_run) {1205 if (!is_threaded_test_pending(info)) {1206 /*1207 * We have nothing to run. This can be due to:1208 */1209 ret = info->last_error;1210 if (ret) {1211 /* 1) Misconfiguration */1212 pr_err("Channel misconfigured, can't continue\n");1213 mutex_unlock(&info->lock);1214 return ret;1215 } else {1216 /* 2) We rely on defaults */1217 pr_info("No channels configured, continue with any\n");1218 if (!is_threaded_test_run(info))1219 stop_threaded_test(info);1220 add_threaded_test(info);1221 }1222 }1223 start_threaded_tests(info);1224 } else {1225 stop_threaded_test(info);1226 }1227 1228 mutex_unlock(&info->lock);1229 1230 return ret;1231}1232 1233static int dmatest_chan_set(const char *val, const struct kernel_param *kp)1234{1235 struct dmatest_info *info = &test_info;1236 struct dmatest_chan *dtc;1237 char chan_reset_val[20];1238 int ret;1239 1240 mutex_lock(&info->lock);1241 ret = param_set_copystring(val, kp);1242 if (ret) {1243 mutex_unlock(&info->lock);1244 return ret;1245 }1246 /*Clear any previously run threads */1247 if (!is_threaded_test_run(info) && !is_threaded_test_pending(info))1248 stop_threaded_test(info);1249 /* Reject channels that are already registered */1250 if (is_threaded_test_pending(info)) {1251 list_for_each_entry(dtc, &info->channels, node) {1252 if (strcmp(dma_chan_name(dtc->chan),1253 strim(test_channel)) == 0) {1254 dtc = list_last_entry(&info->channels,1255 struct dmatest_chan,1256 node);1257 strscpy(chan_reset_val,1258 dma_chan_name(dtc->chan),1259 sizeof(chan_reset_val));1260 ret = -EBUSY;1261 goto add_chan_err;1262 }1263 }1264 }1265 1266 add_threaded_test(info);1267 1268 /* Check if channel was added successfully */1269 if (!list_empty(&info->channels)) {1270 /*1271 * if new channel was not successfully added, revert the1272 * "test_channel" string to the name of the last successfully1273 * added channel. exception for when users issues empty string1274 * to channel parameter.1275 */1276 dtc = list_last_entry(&info->channels, struct dmatest_chan, node);1277 if ((strcmp(dma_chan_name(dtc->chan), strim(test_channel)) != 0)1278 && (strcmp("", strim(test_channel)) != 0)) {1279 ret = -EINVAL;1280 strscpy(chan_reset_val, dma_chan_name(dtc->chan),1281 sizeof(chan_reset_val));1282 goto add_chan_err;1283 }1284 1285 } else {1286 /* Clear test_channel if no channels were added successfully */1287 strscpy(chan_reset_val, "", sizeof(chan_reset_val));1288 ret = -EBUSY;1289 goto add_chan_err;1290 }1291 1292 info->last_error = ret;1293 mutex_unlock(&info->lock);1294 1295 return ret;1296 1297add_chan_err:1298 param_set_copystring(chan_reset_val, kp);1299 info->last_error = ret;1300 mutex_unlock(&info->lock);1301 1302 return ret;1303}1304 1305static int dmatest_chan_get(char *val, const struct kernel_param *kp)1306{1307 struct dmatest_info *info = &test_info;1308 1309 mutex_lock(&info->lock);1310 if (!is_threaded_test_run(info) && !is_threaded_test_pending(info)) {1311 stop_threaded_test(info);1312 strscpy(test_channel, "", sizeof(test_channel));1313 }1314 mutex_unlock(&info->lock);1315 1316 return param_get_string(val, kp);1317}1318 1319static int dmatest_test_list_get(char *val, const struct kernel_param *kp)1320{1321 struct dmatest_info *info = &test_info;1322 struct dmatest_chan *dtc;1323 unsigned int thread_count = 0;1324 1325 list_for_each_entry(dtc, &info->channels, node) {1326 struct dmatest_thread *thread;1327 1328 thread_count = 0;1329 list_for_each_entry(thread, &dtc->threads, node) {1330 thread_count++;1331 }1332 pr_info("%u threads using %s\n",1333 thread_count, dma_chan_name(dtc->chan));1334 }1335 1336 return 0;1337}1338 1339static int __init dmatest_init(void)1340{1341 struct dmatest_info *info = &test_info;1342 struct dmatest_params *params = &info->params;1343 1344 if (dmatest_run) {1345 mutex_lock(&info->lock);1346 add_threaded_test(info);1347 run_pending_tests(info);1348 mutex_unlock(&info->lock);1349 }1350 1351 if (params->iterations && wait)1352 wait_event(thread_wait, !is_threaded_test_run(info));1353 1354 /* module parameters are stable, inittime tests are started,1355 * let userspace take over 'run' control1356 */1357 info->did_init = true;1358 1359 return 0;1360}1361/* when compiled-in wait for drivers to load first */1362late_initcall(dmatest_init);1363 1364static void __exit dmatest_exit(void)1365{1366 struct dmatest_info *info = &test_info;1367 1368 mutex_lock(&info->lock);1369 stop_threaded_test(info);1370 mutex_unlock(&info->lock);1371}1372module_exit(dmatest_exit);1373 1374MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");1375MODULE_DESCRIPTION("DMA Engine test module");1376MODULE_LICENSE("GPL v2");1377