1606 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.4 */5 6/*7 * This code implements the DMA subsystem. It provides a HW-neutral interface8 * for other kernel code to use asynchronous memory copy capabilities,9 * if present, and allows different HW DMA drivers to register as providing10 * this capability.11 *12 * Due to the fact we are accelerating what is already a relatively fast13 * operation, the code goes to great lengths to avoid additional overhead,14 * such as locking.15 *16 * LOCKING:17 *18 * The subsystem keeps a global list of dma_device structs it is protected by a19 * mutex, dma_list_mutex.20 *21 * A subsystem can get access to a channel by calling dmaengine_get() followed22 * by dma_find_channel(), or if it has need for an exclusive channel it can call23 * dma_request_channel(). Once a channel is allocated a reference is taken24 * against its corresponding driver to disable removal.25 *26 * Each device has a channels list, which runs unlocked but is never modified27 * once the device is registered, it's just setup by the driver.28 *29 * See Documentation/driver-api/dmaengine for more details30 */31 32#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt33 34#include <linux/platform_device.h>35#include <linux/dma-mapping.h>36#include <linux/init.h>37#include <linux/module.h>38#include <linux/mm.h>39#include <linux/device.h>40#include <linux/dmaengine.h>41#include <linux/hardirq.h>42#include <linux/spinlock.h>43#include <linux/percpu.h>44#include <linux/rcupdate.h>45#include <linux/mutex.h>46#include <linux/jiffies.h>47#include <linux/rculist.h>48#include <linux/idr.h>49#include <linux/slab.h>50#include <linux/acpi.h>51#include <linux/acpi_dma.h>52#include <linux/of_dma.h>53#include <linux/mempool.h>54#include <linux/numa.h>55 56#include "dmaengine.h"57 58static DEFINE_MUTEX(dma_list_mutex);59static DEFINE_IDA(dma_ida);60static LIST_HEAD(dma_device_list);61static long dmaengine_ref_count;62 63/* --- debugfs implementation --- */64#ifdef CONFIG_DEBUG_FS65#include <linux/debugfs.h>66 67static struct dentry *rootdir;68 69static void dmaengine_debug_register(struct dma_device *dma_dev)70{71 dma_dev->dbg_dev_root = debugfs_create_dir(dev_name(dma_dev->dev),72 rootdir);73 if (IS_ERR(dma_dev->dbg_dev_root))74 dma_dev->dbg_dev_root = NULL;75}76 77static void dmaengine_debug_unregister(struct dma_device *dma_dev)78{79 debugfs_remove_recursive(dma_dev->dbg_dev_root);80 dma_dev->dbg_dev_root = NULL;81}82 83static void dmaengine_dbg_summary_show(struct seq_file *s,84 struct dma_device *dma_dev)85{86 struct dma_chan *chan;87 88 list_for_each_entry(chan, &dma_dev->channels, device_node) {89 if (chan->client_count) {90 seq_printf(s, " %-13s| %s", dma_chan_name(chan),91 chan->dbg_client_name ?: "in-use");92 93 if (chan->router)94 seq_printf(s, " (via router: %s)\n",95 dev_name(chan->router->dev));96 else97 seq_puts(s, "\n");98 }99 }100}101 102static int dmaengine_summary_show(struct seq_file *s, void *data)103{104 struct dma_device *dma_dev = NULL;105 106 mutex_lock(&dma_list_mutex);107 list_for_each_entry(dma_dev, &dma_device_list, global_node) {108 seq_printf(s, "dma%d (%s): number of channels: %u\n",109 dma_dev->dev_id, dev_name(dma_dev->dev),110 dma_dev->chancnt);111 112 if (dma_dev->dbg_summary_show)113 dma_dev->dbg_summary_show(s, dma_dev);114 else115 dmaengine_dbg_summary_show(s, dma_dev);116 117 if (!list_is_last(&dma_dev->global_node, &dma_device_list))118 seq_puts(s, "\n");119 }120 mutex_unlock(&dma_list_mutex);121 122 return 0;123}124DEFINE_SHOW_ATTRIBUTE(dmaengine_summary);125 126static void __init dmaengine_debugfs_init(void)127{128 rootdir = debugfs_create_dir("dmaengine", NULL);129 130 /* /sys/kernel/debug/dmaengine/summary */131 debugfs_create_file("summary", 0444, rootdir, NULL,132 &dmaengine_summary_fops);133}134#else135static inline void dmaengine_debugfs_init(void) { }136static inline int dmaengine_debug_register(struct dma_device *dma_dev)137{138 return 0;139}140 141static inline void dmaengine_debug_unregister(struct dma_device *dma_dev) { }142#endif /* DEBUG_FS */143 144/* --- sysfs implementation --- */145 146#define DMA_SLAVE_NAME "slave"147 148/**149 * dev_to_dma_chan - convert a device pointer to its sysfs container object150 * @dev: device node151 *152 * Must be called under dma_list_mutex.153 */154static struct dma_chan *dev_to_dma_chan(struct device *dev)155{156 struct dma_chan_dev *chan_dev;157 158 chan_dev = container_of(dev, typeof(*chan_dev), device);159 return chan_dev->chan;160}161 162static ssize_t memcpy_count_show(struct device *dev,163 struct device_attribute *attr, char *buf)164{165 struct dma_chan *chan;166 unsigned long count = 0;167 int i;168 int err;169 170 mutex_lock(&dma_list_mutex);171 chan = dev_to_dma_chan(dev);172 if (chan) {173 for_each_possible_cpu(i)174 count += per_cpu_ptr(chan->local, i)->memcpy_count;175 err = sysfs_emit(buf, "%lu\n", count);176 } else177 err = -ENODEV;178 mutex_unlock(&dma_list_mutex);179 180 return err;181}182static DEVICE_ATTR_RO(memcpy_count);183 184static ssize_t bytes_transferred_show(struct device *dev,185 struct device_attribute *attr, char *buf)186{187 struct dma_chan *chan;188 unsigned long count = 0;189 int i;190 int err;191 192 mutex_lock(&dma_list_mutex);193 chan = dev_to_dma_chan(dev);194 if (chan) {195 for_each_possible_cpu(i)196 count += per_cpu_ptr(chan->local, i)->bytes_transferred;197 err = sysfs_emit(buf, "%lu\n", count);198 } else199 err = -ENODEV;200 mutex_unlock(&dma_list_mutex);201 202 return err;203}204static DEVICE_ATTR_RO(bytes_transferred);205 206static ssize_t in_use_show(struct device *dev, struct device_attribute *attr,207 char *buf)208{209 struct dma_chan *chan;210 int err;211 212 mutex_lock(&dma_list_mutex);213 chan = dev_to_dma_chan(dev);214 if (chan)215 err = sysfs_emit(buf, "%d\n", chan->client_count);216 else217 err = -ENODEV;218 mutex_unlock(&dma_list_mutex);219 220 return err;221}222static DEVICE_ATTR_RO(in_use);223 224static struct attribute *dma_dev_attrs[] = {225 &dev_attr_memcpy_count.attr,226 &dev_attr_bytes_transferred.attr,227 &dev_attr_in_use.attr,228 NULL,229};230ATTRIBUTE_GROUPS(dma_dev);231 232static void chan_dev_release(struct device *dev)233{234 struct dma_chan_dev *chan_dev;235 236 chan_dev = container_of(dev, typeof(*chan_dev), device);237 kfree(chan_dev);238}239 240static struct class dma_devclass = {241 .name = "dma",242 .dev_groups = dma_dev_groups,243 .dev_release = chan_dev_release,244};245 246/* --- client and device registration --- */247 248/* enable iteration over all operation types */249static dma_cap_mask_t dma_cap_mask_all;250 251/**252 * struct dma_chan_tbl_ent - tracks channel allocations per core/operation253 * @chan: associated channel for this entry254 */255struct dma_chan_tbl_ent {256 struct dma_chan *chan;257};258 259/* percpu lookup table for memory-to-memory offload providers */260static struct dma_chan_tbl_ent __percpu *channel_table[DMA_TX_TYPE_END];261 262static int __init dma_channel_table_init(void)263{264 enum dma_transaction_type cap;265 int err = 0;266 267 bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END);268 269 /* 'interrupt', 'private', and 'slave' are channel capabilities,270 * but are not associated with an operation so they do not need271 * an entry in the channel_table272 */273 clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits);274 clear_bit(DMA_PRIVATE, dma_cap_mask_all.bits);275 clear_bit(DMA_SLAVE, dma_cap_mask_all.bits);276 277 for_each_dma_cap_mask(cap, dma_cap_mask_all) {278 channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent);279 if (!channel_table[cap]) {280 err = -ENOMEM;281 break;282 }283 }284 285 if (err) {286 pr_err("dmaengine dma_channel_table_init failure: %d\n", err);287 for_each_dma_cap_mask(cap, dma_cap_mask_all)288 free_percpu(channel_table[cap]);289 }290 291 return err;292}293arch_initcall(dma_channel_table_init);294 295/**296 * dma_chan_is_local - checks if the channel is in the same NUMA-node as the CPU297 * @chan: DMA channel to test298 * @cpu: CPU index which the channel should be close to299 *300 * Returns true if the channel is in the same NUMA-node as the CPU.301 */302static bool dma_chan_is_local(struct dma_chan *chan, int cpu)303{304 int node = dev_to_node(chan->device->dev);305 return node == NUMA_NO_NODE ||306 cpumask_test_cpu(cpu, cpumask_of_node(node));307}308 309/**310 * min_chan - finds the channel with min count and in the same NUMA-node as the CPU311 * @cap: capability to match312 * @cpu: CPU index which the channel should be close to313 *314 * If some channels are close to the given CPU, the one with the lowest315 * reference count is returned. Otherwise, CPU is ignored and only the316 * reference count is taken into account.317 *318 * Must be called under dma_list_mutex.319 */320static struct dma_chan *min_chan(enum dma_transaction_type cap, int cpu)321{322 struct dma_device *device;323 struct dma_chan *chan;324 struct dma_chan *min = NULL;325 struct dma_chan *localmin = NULL;326 327 list_for_each_entry(device, &dma_device_list, global_node) {328 if (!dma_has_cap(cap, device->cap_mask) ||329 dma_has_cap(DMA_PRIVATE, device->cap_mask))330 continue;331 list_for_each_entry(chan, &device->channels, device_node) {332 if (!chan->client_count)333 continue;334 if (!min || chan->table_count < min->table_count)335 min = chan;336 337 if (dma_chan_is_local(chan, cpu))338 if (!localmin ||339 chan->table_count < localmin->table_count)340 localmin = chan;341 }342 }343 344 chan = localmin ? localmin : min;345 346 if (chan)347 chan->table_count++;348 349 return chan;350}351 352/**353 * dma_channel_rebalance - redistribute the available channels354 *355 * Optimize for CPU isolation (each CPU gets a dedicated channel for an356 * operation type) in the SMP case, and operation isolation (avoid357 * multi-tasking channels) in the non-SMP case.358 *359 * Must be called under dma_list_mutex.360 */361static void dma_channel_rebalance(void)362{363 struct dma_chan *chan;364 struct dma_device *device;365 int cpu;366 int cap;367 368 /* undo the last distribution */369 for_each_dma_cap_mask(cap, dma_cap_mask_all)370 for_each_possible_cpu(cpu)371 per_cpu_ptr(channel_table[cap], cpu)->chan = NULL;372 373 list_for_each_entry(device, &dma_device_list, global_node) {374 if (dma_has_cap(DMA_PRIVATE, device->cap_mask))375 continue;376 list_for_each_entry(chan, &device->channels, device_node)377 chan->table_count = 0;378 }379 380 /* don't populate the channel_table if no clients are available */381 if (!dmaengine_ref_count)382 return;383 384 /* redistribute available channels */385 for_each_dma_cap_mask(cap, dma_cap_mask_all)386 for_each_online_cpu(cpu) {387 chan = min_chan(cap, cpu);388 per_cpu_ptr(channel_table[cap], cpu)->chan = chan;389 }390}391 392static int dma_device_satisfies_mask(struct dma_device *device,393 const dma_cap_mask_t *want)394{395 dma_cap_mask_t has;396 397 bitmap_and(has.bits, want->bits, device->cap_mask.bits,398 DMA_TX_TYPE_END);399 return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);400}401 402static struct module *dma_chan_to_owner(struct dma_chan *chan)403{404 return chan->device->owner;405}406 407/**408 * balance_ref_count - catch up the channel reference count409 * @chan: channel to balance ->client_count versus dmaengine_ref_count410 *411 * Must be called under dma_list_mutex.412 */413static void balance_ref_count(struct dma_chan *chan)414{415 struct module *owner = dma_chan_to_owner(chan);416 417 while (chan->client_count < dmaengine_ref_count) {418 __module_get(owner);419 chan->client_count++;420 }421}422 423static void dma_device_release(struct kref *ref)424{425 struct dma_device *device = container_of(ref, struct dma_device, ref);426 427 list_del_rcu(&device->global_node);428 dma_channel_rebalance();429 430 if (device->device_release)431 device->device_release(device);432}433 434static void dma_device_put(struct dma_device *device)435{436 lockdep_assert_held(&dma_list_mutex);437 kref_put(&device->ref, dma_device_release);438}439 440/**441 * dma_chan_get - try to grab a DMA channel's parent driver module442 * @chan: channel to grab443 *444 * Must be called under dma_list_mutex.445 */446static int dma_chan_get(struct dma_chan *chan)447{448 struct module *owner = dma_chan_to_owner(chan);449 int ret;450 451 /* The channel is already in use, update client count */452 if (chan->client_count) {453 __module_get(owner);454 chan->client_count++;455 return 0;456 }457 458 if (!try_module_get(owner))459 return -ENODEV;460 461 ret = kref_get_unless_zero(&chan->device->ref);462 if (!ret) {463 ret = -ENODEV;464 goto module_put_out;465 }466 467 /* allocate upon first client reference */468 if (chan->device->device_alloc_chan_resources) {469 ret = chan->device->device_alloc_chan_resources(chan);470 if (ret < 0)471 goto err_out;472 }473 474 chan->client_count++;475 476 if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask))477 balance_ref_count(chan);478 479 return 0;480 481err_out:482 dma_device_put(chan->device);483module_put_out:484 module_put(owner);485 return ret;486}487 488/**489 * dma_chan_put - drop a reference to a DMA channel's parent driver module490 * @chan: channel to release491 *492 * Must be called under dma_list_mutex.493 */494static void dma_chan_put(struct dma_chan *chan)495{496 /* This channel is not in use, bail out */497 if (!chan->client_count)498 return;499 500 chan->client_count--;501 502 /* This channel is not in use anymore, free it */503 if (!chan->client_count && chan->device->device_free_chan_resources) {504 /* Make sure all operations have completed */505 dmaengine_synchronize(chan);506 chan->device->device_free_chan_resources(chan);507 }508 509 /* If the channel is used via a DMA request router, free the mapping */510 if (chan->router && chan->router->route_free) {511 chan->router->route_free(chan->router->dev, chan->route_data);512 chan->router = NULL;513 chan->route_data = NULL;514 }515 516 dma_device_put(chan->device);517 module_put(dma_chan_to_owner(chan));518}519 520enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)521{522 enum dma_status status;523 unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);524 525 dma_async_issue_pending(chan);526 do {527 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);528 if (time_after_eq(jiffies, dma_sync_wait_timeout)) {529 dev_err(chan->device->dev, "%s: timeout!\n", __func__);530 return DMA_ERROR;531 }532 if (status != DMA_IN_PROGRESS)533 break;534 cpu_relax();535 } while (1);536 537 return status;538}539EXPORT_SYMBOL(dma_sync_wait);540 541/**542 * dma_find_channel - find a channel to carry out the operation543 * @tx_type: transaction type544 */545struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)546{547 return this_cpu_read(channel_table[tx_type]->chan);548}549EXPORT_SYMBOL(dma_find_channel);550 551/**552 * dma_issue_pending_all - flush all pending operations across all channels553 */554void dma_issue_pending_all(void)555{556 struct dma_device *device;557 struct dma_chan *chan;558 559 rcu_read_lock();560 list_for_each_entry_rcu(device, &dma_device_list, global_node) {561 if (dma_has_cap(DMA_PRIVATE, device->cap_mask))562 continue;563 list_for_each_entry(chan, &device->channels, device_node)564 if (chan->client_count)565 device->device_issue_pending(chan);566 }567 rcu_read_unlock();568}569EXPORT_SYMBOL(dma_issue_pending_all);570 571int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps)572{573 struct dma_device *device;574 575 if (!chan || !caps)576 return -EINVAL;577 578 device = chan->device;579 580 /* check if the channel supports slave transactions */581 if (!(test_bit(DMA_SLAVE, device->cap_mask.bits) ||582 test_bit(DMA_CYCLIC, device->cap_mask.bits)))583 return -ENXIO;584 585 /*586 * Check whether it reports it uses the generic slave587 * capabilities, if not, that means it doesn't support any588 * kind of slave capabilities reporting.589 */590 if (!device->directions)591 return -ENXIO;592 593 caps->src_addr_widths = device->src_addr_widths;594 caps->dst_addr_widths = device->dst_addr_widths;595 caps->directions = device->directions;596 caps->min_burst = device->min_burst;597 caps->max_burst = device->max_burst;598 caps->max_sg_burst = device->max_sg_burst;599 caps->residue_granularity = device->residue_granularity;600 caps->descriptor_reuse = device->descriptor_reuse;601 caps->cmd_pause = !!device->device_pause;602 caps->cmd_resume = !!device->device_resume;603 caps->cmd_terminate = !!device->device_terminate_all;604 605 /*606 * DMA engine device might be configured with non-uniformly607 * distributed slave capabilities per device channels. In this608 * case the corresponding driver may provide the device_caps609 * callback to override the generic capabilities with610 * channel-specific ones.611 */612 if (device->device_caps)613 device->device_caps(chan, caps);614 615 return 0;616}617EXPORT_SYMBOL_GPL(dma_get_slave_caps);618 619static struct dma_chan *private_candidate(const dma_cap_mask_t *mask,620 struct dma_device *dev,621 dma_filter_fn fn, void *fn_param)622{623 struct dma_chan *chan;624 625 if (mask && !dma_device_satisfies_mask(dev, mask)) {626 dev_dbg(dev->dev, "%s: wrong capabilities\n", __func__);627 return NULL;628 }629 /* devices with multiple channels need special handling as we need to630 * ensure that all channels are either private or public.631 */632 if (dev->chancnt > 1 && !dma_has_cap(DMA_PRIVATE, dev->cap_mask))633 list_for_each_entry(chan, &dev->channels, device_node) {634 /* some channels are already publicly allocated */635 if (chan->client_count)636 return NULL;637 }638 639 list_for_each_entry(chan, &dev->channels, device_node) {640 if (chan->client_count) {641 dev_dbg(dev->dev, "%s: %s busy\n",642 __func__, dma_chan_name(chan));643 continue;644 }645 if (fn && !fn(chan, fn_param)) {646 dev_dbg(dev->dev, "%s: %s filter said false\n",647 __func__, dma_chan_name(chan));648 continue;649 }650 return chan;651 }652 653 return NULL;654}655 656static struct dma_chan *find_candidate(struct dma_device *device,657 const dma_cap_mask_t *mask,658 dma_filter_fn fn, void *fn_param)659{660 struct dma_chan *chan = private_candidate(mask, device, fn, fn_param);661 int err;662 663 if (chan) {664 /* Found a suitable channel, try to grab, prep, and return it.665 * We first set DMA_PRIVATE to disable balance_ref_count as this666 * channel will not be published in the general-purpose667 * allocator668 */669 dma_cap_set(DMA_PRIVATE, device->cap_mask);670 device->privatecnt++;671 err = dma_chan_get(chan);672 673 if (err) {674 if (err == -ENODEV) {675 dev_dbg(device->dev, "%s: %s module removed\n",676 __func__, dma_chan_name(chan));677 list_del_rcu(&device->global_node);678 } else679 dev_dbg(device->dev,680 "%s: failed to get %s: (%d)\n",681 __func__, dma_chan_name(chan), err);682 683 if (--device->privatecnt == 0)684 dma_cap_clear(DMA_PRIVATE, device->cap_mask);685 686 chan = ERR_PTR(err);687 }688 }689 690 return chan ? chan : ERR_PTR(-EPROBE_DEFER);691}692 693/**694 * dma_get_slave_channel - try to get specific channel exclusively695 * @chan: target channel696 */697struct dma_chan *dma_get_slave_channel(struct dma_chan *chan)698{699 /* lock against __dma_request_channel */700 mutex_lock(&dma_list_mutex);701 702 if (chan->client_count == 0) {703 struct dma_device *device = chan->device;704 int err;705 706 dma_cap_set(DMA_PRIVATE, device->cap_mask);707 device->privatecnt++;708 err = dma_chan_get(chan);709 if (err) {710 dev_dbg(chan->device->dev,711 "%s: failed to get %s: (%d)\n",712 __func__, dma_chan_name(chan), err);713 chan = NULL;714 if (--device->privatecnt == 0)715 dma_cap_clear(DMA_PRIVATE, device->cap_mask);716 }717 } else718 chan = NULL;719 720 mutex_unlock(&dma_list_mutex);721 722 723 return chan;724}725EXPORT_SYMBOL_GPL(dma_get_slave_channel);726 727struct dma_chan *dma_get_any_slave_channel(struct dma_device *device)728{729 dma_cap_mask_t mask;730 struct dma_chan *chan;731 732 dma_cap_zero(mask);733 dma_cap_set(DMA_SLAVE, mask);734 735 /* lock against __dma_request_channel */736 mutex_lock(&dma_list_mutex);737 738 chan = find_candidate(device, &mask, NULL, NULL);739 740 mutex_unlock(&dma_list_mutex);741 742 return IS_ERR(chan) ? NULL : chan;743}744EXPORT_SYMBOL_GPL(dma_get_any_slave_channel);745 746/**747 * __dma_request_channel - try to allocate an exclusive channel748 * @mask: capabilities that the channel must satisfy749 * @fn: optional callback to disposition available channels750 * @fn_param: opaque parameter to pass to dma_filter_fn()751 * @np: device node to look for DMA channels752 *753 * Returns pointer to appropriate DMA channel on success or NULL.754 */755struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,756 dma_filter_fn fn, void *fn_param,757 struct device_node *np)758{759 struct dma_device *device, *_d;760 struct dma_chan *chan = NULL;761 762 /* Find a channel */763 mutex_lock(&dma_list_mutex);764 list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {765 /* Finds a DMA controller with matching device node */766 if (np && device->dev->of_node && np != device->dev->of_node)767 continue;768 769 chan = find_candidate(device, mask, fn, fn_param);770 if (!IS_ERR(chan))771 break;772 773 chan = NULL;774 }775 mutex_unlock(&dma_list_mutex);776 777 pr_debug("%s: %s (%s)\n",778 __func__,779 chan ? "success" : "fail",780 chan ? dma_chan_name(chan) : NULL);781 782 return chan;783}784EXPORT_SYMBOL_GPL(__dma_request_channel);785 786static const struct dma_slave_map *dma_filter_match(struct dma_device *device,787 const char *name,788 struct device *dev)789{790 int i;791 792 if (!device->filter.mapcnt)793 return NULL;794 795 for (i = 0; i < device->filter.mapcnt; i++) {796 const struct dma_slave_map *map = &device->filter.map[i];797 798 if (!strcmp(map->devname, dev_name(dev)) &&799 !strcmp(map->slave, name))800 return map;801 }802 803 return NULL;804}805 806/**807 * dma_request_chan - try to allocate an exclusive slave channel808 * @dev: pointer to client device structure809 * @name: slave channel name810 *811 * Returns pointer to appropriate DMA channel on success or an error pointer.812 */813struct dma_chan *dma_request_chan(struct device *dev, const char *name)814{815 struct dma_device *d, *_d;816 struct dma_chan *chan = NULL;817 818 /* If device-tree is present get slave info from here */819 if (dev->of_node)820 chan = of_dma_request_slave_channel(dev->of_node, name);821 822 /* If device was enumerated by ACPI get slave info from here */823 if (has_acpi_companion(dev) && !chan)824 chan = acpi_dma_request_slave_chan_by_name(dev, name);825 826 if (PTR_ERR(chan) == -EPROBE_DEFER)827 return chan;828 829 if (!IS_ERR_OR_NULL(chan))830 goto found;831 832 /* Try to find the channel via the DMA filter map(s) */833 mutex_lock(&dma_list_mutex);834 list_for_each_entry_safe(d, _d, &dma_device_list, global_node) {835 dma_cap_mask_t mask;836 const struct dma_slave_map *map = dma_filter_match(d, name, dev);837 838 if (!map)839 continue;840 841 dma_cap_zero(mask);842 dma_cap_set(DMA_SLAVE, mask);843 844 chan = find_candidate(d, &mask, d->filter.fn, map->param);845 if (!IS_ERR(chan))846 break;847 }848 mutex_unlock(&dma_list_mutex);849 850 if (IS_ERR(chan))851 return chan;852 if (!chan)853 return ERR_PTR(-EPROBE_DEFER);854 855found:856#ifdef CONFIG_DEBUG_FS857 chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev),858 name);859#endif860 861 chan->name = kasprintf(GFP_KERNEL, "dma:%s", name);862 if (!chan->name)863 return chan;864 chan->slave = dev;865 866 if (sysfs_create_link(&chan->dev->device.kobj, &dev->kobj,867 DMA_SLAVE_NAME))868 dev_warn(dev, "Cannot create DMA %s symlink\n", DMA_SLAVE_NAME);869 if (sysfs_create_link(&dev->kobj, &chan->dev->device.kobj, chan->name))870 dev_warn(dev, "Cannot create DMA %s symlink\n", chan->name);871 872 return chan;873}874EXPORT_SYMBOL_GPL(dma_request_chan);875 876/**877 * dma_request_chan_by_mask - allocate a channel satisfying certain capabilities878 * @mask: capabilities that the channel must satisfy879 *880 * Returns pointer to appropriate DMA channel on success or an error pointer.881 */882struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask)883{884 struct dma_chan *chan;885 886 if (!mask)887 return ERR_PTR(-ENODEV);888 889 chan = __dma_request_channel(mask, NULL, NULL, NULL);890 if (!chan) {891 mutex_lock(&dma_list_mutex);892 if (list_empty(&dma_device_list))893 chan = ERR_PTR(-EPROBE_DEFER);894 else895 chan = ERR_PTR(-ENODEV);896 mutex_unlock(&dma_list_mutex);897 }898 899 return chan;900}901EXPORT_SYMBOL_GPL(dma_request_chan_by_mask);902 903void dma_release_channel(struct dma_chan *chan)904{905 mutex_lock(&dma_list_mutex);906 WARN_ONCE(chan->client_count != 1,907 "chan reference count %d != 1\n", chan->client_count);908 dma_chan_put(chan);909 /* drop PRIVATE cap enabled by __dma_request_channel() */910 if (--chan->device->privatecnt == 0)911 dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask);912 913 if (chan->slave) {914 sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME);915 sysfs_remove_link(&chan->slave->kobj, chan->name);916 kfree(chan->name);917 chan->name = NULL;918 chan->slave = NULL;919 }920 921#ifdef CONFIG_DEBUG_FS922 kfree(chan->dbg_client_name);923 chan->dbg_client_name = NULL;924#endif925 mutex_unlock(&dma_list_mutex);926}927EXPORT_SYMBOL_GPL(dma_release_channel);928 929/**930 * dmaengine_get - register interest in dma_channels931 */932void dmaengine_get(void)933{934 struct dma_device *device, *_d;935 struct dma_chan *chan;936 int err;937 938 mutex_lock(&dma_list_mutex);939 dmaengine_ref_count++;940 941 /* try to grab channels */942 list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {943 if (dma_has_cap(DMA_PRIVATE, device->cap_mask))944 continue;945 list_for_each_entry(chan, &device->channels, device_node) {946 err = dma_chan_get(chan);947 if (err == -ENODEV) {948 /* module removed before we could use it */949 list_del_rcu(&device->global_node);950 break;951 } else if (err)952 dev_dbg(chan->device->dev,953 "%s: failed to get %s: (%d)\n",954 __func__, dma_chan_name(chan), err);955 }956 }957 958 /* if this is the first reference and there were channels959 * waiting we need to rebalance to get those channels960 * incorporated into the channel table961 */962 if (dmaengine_ref_count == 1)963 dma_channel_rebalance();964 mutex_unlock(&dma_list_mutex);965}966EXPORT_SYMBOL(dmaengine_get);967 968/**969 * dmaengine_put - let DMA drivers be removed when ref_count == 0970 */971void dmaengine_put(void)972{973 struct dma_device *device, *_d;974 struct dma_chan *chan;975 976 mutex_lock(&dma_list_mutex);977 dmaengine_ref_count--;978 BUG_ON(dmaengine_ref_count < 0);979 /* drop channel references */980 list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {981 if (dma_has_cap(DMA_PRIVATE, device->cap_mask))982 continue;983 list_for_each_entry(chan, &device->channels, device_node)984 dma_chan_put(chan);985 }986 mutex_unlock(&dma_list_mutex);987}988EXPORT_SYMBOL(dmaengine_put);989 990static bool device_has_all_tx_types(struct dma_device *device)991{992 /* A device that satisfies this test has channels that will never cause993 * an async_tx channel switch event as all possible operation types can994 * be handled.995 */996 #ifdef CONFIG_ASYNC_TX_DMA997 if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask))998 return false;999 #endif1000 1001 #if IS_ENABLED(CONFIG_ASYNC_MEMCPY)1002 if (!dma_has_cap(DMA_MEMCPY, device->cap_mask))1003 return false;1004 #endif1005 1006 #if IS_ENABLED(CONFIG_ASYNC_XOR)1007 if (!dma_has_cap(DMA_XOR, device->cap_mask))1008 return false;1009 1010 #ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA1011 if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask))1012 return false;1013 #endif1014 #endif1015 1016 #if IS_ENABLED(CONFIG_ASYNC_PQ)1017 if (!dma_has_cap(DMA_PQ, device->cap_mask))1018 return false;1019 1020 #ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA1021 if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask))1022 return false;1023 #endif1024 #endif1025 1026 return true;1027}1028 1029static int get_dma_id(struct dma_device *device)1030{1031 int rc = ida_alloc(&dma_ida, GFP_KERNEL);1032 1033 if (rc < 0)1034 return rc;1035 device->dev_id = rc;1036 return 0;1037}1038 1039static int __dma_async_device_channel_register(struct dma_device *device,1040 struct dma_chan *chan,1041 const char *name)1042{1043 int rc;1044 1045 chan->local = alloc_percpu(typeof(*chan->local));1046 if (!chan->local)1047 return -ENOMEM;1048 chan->dev = kzalloc(sizeof(*chan->dev), GFP_KERNEL);1049 if (!chan->dev) {1050 rc = -ENOMEM;1051 goto err_free_local;1052 }1053 1054 /*1055 * When the chan_id is a negative value, we are dynamically adding1056 * the channel. Otherwise we are static enumerating.1057 */1058 chan->chan_id = ida_alloc(&device->chan_ida, GFP_KERNEL);1059 if (chan->chan_id < 0) {1060 pr_err("%s: unable to alloc ida for chan: %d\n",1061 __func__, chan->chan_id);1062 rc = chan->chan_id;1063 goto err_free_dev;1064 }1065 1066 chan->dev->device.class = &dma_devclass;1067 chan->dev->device.parent = device->dev;1068 chan->dev->chan = chan;1069 chan->dev->dev_id = device->dev_id;1070 if (!name)1071 dev_set_name(&chan->dev->device, "dma%dchan%d", device->dev_id, chan->chan_id);1072 else1073 dev_set_name(&chan->dev->device, "%s", name);1074 rc = device_register(&chan->dev->device);1075 if (rc)1076 goto err_out_ida;1077 chan->client_count = 0;1078 device->chancnt++;1079 1080 return 0;1081 1082 err_out_ida:1083 ida_free(&device->chan_ida, chan->chan_id);1084 err_free_dev:1085 kfree(chan->dev);1086 err_free_local:1087 free_percpu(chan->local);1088 chan->local = NULL;1089 return rc;1090}1091 1092int dma_async_device_channel_register(struct dma_device *device,1093 struct dma_chan *chan,1094 const char *name)1095{1096 int rc;1097 1098 rc = __dma_async_device_channel_register(device, chan, name);1099 if (rc < 0)1100 return rc;1101 1102 dma_channel_rebalance();1103 return 0;1104}1105EXPORT_SYMBOL_GPL(dma_async_device_channel_register);1106 1107static void __dma_async_device_channel_unregister(struct dma_device *device,1108 struct dma_chan *chan)1109{1110 if (chan->local == NULL)1111 return;1112 1113 WARN_ONCE(!device->device_release && chan->client_count,1114 "%s called while %d clients hold a reference\n",1115 __func__, chan->client_count);1116 mutex_lock(&dma_list_mutex);1117 device->chancnt--;1118 chan->dev->chan = NULL;1119 mutex_unlock(&dma_list_mutex);1120 ida_free(&device->chan_ida, chan->chan_id);1121 device_unregister(&chan->dev->device);1122 free_percpu(chan->local);1123}1124 1125void dma_async_device_channel_unregister(struct dma_device *device,1126 struct dma_chan *chan)1127{1128 __dma_async_device_channel_unregister(device, chan);1129 dma_channel_rebalance();1130}1131EXPORT_SYMBOL_GPL(dma_async_device_channel_unregister);1132 1133/**1134 * dma_async_device_register - registers DMA devices found1135 * @device: pointer to &struct dma_device1136 *1137 * After calling this routine the structure should not be freed except in the1138 * device_release() callback which will be called after1139 * dma_async_device_unregister() is called and no further references are taken.1140 */1141int dma_async_device_register(struct dma_device *device)1142{1143 int rc;1144 struct dma_chan* chan;1145 1146 if (!device)1147 return -ENODEV;1148 1149 /* validate device routines */1150 if (!device->dev) {1151 pr_err("DMAdevice must have dev\n");1152 return -EIO;1153 }1154 1155 device->owner = device->dev->driver->owner;1156 1157#define CHECK_CAP(_name, _type) \1158{ \1159 if (dma_has_cap(_type, device->cap_mask) && !device->device_prep_##_name) { \1160 dev_err(device->dev, \1161 "Device claims capability %s, but op is not defined\n", \1162 __stringify(_type)); \1163 return -EIO; \1164 } \1165}1166 1167 CHECK_CAP(dma_memcpy, DMA_MEMCPY);1168 CHECK_CAP(dma_xor, DMA_XOR);1169 CHECK_CAP(dma_xor_val, DMA_XOR_VAL);1170 CHECK_CAP(dma_pq, DMA_PQ);1171 CHECK_CAP(dma_pq_val, DMA_PQ_VAL);1172 CHECK_CAP(dma_memset, DMA_MEMSET);1173 CHECK_CAP(dma_interrupt, DMA_INTERRUPT);1174 CHECK_CAP(dma_cyclic, DMA_CYCLIC);1175 CHECK_CAP(interleaved_dma, DMA_INTERLEAVE);1176 1177#undef CHECK_CAP1178 1179 if (!device->device_tx_status) {1180 dev_err(device->dev, "Device tx_status is not defined\n");1181 return -EIO;1182 }1183 1184 1185 if (!device->device_issue_pending) {1186 dev_err(device->dev, "Device issue_pending is not defined\n");1187 return -EIO;1188 }1189 1190 if (!device->device_release)1191 dev_dbg(device->dev,1192 "WARN: Device release is not defined so it is not safe to unbind this driver while in use\n");1193 1194 kref_init(&device->ref);1195 1196 /* note: this only matters in the1197 * CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH=n case1198 */1199 if (device_has_all_tx_types(device))1200 dma_cap_set(DMA_ASYNC_TX, device->cap_mask);1201 1202 rc = get_dma_id(device);1203 if (rc != 0)1204 return rc;1205 1206 ida_init(&device->chan_ida);1207 1208 /* represent channels in sysfs. Probably want devs too */1209 list_for_each_entry(chan, &device->channels, device_node) {1210 rc = __dma_async_device_channel_register(device, chan, NULL);1211 if (rc < 0)1212 goto err_out;1213 }1214 1215 mutex_lock(&dma_list_mutex);1216 /* take references on public channels */1217 if (dmaengine_ref_count && !dma_has_cap(DMA_PRIVATE, device->cap_mask))1218 list_for_each_entry(chan, &device->channels, device_node) {1219 /* if clients are already waiting for channels we need1220 * to take references on their behalf1221 */1222 if (dma_chan_get(chan) == -ENODEV) {1223 /* note we can only get here for the first1224 * channel as the remaining channels are1225 * guaranteed to get a reference1226 */1227 rc = -ENODEV;1228 mutex_unlock(&dma_list_mutex);1229 goto err_out;1230 }1231 }1232 list_add_tail_rcu(&device->global_node, &dma_device_list);1233 if (dma_has_cap(DMA_PRIVATE, device->cap_mask))1234 device->privatecnt++; /* Always private */1235 dma_channel_rebalance();1236 mutex_unlock(&dma_list_mutex);1237 1238 dmaengine_debug_register(device);1239 1240 return 0;1241 1242err_out:1243 /* if we never registered a channel just release the idr */1244 if (!device->chancnt) {1245 ida_free(&dma_ida, device->dev_id);1246 return rc;1247 }1248 1249 list_for_each_entry(chan, &device->channels, device_node) {1250 if (chan->local == NULL)1251 continue;1252 mutex_lock(&dma_list_mutex);1253 chan->dev->chan = NULL;1254 mutex_unlock(&dma_list_mutex);1255 device_unregister(&chan->dev->device);1256 free_percpu(chan->local);1257 }1258 return rc;1259}1260EXPORT_SYMBOL(dma_async_device_register);1261 1262/**1263 * dma_async_device_unregister - unregister a DMA device1264 * @device: pointer to &struct dma_device1265 *1266 * This routine is called by dma driver exit routines, dmaengine holds module1267 * references to prevent it being called while channels are in use.1268 */1269void dma_async_device_unregister(struct dma_device *device)1270{1271 struct dma_chan *chan, *n;1272 1273 dmaengine_debug_unregister(device);1274 1275 list_for_each_entry_safe(chan, n, &device->channels, device_node)1276 __dma_async_device_channel_unregister(device, chan);1277 1278 mutex_lock(&dma_list_mutex);1279 /*1280 * setting DMA_PRIVATE ensures the device being torn down will not1281 * be used in the channel_table1282 */1283 dma_cap_set(DMA_PRIVATE, device->cap_mask);1284 dma_channel_rebalance();1285 ida_free(&dma_ida, device->dev_id);1286 dma_device_put(device);1287 mutex_unlock(&dma_list_mutex);1288}1289EXPORT_SYMBOL(dma_async_device_unregister);1290 1291static void dmaenginem_async_device_unregister(void *device)1292{1293 dma_async_device_unregister(device);1294}1295 1296/**1297 * dmaenginem_async_device_register - registers DMA devices found1298 * @device: pointer to &struct dma_device1299 *1300 * The operation is managed and will be undone on driver detach.1301 */1302int dmaenginem_async_device_register(struct dma_device *device)1303{1304 int ret;1305 1306 ret = dma_async_device_register(device);1307 if (ret)1308 return ret;1309 1310 return devm_add_action_or_reset(device->dev, dmaenginem_async_device_unregister, device);1311}1312EXPORT_SYMBOL(dmaenginem_async_device_register);1313 1314struct dmaengine_unmap_pool {1315 struct kmem_cache *cache;1316 const char *name;1317 mempool_t *pool;1318 size_t size;1319};1320 1321#define __UNMAP_POOL(x) { .size = x, .name = "dmaengine-unmap-" __stringify(x) }1322static struct dmaengine_unmap_pool unmap_pool[] = {1323 __UNMAP_POOL(2),1324 #if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)1325 __UNMAP_POOL(16),1326 __UNMAP_POOL(128),1327 __UNMAP_POOL(256),1328 #endif1329};1330 1331static struct dmaengine_unmap_pool *__get_unmap_pool(int nr)1332{1333 int order = get_count_order(nr);1334 1335 switch (order) {1336 case 0 ... 1:1337 return &unmap_pool[0];1338#if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)1339 case 2 ... 4:1340 return &unmap_pool[1];1341 case 5 ... 7:1342 return &unmap_pool[2];1343 case 8:1344 return &unmap_pool[3];1345#endif1346 default:1347 BUG();1348 return NULL;1349 }1350}1351 1352static void dmaengine_unmap(struct kref *kref)1353{1354 struct dmaengine_unmap_data *unmap = container_of(kref, typeof(*unmap), kref);1355 struct device *dev = unmap->dev;1356 int cnt, i;1357 1358 cnt = unmap->to_cnt;1359 for (i = 0; i < cnt; i++)1360 dma_unmap_page(dev, unmap->addr[i], unmap->len,1361 DMA_TO_DEVICE);1362 cnt += unmap->from_cnt;1363 for (; i < cnt; i++)1364 dma_unmap_page(dev, unmap->addr[i], unmap->len,1365 DMA_FROM_DEVICE);1366 cnt += unmap->bidi_cnt;1367 for (; i < cnt; i++) {1368 if (unmap->addr[i] == 0)1369 continue;1370 dma_unmap_page(dev, unmap->addr[i], unmap->len,1371 DMA_BIDIRECTIONAL);1372 }1373 cnt = unmap->map_cnt;1374 mempool_free(unmap, __get_unmap_pool(cnt)->pool);1375}1376 1377void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap)1378{1379 if (unmap)1380 kref_put(&unmap->kref, dmaengine_unmap);1381}1382EXPORT_SYMBOL_GPL(dmaengine_unmap_put);1383 1384static void dmaengine_destroy_unmap_pool(void)1385{1386 int i;1387 1388 for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {1389 struct dmaengine_unmap_pool *p = &unmap_pool[i];1390 1391 mempool_destroy(p->pool);1392 p->pool = NULL;1393 kmem_cache_destroy(p->cache);1394 p->cache = NULL;1395 }1396}1397 1398static int __init dmaengine_init_unmap_pool(void)1399{1400 int i;1401 1402 for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {1403 struct dmaengine_unmap_pool *p = &unmap_pool[i];1404 size_t size;1405 1406 size = sizeof(struct dmaengine_unmap_data) +1407 sizeof(dma_addr_t) * p->size;1408 1409 p->cache = kmem_cache_create(p->name, size, 0,1410 SLAB_HWCACHE_ALIGN, NULL);1411 if (!p->cache)1412 break;1413 p->pool = mempool_create_slab_pool(1, p->cache);1414 if (!p->pool)1415 break;1416 }1417 1418 if (i == ARRAY_SIZE(unmap_pool))1419 return 0;1420 1421 dmaengine_destroy_unmap_pool();1422 return -ENOMEM;1423}1424 1425struct dmaengine_unmap_data *1426dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags)1427{1428 struct dmaengine_unmap_data *unmap;1429 1430 unmap = mempool_alloc(__get_unmap_pool(nr)->pool, flags);1431 if (!unmap)1432 return NULL;1433 1434 memset(unmap, 0, sizeof(*unmap));1435 kref_init(&unmap->kref);1436 unmap->dev = dev;1437 unmap->map_cnt = nr;1438 1439 return unmap;1440}1441EXPORT_SYMBOL(dmaengine_get_unmap_data);1442 1443void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,1444 struct dma_chan *chan)1445{1446 tx->chan = chan;1447 #ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH1448 spin_lock_init(&tx->lock);1449 #endif1450}1451EXPORT_SYMBOL(dma_async_tx_descriptor_init);1452 1453static inline int desc_check_and_set_metadata_mode(1454 struct dma_async_tx_descriptor *desc, enum dma_desc_metadata_mode mode)1455{1456 /* Make sure that the metadata mode is not mixed */1457 if (!desc->desc_metadata_mode) {1458 if (dmaengine_is_metadata_mode_supported(desc->chan, mode))1459 desc->desc_metadata_mode = mode;1460 else1461 return -ENOTSUPP;1462 } else if (desc->desc_metadata_mode != mode) {1463 return -EINVAL;1464 }1465 1466 return 0;1467}1468 1469int dmaengine_desc_attach_metadata(struct dma_async_tx_descriptor *desc,1470 void *data, size_t len)1471{1472 int ret;1473 1474 if (!desc)1475 return -EINVAL;1476 1477 ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_CLIENT);1478 if (ret)1479 return ret;1480 1481 if (!desc->metadata_ops || !desc->metadata_ops->attach)1482 return -ENOTSUPP;1483 1484 return desc->metadata_ops->attach(desc, data, len);1485}1486EXPORT_SYMBOL_GPL(dmaengine_desc_attach_metadata);1487 1488void *dmaengine_desc_get_metadata_ptr(struct dma_async_tx_descriptor *desc,1489 size_t *payload_len, size_t *max_len)1490{1491 int ret;1492 1493 if (!desc)1494 return ERR_PTR(-EINVAL);1495 1496 ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE);1497 if (ret)1498 return ERR_PTR(ret);1499 1500 if (!desc->metadata_ops || !desc->metadata_ops->get_ptr)1501 return ERR_PTR(-ENOTSUPP);1502 1503 return desc->metadata_ops->get_ptr(desc, payload_len, max_len);1504}1505EXPORT_SYMBOL_GPL(dmaengine_desc_get_metadata_ptr);1506 1507int dmaengine_desc_set_metadata_len(struct dma_async_tx_descriptor *desc,1508 size_t payload_len)1509{1510 int ret;1511 1512 if (!desc)1513 return -EINVAL;1514 1515 ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE);1516 if (ret)1517 return ret;1518 1519 if (!desc->metadata_ops || !desc->metadata_ops->set_len)1520 return -ENOTSUPP;1521 1522 return desc->metadata_ops->set_len(desc, payload_len);1523}1524EXPORT_SYMBOL_GPL(dmaengine_desc_set_metadata_len);1525 1526/**1527 * dma_wait_for_async_tx - spin wait for a transaction to complete1528 * @tx: in-flight transaction to wait on1529 */1530enum dma_status1531dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)1532{1533 unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);1534 1535 if (!tx)1536 return DMA_COMPLETE;1537 1538 while (tx->cookie == -EBUSY) {1539 if (time_after_eq(jiffies, dma_sync_wait_timeout)) {1540 dev_err(tx->chan->device->dev,1541 "%s timeout waiting for descriptor submission\n",1542 __func__);1543 return DMA_ERROR;1544 }1545 cpu_relax();1546 }1547 return dma_sync_wait(tx->chan, tx->cookie);1548}1549EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);1550 1551/**1552 * dma_run_dependencies - process dependent operations on the target channel1553 * @tx: transaction with dependencies1554 *1555 * Helper routine for DMA drivers to process (start) dependent operations1556 * on their target channel.1557 */1558void dma_run_dependencies(struct dma_async_tx_descriptor *tx)1559{1560 struct dma_async_tx_descriptor *dep = txd_next(tx);1561 struct dma_async_tx_descriptor *dep_next;1562 struct dma_chan *chan;1563 1564 if (!dep)1565 return;1566 1567 /* we'll submit tx->next now, so clear the link */1568 txd_clear_next(tx);1569 chan = dep->chan;1570 1571 /* keep submitting up until a channel switch is detected1572 * in that case we will be called again as a result of1573 * processing the interrupt from async_tx_channel_switch1574 */1575 for (; dep; dep = dep_next) {1576 txd_lock(dep);1577 txd_clear_parent(dep);1578 dep_next = txd_next(dep);1579 if (dep_next && dep_next->chan == chan)1580 txd_clear_next(dep); /* ->next will be submitted */1581 else1582 dep_next = NULL; /* submit current dep and terminate */1583 txd_unlock(dep);1584 1585 dep->tx_submit(dep);1586 }1587 1588 chan->device->device_issue_pending(chan);1589}1590EXPORT_SYMBOL_GPL(dma_run_dependencies);1591 1592static int __init dma_bus_init(void)1593{1594 int err = dmaengine_init_unmap_pool();1595 1596 if (err)1597 return err;1598 1599 err = class_register(&dma_devclass);1600 if (!err)1601 dmaengine_debugfs_init();1602 1603 return err;1604}1605arch_initcall(dma_bus_init);1606