1464 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * offload engine driver for the Marvell XOR engine4 * Copyright (C) 2007, 2008, Marvell International Ltd.5 */6 7#include <linux/init.h>8#include <linux/slab.h>9#include <linux/delay.h>10#include <linux/dma-mapping.h>11#include <linux/spinlock.h>12#include <linux/interrupt.h>13#include <linux/platform_device.h>14#include <linux/property.h>15#include <linux/memory.h>16#include <linux/clk.h>17#include <linux/of.h>18#include <linux/of_irq.h>19#include <linux/irqdomain.h>20#include <linux/cpumask.h>21#include <linux/platform_data/dma-mv_xor.h>22 23#include "dmaengine.h"24#include "mv_xor.h"25 26enum mv_xor_type {27 XOR_ORION,28 XOR_ARMADA_38X,29 XOR_ARMADA_37XX,30};31 32enum mv_xor_mode {33 XOR_MODE_IN_REG,34 XOR_MODE_IN_DESC,35};36 37static void mv_xor_issue_pending(struct dma_chan *chan);38 39#define to_mv_xor_chan(chan) \40 container_of(chan, struct mv_xor_chan, dmachan)41 42#define to_mv_xor_slot(tx) \43 container_of(tx, struct mv_xor_desc_slot, async_tx)44 45#define mv_chan_to_devp(chan) \46 ((chan)->dmadev.dev)47 48static void mv_desc_init(struct mv_xor_desc_slot *desc,49 dma_addr_t addr, u32 byte_count,50 enum dma_ctrl_flags flags)51{52 struct mv_xor_desc *hw_desc = desc->hw_desc;53 54 hw_desc->status = XOR_DESC_DMA_OWNED;55 hw_desc->phy_next_desc = 0;56 /* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */57 hw_desc->desc_command = (flags & DMA_PREP_INTERRUPT) ?58 XOR_DESC_EOD_INT_EN : 0;59 hw_desc->phy_dest_addr = addr;60 hw_desc->byte_count = byte_count;61}62 63static void mv_desc_set_mode(struct mv_xor_desc_slot *desc)64{65 struct mv_xor_desc *hw_desc = desc->hw_desc;66 67 switch (desc->type) {68 case DMA_XOR:69 case DMA_INTERRUPT:70 hw_desc->desc_command |= XOR_DESC_OPERATION_XOR;71 break;72 case DMA_MEMCPY:73 hw_desc->desc_command |= XOR_DESC_OPERATION_MEMCPY;74 break;75 default:76 BUG();77 return;78 }79}80 81static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc,82 u32 next_desc_addr)83{84 struct mv_xor_desc *hw_desc = desc->hw_desc;85 BUG_ON(hw_desc->phy_next_desc);86 hw_desc->phy_next_desc = next_desc_addr;87}88 89static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc,90 int index, dma_addr_t addr)91{92 struct mv_xor_desc *hw_desc = desc->hw_desc;93 hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr;94 if (desc->type == DMA_XOR)95 hw_desc->desc_command |= (1 << index);96}97 98static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan)99{100 return readl_relaxed(XOR_CURR_DESC(chan));101}102 103static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan,104 u32 next_desc_addr)105{106 writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan));107}108 109static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan)110{111 u32 val = readl_relaxed(XOR_INTR_MASK(chan));112 val |= XOR_INTR_MASK_VALUE << (chan->idx * 16);113 writel_relaxed(val, XOR_INTR_MASK(chan));114}115 116static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan)117{118 u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan));119 intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF;120 return intr_cause;121}122 123static void mv_chan_clear_eoc_cause(struct mv_xor_chan *chan)124{125 u32 val;126 127 val = XOR_INT_END_OF_DESC | XOR_INT_END_OF_CHAIN | XOR_INT_STOPPED;128 val = ~(val << (chan->idx * 16));129 dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val);130 writel_relaxed(val, XOR_INTR_CAUSE(chan));131}132 133static void mv_chan_clear_err_status(struct mv_xor_chan *chan)134{135 u32 val = 0xFFFF0000 >> (chan->idx * 16);136 writel_relaxed(val, XOR_INTR_CAUSE(chan));137}138 139static void mv_chan_set_mode(struct mv_xor_chan *chan,140 u32 op_mode)141{142 u32 config = readl_relaxed(XOR_CONFIG(chan));143 144 config &= ~0x7;145 config |= op_mode;146 147#if defined(__BIG_ENDIAN)148 config |= XOR_DESCRIPTOR_SWAP;149#else150 config &= ~XOR_DESCRIPTOR_SWAP;151#endif152 153 writel_relaxed(config, XOR_CONFIG(chan));154}155 156static void mv_chan_activate(struct mv_xor_chan *chan)157{158 dev_dbg(mv_chan_to_devp(chan), " activate chan.\n");159 160 /* writel ensures all descriptors are flushed before activation */161 writel(BIT(0), XOR_ACTIVATION(chan));162}163 164static char mv_chan_is_busy(struct mv_xor_chan *chan)165{166 u32 state = readl_relaxed(XOR_ACTIVATION(chan));167 168 state = (state >> 4) & 0x3;169 170 return (state == 1) ? 1 : 0;171}172 173/*174 * mv_chan_start_new_chain - program the engine to operate on new175 * chain headed by sw_desc176 * Caller must hold &mv_chan->lock while calling this function177 */178static void mv_chan_start_new_chain(struct mv_xor_chan *mv_chan,179 struct mv_xor_desc_slot *sw_desc)180{181 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: sw_desc %p\n",182 __func__, __LINE__, sw_desc);183 184 /* set the hardware chain */185 mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys);186 187 mv_chan->pending++;188 mv_xor_issue_pending(&mv_chan->dmachan);189}190 191static dma_cookie_t192mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc,193 struct mv_xor_chan *mv_chan,194 dma_cookie_t cookie)195{196 BUG_ON(desc->async_tx.cookie < 0);197 198 if (desc->async_tx.cookie > 0) {199 cookie = desc->async_tx.cookie;200 201 dma_descriptor_unmap(&desc->async_tx);202 /* call the callback (must not sleep or submit new203 * operations to this channel)204 */205 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);206 }207 208 /* run dependent operations */209 dma_run_dependencies(&desc->async_tx);210 211 return cookie;212}213 214static int215mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan)216{217 struct mv_xor_desc_slot *iter, *_iter;218 219 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);220 list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,221 node) {222 223 if (async_tx_test_ack(&iter->async_tx)) {224 list_move_tail(&iter->node, &mv_chan->free_slots);225 if (!list_empty(&iter->sg_tx_list)) {226 list_splice_tail_init(&iter->sg_tx_list,227 &mv_chan->free_slots);228 }229 }230 }231 return 0;232}233 234static int235mv_desc_clean_slot(struct mv_xor_desc_slot *desc,236 struct mv_xor_chan *mv_chan)237{238 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: desc %p flags %d\n",239 __func__, __LINE__, desc, desc->async_tx.flags);240 241 /* the client is allowed to attach dependent operations242 * until 'ack' is set243 */244 if (!async_tx_test_ack(&desc->async_tx)) {245 /* move this slot to the completed_slots */246 list_move_tail(&desc->node, &mv_chan->completed_slots);247 if (!list_empty(&desc->sg_tx_list)) {248 list_splice_tail_init(&desc->sg_tx_list,249 &mv_chan->completed_slots);250 }251 } else {252 list_move_tail(&desc->node, &mv_chan->free_slots);253 if (!list_empty(&desc->sg_tx_list)) {254 list_splice_tail_init(&desc->sg_tx_list,255 &mv_chan->free_slots);256 }257 }258 259 return 0;260}261 262/* This function must be called with the mv_xor_chan spinlock held */263static void mv_chan_slot_cleanup(struct mv_xor_chan *mv_chan)264{265 struct mv_xor_desc_slot *iter, *_iter;266 dma_cookie_t cookie = 0;267 int busy = mv_chan_is_busy(mv_chan);268 u32 current_desc = mv_chan_get_current_desc(mv_chan);269 int current_cleaned = 0;270 struct mv_xor_desc *hw_desc;271 272 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);273 dev_dbg(mv_chan_to_devp(mv_chan), "current_desc %x\n", current_desc);274 mv_chan_clean_completed_slots(mv_chan);275 276 /* free completed slots from the chain starting with277 * the oldest descriptor278 */279 280 list_for_each_entry_safe(iter, _iter, &mv_chan->chain,281 node) {282 283 /* clean finished descriptors */284 hw_desc = iter->hw_desc;285 if (hw_desc->status & XOR_DESC_SUCCESS) {286 cookie = mv_desc_run_tx_complete_actions(iter, mv_chan,287 cookie);288 289 /* done processing desc, clean slot */290 mv_desc_clean_slot(iter, mv_chan);291 292 /* break if we did cleaned the current */293 if (iter->async_tx.phys == current_desc) {294 current_cleaned = 1;295 break;296 }297 } else {298 if (iter->async_tx.phys == current_desc) {299 current_cleaned = 0;300 break;301 }302 }303 }304 305 if ((busy == 0) && !list_empty(&mv_chan->chain)) {306 if (current_cleaned) {307 /*308 * current descriptor cleaned and removed, run309 * from list head310 */311 iter = list_entry(mv_chan->chain.next,312 struct mv_xor_desc_slot,313 node);314 mv_chan_start_new_chain(mv_chan, iter);315 } else {316 if (!list_is_last(&iter->node, &mv_chan->chain)) {317 /*318 * descriptors are still waiting after319 * current, trigger them320 */321 iter = list_entry(iter->node.next,322 struct mv_xor_desc_slot,323 node);324 mv_chan_start_new_chain(mv_chan, iter);325 } else {326 /*327 * some descriptors are still waiting328 * to be cleaned329 */330 tasklet_schedule(&mv_chan->irq_tasklet);331 }332 }333 }334 335 if (cookie > 0)336 mv_chan->dmachan.completed_cookie = cookie;337}338 339static void mv_xor_tasklet(struct tasklet_struct *t)340{341 struct mv_xor_chan *chan = from_tasklet(chan, t, irq_tasklet);342 343 spin_lock(&chan->lock);344 mv_chan_slot_cleanup(chan);345 spin_unlock(&chan->lock);346}347 348static struct mv_xor_desc_slot *349mv_chan_alloc_slot(struct mv_xor_chan *mv_chan)350{351 struct mv_xor_desc_slot *iter;352 353 spin_lock_bh(&mv_chan->lock);354 355 if (!list_empty(&mv_chan->free_slots)) {356 iter = list_first_entry(&mv_chan->free_slots,357 struct mv_xor_desc_slot,358 node);359 360 list_move_tail(&iter->node, &mv_chan->allocated_slots);361 362 spin_unlock_bh(&mv_chan->lock);363 364 /* pre-ack descriptor */365 async_tx_ack(&iter->async_tx);366 iter->async_tx.cookie = -EBUSY;367 368 return iter;369 370 }371 372 spin_unlock_bh(&mv_chan->lock);373 374 /* try to free some slots if the allocation fails */375 tasklet_schedule(&mv_chan->irq_tasklet);376 377 return NULL;378}379 380/************************ DMA engine API functions ****************************/381static dma_cookie_t382mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)383{384 struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx);385 struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan);386 struct mv_xor_desc_slot *old_chain_tail;387 dma_cookie_t cookie;388 int new_hw_chain = 1;389 390 dev_dbg(mv_chan_to_devp(mv_chan),391 "%s sw_desc %p: async_tx %p\n",392 __func__, sw_desc, &sw_desc->async_tx);393 394 spin_lock_bh(&mv_chan->lock);395 cookie = dma_cookie_assign(tx);396 397 if (list_empty(&mv_chan->chain))398 list_move_tail(&sw_desc->node, &mv_chan->chain);399 else {400 new_hw_chain = 0;401 402 old_chain_tail = list_entry(mv_chan->chain.prev,403 struct mv_xor_desc_slot,404 node);405 list_move_tail(&sw_desc->node, &mv_chan->chain);406 407 dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n",408 &old_chain_tail->async_tx.phys);409 410 /* fix up the hardware chain */411 mv_desc_set_next_desc(old_chain_tail, sw_desc->async_tx.phys);412 413 /* if the channel is not busy */414 if (!mv_chan_is_busy(mv_chan)) {415 u32 current_desc = mv_chan_get_current_desc(mv_chan);416 /*417 * and the current desc is the end of the chain before418 * the append, then we need to start the channel419 */420 if (current_desc == old_chain_tail->async_tx.phys)421 new_hw_chain = 1;422 }423 }424 425 if (new_hw_chain)426 mv_chan_start_new_chain(mv_chan, sw_desc);427 428 spin_unlock_bh(&mv_chan->lock);429 430 return cookie;431}432 433/* returns the number of allocated descriptors */434static int mv_xor_alloc_chan_resources(struct dma_chan *chan)435{436 void *virt_desc;437 dma_addr_t dma_desc;438 int idx;439 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);440 struct mv_xor_desc_slot *slot = NULL;441 int num_descs_in_pool = MV_XOR_POOL_SIZE/MV_XOR_SLOT_SIZE;442 443 /* Allocate descriptor slots */444 idx = mv_chan->slots_allocated;445 while (idx < num_descs_in_pool) {446 slot = kzalloc(sizeof(*slot), GFP_KERNEL);447 if (!slot) {448 dev_info(mv_chan_to_devp(mv_chan),449 "channel only initialized %d descriptor slots",450 idx);451 break;452 }453 virt_desc = mv_chan->dma_desc_pool_virt;454 slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE;455 456 dma_async_tx_descriptor_init(&slot->async_tx, chan);457 slot->async_tx.tx_submit = mv_xor_tx_submit;458 INIT_LIST_HEAD(&slot->node);459 INIT_LIST_HEAD(&slot->sg_tx_list);460 dma_desc = mv_chan->dma_desc_pool;461 slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE;462 slot->idx = idx++;463 464 spin_lock_bh(&mv_chan->lock);465 mv_chan->slots_allocated = idx;466 list_add_tail(&slot->node, &mv_chan->free_slots);467 spin_unlock_bh(&mv_chan->lock);468 }469 470 dev_dbg(mv_chan_to_devp(mv_chan),471 "allocated %d descriptor slots\n",472 mv_chan->slots_allocated);473 474 return mv_chan->slots_allocated ? : -ENOMEM;475}476 477/*478 * Check if source or destination is an PCIe/IO address (non-SDRAM) and add479 * a new MBus window if necessary. Use a cache for these check so that480 * the MMIO mapped registers don't have to be accessed for this check481 * to speed up this process.482 */483static int mv_xor_add_io_win(struct mv_xor_chan *mv_chan, u32 addr)484{485 struct mv_xor_device *xordev = mv_chan->xordev;486 void __iomem *base = mv_chan->mmr_high_base;487 u32 win_enable;488 u32 size;489 u8 target, attr;490 int ret;491 int i;492 493 /* Nothing needs to get done for the Armada 3700 */494 if (xordev->xor_type == XOR_ARMADA_37XX)495 return 0;496 497 /*498 * Loop over the cached windows to check, if the requested area499 * is already mapped. If this the case, nothing needs to be done500 * and we can return.501 */502 for (i = 0; i < WINDOW_COUNT; i++) {503 if (addr >= xordev->win_start[i] &&504 addr <= xordev->win_end[i]) {505 /* Window is already mapped */506 return 0;507 }508 }509 510 /*511 * The window is not mapped, so we need to create the new mapping512 */513 514 /* If no IO window is found that addr has to be located in SDRAM */515 ret = mvebu_mbus_get_io_win_info(addr, &size, &target, &attr);516 if (ret < 0)517 return 0;518 519 /*520 * Mask the base addr 'addr' according to 'size' read back from the521 * MBus window. Otherwise we might end up with an address located522 * somewhere in the middle of this area here.523 */524 size -= 1;525 addr &= ~size;526 527 /*528 * Reading one of both enabled register is enough, as they are always529 * programmed to the identical values530 */531 win_enable = readl(base + WINDOW_BAR_ENABLE(0));532 533 /* Set 'i' to the first free window to write the new values to */534 i = ffs(~win_enable) - 1;535 if (i >= WINDOW_COUNT)536 return -ENOMEM;537 538 writel((addr & 0xffff0000) | (attr << 8) | target,539 base + WINDOW_BASE(i));540 writel(size & 0xffff0000, base + WINDOW_SIZE(i));541 542 /* Fill the caching variables for later use */543 xordev->win_start[i] = addr;544 xordev->win_end[i] = addr + size;545 546 win_enable |= (1 << i);547 win_enable |= 3 << (16 + (2 * i));548 writel(win_enable, base + WINDOW_BAR_ENABLE(0));549 writel(win_enable, base + WINDOW_BAR_ENABLE(1));550 551 return 0;552}553 554static struct dma_async_tx_descriptor *555mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,556 unsigned int src_cnt, size_t len, unsigned long flags)557{558 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);559 struct mv_xor_desc_slot *sw_desc;560 int ret;561 562 if (unlikely(len < MV_XOR_MIN_BYTE_COUNT))563 return NULL;564 565 BUG_ON(len > MV_XOR_MAX_BYTE_COUNT);566 567 dev_dbg(mv_chan_to_devp(mv_chan),568 "%s src_cnt: %d len: %zu dest %pad flags: %ld\n",569 __func__, src_cnt, len, &dest, flags);570 571 /* Check if a new window needs to get added for 'dest' */572 ret = mv_xor_add_io_win(mv_chan, dest);573 if (ret)574 return NULL;575 576 sw_desc = mv_chan_alloc_slot(mv_chan);577 if (sw_desc) {578 sw_desc->type = DMA_XOR;579 sw_desc->async_tx.flags = flags;580 mv_desc_init(sw_desc, dest, len, flags);581 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)582 mv_desc_set_mode(sw_desc);583 while (src_cnt--) {584 /* Check if a new window needs to get added for 'src' */585 ret = mv_xor_add_io_win(mv_chan, src[src_cnt]);586 if (ret)587 return NULL;588 mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]);589 }590 }591 592 dev_dbg(mv_chan_to_devp(mv_chan),593 "%s sw_desc %p async_tx %p \n",594 __func__, sw_desc, &sw_desc->async_tx);595 return sw_desc ? &sw_desc->async_tx : NULL;596}597 598static struct dma_async_tx_descriptor *599mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,600 size_t len, unsigned long flags)601{602 /*603 * A MEMCPY operation is identical to an XOR operation with only604 * a single source address.605 */606 return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);607}608 609static struct dma_async_tx_descriptor *610mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)611{612 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);613 dma_addr_t src, dest;614 size_t len;615 616 src = mv_chan->dummy_src_addr;617 dest = mv_chan->dummy_dst_addr;618 len = MV_XOR_MIN_BYTE_COUNT;619 620 /*621 * We implement the DMA_INTERRUPT operation as a minimum sized622 * XOR operation with a single dummy source address.623 */624 return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);625}626 627static void mv_xor_free_chan_resources(struct dma_chan *chan)628{629 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);630 struct mv_xor_desc_slot *iter, *_iter;631 int in_use_descs = 0;632 633 spin_lock_bh(&mv_chan->lock);634 635 mv_chan_slot_cleanup(mv_chan);636 637 list_for_each_entry_safe(iter, _iter, &mv_chan->chain,638 node) {639 in_use_descs++;640 list_move_tail(&iter->node, &mv_chan->free_slots);641 }642 list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,643 node) {644 in_use_descs++;645 list_move_tail(&iter->node, &mv_chan->free_slots);646 }647 list_for_each_entry_safe(iter, _iter, &mv_chan->allocated_slots,648 node) {649 in_use_descs++;650 list_move_tail(&iter->node, &mv_chan->free_slots);651 }652 list_for_each_entry_safe_reverse(653 iter, _iter, &mv_chan->free_slots, node) {654 list_del(&iter->node);655 kfree(iter);656 mv_chan->slots_allocated--;657 }658 659 dev_dbg(mv_chan_to_devp(mv_chan), "%s slots_allocated %d\n",660 __func__, mv_chan->slots_allocated);661 spin_unlock_bh(&mv_chan->lock);662 663 if (in_use_descs)664 dev_err(mv_chan_to_devp(mv_chan),665 "freeing %d in use descriptors!\n", in_use_descs);666}667 668/**669 * mv_xor_status - poll the status of an XOR transaction670 * @chan: XOR channel handle671 * @cookie: XOR transaction identifier672 * @txstate: XOR transactions state holder (or NULL)673 */674static enum dma_status mv_xor_status(struct dma_chan *chan,675 dma_cookie_t cookie,676 struct dma_tx_state *txstate)677{678 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);679 enum dma_status ret;680 681 ret = dma_cookie_status(chan, cookie, txstate);682 if (ret == DMA_COMPLETE)683 return ret;684 685 spin_lock_bh(&mv_chan->lock);686 mv_chan_slot_cleanup(mv_chan);687 spin_unlock_bh(&mv_chan->lock);688 689 return dma_cookie_status(chan, cookie, txstate);690}691 692static void mv_chan_dump_regs(struct mv_xor_chan *chan)693{694 u32 val;695 696 val = readl_relaxed(XOR_CONFIG(chan));697 dev_err(mv_chan_to_devp(chan), "config 0x%08x\n", val);698 699 val = readl_relaxed(XOR_ACTIVATION(chan));700 dev_err(mv_chan_to_devp(chan), "activation 0x%08x\n", val);701 702 val = readl_relaxed(XOR_INTR_CAUSE(chan));703 dev_err(mv_chan_to_devp(chan), "intr cause 0x%08x\n", val);704 705 val = readl_relaxed(XOR_INTR_MASK(chan));706 dev_err(mv_chan_to_devp(chan), "intr mask 0x%08x\n", val);707 708 val = readl_relaxed(XOR_ERROR_CAUSE(chan));709 dev_err(mv_chan_to_devp(chan), "error cause 0x%08x\n", val);710 711 val = readl_relaxed(XOR_ERROR_ADDR(chan));712 dev_err(mv_chan_to_devp(chan), "error addr 0x%08x\n", val);713}714 715static void mv_chan_err_interrupt_handler(struct mv_xor_chan *chan,716 u32 intr_cause)717{718 if (intr_cause & XOR_INT_ERR_DECODE) {719 dev_dbg(mv_chan_to_devp(chan), "ignoring address decode error\n");720 return;721 }722 723 dev_err(mv_chan_to_devp(chan), "error on chan %d. intr cause 0x%08x\n",724 chan->idx, intr_cause);725 726 mv_chan_dump_regs(chan);727 WARN_ON(1);728}729 730static irqreturn_t mv_xor_interrupt_handler(int irq, void *data)731{732 struct mv_xor_chan *chan = data;733 u32 intr_cause = mv_chan_get_intr_cause(chan);734 735 dev_dbg(mv_chan_to_devp(chan), "intr cause %x\n", intr_cause);736 737 if (intr_cause & XOR_INTR_ERRORS)738 mv_chan_err_interrupt_handler(chan, intr_cause);739 740 tasklet_schedule(&chan->irq_tasklet);741 742 mv_chan_clear_eoc_cause(chan);743 744 return IRQ_HANDLED;745}746 747static void mv_xor_issue_pending(struct dma_chan *chan)748{749 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);750 751 if (mv_chan->pending >= MV_XOR_THRESHOLD) {752 mv_chan->pending = 0;753 mv_chan_activate(mv_chan);754 }755}756 757/*758 * Perform a transaction to verify the HW works.759 */760 761static int mv_chan_memcpy_self_test(struct mv_xor_chan *mv_chan)762{763 int i, ret;764 void *src, *dest;765 dma_addr_t src_dma, dest_dma;766 struct dma_chan *dma_chan;767 dma_cookie_t cookie;768 struct dma_async_tx_descriptor *tx;769 struct dmaengine_unmap_data *unmap;770 int err = 0;771 772 src = kmalloc(PAGE_SIZE, GFP_KERNEL);773 if (!src)774 return -ENOMEM;775 776 dest = kzalloc(PAGE_SIZE, GFP_KERNEL);777 if (!dest) {778 kfree(src);779 return -ENOMEM;780 }781 782 /* Fill in src buffer */783 for (i = 0; i < PAGE_SIZE; i++)784 ((u8 *) src)[i] = (u8)i;785 786 dma_chan = &mv_chan->dmachan;787 if (mv_xor_alloc_chan_resources(dma_chan) < 1) {788 err = -ENODEV;789 goto out;790 }791 792 unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL);793 if (!unmap) {794 err = -ENOMEM;795 goto free_resources;796 }797 798 src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src),799 offset_in_page(src), PAGE_SIZE,800 DMA_TO_DEVICE);801 unmap->addr[0] = src_dma;802 803 ret = dma_mapping_error(dma_chan->device->dev, src_dma);804 if (ret) {805 err = -ENOMEM;806 goto free_resources;807 }808 unmap->to_cnt = 1;809 810 dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest),811 offset_in_page(dest), PAGE_SIZE,812 DMA_FROM_DEVICE);813 unmap->addr[1] = dest_dma;814 815 ret = dma_mapping_error(dma_chan->device->dev, dest_dma);816 if (ret) {817 err = -ENOMEM;818 goto free_resources;819 }820 unmap->from_cnt = 1;821 unmap->len = PAGE_SIZE;822 823 tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma,824 PAGE_SIZE, 0);825 if (!tx) {826 dev_err(dma_chan->device->dev,827 "Self-test cannot prepare operation, disabling\n");828 err = -ENODEV;829 goto free_resources;830 }831 832 cookie = mv_xor_tx_submit(tx);833 if (dma_submit_error(cookie)) {834 dev_err(dma_chan->device->dev,835 "Self-test submit error, disabling\n");836 err = -ENODEV;837 goto free_resources;838 }839 840 mv_xor_issue_pending(dma_chan);841 async_tx_ack(tx);842 msleep(1);843 844 if (mv_xor_status(dma_chan, cookie, NULL) !=845 DMA_COMPLETE) {846 dev_err(dma_chan->device->dev,847 "Self-test copy timed out, disabling\n");848 err = -ENODEV;849 goto free_resources;850 }851 852 dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,853 PAGE_SIZE, DMA_FROM_DEVICE);854 if (memcmp(src, dest, PAGE_SIZE)) {855 dev_err(dma_chan->device->dev,856 "Self-test copy failed compare, disabling\n");857 err = -ENODEV;858 goto free_resources;859 }860 861free_resources:862 dmaengine_unmap_put(unmap);863 mv_xor_free_chan_resources(dma_chan);864out:865 kfree(src);866 kfree(dest);867 return err;868}869 870#define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */871static int872mv_chan_xor_self_test(struct mv_xor_chan *mv_chan)873{874 int i, src_idx, ret;875 struct page *dest;876 struct page *xor_srcs[MV_XOR_NUM_SRC_TEST];877 dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST];878 dma_addr_t dest_dma;879 struct dma_async_tx_descriptor *tx;880 struct dmaengine_unmap_data *unmap;881 struct dma_chan *dma_chan;882 dma_cookie_t cookie;883 u8 cmp_byte = 0;884 u32 cmp_word;885 int err = 0;886 int src_count = MV_XOR_NUM_SRC_TEST;887 888 for (src_idx = 0; src_idx < src_count; src_idx++) {889 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);890 if (!xor_srcs[src_idx]) {891 while (src_idx--)892 __free_page(xor_srcs[src_idx]);893 return -ENOMEM;894 }895 }896 897 dest = alloc_page(GFP_KERNEL);898 if (!dest) {899 while (src_idx--)900 __free_page(xor_srcs[src_idx]);901 return -ENOMEM;902 }903 904 /* Fill in src buffers */905 for (src_idx = 0; src_idx < src_count; src_idx++) {906 u8 *ptr = page_address(xor_srcs[src_idx]);907 for (i = 0; i < PAGE_SIZE; i++)908 ptr[i] = (1 << src_idx);909 }910 911 for (src_idx = 0; src_idx < src_count; src_idx++)912 cmp_byte ^= (u8) (1 << src_idx);913 914 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |915 (cmp_byte << 8) | cmp_byte;916 917 memset(page_address(dest), 0, PAGE_SIZE);918 919 dma_chan = &mv_chan->dmachan;920 if (mv_xor_alloc_chan_resources(dma_chan) < 1) {921 err = -ENODEV;922 goto out;923 }924 925 unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1,926 GFP_KERNEL);927 if (!unmap) {928 err = -ENOMEM;929 goto free_resources;930 }931 932 /* test xor */933 for (i = 0; i < src_count; i++) {934 unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],935 0, PAGE_SIZE, DMA_TO_DEVICE);936 dma_srcs[i] = unmap->addr[i];937 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[i]);938 if (ret) {939 err = -ENOMEM;940 goto free_resources;941 }942 unmap->to_cnt++;943 }944 945 unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE,946 DMA_FROM_DEVICE);947 dest_dma = unmap->addr[src_count];948 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[src_count]);949 if (ret) {950 err = -ENOMEM;951 goto free_resources;952 }953 unmap->from_cnt = 1;954 unmap->len = PAGE_SIZE;955 956 tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs,957 src_count, PAGE_SIZE, 0);958 if (!tx) {959 dev_err(dma_chan->device->dev,960 "Self-test cannot prepare operation, disabling\n");961 err = -ENODEV;962 goto free_resources;963 }964 965 cookie = mv_xor_tx_submit(tx);966 if (dma_submit_error(cookie)) {967 dev_err(dma_chan->device->dev,968 "Self-test submit error, disabling\n");969 err = -ENODEV;970 goto free_resources;971 }972 973 mv_xor_issue_pending(dma_chan);974 async_tx_ack(tx);975 msleep(8);976 977 if (mv_xor_status(dma_chan, cookie, NULL) !=978 DMA_COMPLETE) {979 dev_err(dma_chan->device->dev,980 "Self-test xor timed out, disabling\n");981 err = -ENODEV;982 goto free_resources;983 }984 985 dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,986 PAGE_SIZE, DMA_FROM_DEVICE);987 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {988 u32 *ptr = page_address(dest);989 if (ptr[i] != cmp_word) {990 dev_err(dma_chan->device->dev,991 "Self-test xor failed compare, disabling. index %d, data %x, expected %x\n",992 i, ptr[i], cmp_word);993 err = -ENODEV;994 goto free_resources;995 }996 }997 998free_resources:999 dmaengine_unmap_put(unmap);1000 mv_xor_free_chan_resources(dma_chan);1001out:1002 src_idx = src_count;1003 while (src_idx--)1004 __free_page(xor_srcs[src_idx]);1005 __free_page(dest);1006 return err;1007}1008 1009static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan)1010{1011 struct dma_chan *chan, *_chan;1012 struct device *dev = mv_chan->dmadev.dev;1013 1014 dma_async_device_unregister(&mv_chan->dmadev);1015 1016 dma_free_coherent(dev, MV_XOR_POOL_SIZE,1017 mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);1018 dma_unmap_single(dev, mv_chan->dummy_src_addr,1019 MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);1020 dma_unmap_single(dev, mv_chan->dummy_dst_addr,1021 MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);1022 1023 list_for_each_entry_safe(chan, _chan, &mv_chan->dmadev.channels,1024 device_node) {1025 list_del(&chan->device_node);1026 }1027 1028 free_irq(mv_chan->irq, mv_chan);1029 1030 return 0;1031}1032 1033static struct mv_xor_chan *1034mv_xor_channel_add(struct mv_xor_device *xordev,1035 struct platform_device *pdev,1036 int idx, dma_cap_mask_t cap_mask, int irq)1037{1038 int ret = 0;1039 struct mv_xor_chan *mv_chan;1040 struct dma_device *dma_dev;1041 1042 mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL);1043 if (!mv_chan)1044 return ERR_PTR(-ENOMEM);1045 1046 mv_chan->idx = idx;1047 mv_chan->irq = irq;1048 if (xordev->xor_type == XOR_ORION)1049 mv_chan->op_in_desc = XOR_MODE_IN_REG;1050 else1051 mv_chan->op_in_desc = XOR_MODE_IN_DESC;1052 1053 dma_dev = &mv_chan->dmadev;1054 dma_dev->dev = &pdev->dev;1055 mv_chan->xordev = xordev;1056 1057 /*1058 * These source and destination dummy buffers are used to implement1059 * a DMA_INTERRUPT operation as a minimum-sized XOR operation.1060 * Hence, we only need to map the buffers at initialization-time.1061 */1062 mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev,1063 mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);1064 mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev,1065 mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);1066 1067 /* allocate coherent memory for hardware descriptors1068 * note: writecombine gives slightly better performance, but1069 * requires that we explicitly flush the writes1070 */1071 mv_chan->dma_desc_pool_virt =1072 dma_alloc_wc(&pdev->dev, MV_XOR_POOL_SIZE, &mv_chan->dma_desc_pool,1073 GFP_KERNEL);1074 if (!mv_chan->dma_desc_pool_virt)1075 return ERR_PTR(-ENOMEM);1076 1077 /* discover transaction capabilities from the platform data */1078 dma_dev->cap_mask = cap_mask;1079 1080 INIT_LIST_HEAD(&dma_dev->channels);1081 1082 /* set base routines */1083 dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources;1084 dma_dev->device_free_chan_resources = mv_xor_free_chan_resources;1085 dma_dev->device_tx_status = mv_xor_status;1086 dma_dev->device_issue_pending = mv_xor_issue_pending;1087 1088 /* set prep routines based on capability */1089 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))1090 dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt;1091 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))1092 dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy;1093 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {1094 dma_dev->max_xor = 8;1095 dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;1096 }1097 1098 mv_chan->mmr_base = xordev->xor_base;1099 mv_chan->mmr_high_base = xordev->xor_high_base;1100 tasklet_setup(&mv_chan->irq_tasklet, mv_xor_tasklet);1101 1102 /* clear errors before enabling interrupts */1103 mv_chan_clear_err_status(mv_chan);1104 1105 ret = request_irq(mv_chan->irq, mv_xor_interrupt_handler,1106 0, dev_name(&pdev->dev), mv_chan);1107 if (ret)1108 goto err_free_dma;1109 1110 mv_chan_unmask_interrupts(mv_chan);1111 1112 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)1113 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_IN_DESC);1114 else1115 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_XOR);1116 1117 spin_lock_init(&mv_chan->lock);1118 INIT_LIST_HEAD(&mv_chan->chain);1119 INIT_LIST_HEAD(&mv_chan->completed_slots);1120 INIT_LIST_HEAD(&mv_chan->free_slots);1121 INIT_LIST_HEAD(&mv_chan->allocated_slots);1122 mv_chan->dmachan.device = dma_dev;1123 dma_cookie_init(&mv_chan->dmachan);1124 1125 list_add_tail(&mv_chan->dmachan.device_node, &dma_dev->channels);1126 1127 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {1128 ret = mv_chan_memcpy_self_test(mv_chan);1129 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);1130 if (ret)1131 goto err_free_irq;1132 }1133 1134 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {1135 ret = mv_chan_xor_self_test(mv_chan);1136 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);1137 if (ret)1138 goto err_free_irq;1139 }1140 1141 dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n",1142 mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode",1143 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",1144 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",1145 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");1146 1147 ret = dma_async_device_register(dma_dev);1148 if (ret)1149 goto err_free_irq;1150 1151 return mv_chan;1152 1153err_free_irq:1154 free_irq(mv_chan->irq, mv_chan);1155err_free_dma:1156 dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE,1157 mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);1158 return ERR_PTR(ret);1159}1160 1161static void1162mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,1163 const struct mbus_dram_target_info *dram)1164{1165 void __iomem *base = xordev->xor_high_base;1166 u32 win_enable = 0;1167 int i;1168 1169 for (i = 0; i < 8; i++) {1170 writel(0, base + WINDOW_BASE(i));1171 writel(0, base + WINDOW_SIZE(i));1172 if (i < 4)1173 writel(0, base + WINDOW_REMAP_HIGH(i));1174 }1175 1176 for (i = 0; i < dram->num_cs; i++) {1177 const struct mbus_dram_window *cs = dram->cs + i;1178 1179 writel((cs->base & 0xffff0000) |1180 (cs->mbus_attr << 8) |1181 dram->mbus_dram_target_id, base + WINDOW_BASE(i));1182 writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i));1183 1184 /* Fill the caching variables for later use */1185 xordev->win_start[i] = cs->base;1186 xordev->win_end[i] = cs->base + cs->size - 1;1187 1188 win_enable |= (1 << i);1189 win_enable |= 3 << (16 + (2 * i));1190 }1191 1192 writel(win_enable, base + WINDOW_BAR_ENABLE(0));1193 writel(win_enable, base + WINDOW_BAR_ENABLE(1));1194 writel(0, base + WINDOW_OVERRIDE_CTRL(0));1195 writel(0, base + WINDOW_OVERRIDE_CTRL(1));1196}1197 1198static void1199mv_xor_conf_mbus_windows_a3700(struct mv_xor_device *xordev)1200{1201 void __iomem *base = xordev->xor_high_base;1202 u32 win_enable = 0;1203 int i;1204 1205 for (i = 0; i < 8; i++) {1206 writel(0, base + WINDOW_BASE(i));1207 writel(0, base + WINDOW_SIZE(i));1208 if (i < 4)1209 writel(0, base + WINDOW_REMAP_HIGH(i));1210 }1211 /*1212 * For Armada3700 open default 4GB Mbus window. The dram1213 * related configuration are done at AXIS level.1214 */1215 writel(0xffff0000, base + WINDOW_SIZE(0));1216 win_enable |= 1;1217 win_enable |= 3 << 16;1218 1219 writel(win_enable, base + WINDOW_BAR_ENABLE(0));1220 writel(win_enable, base + WINDOW_BAR_ENABLE(1));1221 writel(0, base + WINDOW_OVERRIDE_CTRL(0));1222 writel(0, base + WINDOW_OVERRIDE_CTRL(1));1223}1224 1225/*1226 * Since this XOR driver is basically used only for RAID5, we don't1227 * need to care about synchronizing ->suspend with DMA activity,1228 * because the DMA engine will naturally be quiet due to the block1229 * devices being suspended.1230 */1231static int mv_xor_suspend(struct platform_device *pdev, pm_message_t state)1232{1233 struct mv_xor_device *xordev = platform_get_drvdata(pdev);1234 int i;1235 1236 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {1237 struct mv_xor_chan *mv_chan = xordev->channels[i];1238 1239 if (!mv_chan)1240 continue;1241 1242 mv_chan->saved_config_reg =1243 readl_relaxed(XOR_CONFIG(mv_chan));1244 mv_chan->saved_int_mask_reg =1245 readl_relaxed(XOR_INTR_MASK(mv_chan));1246 }1247 1248 return 0;1249}1250 1251static int mv_xor_resume(struct platform_device *dev)1252{1253 struct mv_xor_device *xordev = platform_get_drvdata(dev);1254 const struct mbus_dram_target_info *dram;1255 int i;1256 1257 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {1258 struct mv_xor_chan *mv_chan = xordev->channels[i];1259 1260 if (!mv_chan)1261 continue;1262 1263 writel_relaxed(mv_chan->saved_config_reg,1264 XOR_CONFIG(mv_chan));1265 writel_relaxed(mv_chan->saved_int_mask_reg,1266 XOR_INTR_MASK(mv_chan));1267 }1268 1269 if (xordev->xor_type == XOR_ARMADA_37XX) {1270 mv_xor_conf_mbus_windows_a3700(xordev);1271 return 0;1272 }1273 1274 dram = mv_mbus_dram_info();1275 if (dram)1276 mv_xor_conf_mbus_windows(xordev, dram);1277 1278 return 0;1279}1280 1281static const struct of_device_id mv_xor_dt_ids[] = {1282 { .compatible = "marvell,orion-xor", .data = (void *)XOR_ORION },1283 { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_ARMADA_38X },1284 { .compatible = "marvell,armada-3700-xor", .data = (void *)XOR_ARMADA_37XX },1285 {},1286};1287 1288static unsigned int mv_xor_engine_count;1289 1290static int mv_xor_probe(struct platform_device *pdev)1291{1292 const struct mbus_dram_target_info *dram;1293 struct mv_xor_device *xordev;1294 struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev);1295 struct resource *res;1296 unsigned int max_engines, max_channels;1297 int i, ret;1298 1299 dev_notice(&pdev->dev, "Marvell shared XOR driver\n");1300 1301 xordev = devm_kzalloc(&pdev->dev, sizeof(*xordev), GFP_KERNEL);1302 if (!xordev)1303 return -ENOMEM;1304 1305 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);1306 if (!res)1307 return -ENODEV;1308 1309 xordev->xor_base = devm_ioremap(&pdev->dev, res->start,1310 resource_size(res));1311 if (!xordev->xor_base)1312 return -EBUSY;1313 1314 res = platform_get_resource(pdev, IORESOURCE_MEM, 1);1315 if (!res)1316 return -ENODEV;1317 1318 xordev->xor_high_base = devm_ioremap(&pdev->dev, res->start,1319 resource_size(res));1320 if (!xordev->xor_high_base)1321 return -EBUSY;1322 1323 platform_set_drvdata(pdev, xordev);1324 1325 1326 /*1327 * We need to know which type of XOR device we use before1328 * setting up. In non-dt case it can only be the legacy one.1329 */1330 xordev->xor_type = XOR_ORION;1331 if (pdev->dev.of_node)1332 xordev->xor_type = (uintptr_t)device_get_match_data(&pdev->dev);1333 1334 /*1335 * (Re-)program MBUS remapping windows if we are asked to.1336 */1337 if (xordev->xor_type == XOR_ARMADA_37XX) {1338 mv_xor_conf_mbus_windows_a3700(xordev);1339 } else {1340 dram = mv_mbus_dram_info();1341 if (dram)1342 mv_xor_conf_mbus_windows(xordev, dram);1343 }1344 1345 /* Not all platforms can gate the clock, so it is not1346 * an error if the clock does not exists.1347 */1348 xordev->clk = clk_get(&pdev->dev, NULL);1349 if (!IS_ERR(xordev->clk))1350 clk_prepare_enable(xordev->clk);1351 1352 /*1353 * We don't want to have more than one channel per CPU in1354 * order for async_tx to perform well. So we limit the number1355 * of engines and channels so that we take into account this1356 * constraint. Note that we also want to use channels from1357 * separate engines when possible. For dual-CPU Armada 37001358 * SoC with single XOR engine allow using its both channels.1359 */1360 max_engines = num_present_cpus();1361 if (xordev->xor_type == XOR_ARMADA_37XX)1362 max_channels = num_present_cpus();1363 else1364 max_channels = min_t(unsigned int,1365 MV_XOR_MAX_CHANNELS,1366 DIV_ROUND_UP(num_present_cpus(), 2));1367 1368 if (mv_xor_engine_count >= max_engines)1369 return 0;1370 1371 if (pdev->dev.of_node) {1372 struct device_node *np;1373 int i = 0;1374 1375 for_each_child_of_node(pdev->dev.of_node, np) {1376 struct mv_xor_chan *chan;1377 dma_cap_mask_t cap_mask;1378 int irq;1379 1380 if (i >= max_channels)1381 continue;1382 1383 dma_cap_zero(cap_mask);1384 dma_cap_set(DMA_MEMCPY, cap_mask);1385 dma_cap_set(DMA_XOR, cap_mask);1386 dma_cap_set(DMA_INTERRUPT, cap_mask);1387 1388 irq = irq_of_parse_and_map(np, 0);1389 if (!irq) {1390 ret = -ENODEV;1391 goto err_channel_add;1392 }1393 1394 chan = mv_xor_channel_add(xordev, pdev, i,1395 cap_mask, irq);1396 if (IS_ERR(chan)) {1397 ret = PTR_ERR(chan);1398 irq_dispose_mapping(irq);1399 goto err_channel_add;1400 }1401 1402 xordev->channels[i] = chan;1403 i++;1404 }1405 } else if (pdata && pdata->channels) {1406 for (i = 0; i < max_channels; i++) {1407 struct mv_xor_channel_data *cd;1408 struct mv_xor_chan *chan;1409 int irq;1410 1411 cd = &pdata->channels[i];1412 irq = platform_get_irq(pdev, i);1413 if (irq < 0) {1414 ret = irq;1415 goto err_channel_add;1416 }1417 1418 chan = mv_xor_channel_add(xordev, pdev, i,1419 cd->cap_mask, irq);1420 if (IS_ERR(chan)) {1421 ret = PTR_ERR(chan);1422 goto err_channel_add;1423 }1424 1425 xordev->channels[i] = chan;1426 }1427 }1428 1429 return 0;1430 1431err_channel_add:1432 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++)1433 if (xordev->channels[i]) {1434 mv_xor_channel_remove(xordev->channels[i]);1435 if (pdev->dev.of_node)1436 irq_dispose_mapping(xordev->channels[i]->irq);1437 }1438 1439 if (!IS_ERR(xordev->clk)) {1440 clk_disable_unprepare(xordev->clk);1441 clk_put(xordev->clk);1442 }1443 1444 return ret;1445}1446 1447static struct platform_driver mv_xor_driver = {1448 .probe = mv_xor_probe,1449 .suspend = mv_xor_suspend,1450 .resume = mv_xor_resume,1451 .driver = {1452 .name = MV_XOR_NAME,1453 .of_match_table = mv_xor_dt_ids,1454 },1455};1456 1457builtin_platform_driver(mv_xor_driver);1458 1459/*1460MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>");1461MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine");1462MODULE_LICENSE("GPL");1463*/1464