3275 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Copyright (c) 2012 Samsung Electronics Co., Ltd.4 * http://www.samsung.com5 *6 * Copyright (C) 2010 Samsung Electronics Co. Ltd.7 * Jaswinder Singh <jassi.brar@samsung.com>8 */9 10#include <linux/debugfs.h>11#include <linux/kernel.h>12#include <linux/io.h>13#include <linux/init.h>14#include <linux/slab.h>15#include <linux/module.h>16#include <linux/string.h>17#include <linux/delay.h>18#include <linux/interrupt.h>19#include <linux/dma-mapping.h>20#include <linux/dmaengine.h>21#include <linux/amba/bus.h>22#include <linux/scatterlist.h>23#include <linux/of.h>24#include <linux/of_dma.h>25#include <linux/err.h>26#include <linux/pm_runtime.h>27#include <linux/bug.h>28#include <linux/reset.h>29 30#include "dmaengine.h"31#define PL330_MAX_CHAN 832#define PL330_MAX_IRQS 3233#define PL330_MAX_PERI 3234#define PL330_MAX_BURST 1635 36#define PL330_QUIRK_BROKEN_NO_FLUSHP BIT(0)37#define PL330_QUIRK_PERIPH_BURST BIT(1)38 39enum pl330_cachectrl {40 CCTRL0, /* Noncacheable and nonbufferable */41 CCTRL1, /* Bufferable only */42 CCTRL2, /* Cacheable, but do not allocate */43 CCTRL3, /* Cacheable and bufferable, but do not allocate */44 INVALID1, /* AWCACHE = 0x1000 */45 INVALID2,46 CCTRL6, /* Cacheable write-through, allocate on writes only */47 CCTRL7, /* Cacheable write-back, allocate on writes only */48};49 50enum pl330_byteswap {51 SWAP_NO,52 SWAP_2,53 SWAP_4,54 SWAP_8,55 SWAP_16,56};57 58/* Register and Bit field Definitions */59#define DS 0x060#define DS_ST_STOP 0x061#define DS_ST_EXEC 0x162#define DS_ST_CMISS 0x263#define DS_ST_UPDTPC 0x364#define DS_ST_WFE 0x465#define DS_ST_ATBRR 0x566#define DS_ST_QBUSY 0x667#define DS_ST_WFP 0x768#define DS_ST_KILL 0x869#define DS_ST_CMPLT 0x970#define DS_ST_FLTCMP 0xe71#define DS_ST_FAULT 0xf72 73#define DPC 0x474#define INTEN 0x2075#define ES 0x2476#define INTSTATUS 0x2877#define INTCLR 0x2c78#define FSM 0x3079#define FSC 0x3480#define FTM 0x3881 82#define _FTC 0x4083#define FTC(n) (_FTC + (n)*0x4)84 85#define _CS 0x10086#define CS(n) (_CS + (n)*0x8)87#define CS_CNS (1 << 21)88 89#define _CPC 0x10490#define CPC(n) (_CPC + (n)*0x8)91 92#define _SA 0x40093#define SA(n) (_SA + (n)*0x20)94 95#define _DA 0x40496#define DA(n) (_DA + (n)*0x20)97 98#define _CC 0x40899#define CC(n) (_CC + (n)*0x20)100 101#define CC_SRCINC (1 << 0)102#define CC_DSTINC (1 << 14)103#define CC_SRCPRI (1 << 8)104#define CC_DSTPRI (1 << 22)105#define CC_SRCNS (1 << 9)106#define CC_DSTNS (1 << 23)107#define CC_SRCIA (1 << 10)108#define CC_DSTIA (1 << 24)109#define CC_SRCBRSTLEN_SHFT 4110#define CC_DSTBRSTLEN_SHFT 18111#define CC_SRCBRSTSIZE_SHFT 1112#define CC_DSTBRSTSIZE_SHFT 15113#define CC_SRCCCTRL_SHFT 11114#define CC_SRCCCTRL_MASK 0x7115#define CC_DSTCCTRL_SHFT 25116#define CC_DRCCCTRL_MASK 0x7117#define CC_SWAP_SHFT 28118 119#define _LC0 0x40c120#define LC0(n) (_LC0 + (n)*0x20)121 122#define _LC1 0x410123#define LC1(n) (_LC1 + (n)*0x20)124 125#define DBGSTATUS 0xd00126#define DBG_BUSY (1 << 0)127 128#define DBGCMD 0xd04129#define DBGINST0 0xd08130#define DBGINST1 0xd0c131 132#define CR0 0xe00133#define CR1 0xe04134#define CR2 0xe08135#define CR3 0xe0c136#define CR4 0xe10137#define CRD 0xe14138 139#define PERIPH_ID 0xfe0140#define PERIPH_REV_SHIFT 20141#define PERIPH_REV_MASK 0xf142#define PERIPH_REV_R0P0 0143#define PERIPH_REV_R1P0 1144#define PERIPH_REV_R1P1 2145 146#define CR0_PERIPH_REQ_SET (1 << 0)147#define CR0_BOOT_EN_SET (1 << 1)148#define CR0_BOOT_MAN_NS (1 << 2)149#define CR0_NUM_CHANS_SHIFT 4150#define CR0_NUM_CHANS_MASK 0x7151#define CR0_NUM_PERIPH_SHIFT 12152#define CR0_NUM_PERIPH_MASK 0x1f153#define CR0_NUM_EVENTS_SHIFT 17154#define CR0_NUM_EVENTS_MASK 0x1f155 156#define CR1_ICACHE_LEN_SHIFT 0157#define CR1_ICACHE_LEN_MASK 0x7158#define CR1_NUM_ICACHELINES_SHIFT 4159#define CR1_NUM_ICACHELINES_MASK 0xf160 161#define CRD_DATA_WIDTH_SHIFT 0162#define CRD_DATA_WIDTH_MASK 0x7163#define CRD_WR_CAP_SHIFT 4164#define CRD_WR_CAP_MASK 0x7165#define CRD_WR_Q_DEP_SHIFT 8166#define CRD_WR_Q_DEP_MASK 0xf167#define CRD_RD_CAP_SHIFT 12168#define CRD_RD_CAP_MASK 0x7169#define CRD_RD_Q_DEP_SHIFT 16170#define CRD_RD_Q_DEP_MASK 0xf171#define CRD_DATA_BUFF_SHIFT 20172#define CRD_DATA_BUFF_MASK 0x3ff173 174#define PART 0x330175#define DESIGNER 0x41176#define REVISION 0x0177#define INTEG_CFG 0x0178#define PERIPH_ID_VAL ((PART << 0) | (DESIGNER << 12))179 180#define PL330_STATE_STOPPED (1 << 0)181#define PL330_STATE_EXECUTING (1 << 1)182#define PL330_STATE_WFE (1 << 2)183#define PL330_STATE_FAULTING (1 << 3)184#define PL330_STATE_COMPLETING (1 << 4)185#define PL330_STATE_WFP (1 << 5)186#define PL330_STATE_KILLING (1 << 6)187#define PL330_STATE_FAULT_COMPLETING (1 << 7)188#define PL330_STATE_CACHEMISS (1 << 8)189#define PL330_STATE_UPDTPC (1 << 9)190#define PL330_STATE_ATBARRIER (1 << 10)191#define PL330_STATE_QUEUEBUSY (1 << 11)192#define PL330_STATE_INVALID (1 << 15)193 194#define PL330_STABLE_STATES (PL330_STATE_STOPPED | PL330_STATE_EXECUTING \195 | PL330_STATE_WFE | PL330_STATE_FAULTING)196 197#define CMD_DMAADDH 0x54198#define CMD_DMAEND 0x00199#define CMD_DMAFLUSHP 0x35200#define CMD_DMAGO 0xa0201#define CMD_DMALD 0x04202#define CMD_DMALDP 0x25203#define CMD_DMALP 0x20204#define CMD_DMALPEND 0x28205#define CMD_DMAKILL 0x01206#define CMD_DMAMOV 0xbc207#define CMD_DMANOP 0x18208#define CMD_DMARMB 0x12209#define CMD_DMASEV 0x34210#define CMD_DMAST 0x08211#define CMD_DMASTP 0x29212#define CMD_DMASTZ 0x0c213#define CMD_DMAWFE 0x36214#define CMD_DMAWFP 0x30215#define CMD_DMAWMB 0x13216 217#define SZ_DMAADDH 3218#define SZ_DMAEND 1219#define SZ_DMAFLUSHP 2220#define SZ_DMALD 1221#define SZ_DMALDP 2222#define SZ_DMALP 2223#define SZ_DMALPEND 2224#define SZ_DMAKILL 1225#define SZ_DMAMOV 6226#define SZ_DMANOP 1227#define SZ_DMARMB 1228#define SZ_DMASEV 2229#define SZ_DMAST 1230#define SZ_DMASTP 2231#define SZ_DMASTZ 1232#define SZ_DMAWFE 2233#define SZ_DMAWFP 2234#define SZ_DMAWMB 1235#define SZ_DMAGO 6236 237#define BRST_LEN(ccr) ((((ccr) >> CC_SRCBRSTLEN_SHFT) & 0xf) + 1)238#define BRST_SIZE(ccr) (1 << (((ccr) >> CC_SRCBRSTSIZE_SHFT) & 0x7))239 240#define BYTE_TO_BURST(b, ccr) ((b) / BRST_SIZE(ccr) / BRST_LEN(ccr))241#define BURST_TO_BYTE(c, ccr) ((c) * BRST_SIZE(ccr) * BRST_LEN(ccr))242 243/*244 * With 256 bytes, we can do more than 2.5MB and 5MB xfers per req245 * at 1byte/burst for P<->M and M<->M respectively.246 * For typical scenario, at 1word/burst, 10MB and 20MB xfers per req247 * should be enough for P<->M and M<->M respectively.248 */249#define MCODE_BUFF_PER_REQ 256250 251/* Use this _only_ to wait on transient states */252#define UNTIL(t, s) while (!(_state(t) & (s))) cpu_relax();253 254#ifdef PL330_DEBUG_MCGEN255static unsigned cmd_line;256#define PL330_DBGCMD_DUMP(off, x...) do { \257 printk("%x:", cmd_line); \258 printk(KERN_CONT x); \259 cmd_line += off; \260 } while (0)261#define PL330_DBGMC_START(addr) (cmd_line = addr)262#else263#define PL330_DBGCMD_DUMP(off, x...) do {} while (0)264#define PL330_DBGMC_START(addr) do {} while (0)265#endif266 267/* The number of default descriptors */268 269#define NR_DEFAULT_DESC 16270 271/* Delay for runtime PM autosuspend, ms */272#define PL330_AUTOSUSPEND_DELAY 20273 274/* Populated by the PL330 core driver for DMA API driver's info */275struct pl330_config {276 u32 periph_id;277#define DMAC_MODE_NS (1 << 0)278 unsigned int mode;279 unsigned int data_bus_width:10; /* In number of bits */280 unsigned int data_buf_dep:11;281 unsigned int num_chan:4;282 unsigned int num_peri:6;283 u32 peri_ns;284 unsigned int num_events:6;285 u32 irq_ns;286};287 288/*289 * Request Configuration.290 * The PL330 core does not modify this and uses the last291 * working configuration if the request doesn't provide any.292 *293 * The Client may want to provide this info only for the294 * first request and a request with new settings.295 */296struct pl330_reqcfg {297 /* Address Incrementing */298 unsigned dst_inc:1;299 unsigned src_inc:1;300 301 /*302 * For now, the SRC & DST protection levels303 * and burst size/length are assumed same.304 */305 bool nonsecure;306 bool privileged;307 bool insnaccess;308 unsigned brst_len:5;309 unsigned brst_size:3; /* in power of 2 */310 311 enum pl330_cachectrl dcctl;312 enum pl330_cachectrl scctl;313 enum pl330_byteswap swap;314 struct pl330_config *pcfg;315};316 317/*318 * One cycle of DMAC operation.319 * There may be more than one xfer in a request.320 */321struct pl330_xfer {322 u32 src_addr;323 u32 dst_addr;324 /* Size to xfer */325 u32 bytes;326};327 328/* The xfer callbacks are made with one of these arguments. */329enum pl330_op_err {330 /* The all xfers in the request were success. */331 PL330_ERR_NONE,332 /* If req aborted due to global error. */333 PL330_ERR_ABORT,334 /* If req failed due to problem with Channel. */335 PL330_ERR_FAIL,336};337 338enum dmamov_dst {339 SAR = 0,340 CCR,341 DAR,342};343 344enum pl330_dst {345 SRC = 0,346 DST,347};348 349enum pl330_cond {350 SINGLE,351 BURST,352 ALWAYS,353};354 355struct dma_pl330_desc;356 357struct _pl330_req {358 u32 mc_bus;359 void *mc_cpu;360 struct dma_pl330_desc *desc;361};362 363/* ToBeDone for tasklet */364struct _pl330_tbd {365 bool reset_dmac;366 bool reset_mngr;367 u8 reset_chan;368};369 370/* A DMAC Thread */371struct pl330_thread {372 u8 id;373 int ev;374 /* If the channel is not yet acquired by any client */375 bool free;376 /* Parent DMAC */377 struct pl330_dmac *dmac;378 /* Only two at a time */379 struct _pl330_req req[2];380 /* Index of the last enqueued request */381 unsigned lstenq;382 /* Index of the last submitted request or -1 if the DMA is stopped */383 int req_running;384};385 386enum pl330_dmac_state {387 UNINIT,388 INIT,389 DYING,390};391 392enum desc_status {393 /* In the DMAC pool */394 FREE,395 /*396 * Allocated to some channel during prep_xxx397 * Also may be sitting on the work_list.398 */399 PREP,400 /*401 * Sitting on the work_list and already submitted402 * to the PL330 core. Not more than two descriptors403 * of a channel can be BUSY at any time.404 */405 BUSY,406 /*407 * Pause was called while descriptor was BUSY. Due to hardware408 * limitations, only termination is possible for descriptors409 * that have been paused.410 */411 PAUSED,412 /*413 * Sitting on the channel work_list but xfer done414 * by PL330 core415 */416 DONE,417};418 419struct dma_pl330_chan {420 /* Schedule desc completion */421 struct tasklet_struct task;422 423 /* DMA-Engine Channel */424 struct dma_chan chan;425 426 /* List of submitted descriptors */427 struct list_head submitted_list;428 /* List of issued descriptors */429 struct list_head work_list;430 /* List of completed descriptors */431 struct list_head completed_list;432 433 /* Pointer to the DMAC that manages this channel,434 * NULL if the channel is available to be acquired.435 * As the parent, this DMAC also provides descriptors436 * to the channel.437 */438 struct pl330_dmac *dmac;439 440 /* To protect channel manipulation */441 spinlock_t lock;442 443 /*444 * Hardware channel thread of PL330 DMAC. NULL if the channel is445 * available.446 */447 struct pl330_thread *thread;448 449 /* For D-to-M and M-to-D channels */450 int burst_sz; /* the peripheral fifo width */451 int burst_len; /* the number of burst */452 phys_addr_t fifo_addr;453 /* DMA-mapped view of the FIFO; may differ if an IOMMU is present */454 dma_addr_t fifo_dma;455 enum dma_data_direction dir;456 struct dma_slave_config slave_config;457 458 /* for cyclic capability */459 bool cyclic;460 461 /* for runtime pm tracking */462 bool active;463};464 465struct pl330_dmac {466 /* DMA-Engine Device */467 struct dma_device ddma;468 469 /* Pool of descriptors available for the DMAC's channels */470 struct list_head desc_pool;471 /* To protect desc_pool manipulation */472 spinlock_t pool_lock;473 474 /* Size of MicroCode buffers for each channel. */475 unsigned mcbufsz;476 /* ioremap'ed address of PL330 registers. */477 void __iomem *base;478 /* Populated by the PL330 core driver during pl330_add */479 struct pl330_config pcfg;480 481 spinlock_t lock;482 /* Maximum possible events/irqs */483 int events[32];484 /* BUS address of MicroCode buffer */485 dma_addr_t mcode_bus;486 /* CPU address of MicroCode buffer */487 void *mcode_cpu;488 /* List of all Channel threads */489 struct pl330_thread *channels;490 /* Pointer to the MANAGER thread */491 struct pl330_thread *manager;492 /* To handle bad news in interrupt */493 struct tasklet_struct tasks;494 struct _pl330_tbd dmac_tbd;495 /* State of DMAC operation */496 enum pl330_dmac_state state;497 /* Holds list of reqs with due callbacks */498 struct list_head req_done;499 500 /* Peripheral channels connected to this DMAC */501 unsigned int num_peripherals;502 struct dma_pl330_chan *peripherals; /* keep at end */503 int quirks;504 505 struct reset_control *rstc;506 struct reset_control *rstc_ocp;507};508 509static struct pl330_of_quirks {510 char *quirk;511 int id;512} of_quirks[] = {513 {514 .quirk = "arm,pl330-broken-no-flushp",515 .id = PL330_QUIRK_BROKEN_NO_FLUSHP,516 },517 {518 .quirk = "arm,pl330-periph-burst",519 .id = PL330_QUIRK_PERIPH_BURST,520 }521};522 523struct dma_pl330_desc {524 /* To attach to a queue as child */525 struct list_head node;526 527 /* Descriptor for the DMA Engine API */528 struct dma_async_tx_descriptor txd;529 530 /* Xfer for PL330 core */531 struct pl330_xfer px;532 533 struct pl330_reqcfg rqcfg;534 535 enum desc_status status;536 537 int bytes_requested;538 bool last;539 540 /* The channel which currently holds this desc */541 struct dma_pl330_chan *pchan;542 543 enum dma_transfer_direction rqtype;544 /* Index of peripheral for the xfer. */545 unsigned peri:5;546 /* Hook to attach to DMAC's list of reqs with due callback */547 struct list_head rqd;548};549 550struct _xfer_spec {551 u32 ccr;552 struct dma_pl330_desc *desc;553};554 555static int pl330_config_write(struct dma_chan *chan,556 struct dma_slave_config *slave_config,557 enum dma_transfer_direction direction);558 559static inline bool _queue_full(struct pl330_thread *thrd)560{561 return thrd->req[0].desc != NULL && thrd->req[1].desc != NULL;562}563 564static inline bool is_manager(struct pl330_thread *thrd)565{566 return thrd->dmac->manager == thrd;567}568 569/* If manager of the thread is in Non-Secure mode */570static inline bool _manager_ns(struct pl330_thread *thrd)571{572 return (thrd->dmac->pcfg.mode & DMAC_MODE_NS) ? true : false;573}574 575static inline u32 get_revision(u32 periph_id)576{577 return (periph_id >> PERIPH_REV_SHIFT) & PERIPH_REV_MASK;578}579 580static inline u32 _emit_END(unsigned dry_run, u8 buf[])581{582 if (dry_run)583 return SZ_DMAEND;584 585 buf[0] = CMD_DMAEND;586 587 PL330_DBGCMD_DUMP(SZ_DMAEND, "\tDMAEND\n");588 589 return SZ_DMAEND;590}591 592static inline u32 _emit_FLUSHP(unsigned dry_run, u8 buf[], u8 peri)593{594 if (dry_run)595 return SZ_DMAFLUSHP;596 597 buf[0] = CMD_DMAFLUSHP;598 599 peri &= 0x1f;600 peri <<= 3;601 buf[1] = peri;602 603 PL330_DBGCMD_DUMP(SZ_DMAFLUSHP, "\tDMAFLUSHP %u\n", peri >> 3);604 605 return SZ_DMAFLUSHP;606}607 608static inline u32 _emit_LD(unsigned dry_run, u8 buf[], enum pl330_cond cond)609{610 if (dry_run)611 return SZ_DMALD;612 613 buf[0] = CMD_DMALD;614 615 if (cond == SINGLE)616 buf[0] |= (0 << 1) | (1 << 0);617 else if (cond == BURST)618 buf[0] |= (1 << 1) | (1 << 0);619 620 PL330_DBGCMD_DUMP(SZ_DMALD, "\tDMALD%c\n",621 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));622 623 return SZ_DMALD;624}625 626static inline u32 _emit_LDP(unsigned dry_run, u8 buf[],627 enum pl330_cond cond, u8 peri)628{629 if (dry_run)630 return SZ_DMALDP;631 632 buf[0] = CMD_DMALDP;633 634 if (cond == BURST)635 buf[0] |= (1 << 1);636 637 peri &= 0x1f;638 peri <<= 3;639 buf[1] = peri;640 641 PL330_DBGCMD_DUMP(SZ_DMALDP, "\tDMALDP%c %u\n",642 cond == SINGLE ? 'S' : 'B', peri >> 3);643 644 return SZ_DMALDP;645}646 647static inline u32 _emit_LP(unsigned dry_run, u8 buf[],648 unsigned loop, u8 cnt)649{650 if (dry_run)651 return SZ_DMALP;652 653 buf[0] = CMD_DMALP;654 655 if (loop)656 buf[0] |= (1 << 1);657 658 cnt--; /* DMAC increments by 1 internally */659 buf[1] = cnt;660 661 PL330_DBGCMD_DUMP(SZ_DMALP, "\tDMALP_%c %u\n", loop ? '1' : '0', cnt);662 663 return SZ_DMALP;664}665 666struct _arg_LPEND {667 enum pl330_cond cond;668 bool forever;669 unsigned loop;670 u8 bjump;671};672 673static inline u32 _emit_LPEND(unsigned dry_run, u8 buf[],674 const struct _arg_LPEND *arg)675{676 enum pl330_cond cond = arg->cond;677 bool forever = arg->forever;678 unsigned loop = arg->loop;679 u8 bjump = arg->bjump;680 681 if (dry_run)682 return SZ_DMALPEND;683 684 buf[0] = CMD_DMALPEND;685 686 if (loop)687 buf[0] |= (1 << 2);688 689 if (!forever)690 buf[0] |= (1 << 4);691 692 if (cond == SINGLE)693 buf[0] |= (0 << 1) | (1 << 0);694 else if (cond == BURST)695 buf[0] |= (1 << 1) | (1 << 0);696 697 buf[1] = bjump;698 699 PL330_DBGCMD_DUMP(SZ_DMALPEND, "\tDMALP%s%c_%c bjmpto_%x\n",700 forever ? "FE" : "END",701 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'),702 loop ? '1' : '0',703 bjump);704 705 return SZ_DMALPEND;706}707 708static inline u32 _emit_KILL(unsigned dry_run, u8 buf[])709{710 if (dry_run)711 return SZ_DMAKILL;712 713 buf[0] = CMD_DMAKILL;714 715 return SZ_DMAKILL;716}717 718static inline u32 _emit_MOV(unsigned dry_run, u8 buf[],719 enum dmamov_dst dst, u32 val)720{721 if (dry_run)722 return SZ_DMAMOV;723 724 buf[0] = CMD_DMAMOV;725 buf[1] = dst;726 buf[2] = val;727 buf[3] = val >> 8;728 buf[4] = val >> 16;729 buf[5] = val >> 24;730 731 PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n",732 dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val);733 734 return SZ_DMAMOV;735}736 737static inline u32 _emit_RMB(unsigned dry_run, u8 buf[])738{739 if (dry_run)740 return SZ_DMARMB;741 742 buf[0] = CMD_DMARMB;743 744 PL330_DBGCMD_DUMP(SZ_DMARMB, "\tDMARMB\n");745 746 return SZ_DMARMB;747}748 749static inline u32 _emit_SEV(unsigned dry_run, u8 buf[], u8 ev)750{751 if (dry_run)752 return SZ_DMASEV;753 754 buf[0] = CMD_DMASEV;755 756 ev &= 0x1f;757 ev <<= 3;758 buf[1] = ev;759 760 PL330_DBGCMD_DUMP(SZ_DMASEV, "\tDMASEV %u\n", ev >> 3);761 762 return SZ_DMASEV;763}764 765static inline u32 _emit_ST(unsigned dry_run, u8 buf[], enum pl330_cond cond)766{767 if (dry_run)768 return SZ_DMAST;769 770 buf[0] = CMD_DMAST;771 772 if (cond == SINGLE)773 buf[0] |= (0 << 1) | (1 << 0);774 else if (cond == BURST)775 buf[0] |= (1 << 1) | (1 << 0);776 777 PL330_DBGCMD_DUMP(SZ_DMAST, "\tDMAST%c\n",778 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));779 780 return SZ_DMAST;781}782 783static inline u32 _emit_STP(unsigned dry_run, u8 buf[],784 enum pl330_cond cond, u8 peri)785{786 if (dry_run)787 return SZ_DMASTP;788 789 buf[0] = CMD_DMASTP;790 791 if (cond == BURST)792 buf[0] |= (1 << 1);793 794 peri &= 0x1f;795 peri <<= 3;796 buf[1] = peri;797 798 PL330_DBGCMD_DUMP(SZ_DMASTP, "\tDMASTP%c %u\n",799 cond == SINGLE ? 'S' : 'B', peri >> 3);800 801 return SZ_DMASTP;802}803 804static inline u32 _emit_WFP(unsigned dry_run, u8 buf[],805 enum pl330_cond cond, u8 peri)806{807 if (dry_run)808 return SZ_DMAWFP;809 810 buf[0] = CMD_DMAWFP;811 812 if (cond == SINGLE)813 buf[0] |= (0 << 1) | (0 << 0);814 else if (cond == BURST)815 buf[0] |= (1 << 1) | (0 << 0);816 else817 buf[0] |= (0 << 1) | (1 << 0);818 819 peri &= 0x1f;820 peri <<= 3;821 buf[1] = peri;822 823 PL330_DBGCMD_DUMP(SZ_DMAWFP, "\tDMAWFP%c %u\n",824 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'P'), peri >> 3);825 826 return SZ_DMAWFP;827}828 829static inline u32 _emit_WMB(unsigned dry_run, u8 buf[])830{831 if (dry_run)832 return SZ_DMAWMB;833 834 buf[0] = CMD_DMAWMB;835 836 PL330_DBGCMD_DUMP(SZ_DMAWMB, "\tDMAWMB\n");837 838 return SZ_DMAWMB;839}840 841struct _arg_GO {842 u8 chan;843 u32 addr;844 unsigned ns;845};846 847static inline u32 _emit_GO(unsigned dry_run, u8 buf[],848 const struct _arg_GO *arg)849{850 u8 chan = arg->chan;851 u32 addr = arg->addr;852 unsigned ns = arg->ns;853 854 if (dry_run)855 return SZ_DMAGO;856 857 buf[0] = CMD_DMAGO;858 buf[0] |= (ns << 1);859 buf[1] = chan & 0x7;860 buf[2] = addr;861 buf[3] = addr >> 8;862 buf[4] = addr >> 16;863 buf[5] = addr >> 24;864 865 return SZ_DMAGO;866}867 868#define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t)869 870/* Returns Time-Out */871static bool _until_dmac_idle(struct pl330_thread *thrd)872{873 void __iomem *regs = thrd->dmac->base;874 unsigned long loops = msecs_to_loops(5);875 876 do {877 /* Until Manager is Idle */878 if (!(readl(regs + DBGSTATUS) & DBG_BUSY))879 break;880 881 cpu_relax();882 } while (--loops);883 884 if (!loops)885 return true;886 887 return false;888}889 890static inline void _execute_DBGINSN(struct pl330_thread *thrd,891 u8 insn[], bool as_manager)892{893 void __iomem *regs = thrd->dmac->base;894 u32 val;895 896 /* If timed out due to halted state-machine */897 if (_until_dmac_idle(thrd)) {898 dev_err(thrd->dmac->ddma.dev, "DMAC halted!\n");899 return;900 }901 902 val = (insn[0] << 16) | (insn[1] << 24);903 if (!as_manager) {904 val |= (1 << 0);905 val |= (thrd->id << 8); /* Channel Number */906 }907 writel(val, regs + DBGINST0);908 909 val = le32_to_cpu(*((__le32 *)&insn[2]));910 writel(val, regs + DBGINST1);911 912 /* Get going */913 writel(0, regs + DBGCMD);914}915 916static inline u32 _state(struct pl330_thread *thrd)917{918 void __iomem *regs = thrd->dmac->base;919 u32 val;920 921 if (is_manager(thrd))922 val = readl(regs + DS) & 0xf;923 else924 val = readl(regs + CS(thrd->id)) & 0xf;925 926 switch (val) {927 case DS_ST_STOP:928 return PL330_STATE_STOPPED;929 case DS_ST_EXEC:930 return PL330_STATE_EXECUTING;931 case DS_ST_CMISS:932 return PL330_STATE_CACHEMISS;933 case DS_ST_UPDTPC:934 return PL330_STATE_UPDTPC;935 case DS_ST_WFE:936 return PL330_STATE_WFE;937 case DS_ST_FAULT:938 return PL330_STATE_FAULTING;939 case DS_ST_ATBRR:940 if (is_manager(thrd))941 return PL330_STATE_INVALID;942 else943 return PL330_STATE_ATBARRIER;944 case DS_ST_QBUSY:945 if (is_manager(thrd))946 return PL330_STATE_INVALID;947 else948 return PL330_STATE_QUEUEBUSY;949 case DS_ST_WFP:950 if (is_manager(thrd))951 return PL330_STATE_INVALID;952 else953 return PL330_STATE_WFP;954 case DS_ST_KILL:955 if (is_manager(thrd))956 return PL330_STATE_INVALID;957 else958 return PL330_STATE_KILLING;959 case DS_ST_CMPLT:960 if (is_manager(thrd))961 return PL330_STATE_INVALID;962 else963 return PL330_STATE_COMPLETING;964 case DS_ST_FLTCMP:965 if (is_manager(thrd))966 return PL330_STATE_INVALID;967 else968 return PL330_STATE_FAULT_COMPLETING;969 default:970 return PL330_STATE_INVALID;971 }972}973 974static void _stop(struct pl330_thread *thrd)975{976 void __iomem *regs = thrd->dmac->base;977 u8 insn[6] = {0, 0, 0, 0, 0, 0};978 u32 inten = readl(regs + INTEN);979 980 if (_state(thrd) == PL330_STATE_FAULT_COMPLETING)981 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);982 983 /* Return if nothing needs to be done */984 if (_state(thrd) == PL330_STATE_COMPLETING985 || _state(thrd) == PL330_STATE_KILLING986 || _state(thrd) == PL330_STATE_STOPPED)987 return;988 989 _emit_KILL(0, insn);990 991 _execute_DBGINSN(thrd, insn, is_manager(thrd));992 993 /* clear the event */994 if (inten & (1 << thrd->ev))995 writel(1 << thrd->ev, regs + INTCLR);996 /* Stop generating interrupts for SEV */997 writel(inten & ~(1 << thrd->ev), regs + INTEN);998}999 1000/* Start doing req 'idx' of thread 'thrd' */1001static bool _trigger(struct pl330_thread *thrd)1002{1003 void __iomem *regs = thrd->dmac->base;1004 struct _pl330_req *req;1005 struct dma_pl330_desc *desc;1006 struct _arg_GO go;1007 unsigned ns;1008 u8 insn[6] = {0, 0, 0, 0, 0, 0};1009 int idx;1010 1011 /* Return if already ACTIVE */1012 if (_state(thrd) != PL330_STATE_STOPPED)1013 return true;1014 1015 idx = 1 - thrd->lstenq;1016 if (thrd->req[idx].desc != NULL) {1017 req = &thrd->req[idx];1018 } else {1019 idx = thrd->lstenq;1020 if (thrd->req[idx].desc != NULL)1021 req = &thrd->req[idx];1022 else1023 req = NULL;1024 }1025 1026 /* Return if no request */1027 if (!req)1028 return true;1029 1030 /* Return if req is running */1031 if (idx == thrd->req_running)1032 return true;1033 1034 desc = req->desc;1035 1036 ns = desc->rqcfg.nonsecure ? 1 : 0;1037 1038 /* See 'Abort Sources' point-4 at Page 2-25 */1039 if (_manager_ns(thrd) && !ns)1040 dev_info(thrd->dmac->ddma.dev, "%s:%d Recipe for ABORT!\n",1041 __func__, __LINE__);1042 1043 go.chan = thrd->id;1044 go.addr = req->mc_bus;1045 go.ns = ns;1046 _emit_GO(0, insn, &go);1047 1048 /* Set to generate interrupts for SEV */1049 writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN);1050 1051 /* Only manager can execute GO */1052 _execute_DBGINSN(thrd, insn, true);1053 1054 thrd->req_running = idx;1055 1056 return true;1057}1058 1059static bool pl330_start_thread(struct pl330_thread *thrd)1060{1061 switch (_state(thrd)) {1062 case PL330_STATE_FAULT_COMPLETING:1063 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);1064 1065 if (_state(thrd) == PL330_STATE_KILLING)1066 UNTIL(thrd, PL330_STATE_STOPPED)1067 fallthrough;1068 1069 case PL330_STATE_FAULTING:1070 _stop(thrd);1071 fallthrough;1072 1073 case PL330_STATE_KILLING:1074 case PL330_STATE_COMPLETING:1075 UNTIL(thrd, PL330_STATE_STOPPED)1076 fallthrough;1077 1078 case PL330_STATE_STOPPED:1079 return _trigger(thrd);1080 1081 case PL330_STATE_WFP:1082 case PL330_STATE_QUEUEBUSY:1083 case PL330_STATE_ATBARRIER:1084 case PL330_STATE_UPDTPC:1085 case PL330_STATE_CACHEMISS:1086 case PL330_STATE_EXECUTING:1087 return true;1088 1089 case PL330_STATE_WFE: /* For RESUME, nothing yet */1090 default:1091 return false;1092 }1093}1094 1095static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],1096 const struct _xfer_spec *pxs, int cyc)1097{1098 int off = 0;1099 struct pl330_config *pcfg = pxs->desc->rqcfg.pcfg;1100 1101 /* check lock-up free version */1102 if (get_revision(pcfg->periph_id) >= PERIPH_REV_R1P0) {1103 while (cyc--) {1104 off += _emit_LD(dry_run, &buf[off], ALWAYS);1105 off += _emit_ST(dry_run, &buf[off], ALWAYS);1106 }1107 } else {1108 while (cyc--) {1109 off += _emit_LD(dry_run, &buf[off], ALWAYS);1110 off += _emit_RMB(dry_run, &buf[off]);1111 off += _emit_ST(dry_run, &buf[off], ALWAYS);1112 off += _emit_WMB(dry_run, &buf[off]);1113 }1114 }1115 1116 return off;1117}1118 1119static u32 _emit_load(unsigned int dry_run, u8 buf[],1120 enum pl330_cond cond, enum dma_transfer_direction direction,1121 u8 peri)1122{1123 int off = 0;1124 1125 switch (direction) {1126 case DMA_MEM_TO_MEM:1127 case DMA_MEM_TO_DEV:1128 off += _emit_LD(dry_run, &buf[off], cond);1129 break;1130 1131 case DMA_DEV_TO_MEM:1132 if (cond == ALWAYS) {1133 off += _emit_LDP(dry_run, &buf[off], SINGLE,1134 peri);1135 off += _emit_LDP(dry_run, &buf[off], BURST,1136 peri);1137 } else {1138 off += _emit_LDP(dry_run, &buf[off], cond,1139 peri);1140 }1141 break;1142 1143 default:1144 /* this code should be unreachable */1145 WARN_ON(1);1146 break;1147 }1148 1149 return off;1150}1151 1152static inline u32 _emit_store(unsigned int dry_run, u8 buf[],1153 enum pl330_cond cond, enum dma_transfer_direction direction,1154 u8 peri)1155{1156 int off = 0;1157 1158 switch (direction) {1159 case DMA_MEM_TO_MEM:1160 case DMA_DEV_TO_MEM:1161 off += _emit_ST(dry_run, &buf[off], cond);1162 break;1163 1164 case DMA_MEM_TO_DEV:1165 if (cond == ALWAYS) {1166 off += _emit_STP(dry_run, &buf[off], SINGLE,1167 peri);1168 off += _emit_STP(dry_run, &buf[off], BURST,1169 peri);1170 } else {1171 off += _emit_STP(dry_run, &buf[off], cond,1172 peri);1173 }1174 break;1175 1176 default:1177 /* this code should be unreachable */1178 WARN_ON(1);1179 break;1180 }1181 1182 return off;1183}1184 1185static inline int _ldst_peripheral(struct pl330_dmac *pl330,1186 unsigned dry_run, u8 buf[],1187 const struct _xfer_spec *pxs, int cyc,1188 enum pl330_cond cond)1189{1190 int off = 0;1191 1192 /*1193 * do FLUSHP at beginning to clear any stale dma requests before the1194 * first WFP.1195 */1196 if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))1197 off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri);1198 while (cyc--) {1199 off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri);1200 off += _emit_load(dry_run, &buf[off], cond, pxs->desc->rqtype,1201 pxs->desc->peri);1202 off += _emit_store(dry_run, &buf[off], cond, pxs->desc->rqtype,1203 pxs->desc->peri);1204 }1205 1206 return off;1207}1208 1209static int _bursts(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[],1210 const struct _xfer_spec *pxs, int cyc)1211{1212 int off = 0;1213 enum pl330_cond cond = BRST_LEN(pxs->ccr) > 1 ? BURST : SINGLE;1214 1215 if (pl330->quirks & PL330_QUIRK_PERIPH_BURST)1216 cond = BURST;1217 1218 switch (pxs->desc->rqtype) {1219 case DMA_MEM_TO_DEV:1220 case DMA_DEV_TO_MEM:1221 off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs, cyc,1222 cond);1223 break;1224 1225 case DMA_MEM_TO_MEM:1226 off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc);1227 break;1228 1229 default:1230 /* this code should be unreachable */1231 WARN_ON(1);1232 break;1233 }1234 1235 return off;1236}1237 1238/*1239 * only the unaligned burst transfers have the dregs.1240 * so, still transfer dregs with a reduced size burst1241 * for mem-to-mem, mem-to-dev or dev-to-mem.1242 */1243static int _dregs(struct pl330_dmac *pl330, unsigned int dry_run, u8 buf[],1244 const struct _xfer_spec *pxs, int transfer_length)1245{1246 int off = 0;1247 int dregs_ccr;1248 1249 if (transfer_length == 0)1250 return off;1251 1252 /*1253 * dregs_len = (total bytes - BURST_TO_BYTE(bursts, ccr)) /1254 * BRST_SIZE(ccr)1255 * the dregs len must be smaller than burst len,1256 * so, for higher efficiency, we can modify CCR1257 * to use a reduced size burst len for the dregs.1258 */1259 dregs_ccr = pxs->ccr;1260 dregs_ccr &= ~((0xf << CC_SRCBRSTLEN_SHFT) |1261 (0xf << CC_DSTBRSTLEN_SHFT));1262 dregs_ccr |= (((transfer_length - 1) & 0xf) <<1263 CC_SRCBRSTLEN_SHFT);1264 dregs_ccr |= (((transfer_length - 1) & 0xf) <<1265 CC_DSTBRSTLEN_SHFT);1266 1267 switch (pxs->desc->rqtype) {1268 case DMA_MEM_TO_DEV:1269 case DMA_DEV_TO_MEM:1270 off += _emit_MOV(dry_run, &buf[off], CCR, dregs_ccr);1271 off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs, 1,1272 BURST);1273 break;1274 1275 case DMA_MEM_TO_MEM:1276 off += _emit_MOV(dry_run, &buf[off], CCR, dregs_ccr);1277 off += _ldst_memtomem(dry_run, &buf[off], pxs, 1);1278 break;1279 1280 default:1281 /* this code should be unreachable */1282 WARN_ON(1);1283 break;1284 }1285 1286 return off;1287}1288 1289/* Returns bytes consumed and updates bursts */1290static inline int _loop(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[],1291 unsigned long *bursts, const struct _xfer_spec *pxs)1292{1293 int cyc, cycmax, szlp, szlpend, szbrst, off;1294 unsigned lcnt0, lcnt1, ljmp0, ljmp1;1295 struct _arg_LPEND lpend;1296 1297 if (*bursts == 1)1298 return _bursts(pl330, dry_run, buf, pxs, 1);1299 1300 /* Max iterations possible in DMALP is 256 */1301 if (*bursts >= 256*256) {1302 lcnt1 = 256;1303 lcnt0 = 256;1304 cyc = *bursts / lcnt1 / lcnt0;1305 } else if (*bursts > 256) {1306 lcnt1 = 256;1307 lcnt0 = *bursts / lcnt1;1308 cyc = 1;1309 } else {1310 lcnt1 = *bursts;1311 lcnt0 = 0;1312 cyc = 1;1313 }1314 1315 szlp = _emit_LP(1, buf, 0, 0);1316 szbrst = _bursts(pl330, 1, buf, pxs, 1);1317 1318 lpend.cond = ALWAYS;1319 lpend.forever = false;1320 lpend.loop = 0;1321 lpend.bjump = 0;1322 szlpend = _emit_LPEND(1, buf, &lpend);1323 1324 if (lcnt0) {1325 szlp *= 2;1326 szlpend *= 2;1327 }1328 1329 /*1330 * Max bursts that we can unroll due to limit on the1331 * size of backward jump that can be encoded in DMALPEND1332 * which is 8-bits and hence 2551333 */1334 cycmax = (255 - (szlp + szlpend)) / szbrst;1335 1336 cyc = (cycmax < cyc) ? cycmax : cyc;1337 1338 off = 0;1339 1340 if (lcnt0) {1341 off += _emit_LP(dry_run, &buf[off], 0, lcnt0);1342 ljmp0 = off;1343 }1344 1345 off += _emit_LP(dry_run, &buf[off], 1, lcnt1);1346 ljmp1 = off;1347 1348 off += _bursts(pl330, dry_run, &buf[off], pxs, cyc);1349 1350 lpend.cond = ALWAYS;1351 lpend.forever = false;1352 lpend.loop = 1;1353 lpend.bjump = off - ljmp1;1354 off += _emit_LPEND(dry_run, &buf[off], &lpend);1355 1356 if (lcnt0) {1357 lpend.cond = ALWAYS;1358 lpend.forever = false;1359 lpend.loop = 0;1360 lpend.bjump = off - ljmp0;1361 off += _emit_LPEND(dry_run, &buf[off], &lpend);1362 }1363 1364 *bursts = lcnt1 * cyc;1365 if (lcnt0)1366 *bursts *= lcnt0;1367 1368 return off;1369}1370 1371static inline int _setup_loops(struct pl330_dmac *pl330,1372 unsigned dry_run, u8 buf[],1373 const struct _xfer_spec *pxs)1374{1375 struct pl330_xfer *x = &pxs->desc->px;1376 u32 ccr = pxs->ccr;1377 unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr);1378 int num_dregs = (x->bytes - BURST_TO_BYTE(bursts, ccr)) /1379 BRST_SIZE(ccr);1380 int off = 0;1381 1382 while (bursts) {1383 c = bursts;1384 off += _loop(pl330, dry_run, &buf[off], &c, pxs);1385 bursts -= c;1386 }1387 off += _dregs(pl330, dry_run, &buf[off], pxs, num_dregs);1388 1389 return off;1390}1391 1392static inline int _setup_xfer(struct pl330_dmac *pl330,1393 unsigned dry_run, u8 buf[],1394 const struct _xfer_spec *pxs)1395{1396 struct pl330_xfer *x = &pxs->desc->px;1397 int off = 0;1398 1399 /* DMAMOV SAR, x->src_addr */1400 off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr);1401 /* DMAMOV DAR, x->dst_addr */1402 off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr);1403 1404 /* Setup Loop(s) */1405 off += _setup_loops(pl330, dry_run, &buf[off], pxs);1406 1407 return off;1408}1409 1410/*1411 * A req is a sequence of one or more xfer units.1412 * Returns the number of bytes taken to setup the MC for the req.1413 */1414static int _setup_req(struct pl330_dmac *pl330, unsigned dry_run,1415 struct pl330_thread *thrd, unsigned index,1416 struct _xfer_spec *pxs)1417{1418 struct _pl330_req *req = &thrd->req[index];1419 u8 *buf = req->mc_cpu;1420 int off = 0;1421 1422 PL330_DBGMC_START(req->mc_bus);1423 1424 /* DMAMOV CCR, ccr */1425 off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr);1426 1427 off += _setup_xfer(pl330, dry_run, &buf[off], pxs);1428 1429 /* DMASEV peripheral/event */1430 off += _emit_SEV(dry_run, &buf[off], thrd->ev);1431 /* DMAEND */1432 off += _emit_END(dry_run, &buf[off]);1433 1434 return off;1435}1436 1437static inline u32 _prepare_ccr(const struct pl330_reqcfg *rqc)1438{1439 u32 ccr = 0;1440 1441 if (rqc->src_inc)1442 ccr |= CC_SRCINC;1443 1444 if (rqc->dst_inc)1445 ccr |= CC_DSTINC;1446 1447 /* We set same protection levels for Src and DST for now */1448 if (rqc->privileged)1449 ccr |= CC_SRCPRI | CC_DSTPRI;1450 if (rqc->nonsecure)1451 ccr |= CC_SRCNS | CC_DSTNS;1452 if (rqc->insnaccess)1453 ccr |= CC_SRCIA | CC_DSTIA;1454 1455 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_SRCBRSTLEN_SHFT);1456 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_DSTBRSTLEN_SHFT);1457 1458 ccr |= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT);1459 ccr |= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT);1460 1461 ccr |= (rqc->scctl << CC_SRCCCTRL_SHFT);1462 ccr |= (rqc->dcctl << CC_DSTCCTRL_SHFT);1463 1464 ccr |= (rqc->swap << CC_SWAP_SHFT);1465 1466 return ccr;1467}1468 1469/*1470 * Submit a list of xfers after which the client wants notification.1471 * Client is not notified after each xfer unit, just once after all1472 * xfer units are done or some error occurs.1473 */1474static int pl330_submit_req(struct pl330_thread *thrd,1475 struct dma_pl330_desc *desc)1476{1477 struct pl330_dmac *pl330 = thrd->dmac;1478 struct _xfer_spec xs;1479 unsigned long flags;1480 unsigned idx;1481 u32 ccr;1482 int ret = 0;1483 1484 switch (desc->rqtype) {1485 case DMA_MEM_TO_DEV:1486 break;1487 1488 case DMA_DEV_TO_MEM:1489 break;1490 1491 case DMA_MEM_TO_MEM:1492 break;1493 1494 default:1495 return -ENOTSUPP;1496 }1497 1498 if (pl330->state == DYING1499 || pl330->dmac_tbd.reset_chan & (1 << thrd->id)) {1500 dev_info(thrd->dmac->ddma.dev, "%s:%d\n",1501 __func__, __LINE__);1502 return -EAGAIN;1503 }1504 1505 /* If request for non-existing peripheral */1506 if (desc->rqtype != DMA_MEM_TO_MEM &&1507 desc->peri >= pl330->pcfg.num_peri) {1508 dev_info(thrd->dmac->ddma.dev,1509 "%s:%d Invalid peripheral(%u)!\n",1510 __func__, __LINE__, desc->peri);1511 return -EINVAL;1512 }1513 1514 spin_lock_irqsave(&pl330->lock, flags);1515 1516 if (_queue_full(thrd)) {1517 ret = -EAGAIN;1518 goto xfer_exit;1519 }1520 1521 /* Prefer Secure Channel */1522 if (!_manager_ns(thrd))1523 desc->rqcfg.nonsecure = 0;1524 else1525 desc->rqcfg.nonsecure = 1;1526 1527 ccr = _prepare_ccr(&desc->rqcfg);1528 1529 idx = thrd->req[0].desc == NULL ? 0 : 1;1530 1531 xs.ccr = ccr;1532 xs.desc = desc;1533 1534 /* First dry run to check if req is acceptable */1535 ret = _setup_req(pl330, 1, thrd, idx, &xs);1536 1537 if (ret > pl330->mcbufsz / 2) {1538 dev_info(pl330->ddma.dev, "%s:%d Try increasing mcbufsz (%i/%i)\n",1539 __func__, __LINE__, ret, pl330->mcbufsz / 2);1540 ret = -ENOMEM;1541 goto xfer_exit;1542 }1543 1544 /* Hook the request */1545 thrd->lstenq = idx;1546 thrd->req[idx].desc = desc;1547 _setup_req(pl330, 0, thrd, idx, &xs);1548 1549 ret = 0;1550 1551xfer_exit:1552 spin_unlock_irqrestore(&pl330->lock, flags);1553 1554 return ret;1555}1556 1557static void dma_pl330_rqcb(struct dma_pl330_desc *desc, enum pl330_op_err err)1558{1559 struct dma_pl330_chan *pch;1560 unsigned long flags;1561 1562 if (!desc)1563 return;1564 1565 pch = desc->pchan;1566 1567 /* If desc aborted */1568 if (!pch)1569 return;1570 1571 spin_lock_irqsave(&pch->lock, flags);1572 1573 desc->status = DONE;1574 1575 spin_unlock_irqrestore(&pch->lock, flags);1576 1577 tasklet_schedule(&pch->task);1578}1579 1580static void pl330_dotask(struct tasklet_struct *t)1581{1582 struct pl330_dmac *pl330 = from_tasklet(pl330, t, tasks);1583 unsigned long flags;1584 int i;1585 1586 spin_lock_irqsave(&pl330->lock, flags);1587 1588 /* The DMAC itself gone nuts */1589 if (pl330->dmac_tbd.reset_dmac) {1590 pl330->state = DYING;1591 /* Reset the manager too */1592 pl330->dmac_tbd.reset_mngr = true;1593 /* Clear the reset flag */1594 pl330->dmac_tbd.reset_dmac = false;1595 }1596 1597 if (pl330->dmac_tbd.reset_mngr) {1598 _stop(pl330->manager);1599 /* Reset all channels */1600 pl330->dmac_tbd.reset_chan = (1 << pl330->pcfg.num_chan) - 1;1601 /* Clear the reset flag */1602 pl330->dmac_tbd.reset_mngr = false;1603 }1604 1605 for (i = 0; i < pl330->pcfg.num_chan; i++) {1606 1607 if (pl330->dmac_tbd.reset_chan & (1 << i)) {1608 struct pl330_thread *thrd = &pl330->channels[i];1609 void __iomem *regs = pl330->base;1610 enum pl330_op_err err;1611 1612 _stop(thrd);1613 1614 if (readl(regs + FSC) & (1 << thrd->id))1615 err = PL330_ERR_FAIL;1616 else1617 err = PL330_ERR_ABORT;1618 1619 spin_unlock_irqrestore(&pl330->lock, flags);1620 dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, err);1621 dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, err);1622 spin_lock_irqsave(&pl330->lock, flags);1623 1624 thrd->req[0].desc = NULL;1625 thrd->req[1].desc = NULL;1626 thrd->req_running = -1;1627 1628 /* Clear the reset flag */1629 pl330->dmac_tbd.reset_chan &= ~(1 << i);1630 }1631 }1632 1633 spin_unlock_irqrestore(&pl330->lock, flags);1634 1635 return;1636}1637 1638/* Returns 1 if state was updated, 0 otherwise */1639static int pl330_update(struct pl330_dmac *pl330)1640{1641 struct dma_pl330_desc *descdone;1642 unsigned long flags;1643 void __iomem *regs;1644 u32 val;1645 int id, ev, ret = 0;1646 1647 regs = pl330->base;1648 1649 spin_lock_irqsave(&pl330->lock, flags);1650 1651 val = readl(regs + FSM) & 0x1;1652 if (val)1653 pl330->dmac_tbd.reset_mngr = true;1654 else1655 pl330->dmac_tbd.reset_mngr = false;1656 1657 val = readl(regs + FSC) & ((1 << pl330->pcfg.num_chan) - 1);1658 pl330->dmac_tbd.reset_chan |= val;1659 if (val) {1660 int i = 0;1661 while (i < pl330->pcfg.num_chan) {1662 if (val & (1 << i)) {1663 dev_info(pl330->ddma.dev,1664 "Reset Channel-%d\t CS-%x FTC-%x\n",1665 i, readl(regs + CS(i)),1666 readl(regs + FTC(i)));1667 _stop(&pl330->channels[i]);1668 }1669 i++;1670 }1671 }1672 1673 /* Check which event happened i.e, thread notified */1674 val = readl(regs + ES);1675 if (pl330->pcfg.num_events < 321676 && val & ~((1 << pl330->pcfg.num_events) - 1)) {1677 pl330->dmac_tbd.reset_dmac = true;1678 dev_err(pl330->ddma.dev, "%s:%d Unexpected!\n", __func__,1679 __LINE__);1680 ret = 1;1681 goto updt_exit;1682 }1683 1684 for (ev = 0; ev < pl330->pcfg.num_events; ev++) {1685 if (val & (1 << ev)) { /* Event occurred */1686 struct pl330_thread *thrd;1687 u32 inten = readl(regs + INTEN);1688 int active;1689 1690 /* Clear the event */1691 if (inten & (1 << ev))1692 writel(1 << ev, regs + INTCLR);1693 1694 ret = 1;1695 1696 id = pl330->events[ev];1697 1698 thrd = &pl330->channels[id];1699 1700 active = thrd->req_running;1701 if (active == -1) /* Aborted */1702 continue;1703 1704 /* Detach the req */1705 descdone = thrd->req[active].desc;1706 thrd->req[active].desc = NULL;1707 1708 thrd->req_running = -1;1709 1710 /* Get going again ASAP */1711 pl330_start_thread(thrd);1712 1713 /* For now, just make a list of callbacks to be done */1714 list_add_tail(&descdone->rqd, &pl330->req_done);1715 }1716 }1717 1718 /* Now that we are in no hurry, do the callbacks */1719 while (!list_empty(&pl330->req_done)) {1720 descdone = list_first_entry(&pl330->req_done,1721 struct dma_pl330_desc, rqd);1722 list_del(&descdone->rqd);1723 spin_unlock_irqrestore(&pl330->lock, flags);1724 dma_pl330_rqcb(descdone, PL330_ERR_NONE);1725 spin_lock_irqsave(&pl330->lock, flags);1726 }1727 1728updt_exit:1729 spin_unlock_irqrestore(&pl330->lock, flags);1730 1731 if (pl330->dmac_tbd.reset_dmac1732 || pl330->dmac_tbd.reset_mngr1733 || pl330->dmac_tbd.reset_chan) {1734 ret = 1;1735 tasklet_schedule(&pl330->tasks);1736 }1737 1738 return ret;1739}1740 1741/* Reserve an event */1742static inline int _alloc_event(struct pl330_thread *thrd)1743{1744 struct pl330_dmac *pl330 = thrd->dmac;1745 int ev;1746 1747 for (ev = 0; ev < pl330->pcfg.num_events; ev++)1748 if (pl330->events[ev] == -1) {1749 pl330->events[ev] = thrd->id;1750 return ev;1751 }1752 1753 return -1;1754}1755 1756static bool _chan_ns(const struct pl330_dmac *pl330, int i)1757{1758 return pl330->pcfg.irq_ns & (1 << i);1759}1760 1761/* Upon success, returns IdentityToken for the1762 * allocated channel, NULL otherwise.1763 */1764static struct pl330_thread *pl330_request_channel(struct pl330_dmac *pl330)1765{1766 struct pl330_thread *thrd = NULL;1767 int chans, i;1768 1769 if (pl330->state == DYING)1770 return NULL;1771 1772 chans = pl330->pcfg.num_chan;1773 1774 for (i = 0; i < chans; i++) {1775 thrd = &pl330->channels[i];1776 if ((thrd->free) && (!_manager_ns(thrd) ||1777 _chan_ns(pl330, i))) {1778 thrd->ev = _alloc_event(thrd);1779 if (thrd->ev >= 0) {1780 thrd->free = false;1781 thrd->lstenq = 1;1782 thrd->req[0].desc = NULL;1783 thrd->req[1].desc = NULL;1784 thrd->req_running = -1;1785 break;1786 }1787 }1788 thrd = NULL;1789 }1790 1791 return thrd;1792}1793 1794/* Release an event */1795static inline void _free_event(struct pl330_thread *thrd, int ev)1796{1797 struct pl330_dmac *pl330 = thrd->dmac;1798 1799 /* If the event is valid and was held by the thread */1800 if (ev >= 0 && ev < pl330->pcfg.num_events1801 && pl330->events[ev] == thrd->id)1802 pl330->events[ev] = -1;1803}1804 1805static void pl330_release_channel(struct pl330_thread *thrd)1806{1807 if (!thrd || thrd->free)1808 return;1809 1810 _stop(thrd);1811 1812 dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, PL330_ERR_ABORT);1813 dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, PL330_ERR_ABORT);1814 1815 _free_event(thrd, thrd->ev);1816 thrd->free = true;1817}1818 1819/* Initialize the structure for PL330 configuration, that can be used1820 * by the client driver the make best use of the DMAC1821 */1822static void read_dmac_config(struct pl330_dmac *pl330)1823{1824 void __iomem *regs = pl330->base;1825 u32 val;1826 1827 val = readl(regs + CRD) >> CRD_DATA_WIDTH_SHIFT;1828 val &= CRD_DATA_WIDTH_MASK;1829 pl330->pcfg.data_bus_width = 8 * (1 << val);1830 1831 val = readl(regs + CRD) >> CRD_DATA_BUFF_SHIFT;1832 val &= CRD_DATA_BUFF_MASK;1833 pl330->pcfg.data_buf_dep = val + 1;1834 1835 val = readl(regs + CR0) >> CR0_NUM_CHANS_SHIFT;1836 val &= CR0_NUM_CHANS_MASK;1837 val += 1;1838 pl330->pcfg.num_chan = val;1839 1840 val = readl(regs + CR0);1841 if (val & CR0_PERIPH_REQ_SET) {1842 val = (val >> CR0_NUM_PERIPH_SHIFT) & CR0_NUM_PERIPH_MASK;1843 val += 1;1844 pl330->pcfg.num_peri = val;1845 pl330->pcfg.peri_ns = readl(regs + CR4);1846 } else {1847 pl330->pcfg.num_peri = 0;1848 }1849 1850 val = readl(regs + CR0);1851 if (val & CR0_BOOT_MAN_NS)1852 pl330->pcfg.mode |= DMAC_MODE_NS;1853 else1854 pl330->pcfg.mode &= ~DMAC_MODE_NS;1855 1856 val = readl(regs + CR0) >> CR0_NUM_EVENTS_SHIFT;1857 val &= CR0_NUM_EVENTS_MASK;1858 val += 1;1859 pl330->pcfg.num_events = val;1860 1861 pl330->pcfg.irq_ns = readl(regs + CR3);1862}1863 1864static inline void _reset_thread(struct pl330_thread *thrd)1865{1866 struct pl330_dmac *pl330 = thrd->dmac;1867 1868 thrd->req[0].mc_cpu = pl330->mcode_cpu1869 + (thrd->id * pl330->mcbufsz);1870 thrd->req[0].mc_bus = pl330->mcode_bus1871 + (thrd->id * pl330->mcbufsz);1872 thrd->req[0].desc = NULL;1873 1874 thrd->req[1].mc_cpu = thrd->req[0].mc_cpu1875 + pl330->mcbufsz / 2;1876 thrd->req[1].mc_bus = thrd->req[0].mc_bus1877 + pl330->mcbufsz / 2;1878 thrd->req[1].desc = NULL;1879 1880 thrd->req_running = -1;1881}1882 1883static int dmac_alloc_threads(struct pl330_dmac *pl330)1884{1885 int chans = pl330->pcfg.num_chan;1886 struct pl330_thread *thrd;1887 int i;1888 1889 /* Allocate 1 Manager and 'chans' Channel threads */1890 pl330->channels = kcalloc(1 + chans, sizeof(*thrd),1891 GFP_KERNEL);1892 if (!pl330->channels)1893 return -ENOMEM;1894 1895 /* Init Channel threads */1896 for (i = 0; i < chans; i++) {1897 thrd = &pl330->channels[i];1898 thrd->id = i;1899 thrd->dmac = pl330;1900 _reset_thread(thrd);1901 thrd->free = true;1902 }1903 1904 /* MANAGER is indexed at the end */1905 thrd = &pl330->channels[chans];1906 thrd->id = chans;1907 thrd->dmac = pl330;1908 thrd->free = false;1909 pl330->manager = thrd;1910 1911 return 0;1912}1913 1914static int dmac_alloc_resources(struct pl330_dmac *pl330)1915{1916 int chans = pl330->pcfg.num_chan;1917 int ret;1918 1919 /*1920 * Alloc MicroCode buffer for 'chans' Channel threads.1921 * A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN)1922 */1923 pl330->mcode_cpu = dma_alloc_attrs(pl330->ddma.dev,1924 chans * pl330->mcbufsz,1925 &pl330->mcode_bus, GFP_KERNEL,1926 DMA_ATTR_PRIVILEGED);1927 if (!pl330->mcode_cpu) {1928 dev_err(pl330->ddma.dev, "%s:%d Can't allocate memory!\n",1929 __func__, __LINE__);1930 return -ENOMEM;1931 }1932 1933 ret = dmac_alloc_threads(pl330);1934 if (ret) {1935 dev_err(pl330->ddma.dev, "%s:%d Can't to create channels for DMAC!\n",1936 __func__, __LINE__);1937 dma_free_attrs(pl330->ddma.dev,1938 chans * pl330->mcbufsz,1939 pl330->mcode_cpu, pl330->mcode_bus,1940 DMA_ATTR_PRIVILEGED);1941 return ret;1942 }1943 1944 return 0;1945}1946 1947static int pl330_add(struct pl330_dmac *pl330)1948{1949 int i, ret;1950 1951 /* Check if we can handle this DMAC */1952 if ((pl330->pcfg.periph_id & 0xfffff) != PERIPH_ID_VAL) {1953 dev_err(pl330->ddma.dev, "PERIPH_ID 0x%x !\n",1954 pl330->pcfg.periph_id);1955 return -EINVAL;1956 }1957 1958 /* Read the configuration of the DMAC */1959 read_dmac_config(pl330);1960 1961 if (pl330->pcfg.num_events == 0) {1962 dev_err(pl330->ddma.dev, "%s:%d Can't work without events!\n",1963 __func__, __LINE__);1964 return -EINVAL;1965 }1966 1967 spin_lock_init(&pl330->lock);1968 1969 INIT_LIST_HEAD(&pl330->req_done);1970 1971 /* Use default MC buffer size if not provided */1972 if (!pl330->mcbufsz)1973 pl330->mcbufsz = MCODE_BUFF_PER_REQ * 2;1974 1975 /* Mark all events as free */1976 for (i = 0; i < pl330->pcfg.num_events; i++)1977 pl330->events[i] = -1;1978 1979 /* Allocate resources needed by the DMAC */1980 ret = dmac_alloc_resources(pl330);1981 if (ret) {1982 dev_err(pl330->ddma.dev, "Unable to create channels for DMAC\n");1983 return ret;1984 }1985 1986 tasklet_setup(&pl330->tasks, pl330_dotask);1987 1988 pl330->state = INIT;1989 1990 return 0;1991}1992 1993static int dmac_free_threads(struct pl330_dmac *pl330)1994{1995 struct pl330_thread *thrd;1996 int i;1997 1998 /* Release Channel threads */1999 for (i = 0; i < pl330->pcfg.num_chan; i++) {2000 thrd = &pl330->channels[i];2001 pl330_release_channel(thrd);2002 }2003 2004 /* Free memory */2005 kfree(pl330->channels);2006 2007 return 0;2008}2009 2010static void pl330_del(struct pl330_dmac *pl330)2011{2012 pl330->state = UNINIT;2013 2014 tasklet_kill(&pl330->tasks);2015 2016 /* Free DMAC resources */2017 dmac_free_threads(pl330);2018 2019 dma_free_attrs(pl330->ddma.dev,2020 pl330->pcfg.num_chan * pl330->mcbufsz, pl330->mcode_cpu,2021 pl330->mcode_bus, DMA_ATTR_PRIVILEGED);2022}2023 2024/* forward declaration */2025static struct amba_driver pl330_driver;2026 2027static inline struct dma_pl330_chan *2028to_pchan(struct dma_chan *ch)2029{2030 if (!ch)2031 return NULL;2032 2033 return container_of(ch, struct dma_pl330_chan, chan);2034}2035 2036static inline struct dma_pl330_desc *2037to_desc(struct dma_async_tx_descriptor *tx)2038{2039 return container_of(tx, struct dma_pl330_desc, txd);2040}2041 2042static inline void fill_queue(struct dma_pl330_chan *pch)2043{2044 struct dma_pl330_desc *desc;2045 int ret;2046 2047 list_for_each_entry(desc, &pch->work_list, node) {2048 2049 /* If already submitted */2050 if (desc->status == BUSY || desc->status == PAUSED)2051 continue;2052 2053 ret = pl330_submit_req(pch->thread, desc);2054 if (!ret) {2055 desc->status = BUSY;2056 } else if (ret == -EAGAIN) {2057 /* QFull or DMAC Dying */2058 break;2059 } else {2060 /* Unacceptable request */2061 desc->status = DONE;2062 dev_err(pch->dmac->ddma.dev, "%s:%d Bad Desc(%d)\n",2063 __func__, __LINE__, desc->txd.cookie);2064 tasklet_schedule(&pch->task);2065 }2066 }2067}2068 2069static void pl330_tasklet(struct tasklet_struct *t)2070{2071 struct dma_pl330_chan *pch = from_tasklet(pch, t, task);2072 struct dma_pl330_desc *desc, *_dt;2073 unsigned long flags;2074 bool power_down = false;2075 2076 spin_lock_irqsave(&pch->lock, flags);2077 2078 /* Pick up ripe tomatoes */2079 list_for_each_entry_safe(desc, _dt, &pch->work_list, node)2080 if (desc->status == DONE) {2081 if (!pch->cyclic)2082 dma_cookie_complete(&desc->txd);2083 list_move_tail(&desc->node, &pch->completed_list);2084 }2085 2086 /* Try to submit a req imm. next to the last completed cookie */2087 fill_queue(pch);2088 2089 if (list_empty(&pch->work_list)) {2090 spin_lock(&pch->thread->dmac->lock);2091 _stop(pch->thread);2092 spin_unlock(&pch->thread->dmac->lock);2093 power_down = true;2094 pch->active = false;2095 } else {2096 /* Make sure the PL330 Channel thread is active */2097 spin_lock(&pch->thread->dmac->lock);2098 pl330_start_thread(pch->thread);2099 spin_unlock(&pch->thread->dmac->lock);2100 }2101 2102 while (!list_empty(&pch->completed_list)) {2103 struct dmaengine_desc_callback cb;2104 2105 desc = list_first_entry(&pch->completed_list,2106 struct dma_pl330_desc, node);2107 2108 dmaengine_desc_get_callback(&desc->txd, &cb);2109 2110 if (pch->cyclic) {2111 desc->status = PREP;2112 list_move_tail(&desc->node, &pch->work_list);2113 if (power_down) {2114 pch->active = true;2115 spin_lock(&pch->thread->dmac->lock);2116 pl330_start_thread(pch->thread);2117 spin_unlock(&pch->thread->dmac->lock);2118 power_down = false;2119 }2120 } else {2121 desc->status = FREE;2122 list_move_tail(&desc->node, &pch->dmac->desc_pool);2123 }2124 2125 dma_descriptor_unmap(&desc->txd);2126 2127 if (dmaengine_desc_callback_valid(&cb)) {2128 spin_unlock_irqrestore(&pch->lock, flags);2129 dmaengine_desc_callback_invoke(&cb, NULL);2130 spin_lock_irqsave(&pch->lock, flags);2131 }2132 }2133 spin_unlock_irqrestore(&pch->lock, flags);2134 2135 /* If work list empty, power down */2136 if (power_down) {2137 pm_runtime_mark_last_busy(pch->dmac->ddma.dev);2138 pm_runtime_put_autosuspend(pch->dmac->ddma.dev);2139 }2140}2141 2142static struct dma_chan *of_dma_pl330_xlate(struct of_phandle_args *dma_spec,2143 struct of_dma *ofdma)2144{2145 int count = dma_spec->args_count;2146 struct pl330_dmac *pl330 = ofdma->of_dma_data;2147 unsigned int chan_id;2148 2149 if (!pl330)2150 return NULL;2151 2152 if (count != 1)2153 return NULL;2154 2155 chan_id = dma_spec->args[0];2156 if (chan_id >= pl330->num_peripherals)2157 return NULL;2158 2159 return dma_get_slave_channel(&pl330->peripherals[chan_id].chan);2160}2161 2162static int pl330_alloc_chan_resources(struct dma_chan *chan)2163{2164 struct dma_pl330_chan *pch = to_pchan(chan);2165 struct pl330_dmac *pl330 = pch->dmac;2166 unsigned long flags;2167 2168 spin_lock_irqsave(&pl330->lock, flags);2169 2170 dma_cookie_init(chan);2171 pch->cyclic = false;2172 2173 pch->thread = pl330_request_channel(pl330);2174 if (!pch->thread) {2175 spin_unlock_irqrestore(&pl330->lock, flags);2176 return -ENOMEM;2177 }2178 2179 tasklet_setup(&pch->task, pl330_tasklet);2180 2181 spin_unlock_irqrestore(&pl330->lock, flags);2182 2183 return 1;2184}2185 2186/*2187 * We need the data direction between the DMAC (the dma-mapping "device") and2188 * the FIFO (the dmaengine "dev"), from the FIFO's point of view. Confusing!2189 */2190static enum dma_data_direction2191pl330_dma_slave_map_dir(enum dma_transfer_direction dir)2192{2193 switch (dir) {2194 case DMA_MEM_TO_DEV:2195 return DMA_FROM_DEVICE;2196 case DMA_DEV_TO_MEM:2197 return DMA_TO_DEVICE;2198 case DMA_DEV_TO_DEV:2199 return DMA_BIDIRECTIONAL;2200 default:2201 return DMA_NONE;2202 }2203}2204 2205static void pl330_unprep_slave_fifo(struct dma_pl330_chan *pch)2206{2207 if (pch->dir != DMA_NONE)2208 dma_unmap_resource(pch->chan.device->dev, pch->fifo_dma,2209 1 << pch->burst_sz, pch->dir, 0);2210 pch->dir = DMA_NONE;2211}2212 2213 2214static bool pl330_prep_slave_fifo(struct dma_pl330_chan *pch,2215 enum dma_transfer_direction dir)2216{2217 struct device *dev = pch->chan.device->dev;2218 enum dma_data_direction dma_dir = pl330_dma_slave_map_dir(dir);2219 2220 /* Already mapped for this config? */2221 if (pch->dir == dma_dir)2222 return true;2223 2224 pl330_unprep_slave_fifo(pch);2225 pch->fifo_dma = dma_map_resource(dev, pch->fifo_addr,2226 1 << pch->burst_sz, dma_dir, 0);2227 if (dma_mapping_error(dev, pch->fifo_dma))2228 return false;2229 2230 pch->dir = dma_dir;2231 return true;2232}2233 2234static int fixup_burst_len(int max_burst_len, int quirks)2235{2236 if (max_burst_len > PL330_MAX_BURST)2237 return PL330_MAX_BURST;2238 else if (max_burst_len < 1)2239 return 1;2240 else2241 return max_burst_len;2242}2243 2244static int pl330_config_write(struct dma_chan *chan,2245 struct dma_slave_config *slave_config,2246 enum dma_transfer_direction direction)2247{2248 struct dma_pl330_chan *pch = to_pchan(chan);2249 2250 pl330_unprep_slave_fifo(pch);2251 if (direction == DMA_MEM_TO_DEV) {2252 if (slave_config->dst_addr)2253 pch->fifo_addr = slave_config->dst_addr;2254 if (slave_config->dst_addr_width)2255 pch->burst_sz = __ffs(slave_config->dst_addr_width);2256 pch->burst_len = fixup_burst_len(slave_config->dst_maxburst,2257 pch->dmac->quirks);2258 } else if (direction == DMA_DEV_TO_MEM) {2259 if (slave_config->src_addr)2260 pch->fifo_addr = slave_config->src_addr;2261 if (slave_config->src_addr_width)2262 pch->burst_sz = __ffs(slave_config->src_addr_width);2263 pch->burst_len = fixup_burst_len(slave_config->src_maxburst,2264 pch->dmac->quirks);2265 }2266 2267 return 0;2268}2269 2270static int pl330_config(struct dma_chan *chan,2271 struct dma_slave_config *slave_config)2272{2273 struct dma_pl330_chan *pch = to_pchan(chan);2274 2275 memcpy(&pch->slave_config, slave_config, sizeof(*slave_config));2276 2277 return 0;2278}2279 2280static int pl330_terminate_all(struct dma_chan *chan)2281{2282 struct dma_pl330_chan *pch = to_pchan(chan);2283 struct dma_pl330_desc *desc;2284 unsigned long flags;2285 struct pl330_dmac *pl330 = pch->dmac;2286 bool power_down = false;2287 2288 pm_runtime_get_sync(pl330->ddma.dev);2289 spin_lock_irqsave(&pch->lock, flags);2290 2291 spin_lock(&pl330->lock);2292 _stop(pch->thread);2293 pch->thread->req[0].desc = NULL;2294 pch->thread->req[1].desc = NULL;2295 pch->thread->req_running = -1;2296 spin_unlock(&pl330->lock);2297 2298 power_down = pch->active;2299 pch->active = false;2300 2301 /* Mark all desc done */2302 list_for_each_entry(desc, &pch->submitted_list, node) {2303 desc->status = FREE;2304 dma_cookie_complete(&desc->txd);2305 }2306 2307 list_for_each_entry(desc, &pch->work_list , node) {2308 desc->status = FREE;2309 dma_cookie_complete(&desc->txd);2310 }2311 2312 list_splice_tail_init(&pch->submitted_list, &pl330->desc_pool);2313 list_splice_tail_init(&pch->work_list, &pl330->desc_pool);2314 list_splice_tail_init(&pch->completed_list, &pl330->desc_pool);2315 spin_unlock_irqrestore(&pch->lock, flags);2316 pm_runtime_mark_last_busy(pl330->ddma.dev);2317 if (power_down)2318 pm_runtime_put_autosuspend(pl330->ddma.dev);2319 pm_runtime_put_autosuspend(pl330->ddma.dev);2320 2321 return 0;2322}2323 2324/*2325 * We don't support DMA_RESUME command because of hardware2326 * limitations, so after pausing the channel we cannot restore2327 * it to active state. We have to terminate channel and setup2328 * DMA transfer again. This pause feature was implemented to2329 * allow safely read residue before channel termination.2330 */2331static int pl330_pause(struct dma_chan *chan)2332{2333 struct dma_pl330_chan *pch = to_pchan(chan);2334 struct pl330_dmac *pl330 = pch->dmac;2335 struct dma_pl330_desc *desc;2336 unsigned long flags;2337 2338 pm_runtime_get_sync(pl330->ddma.dev);2339 spin_lock_irqsave(&pch->lock, flags);2340 2341 spin_lock(&pl330->lock);2342 _stop(pch->thread);2343 spin_unlock(&pl330->lock);2344 2345 list_for_each_entry(desc, &pch->work_list, node) {2346 if (desc->status == BUSY)2347 desc->status = PAUSED;2348 }2349 spin_unlock_irqrestore(&pch->lock, flags);2350 pm_runtime_mark_last_busy(pl330->ddma.dev);2351 pm_runtime_put_autosuspend(pl330->ddma.dev);2352 2353 return 0;2354}2355 2356static void pl330_free_chan_resources(struct dma_chan *chan)2357{2358 struct dma_pl330_chan *pch = to_pchan(chan);2359 struct pl330_dmac *pl330 = pch->dmac;2360 unsigned long flags;2361 2362 tasklet_kill(&pch->task);2363 2364 pm_runtime_get_sync(pch->dmac->ddma.dev);2365 spin_lock_irqsave(&pl330->lock, flags);2366 2367 pl330_release_channel(pch->thread);2368 pch->thread = NULL;2369 2370 if (pch->cyclic)2371 list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);2372 2373 spin_unlock_irqrestore(&pl330->lock, flags);2374 pm_runtime_mark_last_busy(pch->dmac->ddma.dev);2375 pm_runtime_put_autosuspend(pch->dmac->ddma.dev);2376 pl330_unprep_slave_fifo(pch);2377}2378 2379static int pl330_get_current_xferred_count(struct dma_pl330_chan *pch,2380 struct dma_pl330_desc *desc)2381{2382 struct pl330_thread *thrd = pch->thread;2383 struct pl330_dmac *pl330 = pch->dmac;2384 void __iomem *regs = thrd->dmac->base;2385 u32 val, addr;2386 2387 pm_runtime_get_sync(pl330->ddma.dev);2388 val = addr = 0;2389 if (desc->rqcfg.src_inc) {2390 val = readl(regs + SA(thrd->id));2391 addr = desc->px.src_addr;2392 } else {2393 val = readl(regs + DA(thrd->id));2394 addr = desc->px.dst_addr;2395 }2396 pm_runtime_mark_last_busy(pch->dmac->ddma.dev);2397 pm_runtime_put_autosuspend(pl330->ddma.dev);2398 2399 /* If DMAMOV hasn't finished yet, SAR/DAR can be zero */2400 if (!val)2401 return 0;2402 2403 return val - addr;2404}2405 2406static enum dma_status2407pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,2408 struct dma_tx_state *txstate)2409{2410 enum dma_status ret;2411 unsigned long flags;2412 struct dma_pl330_desc *desc, *running = NULL, *last_enq = NULL;2413 struct dma_pl330_chan *pch = to_pchan(chan);2414 unsigned int transferred, residual = 0;2415 2416 ret = dma_cookie_status(chan, cookie, txstate);2417 2418 if (!txstate)2419 return ret;2420 2421 if (ret == DMA_COMPLETE)2422 goto out;2423 2424 spin_lock_irqsave(&pch->lock, flags);2425 spin_lock(&pch->thread->dmac->lock);2426 2427 if (pch->thread->req_running != -1)2428 running = pch->thread->req[pch->thread->req_running].desc;2429 2430 last_enq = pch->thread->req[pch->thread->lstenq].desc;2431 2432 /* Check in pending list */2433 list_for_each_entry(desc, &pch->work_list, node) {2434 if (desc->status == DONE)2435 transferred = desc->bytes_requested;2436 else if (running && desc == running)2437 transferred =2438 pl330_get_current_xferred_count(pch, desc);2439 else if (desc->status == BUSY || desc->status == PAUSED)2440 /*2441 * Busy but not running means either just enqueued,2442 * or finished and not yet marked done2443 */2444 if (desc == last_enq)2445 transferred = 0;2446 else2447 transferred = desc->bytes_requested;2448 else2449 transferred = 0;2450 residual += desc->bytes_requested - transferred;2451 if (desc->txd.cookie == cookie) {2452 switch (desc->status) {2453 case DONE:2454 ret = DMA_COMPLETE;2455 break;2456 case PAUSED:2457 ret = DMA_PAUSED;2458 break;2459 case PREP:2460 case BUSY:2461 ret = DMA_IN_PROGRESS;2462 break;2463 default:2464 WARN_ON(1);2465 }2466 break;2467 }2468 if (desc->last)2469 residual = 0;2470 }2471 spin_unlock(&pch->thread->dmac->lock);2472 spin_unlock_irqrestore(&pch->lock, flags);2473 2474out:2475 dma_set_residue(txstate, residual);2476 2477 return ret;2478}2479 2480static void pl330_issue_pending(struct dma_chan *chan)2481{2482 struct dma_pl330_chan *pch = to_pchan(chan);2483 unsigned long flags;2484 2485 spin_lock_irqsave(&pch->lock, flags);2486 if (list_empty(&pch->work_list)) {2487 /*2488 * Warn on nothing pending. Empty submitted_list may2489 * break our pm_runtime usage counter as it is2490 * updated on work_list emptiness status.2491 */2492 WARN_ON(list_empty(&pch->submitted_list));2493 pch->active = true;2494 pm_runtime_get_sync(pch->dmac->ddma.dev);2495 }2496 list_splice_tail_init(&pch->submitted_list, &pch->work_list);2497 spin_unlock_irqrestore(&pch->lock, flags);2498 2499 pl330_tasklet(&pch->task);2500}2501 2502/*2503 * We returned the last one of the circular list of descriptor(s)2504 * from prep_xxx, so the argument to submit corresponds to the last2505 * descriptor of the list.2506 */2507static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx)2508{2509 struct dma_pl330_desc *desc, *last = to_desc(tx);2510 struct dma_pl330_chan *pch = to_pchan(tx->chan);2511 dma_cookie_t cookie;2512 unsigned long flags;2513 2514 spin_lock_irqsave(&pch->lock, flags);2515 2516 /* Assign cookies to all nodes */2517 while (!list_empty(&last->node)) {2518 desc = list_entry(last->node.next, struct dma_pl330_desc, node);2519 if (pch->cyclic) {2520 desc->txd.callback = last->txd.callback;2521 desc->txd.callback_param = last->txd.callback_param;2522 }2523 desc->last = false;2524 2525 dma_cookie_assign(&desc->txd);2526 2527 list_move_tail(&desc->node, &pch->submitted_list);2528 }2529 2530 last->last = true;2531 cookie = dma_cookie_assign(&last->txd);2532 list_add_tail(&last->node, &pch->submitted_list);2533 spin_unlock_irqrestore(&pch->lock, flags);2534 2535 return cookie;2536}2537 2538static inline void _init_desc(struct dma_pl330_desc *desc)2539{2540 desc->rqcfg.swap = SWAP_NO;2541 desc->rqcfg.scctl = CCTRL0;2542 desc->rqcfg.dcctl = CCTRL0;2543 desc->txd.tx_submit = pl330_tx_submit;2544 2545 INIT_LIST_HEAD(&desc->node);2546}2547 2548/* Returns the number of descriptors added to the DMAC pool */2549static int add_desc(struct list_head *pool, spinlock_t *lock,2550 gfp_t flg, int count)2551{2552 struct dma_pl330_desc *desc;2553 unsigned long flags;2554 int i;2555 2556 desc = kcalloc(count, sizeof(*desc), flg);2557 if (!desc)2558 return 0;2559 2560 spin_lock_irqsave(lock, flags);2561 2562 for (i = 0; i < count; i++) {2563 _init_desc(&desc[i]);2564 list_add_tail(&desc[i].node, pool);2565 }2566 2567 spin_unlock_irqrestore(lock, flags);2568 2569 return count;2570}2571 2572static struct dma_pl330_desc *pluck_desc(struct list_head *pool,2573 spinlock_t *lock)2574{2575 struct dma_pl330_desc *desc = NULL;2576 unsigned long flags;2577 2578 spin_lock_irqsave(lock, flags);2579 2580 if (!list_empty(pool)) {2581 desc = list_entry(pool->next,2582 struct dma_pl330_desc, node);2583 2584 list_del_init(&desc->node);2585 2586 desc->status = PREP;2587 desc->txd.callback = NULL;2588 desc->txd.callback_result = NULL;2589 }2590 2591 spin_unlock_irqrestore(lock, flags);2592 2593 return desc;2594}2595 2596static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)2597{2598 struct pl330_dmac *pl330 = pch->dmac;2599 u8 *peri_id = pch->chan.private;2600 struct dma_pl330_desc *desc;2601 2602 /* Pluck one desc from the pool of DMAC */2603 desc = pluck_desc(&pl330->desc_pool, &pl330->pool_lock);2604 2605 /* If the DMAC pool is empty, alloc new */2606 if (!desc) {2607 static DEFINE_SPINLOCK(lock);2608 LIST_HEAD(pool);2609 2610 if (!add_desc(&pool, &lock, GFP_ATOMIC, 1))2611 return NULL;2612 2613 desc = pluck_desc(&pool, &lock);2614 WARN_ON(!desc || !list_empty(&pool));2615 }2616 2617 /* Initialize the descriptor */2618 desc->pchan = pch;2619 desc->txd.cookie = 0;2620 async_tx_ack(&desc->txd);2621 2622 desc->peri = peri_id ? pch->chan.chan_id : 0;2623 desc->rqcfg.pcfg = &pch->dmac->pcfg;2624 2625 dma_async_tx_descriptor_init(&desc->txd, &pch->chan);2626 2627 return desc;2628}2629 2630static inline void fill_px(struct pl330_xfer *px,2631 dma_addr_t dst, dma_addr_t src, size_t len)2632{2633 px->bytes = len;2634 px->dst_addr = dst;2635 px->src_addr = src;2636}2637 2638static struct dma_pl330_desc *2639__pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,2640 dma_addr_t src, size_t len)2641{2642 struct dma_pl330_desc *desc = pl330_get_desc(pch);2643 2644 if (!desc) {2645 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",2646 __func__, __LINE__);2647 return NULL;2648 }2649 2650 /*2651 * Ideally we should lookout for reqs bigger than2652 * those that can be programmed with 256 bytes of2653 * MC buffer, but considering a req size is seldom2654 * going to be word-unaligned and more than 200MB,2655 * we take it easy.2656 * Also, should the limit is reached we'd rather2657 * have the platform increase MC buffer size than2658 * complicating this API driver.2659 */2660 fill_px(&desc->px, dst, src, len);2661 2662 return desc;2663}2664 2665/* Call after fixing burst size */2666static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)2667{2668 struct dma_pl330_chan *pch = desc->pchan;2669 struct pl330_dmac *pl330 = pch->dmac;2670 int burst_len;2671 2672 burst_len = pl330->pcfg.data_bus_width / 8;2673 burst_len *= pl330->pcfg.data_buf_dep / pl330->pcfg.num_chan;2674 burst_len >>= desc->rqcfg.brst_size;2675 2676 /* src/dst_burst_len can't be more than 16 */2677 if (burst_len > PL330_MAX_BURST)2678 burst_len = PL330_MAX_BURST;2679 2680 return burst_len;2681}2682 2683static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(2684 struct dma_chan *chan, dma_addr_t dma_addr, size_t len,2685 size_t period_len, enum dma_transfer_direction direction,2686 unsigned long flags)2687{2688 struct dma_pl330_desc *desc = NULL, *first = NULL;2689 struct dma_pl330_chan *pch = to_pchan(chan);2690 struct pl330_dmac *pl330 = pch->dmac;2691 unsigned int i;2692 dma_addr_t dst;2693 dma_addr_t src;2694 2695 if (len % period_len != 0)2696 return NULL;2697 2698 if (!is_slave_direction(direction)) {2699 dev_err(pch->dmac->ddma.dev, "%s:%d Invalid dma direction\n",2700 __func__, __LINE__);2701 return NULL;2702 }2703 2704 pl330_config_write(chan, &pch->slave_config, direction);2705 2706 if (!pl330_prep_slave_fifo(pch, direction))2707 return NULL;2708 2709 for (i = 0; i < len / period_len; i++) {2710 desc = pl330_get_desc(pch);2711 if (!desc) {2712 unsigned long iflags;2713 2714 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",2715 __func__, __LINE__);2716 2717 if (!first)2718 return NULL;2719 2720 spin_lock_irqsave(&pl330->pool_lock, iflags);2721 2722 while (!list_empty(&first->node)) {2723 desc = list_entry(first->node.next,2724 struct dma_pl330_desc, node);2725 list_move_tail(&desc->node, &pl330->desc_pool);2726 }2727 2728 list_move_tail(&first->node, &pl330->desc_pool);2729 2730 spin_unlock_irqrestore(&pl330->pool_lock, iflags);2731 2732 return NULL;2733 }2734 2735 switch (direction) {2736 case DMA_MEM_TO_DEV:2737 desc->rqcfg.src_inc = 1;2738 desc->rqcfg.dst_inc = 0;2739 src = dma_addr;2740 dst = pch->fifo_dma;2741 break;2742 case DMA_DEV_TO_MEM:2743 desc->rqcfg.src_inc = 0;2744 desc->rqcfg.dst_inc = 1;2745 src = pch->fifo_dma;2746 dst = dma_addr;2747 break;2748 default:2749 break;2750 }2751 2752 desc->rqtype = direction;2753 desc->rqcfg.brst_size = pch->burst_sz;2754 desc->rqcfg.brst_len = pch->burst_len;2755 desc->bytes_requested = period_len;2756 fill_px(&desc->px, dst, src, period_len);2757 2758 if (!first)2759 first = desc;2760 else2761 list_add_tail(&desc->node, &first->node);2762 2763 dma_addr += period_len;2764 }2765 2766 if (!desc)2767 return NULL;2768 2769 pch->cyclic = true;2770 2771 return &desc->txd;2772}2773 2774static struct dma_async_tx_descriptor *2775pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,2776 dma_addr_t src, size_t len, unsigned long flags)2777{2778 struct dma_pl330_desc *desc;2779 struct dma_pl330_chan *pch = to_pchan(chan);2780 struct pl330_dmac *pl330;2781 int burst;2782 2783 if (unlikely(!pch || !len))2784 return NULL;2785 2786 pl330 = pch->dmac;2787 2788 desc = __pl330_prep_dma_memcpy(pch, dst, src, len);2789 if (!desc)2790 return NULL;2791 2792 desc->rqcfg.src_inc = 1;2793 desc->rqcfg.dst_inc = 1;2794 desc->rqtype = DMA_MEM_TO_MEM;2795 2796 /* Select max possible burst size */2797 burst = pl330->pcfg.data_bus_width / 8;2798 2799 /*2800 * Make sure we use a burst size that aligns with all the memcpy2801 * parameters because our DMA programming algorithm doesn't cope with2802 * transfers which straddle an entry in the DMA device's MFIFO.2803 */2804 while ((src | dst | len) & (burst - 1))2805 burst /= 2;2806 2807 desc->rqcfg.brst_size = 0;2808 while (burst != (1 << desc->rqcfg.brst_size))2809 desc->rqcfg.brst_size++;2810 2811 desc->rqcfg.brst_len = get_burst_len(desc, len);2812 /*2813 * If burst size is smaller than bus width then make sure we only2814 * transfer one at a time to avoid a burst stradling an MFIFO entry.2815 */2816 if (burst * 8 < pl330->pcfg.data_bus_width)2817 desc->rqcfg.brst_len = 1;2818 2819 desc->bytes_requested = len;2820 2821 return &desc->txd;2822}2823 2824static void __pl330_giveback_desc(struct pl330_dmac *pl330,2825 struct dma_pl330_desc *first)2826{2827 unsigned long flags;2828 struct dma_pl330_desc *desc;2829 2830 if (!first)2831 return;2832 2833 spin_lock_irqsave(&pl330->pool_lock, flags);2834 2835 while (!list_empty(&first->node)) {2836 desc = list_entry(first->node.next,2837 struct dma_pl330_desc, node);2838 list_move_tail(&desc->node, &pl330->desc_pool);2839 }2840 2841 list_move_tail(&first->node, &pl330->desc_pool);2842 2843 spin_unlock_irqrestore(&pl330->pool_lock, flags);2844}2845 2846static struct dma_async_tx_descriptor *2847pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,2848 unsigned int sg_len, enum dma_transfer_direction direction,2849 unsigned long flg, void *context)2850{2851 struct dma_pl330_desc *first, *desc = NULL;2852 struct dma_pl330_chan *pch = to_pchan(chan);2853 struct scatterlist *sg;2854 int i;2855 2856 if (unlikely(!pch || !sgl || !sg_len))2857 return NULL;2858 2859 pl330_config_write(chan, &pch->slave_config, direction);2860 2861 if (!pl330_prep_slave_fifo(pch, direction))2862 return NULL;2863 2864 first = NULL;2865 2866 for_each_sg(sgl, sg, sg_len, i) {2867 2868 desc = pl330_get_desc(pch);2869 if (!desc) {2870 struct pl330_dmac *pl330 = pch->dmac;2871 2872 dev_err(pch->dmac->ddma.dev,2873 "%s:%d Unable to fetch desc\n",2874 __func__, __LINE__);2875 __pl330_giveback_desc(pl330, first);2876 2877 return NULL;2878 }2879 2880 if (!first)2881 first = desc;2882 else2883 list_add_tail(&desc->node, &first->node);2884 2885 if (direction == DMA_MEM_TO_DEV) {2886 desc->rqcfg.src_inc = 1;2887 desc->rqcfg.dst_inc = 0;2888 fill_px(&desc->px, pch->fifo_dma, sg_dma_address(sg),2889 sg_dma_len(sg));2890 } else {2891 desc->rqcfg.src_inc = 0;2892 desc->rqcfg.dst_inc = 1;2893 fill_px(&desc->px, sg_dma_address(sg), pch->fifo_dma,2894 sg_dma_len(sg));2895 }2896 2897 desc->rqcfg.brst_size = pch->burst_sz;2898 desc->rqcfg.brst_len = pch->burst_len;2899 desc->rqtype = direction;2900 desc->bytes_requested = sg_dma_len(sg);2901 }2902 2903 /* Return the last desc in the chain */2904 return &desc->txd;2905}2906 2907static irqreturn_t pl330_irq_handler(int irq, void *data)2908{2909 if (pl330_update(data))2910 return IRQ_HANDLED;2911 else2912 return IRQ_NONE;2913}2914 2915#define PL330_DMA_BUSWIDTHS \2916 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \2917 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \2918 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \2919 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \2920 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)2921 2922#ifdef CONFIG_DEBUG_FS2923static int pl330_debugfs_show(struct seq_file *s, void *data)2924{2925 struct pl330_dmac *pl330 = s->private;2926 int chans, pchs, ch, pr;2927 2928 chans = pl330->pcfg.num_chan;2929 pchs = pl330->num_peripherals;2930 2931 seq_puts(s, "PL330 physical channels:\n");2932 seq_puts(s, "THREAD:\t\tCHANNEL:\n");2933 seq_puts(s, "--------\t-----\n");2934 for (ch = 0; ch < chans; ch++) {2935 struct pl330_thread *thrd = &pl330->channels[ch];2936 int found = -1;2937 2938 for (pr = 0; pr < pchs; pr++) {2939 struct dma_pl330_chan *pch = &pl330->peripherals[pr];2940 2941 if (!pch->thread || thrd->id != pch->thread->id)2942 continue;2943 2944 found = pr;2945 }2946 2947 seq_printf(s, "%d\t\t", thrd->id);2948 if (found == -1)2949 seq_puts(s, "--\n");2950 else2951 seq_printf(s, "%d\n", found);2952 }2953 2954 return 0;2955}2956 2957DEFINE_SHOW_ATTRIBUTE(pl330_debugfs);2958 2959static inline void init_pl330_debugfs(struct pl330_dmac *pl330)2960{2961 debugfs_create_file(dev_name(pl330->ddma.dev),2962 S_IFREG | 0444, NULL, pl330,2963 &pl330_debugfs_fops);2964}2965#else2966static inline void init_pl330_debugfs(struct pl330_dmac *pl330)2967{2968}2969#endif2970 2971/*2972 * Runtime PM callbacks are provided by amba/bus.c driver.2973 *2974 * It is assumed here that IRQ safe runtime PM is chosen in probe and amba2975 * bus driver will only disable/enable the clock in runtime PM callbacks.2976 */2977static int __maybe_unused pl330_suspend(struct device *dev)2978{2979 struct amba_device *pcdev = to_amba_device(dev);2980 2981 pm_runtime_force_suspend(dev);2982 clk_unprepare(pcdev->pclk);2983 2984 return 0;2985}2986 2987static int __maybe_unused pl330_resume(struct device *dev)2988{2989 struct amba_device *pcdev = to_amba_device(dev);2990 int ret;2991 2992 ret = clk_prepare(pcdev->pclk);2993 if (ret)2994 return ret;2995 2996 pm_runtime_force_resume(dev);2997 2998 return ret;2999}3000 3001static const struct dev_pm_ops pl330_pm = {3002 SET_LATE_SYSTEM_SLEEP_PM_OPS(pl330_suspend, pl330_resume)3003};3004 3005static int3006pl330_probe(struct amba_device *adev, const struct amba_id *id)3007{3008 struct pl330_config *pcfg;3009 struct pl330_dmac *pl330;3010 struct dma_pl330_chan *pch, *_p;3011 struct dma_device *pd;3012 struct resource *res;3013 int i, ret, irq;3014 int num_chan;3015 struct device_node *np = adev->dev.of_node;3016 3017 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32));3018 if (ret)3019 return ret;3020 3021 /* Allocate a new DMAC and its Channels */3022 pl330 = devm_kzalloc(&adev->dev, sizeof(*pl330), GFP_KERNEL);3023 if (!pl330)3024 return -ENOMEM;3025 3026 pd = &pl330->ddma;3027 pd->dev = &adev->dev;3028 3029 pl330->mcbufsz = 0;3030 3031 /* get quirk */3032 for (i = 0; i < ARRAY_SIZE(of_quirks); i++)3033 if (of_property_read_bool(np, of_quirks[i].quirk))3034 pl330->quirks |= of_quirks[i].id;3035 3036 res = &adev->res;3037 pl330->base = devm_ioremap_resource(&adev->dev, res);3038 if (IS_ERR(pl330->base))3039 return PTR_ERR(pl330->base);3040 3041 amba_set_drvdata(adev, pl330);3042 3043 pl330->rstc = devm_reset_control_get_optional(&adev->dev, "dma");3044 if (IS_ERR(pl330->rstc)) {3045 return dev_err_probe(&adev->dev, PTR_ERR(pl330->rstc), "Failed to get reset!\n");3046 } else {3047 ret = reset_control_deassert(pl330->rstc);3048 if (ret) {3049 dev_err(&adev->dev, "Couldn't deassert the device from reset!\n");3050 return ret;3051 }3052 }3053 3054 pl330->rstc_ocp = devm_reset_control_get_optional(&adev->dev, "dma-ocp");3055 if (IS_ERR(pl330->rstc_ocp)) {3056 return dev_err_probe(&adev->dev, PTR_ERR(pl330->rstc_ocp),3057 "Failed to get OCP reset!\n");3058 } else {3059 ret = reset_control_deassert(pl330->rstc_ocp);3060 if (ret) {3061 dev_err(&adev->dev, "Couldn't deassert the device from OCP reset!\n");3062 return ret;3063 }3064 }3065 3066 for (i = 0; i < AMBA_NR_IRQS; i++) {3067 irq = adev->irq[i];3068 if (irq) {3069 ret = devm_request_irq(&adev->dev, irq,3070 pl330_irq_handler, 0,3071 dev_name(&adev->dev), pl330);3072 if (ret)3073 return ret;3074 } else {3075 break;3076 }3077 }3078 3079 pcfg = &pl330->pcfg;3080 3081 pcfg->periph_id = adev->periphid;3082 ret = pl330_add(pl330);3083 if (ret)3084 return ret;3085 3086 INIT_LIST_HEAD(&pl330->desc_pool);3087 spin_lock_init(&pl330->pool_lock);3088 3089 /* Create a descriptor pool of default size */3090 if (!add_desc(&pl330->desc_pool, &pl330->pool_lock,3091 GFP_KERNEL, NR_DEFAULT_DESC))3092 dev_warn(&adev->dev, "unable to allocate desc\n");3093 3094 INIT_LIST_HEAD(&pd->channels);3095 3096 /* Initialize channel parameters */3097 num_chan = max_t(int, pcfg->num_peri, pcfg->num_chan);3098 3099 pl330->num_peripherals = num_chan;3100 3101 pl330->peripherals = kcalloc(num_chan, sizeof(*pch), GFP_KERNEL);3102 if (!pl330->peripherals) {3103 ret = -ENOMEM;3104 goto probe_err2;3105 }3106 3107 for (i = 0; i < num_chan; i++) {3108 pch = &pl330->peripherals[i];3109 3110 pch->chan.private = adev->dev.of_node;3111 INIT_LIST_HEAD(&pch->submitted_list);3112 INIT_LIST_HEAD(&pch->work_list);3113 INIT_LIST_HEAD(&pch->completed_list);3114 spin_lock_init(&pch->lock);3115 pch->thread = NULL;3116 pch->chan.device = pd;3117 pch->dmac = pl330;3118 pch->dir = DMA_NONE;3119 3120 /* Add the channel to the DMAC list */3121 list_add_tail(&pch->chan.device_node, &pd->channels);3122 }3123 3124 dma_cap_set(DMA_MEMCPY, pd->cap_mask);3125 if (pcfg->num_peri) {3126 dma_cap_set(DMA_SLAVE, pd->cap_mask);3127 dma_cap_set(DMA_CYCLIC, pd->cap_mask);3128 dma_cap_set(DMA_PRIVATE, pd->cap_mask);3129 }3130 3131 pd->device_alloc_chan_resources = pl330_alloc_chan_resources;3132 pd->device_free_chan_resources = pl330_free_chan_resources;3133 pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy;3134 pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic;3135 pd->device_tx_status = pl330_tx_status;3136 pd->device_prep_slave_sg = pl330_prep_slave_sg;3137 pd->device_config = pl330_config;3138 pd->device_pause = pl330_pause;3139 pd->device_terminate_all = pl330_terminate_all;3140 pd->device_issue_pending = pl330_issue_pending;3141 pd->src_addr_widths = PL330_DMA_BUSWIDTHS;3142 pd->dst_addr_widths = PL330_DMA_BUSWIDTHS;3143 pd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);3144 pd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;3145 pd->max_burst = PL330_MAX_BURST;3146 3147 ret = dma_async_device_register(pd);3148 if (ret) {3149 dev_err(&adev->dev, "unable to register DMAC\n");3150 goto probe_err3;3151 }3152 3153 if (adev->dev.of_node) {3154 ret = of_dma_controller_register(adev->dev.of_node,3155 of_dma_pl330_xlate, pl330);3156 if (ret) {3157 dev_err(&adev->dev,3158 "unable to register DMA to the generic DT DMA helpers\n");3159 }3160 }3161 3162 /*3163 * This is the limit for transfers with a buswidth of 1, larger3164 * buswidths will have larger limits.3165 */3166 dma_set_max_seg_size(&adev->dev, 1900800);3167 3168 init_pl330_debugfs(pl330);3169 dev_info(&adev->dev,3170 "Loaded driver for PL330 DMAC-%x\n", adev->periphid);3171 dev_info(&adev->dev,3172 "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n",3173 pcfg->data_buf_dep, pcfg->data_bus_width / 8, pcfg->num_chan,3174 pcfg->num_peri, pcfg->num_events);3175 3176 pm_runtime_irq_safe(&adev->dev);3177 pm_runtime_use_autosuspend(&adev->dev);3178 pm_runtime_set_autosuspend_delay(&adev->dev, PL330_AUTOSUSPEND_DELAY);3179 pm_runtime_mark_last_busy(&adev->dev);3180 pm_runtime_put_autosuspend(&adev->dev);3181 3182 return 0;3183probe_err3:3184 /* Idle the DMAC */3185 list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,3186 chan.device_node) {3187 3188 /* Remove the channel */3189 list_del(&pch->chan.device_node);3190 3191 /* Flush the channel */3192 if (pch->thread) {3193 pl330_terminate_all(&pch->chan);3194 pl330_free_chan_resources(&pch->chan);3195 }3196 }3197probe_err2:3198 pl330_del(pl330);3199 3200 if (pl330->rstc_ocp)3201 reset_control_assert(pl330->rstc_ocp);3202 3203 if (pl330->rstc)3204 reset_control_assert(pl330->rstc);3205 return ret;3206}3207 3208static void pl330_remove(struct amba_device *adev)3209{3210 struct pl330_dmac *pl330 = amba_get_drvdata(adev);3211 struct dma_pl330_chan *pch, *_p;3212 int i, irq;3213 3214 pm_runtime_get_noresume(pl330->ddma.dev);3215 3216 if (adev->dev.of_node)3217 of_dma_controller_free(adev->dev.of_node);3218 3219 for (i = 0; i < AMBA_NR_IRQS; i++) {3220 irq = adev->irq[i];3221 if (irq)3222 devm_free_irq(&adev->dev, irq, pl330);3223 }3224 3225 dma_async_device_unregister(&pl330->ddma);3226 3227 /* Idle the DMAC */3228 list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,3229 chan.device_node) {3230 3231 /* Remove the channel */3232 list_del(&pch->chan.device_node);3233 3234 /* Flush the channel */3235 if (pch->thread) {3236 pl330_terminate_all(&pch->chan);3237 pl330_free_chan_resources(&pch->chan);3238 }3239 }3240 3241 pl330_del(pl330);3242 3243 if (pl330->rstc_ocp)3244 reset_control_assert(pl330->rstc_ocp);3245 3246 if (pl330->rstc)3247 reset_control_assert(pl330->rstc);3248}3249 3250static const struct amba_id pl330_ids[] = {3251 {3252 .id = 0x00041330,3253 .mask = 0x000fffff,3254 },3255 { 0, 0 },3256};3257 3258MODULE_DEVICE_TABLE(amba, pl330_ids);3259 3260static struct amba_driver pl330_driver = {3261 .drv = {3262 .name = "dma-pl330",3263 .pm = &pl330_pm,3264 },3265 .id_table = pl330_ids,3266 .probe = pl330_probe,3267 .remove = pl330_remove,3268};3269 3270module_amba_driver(pl330_driver);3271 3272MODULE_AUTHOR("Jaswinder Singh <jassisinghbrar@gmail.com>");3273MODULE_DESCRIPTION("API Driver for PL330 DMAC");3274MODULE_LICENSE("GPL");3275