2313 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.4 * Copyright (c) 2020, Linaro Limited5 */6 7#include <dt-bindings/dma/qcom-gpi.h>8#include <linux/bitfield.h>9#include <linux/dma-mapping.h>10#include <linux/dmaengine.h>11#include <linux/module.h>12#include <linux/of_dma.h>13#include <linux/platform_device.h>14#include <linux/dma/qcom-gpi-dma.h>15#include <linux/scatterlist.h>16#include <linux/slab.h>17#include "../dmaengine.h"18#include "../virt-dma.h"19 20#define TRE_TYPE_DMA 0x1021#define TRE_TYPE_GO 0x2022#define TRE_TYPE_CONFIG0 0x2223 24/* TRE flags */25#define TRE_FLAGS_CHAIN BIT(0)26#define TRE_FLAGS_IEOB BIT(8)27#define TRE_FLAGS_IEOT BIT(9)28#define TRE_FLAGS_BEI BIT(10)29#define TRE_FLAGS_LINK BIT(11)30#define TRE_FLAGS_TYPE GENMASK(23, 16)31 32/* SPI CONFIG0 WD0 */33#define TRE_SPI_C0_WORD_SZ GENMASK(4, 0)34#define TRE_SPI_C0_LOOPBACK BIT(8)35#define TRE_SPI_C0_CS BIT(11)36#define TRE_SPI_C0_CPHA BIT(12)37#define TRE_SPI_C0_CPOL BIT(13)38#define TRE_SPI_C0_TX_PACK BIT(24)39#define TRE_SPI_C0_RX_PACK BIT(25)40 41/* CONFIG0 WD2 */42#define TRE_C0_CLK_DIV GENMASK(11, 0)43#define TRE_C0_CLK_SRC GENMASK(19, 16)44 45/* SPI GO WD0 */46#define TRE_SPI_GO_CMD GENMASK(4, 0)47#define TRE_SPI_GO_CS GENMASK(10, 8)48#define TRE_SPI_GO_FRAG BIT(26)49 50/* GO WD2 */51#define TRE_RX_LEN GENMASK(23, 0)52 53/* I2C Config0 WD0 */54#define TRE_I2C_C0_TLOW GENMASK(7, 0)55#define TRE_I2C_C0_THIGH GENMASK(15, 8)56#define TRE_I2C_C0_TCYL GENMASK(23, 16)57#define TRE_I2C_C0_TX_PACK BIT(24)58#define TRE_I2C_C0_RX_PACK BIT(25)59 60/* I2C GO WD0 */61#define TRE_I2C_GO_CMD GENMASK(4, 0)62#define TRE_I2C_GO_ADDR GENMASK(14, 8)63#define TRE_I2C_GO_STRETCH BIT(26)64 65/* DMA TRE */66#define TRE_DMA_LEN GENMASK(23, 0)67 68/* Register offsets from gpi-top */69#define GPII_n_CH_k_CNTXT_0_OFFS(n, k) (0x20000 + (0x4000 * (n)) + (0x80 * (k)))70#define GPII_n_CH_k_CNTXT_0_EL_SIZE GENMASK(31, 24)71#define GPII_n_CH_k_CNTXT_0_CHSTATE GENMASK(23, 20)72#define GPII_n_CH_k_CNTXT_0_ERIDX GENMASK(18, 14)73#define GPII_n_CH_k_CNTXT_0_DIR BIT(3)74#define GPII_n_CH_k_CNTXT_0_PROTO GENMASK(2, 0)75 76#define GPII_n_CH_k_CNTXT_0(el_size, erindex, dir, chtype_proto) \77 (FIELD_PREP(GPII_n_CH_k_CNTXT_0_EL_SIZE, el_size) | \78 FIELD_PREP(GPII_n_CH_k_CNTXT_0_ERIDX, erindex) | \79 FIELD_PREP(GPII_n_CH_k_CNTXT_0_DIR, dir) | \80 FIELD_PREP(GPII_n_CH_k_CNTXT_0_PROTO, chtype_proto))81 82#define GPI_CHTYPE_DIR_IN (0)83#define GPI_CHTYPE_DIR_OUT (1)84 85#define GPI_CHTYPE_PROTO_GPI (0x2)86 87#define GPII_n_CH_k_DOORBELL_0_OFFS(n, k) (0x22000 + (0x4000 * (n)) + (0x8 * (k)))88#define GPII_n_CH_CMD_OFFS(n) (0x23008 + (0x4000 * (n)))89#define GPII_n_CH_CMD_OPCODE GENMASK(31, 24)90#define GPII_n_CH_CMD_CHID GENMASK(7, 0)91#define GPII_n_CH_CMD(opcode, chid) \92 (FIELD_PREP(GPII_n_CH_CMD_OPCODE, opcode) | \93 FIELD_PREP(GPII_n_CH_CMD_CHID, chid))94 95#define GPII_n_CH_CMD_ALLOCATE (0)96#define GPII_n_CH_CMD_START (1)97#define GPII_n_CH_CMD_STOP (2)98#define GPII_n_CH_CMD_RESET (9)99#define GPII_n_CH_CMD_DE_ALLOC (10)100#define GPII_n_CH_CMD_UART_SW_STALE (32)101#define GPII_n_CH_CMD_UART_RFR_READY (33)102#define GPII_n_CH_CMD_UART_RFR_NOT_READY (34)103 104/* EV Context Array */105#define GPII_n_EV_CH_k_CNTXT_0_OFFS(n, k) (0x21000 + (0x4000 * (n)) + (0x80 * (k)))106#define GPII_n_EV_k_CNTXT_0_EL_SIZE GENMASK(31, 24)107#define GPII_n_EV_k_CNTXT_0_CHSTATE GENMASK(23, 20)108#define GPII_n_EV_k_CNTXT_0_INTYPE BIT(16)109#define GPII_n_EV_k_CNTXT_0_CHTYPE GENMASK(3, 0)110 111#define GPII_n_EV_k_CNTXT_0(el_size, inttype, chtype) \112 (FIELD_PREP(GPII_n_EV_k_CNTXT_0_EL_SIZE, el_size) | \113 FIELD_PREP(GPII_n_EV_k_CNTXT_0_INTYPE, inttype) | \114 FIELD_PREP(GPII_n_EV_k_CNTXT_0_CHTYPE, chtype))115 116#define GPI_INTTYPE_IRQ (1)117#define GPI_CHTYPE_GPI_EV (0x2)118 119enum CNTXT_OFFS {120 CNTXT_0_CONFIG = 0x0,121 CNTXT_1_R_LENGTH = 0x4,122 CNTXT_2_RING_BASE_LSB = 0x8,123 CNTXT_3_RING_BASE_MSB = 0xC,124 CNTXT_4_RING_RP_LSB = 0x10,125 CNTXT_5_RING_RP_MSB = 0x14,126 CNTXT_6_RING_WP_LSB = 0x18,127 CNTXT_7_RING_WP_MSB = 0x1C,128 CNTXT_8_RING_INT_MOD = 0x20,129 CNTXT_9_RING_INTVEC = 0x24,130 CNTXT_10_RING_MSI_LSB = 0x28,131 CNTXT_11_RING_MSI_MSB = 0x2C,132 CNTXT_12_RING_RP_UPDATE_LSB = 0x30,133 CNTXT_13_RING_RP_UPDATE_MSB = 0x34,134};135 136#define GPII_n_EV_CH_k_DOORBELL_0_OFFS(n, k) (0x22100 + (0x4000 * (n)) + (0x8 * (k)))137#define GPII_n_EV_CH_CMD_OFFS(n) (0x23010 + (0x4000 * (n)))138#define GPII_n_EV_CMD_OPCODE GENMASK(31, 24)139#define GPII_n_EV_CMD_CHID GENMASK(7, 0)140#define GPII_n_EV_CMD(opcode, chid) \141 (FIELD_PREP(GPII_n_EV_CMD_OPCODE, opcode) | \142 FIELD_PREP(GPII_n_EV_CMD_CHID, chid))143 144#define GPII_n_EV_CH_CMD_ALLOCATE (0x00)145#define GPII_n_EV_CH_CMD_RESET (0x09)146#define GPII_n_EV_CH_CMD_DE_ALLOC (0x0A)147 148#define GPII_n_CNTXT_TYPE_IRQ_OFFS(n) (0x23080 + (0x4000 * (n)))149 150/* mask type register */151#define GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(n) (0x23088 + (0x4000 * (n)))152#define GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK GENMASK(6, 0)153#define GPII_n_CNTXT_TYPE_IRQ_MSK_GENERAL BIT(6)154#define GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB BIT(3)155#define GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB BIT(2)156#define GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL BIT(1)157#define GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL BIT(0)158 159#define GPII_n_CNTXT_SRC_GPII_CH_IRQ_OFFS(n) (0x23090 + (0x4000 * (n)))160#define GPII_n_CNTXT_SRC_EV_CH_IRQ_OFFS(n) (0x23094 + (0x4000 * (n)))161 162/* Mask channel control interrupt register */163#define GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(n) (0x23098 + (0x4000 * (n)))164#define GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK GENMASK(1, 0)165 166/* Mask event control interrupt register */167#define GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(n) (0x2309C + (0x4000 * (n)))168#define GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK BIT(0)169 170#define GPII_n_CNTXT_SRC_CH_IRQ_CLR_OFFS(n) (0x230A0 + (0x4000 * (n)))171#define GPII_n_CNTXT_SRC_EV_CH_IRQ_CLR_OFFS(n) (0x230A4 + (0x4000 * (n)))172 173/* Mask event interrupt register */174#define GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(n) (0x230B8 + (0x4000 * (n)))175#define GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK BIT(0)176 177#define GPII_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(n) (0x230C0 + (0x4000 * (n)))178#define GPII_n_CNTXT_GLOB_IRQ_STTS_OFFS(n) (0x23100 + (0x4000 * (n)))179#define GPI_GLOB_IRQ_ERROR_INT_MSK BIT(0)180 181/* GPII specific Global - Enable bit register */182#define GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(n) (0x23108 + (0x4000 * (n)))183#define GPII_n_CNTXT_GLOB_IRQ_CLR_OFFS(n) (0x23110 + (0x4000 * (n)))184#define GPII_n_CNTXT_GPII_IRQ_STTS_OFFS(n) (0x23118 + (0x4000 * (n)))185 186/* GPII general interrupt - Enable bit register */187#define GPII_n_CNTXT_GPII_IRQ_EN_OFFS(n) (0x23120 + (0x4000 * (n)))188#define GPII_n_CNTXT_GPII_IRQ_EN_BMSK GENMASK(3, 0)189 190#define GPII_n_CNTXT_GPII_IRQ_CLR_OFFS(n) (0x23128 + (0x4000 * (n)))191 192/* GPII Interrupt Type register */193#define GPII_n_CNTXT_INTSET_OFFS(n) (0x23180 + (0x4000 * (n)))194#define GPII_n_CNTXT_INTSET_BMSK BIT(0)195 196#define GPII_n_CNTXT_MSI_BASE_LSB_OFFS(n) (0x23188 + (0x4000 * (n)))197#define GPII_n_CNTXT_MSI_BASE_MSB_OFFS(n) (0x2318C + (0x4000 * (n)))198#define GPII_n_CNTXT_SCRATCH_0_OFFS(n) (0x23400 + (0x4000 * (n)))199#define GPII_n_CNTXT_SCRATCH_1_OFFS(n) (0x23404 + (0x4000 * (n)))200 201#define GPII_n_ERROR_LOG_OFFS(n) (0x23200 + (0x4000 * (n)))202 203/* QOS Registers */204#define GPII_n_CH_k_QOS_OFFS(n, k) (0x2005C + (0x4000 * (n)) + (0x80 * (k)))205 206/* Scratch registers */207#define GPII_n_CH_k_SCRATCH_0_OFFS(n, k) (0x20060 + (0x4000 * (n)) + (0x80 * (k)))208#define GPII_n_CH_k_SCRATCH_0_SEID GENMASK(2, 0)209#define GPII_n_CH_k_SCRATCH_0_PROTO GENMASK(7, 4)210#define GPII_n_CH_k_SCRATCH_0_PAIR GENMASK(20, 16)211#define GPII_n_CH_k_SCRATCH_0(pair, proto, seid) \212 (FIELD_PREP(GPII_n_CH_k_SCRATCH_0_PAIR, pair) | \213 FIELD_PREP(GPII_n_CH_k_SCRATCH_0_PROTO, proto) | \214 FIELD_PREP(GPII_n_CH_k_SCRATCH_0_SEID, seid))215#define GPII_n_CH_k_SCRATCH_1_OFFS(n, k) (0x20064 + (0x4000 * (n)) + (0x80 * (k)))216#define GPII_n_CH_k_SCRATCH_2_OFFS(n, k) (0x20068 + (0x4000 * (n)) + (0x80 * (k)))217#define GPII_n_CH_k_SCRATCH_3_OFFS(n, k) (0x2006C + (0x4000 * (n)) + (0x80 * (k)))218 219struct __packed gpi_tre {220 u32 dword[4];221};222 223enum msm_gpi_tce_code {224 MSM_GPI_TCE_SUCCESS = 1,225 MSM_GPI_TCE_EOT = 2,226 MSM_GPI_TCE_EOB = 4,227 MSM_GPI_TCE_UNEXP_ERR = 16,228};229 230#define CMD_TIMEOUT_MS (250)231 232#define MAX_CHANNELS_PER_GPII (2)233#define GPI_TX_CHAN (0)234#define GPI_RX_CHAN (1)235#define STATE_IGNORE (U32_MAX)236#define EV_FACTOR (2)237#define REQ_OF_DMA_ARGS (5) /* # of arguments required from client */238#define CHAN_TRES 64239 240struct __packed xfer_compl_event {241 u64 ptr;242 u32 length:24;243 u8 code;244 u16 status;245 u8 type;246 u8 chid;247};248 249struct __packed immediate_data_event {250 u8 data_bytes[8];251 u8 length:4;252 u8 resvd:4;253 u16 tre_index;254 u8 code;255 u16 status;256 u8 type;257 u8 chid;258};259 260struct __packed qup_notif_event {261 u32 status;262 u32 time;263 u32 count:24;264 u8 resvd;265 u16 resvd1;266 u8 type;267 u8 chid;268};269 270struct __packed gpi_ere {271 u32 dword[4];272};273 274enum GPI_EV_TYPE {275 XFER_COMPLETE_EV_TYPE = 0x22,276 IMMEDIATE_DATA_EV_TYPE = 0x30,277 QUP_NOTIF_EV_TYPE = 0x31,278 STALE_EV_TYPE = 0xFF,279};280 281union __packed gpi_event {282 struct __packed xfer_compl_event xfer_compl_event;283 struct __packed immediate_data_event immediate_data_event;284 struct __packed qup_notif_event qup_notif_event;285 struct __packed gpi_ere gpi_ere;286};287 288enum gpii_irq_settings {289 DEFAULT_IRQ_SETTINGS,290 MASK_IEOB_SETTINGS,291};292 293enum gpi_ev_state {294 DEFAULT_EV_CH_STATE = 0,295 EV_STATE_NOT_ALLOCATED = DEFAULT_EV_CH_STATE,296 EV_STATE_ALLOCATED,297 MAX_EV_STATES298};299 300static const char *const gpi_ev_state_str[MAX_EV_STATES] = {301 [EV_STATE_NOT_ALLOCATED] = "NOT ALLOCATED",302 [EV_STATE_ALLOCATED] = "ALLOCATED",303};304 305#define TO_GPI_EV_STATE_STR(_state) (((_state) >= MAX_EV_STATES) ? \306 "INVALID" : gpi_ev_state_str[(_state)])307 308enum gpi_ch_state {309 DEFAULT_CH_STATE = 0x0,310 CH_STATE_NOT_ALLOCATED = DEFAULT_CH_STATE,311 CH_STATE_ALLOCATED = 0x1,312 CH_STATE_STARTED = 0x2,313 CH_STATE_STOPPED = 0x3,314 CH_STATE_STOP_IN_PROC = 0x4,315 CH_STATE_ERROR = 0xf,316 MAX_CH_STATES317};318 319enum gpi_cmd {320 GPI_CH_CMD_BEGIN,321 GPI_CH_CMD_ALLOCATE = GPI_CH_CMD_BEGIN,322 GPI_CH_CMD_START,323 GPI_CH_CMD_STOP,324 GPI_CH_CMD_RESET,325 GPI_CH_CMD_DE_ALLOC,326 GPI_CH_CMD_UART_SW_STALE,327 GPI_CH_CMD_UART_RFR_READY,328 GPI_CH_CMD_UART_RFR_NOT_READY,329 GPI_CH_CMD_END = GPI_CH_CMD_UART_RFR_NOT_READY,330 GPI_EV_CMD_BEGIN,331 GPI_EV_CMD_ALLOCATE = GPI_EV_CMD_BEGIN,332 GPI_EV_CMD_RESET,333 GPI_EV_CMD_DEALLOC,334 GPI_EV_CMD_END = GPI_EV_CMD_DEALLOC,335 GPI_MAX_CMD,336};337 338#define IS_CHAN_CMD(_cmd) ((_cmd) <= GPI_CH_CMD_END)339 340static const char *const gpi_cmd_str[GPI_MAX_CMD] = {341 [GPI_CH_CMD_ALLOCATE] = "CH ALLOCATE",342 [GPI_CH_CMD_START] = "CH START",343 [GPI_CH_CMD_STOP] = "CH STOP",344 [GPI_CH_CMD_RESET] = "CH_RESET",345 [GPI_CH_CMD_DE_ALLOC] = "DE ALLOC",346 [GPI_CH_CMD_UART_SW_STALE] = "UART SW STALE",347 [GPI_CH_CMD_UART_RFR_READY] = "UART RFR READY",348 [GPI_CH_CMD_UART_RFR_NOT_READY] = "UART RFR NOT READY",349 [GPI_EV_CMD_ALLOCATE] = "EV ALLOCATE",350 [GPI_EV_CMD_RESET] = "EV RESET",351 [GPI_EV_CMD_DEALLOC] = "EV DEALLOC",352};353 354#define TO_GPI_CMD_STR(_cmd) (((_cmd) >= GPI_MAX_CMD) ? "INVALID" : \355 gpi_cmd_str[(_cmd)])356 357/*358 * @DISABLE_STATE: no register access allowed359 * @CONFIG_STATE: client has configured the channel360 * @PREP_HARDWARE: register access is allowed361 * however, no processing EVENTS362 * @ACTIVE_STATE: channels are fully operational363 * @PREPARE_TERMINATE: graceful termination of channels364 * register access is allowed365 * @PAUSE_STATE: channels are active, but not processing any events366 */367enum gpi_pm_state {368 DISABLE_STATE,369 CONFIG_STATE,370 PREPARE_HARDWARE,371 ACTIVE_STATE,372 PREPARE_TERMINATE,373 PAUSE_STATE,374 MAX_PM_STATE375};376 377#define REG_ACCESS_VALID(_pm_state) ((_pm_state) >= PREPARE_HARDWARE)378 379static const char *const gpi_pm_state_str[MAX_PM_STATE] = {380 [DISABLE_STATE] = "DISABLE",381 [CONFIG_STATE] = "CONFIG",382 [PREPARE_HARDWARE] = "PREPARE HARDWARE",383 [ACTIVE_STATE] = "ACTIVE",384 [PREPARE_TERMINATE] = "PREPARE TERMINATE",385 [PAUSE_STATE] = "PAUSE",386};387 388#define TO_GPI_PM_STR(_state) (((_state) >= MAX_PM_STATE) ? \389 "INVALID" : gpi_pm_state_str[(_state)])390 391static const struct {392 enum gpi_cmd gpi_cmd;393 u32 opcode;394 u32 state;395} gpi_cmd_info[GPI_MAX_CMD] = {396 {397 GPI_CH_CMD_ALLOCATE,398 GPII_n_CH_CMD_ALLOCATE,399 CH_STATE_ALLOCATED,400 },401 {402 GPI_CH_CMD_START,403 GPII_n_CH_CMD_START,404 CH_STATE_STARTED,405 },406 {407 GPI_CH_CMD_STOP,408 GPII_n_CH_CMD_STOP,409 CH_STATE_STOPPED,410 },411 {412 GPI_CH_CMD_RESET,413 GPII_n_CH_CMD_RESET,414 CH_STATE_ALLOCATED,415 },416 {417 GPI_CH_CMD_DE_ALLOC,418 GPII_n_CH_CMD_DE_ALLOC,419 CH_STATE_NOT_ALLOCATED,420 },421 {422 GPI_CH_CMD_UART_SW_STALE,423 GPII_n_CH_CMD_UART_SW_STALE,424 STATE_IGNORE,425 },426 {427 GPI_CH_CMD_UART_RFR_READY,428 GPII_n_CH_CMD_UART_RFR_READY,429 STATE_IGNORE,430 },431 {432 GPI_CH_CMD_UART_RFR_NOT_READY,433 GPII_n_CH_CMD_UART_RFR_NOT_READY,434 STATE_IGNORE,435 },436 {437 GPI_EV_CMD_ALLOCATE,438 GPII_n_EV_CH_CMD_ALLOCATE,439 EV_STATE_ALLOCATED,440 },441 {442 GPI_EV_CMD_RESET,443 GPII_n_EV_CH_CMD_RESET,444 EV_STATE_ALLOCATED,445 },446 {447 GPI_EV_CMD_DEALLOC,448 GPII_n_EV_CH_CMD_DE_ALLOC,449 EV_STATE_NOT_ALLOCATED,450 },451};452 453struct gpi_ring {454 void *pre_aligned;455 size_t alloc_size;456 phys_addr_t phys_addr;457 dma_addr_t dma_handle;458 void *base;459 void *wp;460 void *rp;461 u32 len;462 u32 el_size;463 u32 elements;464 bool configured;465};466 467struct gpi_dev {468 struct dma_device dma_device;469 struct device *dev;470 struct resource *res;471 void __iomem *regs;472 void __iomem *ee_base; /*ee register base address*/473 u32 max_gpii; /* maximum # of gpii instances available per gpi block */474 u32 gpii_mask; /* gpii instances available for apps */475 u32 ev_factor; /* ev ring length factor */476 struct gpii *gpiis;477};478 479struct gchan {480 struct virt_dma_chan vc;481 u32 chid;482 u32 seid;483 u32 protocol;484 struct gpii *gpii;485 enum gpi_ch_state ch_state;486 enum gpi_pm_state pm_state;487 void __iomem *ch_cntxt_base_reg;488 void __iomem *ch_cntxt_db_reg;489 void __iomem *ch_cmd_reg;490 u32 dir;491 struct gpi_ring ch_ring;492 void *config;493};494 495struct gpii {496 u32 gpii_id;497 struct gchan gchan[MAX_CHANNELS_PER_GPII];498 struct gpi_dev *gpi_dev;499 int irq;500 void __iomem *regs; /* points to gpi top */501 void __iomem *ev_cntxt_base_reg;502 void __iomem *ev_cntxt_db_reg;503 void __iomem *ev_ring_rp_lsb_reg;504 void __iomem *ev_cmd_reg;505 void __iomem *ieob_clr_reg;506 struct mutex ctrl_lock;507 enum gpi_ev_state ev_state;508 bool configured_irq;509 enum gpi_pm_state pm_state;510 rwlock_t pm_lock;511 struct gpi_ring ev_ring;512 struct tasklet_struct ev_task; /* event processing tasklet */513 struct completion cmd_completion;514 enum gpi_cmd gpi_cmd;515 u32 cntxt_type_irq_msk;516 bool ieob_set;517};518 519#define MAX_TRE 3520 521struct gpi_desc {522 struct virt_dma_desc vd;523 size_t len;524 void *db; /* DB register to program */525 struct gchan *gchan;526 struct gpi_tre tre[MAX_TRE];527 u32 num_tre;528};529 530static const u32 GPII_CHAN_DIR[MAX_CHANNELS_PER_GPII] = {531 GPI_CHTYPE_DIR_OUT, GPI_CHTYPE_DIR_IN532};533 534static irqreturn_t gpi_handle_irq(int irq, void *data);535static void gpi_ring_recycle_ev_element(struct gpi_ring *ring);536static int gpi_ring_add_element(struct gpi_ring *ring, void **wp);537static void gpi_process_events(struct gpii *gpii);538 539static inline struct gchan *to_gchan(struct dma_chan *dma_chan)540{541 return container_of(dma_chan, struct gchan, vc.chan);542}543 544static inline struct gpi_desc *to_gpi_desc(struct virt_dma_desc *vd)545{546 return container_of(vd, struct gpi_desc, vd);547}548 549static inline phys_addr_t to_physical(const struct gpi_ring *const ring,550 void *addr)551{552 return ring->phys_addr + (addr - ring->base);553}554 555static inline void *to_virtual(const struct gpi_ring *const ring, phys_addr_t addr)556{557 return ring->base + (addr - ring->phys_addr);558}559 560static inline u32 gpi_read_reg(struct gpii *gpii, void __iomem *addr)561{562 return readl_relaxed(addr);563}564 565static inline void gpi_write_reg(struct gpii *gpii, void __iomem *addr, u32 val)566{567 writel_relaxed(val, addr);568}569 570/* gpi_write_reg_field - write to specific bit field */571static inline void gpi_write_reg_field(struct gpii *gpii, void __iomem *addr,572 u32 mask, u32 shift, u32 val)573{574 u32 tmp = gpi_read_reg(gpii, addr);575 576 tmp &= ~mask;577 val = tmp | ((val << shift) & mask);578 gpi_write_reg(gpii, addr, val);579}580 581static __always_inline void582gpi_update_reg(struct gpii *gpii, u32 offset, u32 mask, u32 val)583{584 void __iomem *addr = gpii->regs + offset;585 u32 tmp = gpi_read_reg(gpii, addr);586 587 tmp &= ~mask;588 tmp |= u32_encode_bits(val, mask);589 590 gpi_write_reg(gpii, addr, tmp);591}592 593static void gpi_disable_interrupts(struct gpii *gpii)594{595 gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id),596 GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK, 0);597 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(gpii->gpii_id),598 GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK, 0);599 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(gpii->gpii_id),600 GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK, 0);601 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(gpii->gpii_id),602 GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK, 0);603 gpi_update_reg(gpii, GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(gpii->gpii_id),604 GPII_n_CNTXT_GPII_IRQ_EN_BMSK, 0);605 gpi_update_reg(gpii, GPII_n_CNTXT_GPII_IRQ_EN_OFFS(gpii->gpii_id),606 GPII_n_CNTXT_GPII_IRQ_EN_BMSK, 0);607 gpi_update_reg(gpii, GPII_n_CNTXT_INTSET_OFFS(gpii->gpii_id),608 GPII_n_CNTXT_INTSET_BMSK, 0);609 610 gpii->cntxt_type_irq_msk = 0;611 devm_free_irq(gpii->gpi_dev->dev, gpii->irq, gpii);612 gpii->configured_irq = false;613}614 615/* configure and enable interrupts */616static int gpi_config_interrupts(struct gpii *gpii, enum gpii_irq_settings settings, bool mask)617{618 const u32 enable = (GPII_n_CNTXT_TYPE_IRQ_MSK_GENERAL |619 GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB |620 GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB |621 GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL |622 GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL);623 int ret;624 625 if (!gpii->configured_irq) {626 ret = devm_request_irq(gpii->gpi_dev->dev, gpii->irq,627 gpi_handle_irq, IRQF_TRIGGER_HIGH,628 "gpi-dma", gpii);629 if (ret < 0) {630 dev_err(gpii->gpi_dev->dev, "error request irq:%d ret:%d\n",631 gpii->irq, ret);632 return ret;633 }634 }635 636 if (settings == MASK_IEOB_SETTINGS) {637 /*638 * GPII only uses one EV ring per gpii so we can globally639 * enable/disable IEOB interrupt640 */641 if (mask)642 gpii->cntxt_type_irq_msk |= GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB;643 else644 gpii->cntxt_type_irq_msk &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB);645 gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id),646 GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK, gpii->cntxt_type_irq_msk);647 } else {648 gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id),649 GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK, enable);650 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(gpii->gpii_id),651 GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK,652 GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_BMSK);653 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(gpii->gpii_id),654 GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK,655 GPII_n_CNTXT_SRC_CH_IRQ_MSK_BMSK);656 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(gpii->gpii_id),657 GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK,658 GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_BMSK);659 gpi_update_reg(gpii, GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(gpii->gpii_id),660 GPII_n_CNTXT_GPII_IRQ_EN_BMSK,661 GPII_n_CNTXT_GPII_IRQ_EN_BMSK);662 gpi_update_reg(gpii, GPII_n_CNTXT_GPII_IRQ_EN_OFFS(gpii->gpii_id),663 GPII_n_CNTXT_GPII_IRQ_EN_BMSK, GPII_n_CNTXT_GPII_IRQ_EN_BMSK);664 gpi_update_reg(gpii, GPII_n_CNTXT_MSI_BASE_LSB_OFFS(gpii->gpii_id), U32_MAX, 0);665 gpi_update_reg(gpii, GPII_n_CNTXT_MSI_BASE_MSB_OFFS(gpii->gpii_id), U32_MAX, 0);666 gpi_update_reg(gpii, GPII_n_CNTXT_SCRATCH_0_OFFS(gpii->gpii_id), U32_MAX, 0);667 gpi_update_reg(gpii, GPII_n_CNTXT_SCRATCH_1_OFFS(gpii->gpii_id), U32_MAX, 0);668 gpi_update_reg(gpii, GPII_n_CNTXT_INTSET_OFFS(gpii->gpii_id),669 GPII_n_CNTXT_INTSET_BMSK, 1);670 gpi_update_reg(gpii, GPII_n_ERROR_LOG_OFFS(gpii->gpii_id), U32_MAX, 0);671 672 gpii->cntxt_type_irq_msk = enable;673 }674 675 gpii->configured_irq = true;676 return 0;677}678 679/* Sends gpii event or channel command */680static int gpi_send_cmd(struct gpii *gpii, struct gchan *gchan,681 enum gpi_cmd gpi_cmd)682{683 u32 chid = MAX_CHANNELS_PER_GPII;684 unsigned long timeout;685 void __iomem *cmd_reg;686 u32 cmd;687 688 if (gpi_cmd >= GPI_MAX_CMD)689 return -EINVAL;690 if (IS_CHAN_CMD(gpi_cmd))691 chid = gchan->chid;692 693 dev_dbg(gpii->gpi_dev->dev,694 "sending cmd: %s:%u\n", TO_GPI_CMD_STR(gpi_cmd), chid);695 696 /* send opcode and wait for completion */697 reinit_completion(&gpii->cmd_completion);698 gpii->gpi_cmd = gpi_cmd;699 700 cmd_reg = IS_CHAN_CMD(gpi_cmd) ? gchan->ch_cmd_reg : gpii->ev_cmd_reg;701 cmd = IS_CHAN_CMD(gpi_cmd) ? GPII_n_CH_CMD(gpi_cmd_info[gpi_cmd].opcode, chid) :702 GPII_n_EV_CMD(gpi_cmd_info[gpi_cmd].opcode, 0);703 gpi_write_reg(gpii, cmd_reg, cmd);704 timeout = wait_for_completion_timeout(&gpii->cmd_completion,705 msecs_to_jiffies(CMD_TIMEOUT_MS));706 if (!timeout) {707 dev_err(gpii->gpi_dev->dev, "cmd: %s completion timeout:%u\n",708 TO_GPI_CMD_STR(gpi_cmd), chid);709 return -EIO;710 }711 712 /* confirm new ch state is correct , if the cmd is a state change cmd */713 if (gpi_cmd_info[gpi_cmd].state == STATE_IGNORE)714 return 0;715 716 if (IS_CHAN_CMD(gpi_cmd) && gchan->ch_state == gpi_cmd_info[gpi_cmd].state)717 return 0;718 719 if (!IS_CHAN_CMD(gpi_cmd) && gpii->ev_state == gpi_cmd_info[gpi_cmd].state)720 return 0;721 722 return -EIO;723}724 725/* program transfer ring DB register */726static inline void gpi_write_ch_db(struct gchan *gchan,727 struct gpi_ring *ring, void *wp)728{729 struct gpii *gpii = gchan->gpii;730 phys_addr_t p_wp;731 732 p_wp = to_physical(ring, wp);733 gpi_write_reg(gpii, gchan->ch_cntxt_db_reg, p_wp);734}735 736/* program event ring DB register */737static inline void gpi_write_ev_db(struct gpii *gpii,738 struct gpi_ring *ring, void *wp)739{740 phys_addr_t p_wp;741 742 p_wp = ring->phys_addr + (wp - ring->base);743 gpi_write_reg(gpii, gpii->ev_cntxt_db_reg, p_wp);744}745 746/* process transfer completion interrupt */747static void gpi_process_ieob(struct gpii *gpii)748{749 gpi_write_reg(gpii, gpii->ieob_clr_reg, BIT(0));750 751 gpi_config_interrupts(gpii, MASK_IEOB_SETTINGS, 0);752 tasklet_hi_schedule(&gpii->ev_task);753}754 755/* process channel control interrupt */756static void gpi_process_ch_ctrl_irq(struct gpii *gpii)757{758 u32 gpii_id = gpii->gpii_id;759 u32 offset = GPII_n_CNTXT_SRC_GPII_CH_IRQ_OFFS(gpii_id);760 u32 ch_irq = gpi_read_reg(gpii, gpii->regs + offset);761 struct gchan *gchan;762 u32 chid, state;763 764 /* clear the status */765 offset = GPII_n_CNTXT_SRC_CH_IRQ_CLR_OFFS(gpii_id);766 gpi_write_reg(gpii, gpii->regs + offset, (u32)ch_irq);767 768 for (chid = 0; chid < MAX_CHANNELS_PER_GPII; chid++) {769 if (!(BIT(chid) & ch_irq))770 continue;771 772 gchan = &gpii->gchan[chid];773 state = gpi_read_reg(gpii, gchan->ch_cntxt_base_reg +774 CNTXT_0_CONFIG);775 state = FIELD_GET(GPII_n_CH_k_CNTXT_0_CHSTATE, state);776 777 /*778 * CH_CMD_DEALLOC cmd always successful. However cmd does779 * not change hardware status. So overwriting software state780 * to default state.781 */782 if (gpii->gpi_cmd == GPI_CH_CMD_DE_ALLOC)783 state = DEFAULT_CH_STATE;784 gchan->ch_state = state;785 786 /*787 * Triggering complete all if ch_state is not a stop in process.788 * Stop in process is a transition state and we will wait for789 * stop interrupt before notifying.790 */791 if (gchan->ch_state != CH_STATE_STOP_IN_PROC)792 complete_all(&gpii->cmd_completion);793 }794}795 796/* processing gpi general error interrupts */797static void gpi_process_gen_err_irq(struct gpii *gpii)798{799 u32 gpii_id = gpii->gpii_id;800 u32 offset = GPII_n_CNTXT_GPII_IRQ_STTS_OFFS(gpii_id);801 u32 irq_stts = gpi_read_reg(gpii, gpii->regs + offset);802 803 /* clear the status */804 dev_dbg(gpii->gpi_dev->dev, "irq_stts:0x%x\n", irq_stts);805 806 /* Clear the register */807 offset = GPII_n_CNTXT_GPII_IRQ_CLR_OFFS(gpii_id);808 gpi_write_reg(gpii, gpii->regs + offset, irq_stts);809}810 811/* processing gpi level error interrupts */812static void gpi_process_glob_err_irq(struct gpii *gpii)813{814 u32 gpii_id = gpii->gpii_id;815 u32 offset = GPII_n_CNTXT_GLOB_IRQ_STTS_OFFS(gpii_id);816 u32 irq_stts = gpi_read_reg(gpii, gpii->regs + offset);817 818 offset = GPII_n_CNTXT_GLOB_IRQ_CLR_OFFS(gpii_id);819 gpi_write_reg(gpii, gpii->regs + offset, irq_stts);820 821 /* only error interrupt should be set */822 if (irq_stts & ~GPI_GLOB_IRQ_ERROR_INT_MSK) {823 dev_err(gpii->gpi_dev->dev, "invalid error status:0x%x\n", irq_stts);824 return;825 }826 827 offset = GPII_n_ERROR_LOG_OFFS(gpii_id);828 gpi_write_reg(gpii, gpii->regs + offset, 0);829}830 831/* gpii interrupt handler */832static irqreturn_t gpi_handle_irq(int irq, void *data)833{834 struct gpii *gpii = data;835 u32 gpii_id = gpii->gpii_id;836 u32 type, offset;837 unsigned long flags;838 839 read_lock_irqsave(&gpii->pm_lock, flags);840 841 /*842 * States are out of sync to receive interrupt843 * while software state is in DISABLE state, bailing out.844 */845 if (!REG_ACCESS_VALID(gpii->pm_state)) {846 dev_err(gpii->gpi_dev->dev, "receive interrupt while in %s state\n",847 TO_GPI_PM_STR(gpii->pm_state));848 goto exit_irq;849 }850 851 offset = GPII_n_CNTXT_TYPE_IRQ_OFFS(gpii->gpii_id);852 type = gpi_read_reg(gpii, gpii->regs + offset);853 854 do {855 /* global gpii error */856 if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB) {857 gpi_process_glob_err_irq(gpii);858 type &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_GLOB);859 }860 861 /* transfer complete interrupt */862 if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB) {863 gpi_process_ieob(gpii);864 type &= ~GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB;865 }866 867 /* event control irq */868 if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL) {869 u32 ev_state;870 u32 ev_ch_irq;871 872 dev_dbg(gpii->gpi_dev->dev,873 "processing EV CTRL interrupt\n");874 offset = GPII_n_CNTXT_SRC_EV_CH_IRQ_OFFS(gpii_id);875 ev_ch_irq = gpi_read_reg(gpii, gpii->regs + offset);876 877 offset = GPII_n_CNTXT_SRC_EV_CH_IRQ_CLR_OFFS878 (gpii_id);879 gpi_write_reg(gpii, gpii->regs + offset, ev_ch_irq);880 ev_state = gpi_read_reg(gpii, gpii->ev_cntxt_base_reg +881 CNTXT_0_CONFIG);882 ev_state = FIELD_GET(GPII_n_EV_k_CNTXT_0_CHSTATE, ev_state);883 884 /*885 * CMD EV_CMD_DEALLOC is always successful. However886 * cmd does not change hardware status. So overwriting887 * software state to default state.888 */889 if (gpii->gpi_cmd == GPI_EV_CMD_DEALLOC)890 ev_state = DEFAULT_EV_CH_STATE;891 892 gpii->ev_state = ev_state;893 dev_dbg(gpii->gpi_dev->dev, "setting EV state to %s\n",894 TO_GPI_EV_STATE_STR(gpii->ev_state));895 complete_all(&gpii->cmd_completion);896 type &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_EV_CTRL);897 }898 899 /* channel control irq */900 if (type & GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL) {901 dev_dbg(gpii->gpi_dev->dev, "process CH CTRL interrupts\n");902 gpi_process_ch_ctrl_irq(gpii);903 type &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_CH_CTRL);904 }905 906 if (type) {907 dev_err(gpii->gpi_dev->dev, "Unhandled interrupt status:0x%x\n", type);908 gpi_process_gen_err_irq(gpii);909 goto exit_irq;910 }911 912 offset = GPII_n_CNTXT_TYPE_IRQ_OFFS(gpii->gpii_id);913 type = gpi_read_reg(gpii, gpii->regs + offset);914 } while (type);915 916exit_irq:917 read_unlock_irqrestore(&gpii->pm_lock, flags);918 919 return IRQ_HANDLED;920}921 922/* process DMA Immediate completion data events */923static void gpi_process_imed_data_event(struct gchan *gchan,924 struct immediate_data_event *imed_event)925{926 struct gpii *gpii = gchan->gpii;927 struct gpi_ring *ch_ring = &gchan->ch_ring;928 void *tre = ch_ring->base + (ch_ring->el_size * imed_event->tre_index);929 struct dmaengine_result result;930 struct gpi_desc *gpi_desc;931 struct virt_dma_desc *vd;932 unsigned long flags;933 u32 chid;934 935 /*936 * If channel not active don't process event937 */938 if (gchan->pm_state != ACTIVE_STATE) {939 dev_err(gpii->gpi_dev->dev, "skipping processing event because ch @ %s state\n",940 TO_GPI_PM_STR(gchan->pm_state));941 return;942 }943 944 spin_lock_irqsave(&gchan->vc.lock, flags);945 vd = vchan_next_desc(&gchan->vc);946 if (!vd) {947 struct gpi_ere *gpi_ere;948 struct gpi_tre *gpi_tre;949 950 spin_unlock_irqrestore(&gchan->vc.lock, flags);951 dev_dbg(gpii->gpi_dev->dev, "event without a pending descriptor!\n");952 gpi_ere = (struct gpi_ere *)imed_event;953 dev_dbg(gpii->gpi_dev->dev,954 "Event: %08x %08x %08x %08x\n",955 gpi_ere->dword[0], gpi_ere->dword[1],956 gpi_ere->dword[2], gpi_ere->dword[3]);957 gpi_tre = tre;958 dev_dbg(gpii->gpi_dev->dev,959 "Pending TRE: %08x %08x %08x %08x\n",960 gpi_tre->dword[0], gpi_tre->dword[1],961 gpi_tre->dword[2], gpi_tre->dword[3]);962 return;963 }964 gpi_desc = to_gpi_desc(vd);965 spin_unlock_irqrestore(&gchan->vc.lock, flags);966 967 /*968 * RP pointed by Event is to last TRE processed,969 * we need to update ring rp to tre + 1970 */971 tre += ch_ring->el_size;972 if (tre >= (ch_ring->base + ch_ring->len))973 tre = ch_ring->base;974 ch_ring->rp = tre;975 976 /* make sure rp updates are immediately visible to all cores */977 smp_wmb();978 979 chid = imed_event->chid;980 if (imed_event->code == MSM_GPI_TCE_EOT && gpii->ieob_set) {981 if (chid == GPI_RX_CHAN)982 goto gpi_free_desc;983 else984 return;985 }986 987 if (imed_event->code == MSM_GPI_TCE_UNEXP_ERR)988 result.result = DMA_TRANS_ABORTED;989 else990 result.result = DMA_TRANS_NOERROR;991 result.residue = gpi_desc->len - imed_event->length;992 993 dma_cookie_complete(&vd->tx);994 dmaengine_desc_get_callback_invoke(&vd->tx, &result);995 996gpi_free_desc:997 spin_lock_irqsave(&gchan->vc.lock, flags);998 list_del(&vd->node);999 spin_unlock_irqrestore(&gchan->vc.lock, flags);1000 kfree(gpi_desc);1001 gpi_desc = NULL;1002}1003 1004/* processing transfer completion events */1005static void gpi_process_xfer_compl_event(struct gchan *gchan,1006 struct xfer_compl_event *compl_event)1007{1008 struct gpii *gpii = gchan->gpii;1009 struct gpi_ring *ch_ring = &gchan->ch_ring;1010 void *ev_rp = to_virtual(ch_ring, compl_event->ptr);1011 struct virt_dma_desc *vd;1012 struct gpi_desc *gpi_desc;1013 struct dmaengine_result result;1014 unsigned long flags;1015 u32 chid;1016 1017 /* only process events on active channel */1018 if (unlikely(gchan->pm_state != ACTIVE_STATE)) {1019 dev_err(gpii->gpi_dev->dev, "skipping processing event because ch @ %s state\n",1020 TO_GPI_PM_STR(gchan->pm_state));1021 return;1022 }1023 1024 spin_lock_irqsave(&gchan->vc.lock, flags);1025 vd = vchan_next_desc(&gchan->vc);1026 if (!vd) {1027 struct gpi_ere *gpi_ere;1028 1029 spin_unlock_irqrestore(&gchan->vc.lock, flags);1030 dev_err(gpii->gpi_dev->dev, "Event without a pending descriptor!\n");1031 gpi_ere = (struct gpi_ere *)compl_event;1032 dev_err(gpii->gpi_dev->dev,1033 "Event: %08x %08x %08x %08x\n",1034 gpi_ere->dword[0], gpi_ere->dword[1],1035 gpi_ere->dword[2], gpi_ere->dword[3]);1036 return;1037 }1038 1039 gpi_desc = to_gpi_desc(vd);1040 spin_unlock_irqrestore(&gchan->vc.lock, flags);1041 1042 /*1043 * RP pointed by Event is to last TRE processed,1044 * we need to update ring rp to ev_rp + 11045 */1046 ev_rp += ch_ring->el_size;1047 if (ev_rp >= (ch_ring->base + ch_ring->len))1048 ev_rp = ch_ring->base;1049 ch_ring->rp = ev_rp;1050 1051 /* update must be visible to other cores */1052 smp_wmb();1053 1054 chid = compl_event->chid;1055 if (compl_event->code == MSM_GPI_TCE_EOT && gpii->ieob_set) {1056 if (chid == GPI_RX_CHAN)1057 goto gpi_free_desc;1058 else1059 return;1060 }1061 1062 if (compl_event->code == MSM_GPI_TCE_UNEXP_ERR) {1063 dev_err(gpii->gpi_dev->dev, "Error in Transaction\n");1064 result.result = DMA_TRANS_ABORTED;1065 } else {1066 dev_dbg(gpii->gpi_dev->dev, "Transaction Success\n");1067 result.result = DMA_TRANS_NOERROR;1068 }1069 result.residue = gpi_desc->len - compl_event->length;1070 dev_dbg(gpii->gpi_dev->dev, "Residue %d\n", result.residue);1071 1072 dma_cookie_complete(&vd->tx);1073 dmaengine_desc_get_callback_invoke(&vd->tx, &result);1074 1075gpi_free_desc:1076 spin_lock_irqsave(&gchan->vc.lock, flags);1077 list_del(&vd->node);1078 spin_unlock_irqrestore(&gchan->vc.lock, flags);1079 kfree(gpi_desc);1080 gpi_desc = NULL;1081}1082 1083/* process all events */1084static void gpi_process_events(struct gpii *gpii)1085{1086 struct gpi_ring *ev_ring = &gpii->ev_ring;1087 phys_addr_t cntxt_rp;1088 void *rp;1089 union gpi_event *gpi_event;1090 struct gchan *gchan;1091 u32 chid, type;1092 1093 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);1094 rp = to_virtual(ev_ring, cntxt_rp);1095 1096 do {1097 while (rp != ev_ring->rp) {1098 gpi_event = ev_ring->rp;1099 chid = gpi_event->xfer_compl_event.chid;1100 type = gpi_event->xfer_compl_event.type;1101 1102 dev_dbg(gpii->gpi_dev->dev,1103 "Event: CHID:%u, type:%x %08x %08x %08x %08x\n",1104 chid, type, gpi_event->gpi_ere.dword[0],1105 gpi_event->gpi_ere.dword[1], gpi_event->gpi_ere.dword[2],1106 gpi_event->gpi_ere.dword[3]);1107 1108 switch (type) {1109 case XFER_COMPLETE_EV_TYPE:1110 gchan = &gpii->gchan[chid];1111 gpi_process_xfer_compl_event(gchan,1112 &gpi_event->xfer_compl_event);1113 break;1114 case STALE_EV_TYPE:1115 dev_dbg(gpii->gpi_dev->dev, "stale event, not processing\n");1116 break;1117 case IMMEDIATE_DATA_EV_TYPE:1118 gchan = &gpii->gchan[chid];1119 gpi_process_imed_data_event(gchan,1120 &gpi_event->immediate_data_event);1121 break;1122 case QUP_NOTIF_EV_TYPE:1123 dev_dbg(gpii->gpi_dev->dev, "QUP_NOTIF_EV_TYPE\n");1124 break;1125 default:1126 dev_dbg(gpii->gpi_dev->dev,1127 "not supported event type:0x%x\n", type);1128 }1129 gpi_ring_recycle_ev_element(ev_ring);1130 }1131 gpi_write_ev_db(gpii, ev_ring, ev_ring->wp);1132 1133 /* clear pending IEOB events */1134 gpi_write_reg(gpii, gpii->ieob_clr_reg, BIT(0));1135 1136 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);1137 rp = to_virtual(ev_ring, cntxt_rp);1138 1139 } while (rp != ev_ring->rp);1140}1141 1142/* processing events using tasklet */1143static void gpi_ev_tasklet(unsigned long data)1144{1145 struct gpii *gpii = (struct gpii *)data;1146 1147 read_lock(&gpii->pm_lock);1148 if (!REG_ACCESS_VALID(gpii->pm_state)) {1149 read_unlock(&gpii->pm_lock);1150 dev_err(gpii->gpi_dev->dev, "not processing any events, pm_state:%s\n",1151 TO_GPI_PM_STR(gpii->pm_state));1152 return;1153 }1154 1155 /* process the events */1156 gpi_process_events(gpii);1157 1158 /* enable IEOB, switching back to interrupts */1159 gpi_config_interrupts(gpii, MASK_IEOB_SETTINGS, 1);1160 read_unlock(&gpii->pm_lock);1161}1162 1163/* marks all pending events for the channel as stale */1164static void gpi_mark_stale_events(struct gchan *gchan)1165{1166 struct gpii *gpii = gchan->gpii;1167 struct gpi_ring *ev_ring = &gpii->ev_ring;1168 u32 cntxt_rp, local_rp;1169 void *ev_rp;1170 1171 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);1172 1173 ev_rp = ev_ring->rp;1174 local_rp = (u32)to_physical(ev_ring, ev_rp);1175 while (local_rp != cntxt_rp) {1176 union gpi_event *gpi_event = ev_rp;1177 u32 chid = gpi_event->xfer_compl_event.chid;1178 1179 if (chid == gchan->chid)1180 gpi_event->xfer_compl_event.type = STALE_EV_TYPE;1181 ev_rp += ev_ring->el_size;1182 if (ev_rp >= (ev_ring->base + ev_ring->len))1183 ev_rp = ev_ring->base;1184 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);1185 local_rp = (u32)to_physical(ev_ring, ev_rp);1186 }1187}1188 1189/* reset sw state and issue channel reset or de-alloc */1190static int gpi_reset_chan(struct gchan *gchan, enum gpi_cmd gpi_cmd)1191{1192 struct gpii *gpii = gchan->gpii;1193 struct gpi_ring *ch_ring = &gchan->ch_ring;1194 LIST_HEAD(list);1195 int ret;1196 1197 ret = gpi_send_cmd(gpii, gchan, gpi_cmd);1198 if (ret) {1199 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",1200 TO_GPI_CMD_STR(gpi_cmd), ret);1201 return ret;1202 }1203 1204 /* initialize the local ring ptrs */1205 ch_ring->rp = ch_ring->base;1206 ch_ring->wp = ch_ring->base;1207 1208 /* visible to other cores */1209 smp_wmb();1210 1211 /* check event ring for any stale events */1212 write_lock_irq(&gpii->pm_lock);1213 gpi_mark_stale_events(gchan);1214 1215 /* remove all async descriptors */1216 spin_lock(&gchan->vc.lock);1217 vchan_get_all_descriptors(&gchan->vc, &list);1218 spin_unlock(&gchan->vc.lock);1219 write_unlock_irq(&gpii->pm_lock);1220 vchan_dma_desc_free_list(&gchan->vc, &list);1221 1222 return 0;1223}1224 1225static int gpi_start_chan(struct gchan *gchan)1226{1227 struct gpii *gpii = gchan->gpii;1228 int ret;1229 1230 ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_START);1231 if (ret) {1232 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",1233 TO_GPI_CMD_STR(GPI_CH_CMD_START), ret);1234 return ret;1235 }1236 1237 /* gpii CH is active now */1238 write_lock_irq(&gpii->pm_lock);1239 gchan->pm_state = ACTIVE_STATE;1240 write_unlock_irq(&gpii->pm_lock);1241 1242 return 0;1243}1244 1245static int gpi_stop_chan(struct gchan *gchan)1246{1247 struct gpii *gpii = gchan->gpii;1248 int ret;1249 1250 ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_STOP);1251 if (ret) {1252 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",1253 TO_GPI_CMD_STR(GPI_CH_CMD_STOP), ret);1254 return ret;1255 }1256 1257 return 0;1258}1259 1260/* allocate and configure the transfer channel */1261static int gpi_alloc_chan(struct gchan *chan, bool send_alloc_cmd)1262{1263 struct gpii *gpii = chan->gpii;1264 struct gpi_ring *ring = &chan->ch_ring;1265 int ret;1266 u32 id = gpii->gpii_id;1267 u32 chid = chan->chid;1268 u32 pair_chid = !chid;1269 1270 if (send_alloc_cmd) {1271 ret = gpi_send_cmd(gpii, chan, GPI_CH_CMD_ALLOCATE);1272 if (ret) {1273 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",1274 TO_GPI_CMD_STR(GPI_CH_CMD_ALLOCATE), ret);1275 return ret;1276 }1277 }1278 1279 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_0_CONFIG,1280 GPII_n_CH_k_CNTXT_0(ring->el_size, 0, chan->dir, GPI_CHTYPE_PROTO_GPI));1281 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_1_R_LENGTH, ring->len);1282 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_2_RING_BASE_LSB, ring->phys_addr);1283 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_3_RING_BASE_MSB,1284 upper_32_bits(ring->phys_addr));1285 gpi_write_reg(gpii, chan->ch_cntxt_db_reg + CNTXT_5_RING_RP_MSB - CNTXT_4_RING_RP_LSB,1286 upper_32_bits(ring->phys_addr));1287 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_0_OFFS(id, chid),1288 GPII_n_CH_k_SCRATCH_0(pair_chid, chan->protocol, chan->seid));1289 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_1_OFFS(id, chid), 0);1290 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_2_OFFS(id, chid), 0);1291 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_3_OFFS(id, chid), 0);1292 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_QOS_OFFS(id, chid), 1);1293 1294 /* flush all the writes */1295 wmb();1296 return 0;1297}1298 1299/* allocate and configure event ring */1300static int gpi_alloc_ev_chan(struct gpii *gpii)1301{1302 struct gpi_ring *ring = &gpii->ev_ring;1303 void __iomem *base = gpii->ev_cntxt_base_reg;1304 int ret;1305 1306 ret = gpi_send_cmd(gpii, NULL, GPI_EV_CMD_ALLOCATE);1307 if (ret) {1308 dev_err(gpii->gpi_dev->dev, "error with cmd:%s ret:%d\n",1309 TO_GPI_CMD_STR(GPI_EV_CMD_ALLOCATE), ret);1310 return ret;1311 }1312 1313 /* program event context */1314 gpi_write_reg(gpii, base + CNTXT_0_CONFIG,1315 GPII_n_EV_k_CNTXT_0(ring->el_size, GPI_INTTYPE_IRQ, GPI_CHTYPE_GPI_EV));1316 gpi_write_reg(gpii, base + CNTXT_1_R_LENGTH, ring->len);1317 gpi_write_reg(gpii, base + CNTXT_2_RING_BASE_LSB, lower_32_bits(ring->phys_addr));1318 gpi_write_reg(gpii, base + CNTXT_3_RING_BASE_MSB, upper_32_bits(ring->phys_addr));1319 gpi_write_reg(gpii, gpii->ev_cntxt_db_reg + CNTXT_5_RING_RP_MSB - CNTXT_4_RING_RP_LSB,1320 upper_32_bits(ring->phys_addr));1321 gpi_write_reg(gpii, base + CNTXT_8_RING_INT_MOD, 0);1322 gpi_write_reg(gpii, base + CNTXT_10_RING_MSI_LSB, 0);1323 gpi_write_reg(gpii, base + CNTXT_11_RING_MSI_MSB, 0);1324 gpi_write_reg(gpii, base + CNTXT_8_RING_INT_MOD, 0);1325 gpi_write_reg(gpii, base + CNTXT_12_RING_RP_UPDATE_LSB, 0);1326 gpi_write_reg(gpii, base + CNTXT_13_RING_RP_UPDATE_MSB, 0);1327 1328 /* add events to ring */1329 ring->wp = (ring->base + ring->len - ring->el_size);1330 1331 /* flush all the writes */1332 wmb();1333 1334 /* gpii is active now */1335 write_lock_irq(&gpii->pm_lock);1336 gpii->pm_state = ACTIVE_STATE;1337 write_unlock_irq(&gpii->pm_lock);1338 gpi_write_ev_db(gpii, ring, ring->wp);1339 1340 return 0;1341}1342 1343/* calculate # of ERE/TRE available to queue */1344static int gpi_ring_num_elements_avail(const struct gpi_ring * const ring)1345{1346 int elements = 0;1347 1348 if (ring->wp < ring->rp) {1349 elements = ((ring->rp - ring->wp) / ring->el_size) - 1;1350 } else {1351 elements = (ring->rp - ring->base) / ring->el_size;1352 elements += ((ring->base + ring->len - ring->wp) / ring->el_size) - 1;1353 }1354 1355 return elements;1356}1357 1358static int gpi_ring_add_element(struct gpi_ring *ring, void **wp)1359{1360 if (gpi_ring_num_elements_avail(ring) <= 0)1361 return -ENOMEM;1362 1363 *wp = ring->wp;1364 ring->wp += ring->el_size;1365 if (ring->wp >= (ring->base + ring->len))1366 ring->wp = ring->base;1367 1368 /* visible to other cores */1369 smp_wmb();1370 1371 return 0;1372}1373 1374static void gpi_ring_recycle_ev_element(struct gpi_ring *ring)1375{1376 /* Update the WP */1377 ring->wp += ring->el_size;1378 if (ring->wp >= (ring->base + ring->len))1379 ring->wp = ring->base;1380 1381 /* Update the RP */1382 ring->rp += ring->el_size;1383 if (ring->rp >= (ring->base + ring->len))1384 ring->rp = ring->base;1385 1386 /* visible to other cores */1387 smp_wmb();1388}1389 1390static void gpi_free_ring(struct gpi_ring *ring,1391 struct gpii *gpii)1392{1393 dma_free_coherent(gpii->gpi_dev->dev, ring->alloc_size,1394 ring->pre_aligned, ring->dma_handle);1395 memset(ring, 0, sizeof(*ring));1396}1397 1398/* allocate memory for transfer and event rings */1399static int gpi_alloc_ring(struct gpi_ring *ring, u32 elements,1400 u32 el_size, struct gpii *gpii)1401{1402 u64 len = elements * el_size;1403 int bit;1404 1405 /* ring len must be power of 2 */1406 bit = find_last_bit((unsigned long *)&len, 32);1407 if (((1 << bit) - 1) & len)1408 bit++;1409 len = 1 << bit;1410 ring->alloc_size = (len + (len - 1));1411 dev_dbg(gpii->gpi_dev->dev,1412 "#el:%u el_size:%u len:%u actual_len:%llu alloc_size:%zu\n",1413 elements, el_size, (elements * el_size), len,1414 ring->alloc_size);1415 1416 ring->pre_aligned = dma_alloc_coherent(gpii->gpi_dev->dev,1417 ring->alloc_size,1418 &ring->dma_handle, GFP_KERNEL);1419 if (!ring->pre_aligned) {1420 dev_err(gpii->gpi_dev->dev, "could not alloc size:%zu mem for ring\n",1421 ring->alloc_size);1422 return -ENOMEM;1423 }1424 1425 /* align the physical mem */1426 ring->phys_addr = (ring->dma_handle + (len - 1)) & ~(len - 1);1427 ring->base = ring->pre_aligned + (ring->phys_addr - ring->dma_handle);1428 ring->rp = ring->base;1429 ring->wp = ring->base;1430 ring->len = len;1431 ring->el_size = el_size;1432 ring->elements = ring->len / ring->el_size;1433 memset(ring->base, 0, ring->len);1434 ring->configured = true;1435 1436 /* update to other cores */1437 smp_wmb();1438 1439 dev_dbg(gpii->gpi_dev->dev,1440 "phy_pre:%pad phy_alig:%pa len:%u el_size:%u elements:%u\n",1441 &ring->dma_handle, &ring->phys_addr, ring->len,1442 ring->el_size, ring->elements);1443 1444 return 0;1445}1446 1447/* copy tre into transfer ring */1448static void gpi_queue_xfer(struct gpii *gpii, struct gchan *gchan,1449 struct gpi_tre *gpi_tre, void **wp)1450{1451 struct gpi_tre *ch_tre;1452 int ret;1453 1454 /* get next tre location we can copy */1455 ret = gpi_ring_add_element(&gchan->ch_ring, (void **)&ch_tre);1456 if (unlikely(ret)) {1457 dev_err(gpii->gpi_dev->dev, "Error adding ring element to xfer ring\n");1458 return;1459 }1460 1461 /* copy the tre info */1462 memcpy(ch_tre, gpi_tre, sizeof(*ch_tre));1463 *wp = ch_tre;1464}1465 1466/* reset and restart transfer channel */1467static int gpi_terminate_all(struct dma_chan *chan)1468{1469 struct gchan *gchan = to_gchan(chan);1470 struct gpii *gpii = gchan->gpii;1471 int schid, echid, i;1472 int ret = 0;1473 1474 mutex_lock(&gpii->ctrl_lock);1475 1476 /*1477 * treat both channels as a group if its protocol is not UART1478 * STOP, RESET, or START needs to be in lockstep1479 */1480 schid = (gchan->protocol == QCOM_GPI_UART) ? gchan->chid : 0;1481 echid = (gchan->protocol == QCOM_GPI_UART) ? schid + 1 : MAX_CHANNELS_PER_GPII;1482 1483 /* stop the channel */1484 for (i = schid; i < echid; i++) {1485 gchan = &gpii->gchan[i];1486 1487 /* disable ch state so no more TRE processing */1488 write_lock_irq(&gpii->pm_lock);1489 gchan->pm_state = PREPARE_TERMINATE;1490 write_unlock_irq(&gpii->pm_lock);1491 1492 /* send command to Stop the channel */1493 ret = gpi_stop_chan(gchan);1494 }1495 1496 /* reset the channels (clears any pending tre) */1497 for (i = schid; i < echid; i++) {1498 gchan = &gpii->gchan[i];1499 1500 ret = gpi_reset_chan(gchan, GPI_CH_CMD_RESET);1501 if (ret) {1502 dev_err(gpii->gpi_dev->dev, "Error resetting channel ret:%d\n", ret);1503 goto terminate_exit;1504 }1505 1506 /* reprogram channel CNTXT */1507 ret = gpi_alloc_chan(gchan, false);1508 if (ret) {1509 dev_err(gpii->gpi_dev->dev, "Error alloc_channel ret:%d\n", ret);1510 goto terminate_exit;1511 }1512 }1513 1514 /* restart the channels */1515 for (i = schid; i < echid; i++) {1516 gchan = &gpii->gchan[i];1517 1518 ret = gpi_start_chan(gchan);1519 if (ret) {1520 dev_err(gpii->gpi_dev->dev, "Error Starting Channel ret:%d\n", ret);1521 goto terminate_exit;1522 }1523 }1524 1525terminate_exit:1526 mutex_unlock(&gpii->ctrl_lock);1527 return ret;1528}1529 1530/* pause dma transfer for all channels */1531static int gpi_pause(struct dma_chan *chan)1532{1533 struct gchan *gchan = to_gchan(chan);1534 struct gpii *gpii = gchan->gpii;1535 int i, ret;1536 1537 mutex_lock(&gpii->ctrl_lock);1538 1539 /*1540 * pause/resume are per gpii not per channel, so1541 * client needs to call pause only once1542 */1543 if (gpii->pm_state == PAUSE_STATE) {1544 dev_dbg(gpii->gpi_dev->dev, "channel is already paused\n");1545 mutex_unlock(&gpii->ctrl_lock);1546 return 0;1547 }1548 1549 /* send stop command to stop the channels */1550 for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) {1551 ret = gpi_stop_chan(&gpii->gchan[i]);1552 if (ret) {1553 mutex_unlock(&gpii->ctrl_lock);1554 return ret;1555 }1556 }1557 1558 disable_irq(gpii->irq);1559 1560 /* Wait for threads to complete out */1561 tasklet_kill(&gpii->ev_task);1562 1563 write_lock_irq(&gpii->pm_lock);1564 gpii->pm_state = PAUSE_STATE;1565 write_unlock_irq(&gpii->pm_lock);1566 mutex_unlock(&gpii->ctrl_lock);1567 1568 return 0;1569}1570 1571/* resume dma transfer */1572static int gpi_resume(struct dma_chan *chan)1573{1574 struct gchan *gchan = to_gchan(chan);1575 struct gpii *gpii = gchan->gpii;1576 int i, ret;1577 1578 mutex_lock(&gpii->ctrl_lock);1579 if (gpii->pm_state == ACTIVE_STATE) {1580 dev_dbg(gpii->gpi_dev->dev, "channel is already active\n");1581 mutex_unlock(&gpii->ctrl_lock);1582 return 0;1583 }1584 1585 enable_irq(gpii->irq);1586 1587 /* send start command to start the channels */1588 for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) {1589 ret = gpi_send_cmd(gpii, &gpii->gchan[i], GPI_CH_CMD_START);1590 if (ret) {1591 dev_err(gpii->gpi_dev->dev, "Error starting chan, ret:%d\n", ret);1592 mutex_unlock(&gpii->ctrl_lock);1593 return ret;1594 }1595 }1596 1597 write_lock_irq(&gpii->pm_lock);1598 gpii->pm_state = ACTIVE_STATE;1599 write_unlock_irq(&gpii->pm_lock);1600 mutex_unlock(&gpii->ctrl_lock);1601 1602 return 0;1603}1604 1605static void gpi_desc_free(struct virt_dma_desc *vd)1606{1607 struct gpi_desc *gpi_desc = to_gpi_desc(vd);1608 1609 kfree(gpi_desc);1610 gpi_desc = NULL;1611}1612 1613static int1614gpi_peripheral_config(struct dma_chan *chan, struct dma_slave_config *config)1615{1616 struct gchan *gchan = to_gchan(chan);1617 1618 if (!config->peripheral_config)1619 return -EINVAL;1620 1621 gchan->config = krealloc(gchan->config, config->peripheral_size, GFP_NOWAIT);1622 if (!gchan->config)1623 return -ENOMEM;1624 1625 memcpy(gchan->config, config->peripheral_config, config->peripheral_size);1626 1627 return 0;1628}1629 1630static int gpi_create_i2c_tre(struct gchan *chan, struct gpi_desc *desc,1631 struct scatterlist *sgl, enum dma_transfer_direction direction)1632{1633 struct gpi_i2c_config *i2c = chan->config;1634 struct device *dev = chan->gpii->gpi_dev->dev;1635 unsigned int tre_idx = 0;1636 dma_addr_t address;1637 struct gpi_tre *tre;1638 unsigned int i;1639 1640 /* first create config tre if applicable */1641 if (i2c->set_config) {1642 tre = &desc->tre[tre_idx];1643 tre_idx++;1644 1645 tre->dword[0] = u32_encode_bits(i2c->low_count, TRE_I2C_C0_TLOW);1646 tre->dword[0] |= u32_encode_bits(i2c->high_count, TRE_I2C_C0_THIGH);1647 tre->dword[0] |= u32_encode_bits(i2c->cycle_count, TRE_I2C_C0_TCYL);1648 tre->dword[0] |= u32_encode_bits(i2c->pack_enable, TRE_I2C_C0_TX_PACK);1649 tre->dword[0] |= u32_encode_bits(i2c->pack_enable, TRE_I2C_C0_RX_PACK);1650 1651 tre->dword[1] = 0;1652 1653 tre->dword[2] = u32_encode_bits(i2c->clk_div, TRE_C0_CLK_DIV);1654 1655 tre->dword[3] = u32_encode_bits(TRE_TYPE_CONFIG0, TRE_FLAGS_TYPE);1656 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);1657 }1658 1659 /* create the GO tre for Tx */1660 if (i2c->op == I2C_WRITE) {1661 tre = &desc->tre[tre_idx];1662 tre_idx++;1663 1664 if (i2c->multi_msg)1665 tre->dword[0] = u32_encode_bits(I2C_READ, TRE_I2C_GO_CMD);1666 else1667 tre->dword[0] = u32_encode_bits(i2c->op, TRE_I2C_GO_CMD);1668 1669 tre->dword[0] |= u32_encode_bits(i2c->addr, TRE_I2C_GO_ADDR);1670 tre->dword[0] |= u32_encode_bits(i2c->stretch, TRE_I2C_GO_STRETCH);1671 1672 tre->dword[1] = 0;1673 tre->dword[2] = u32_encode_bits(i2c->rx_len, TRE_RX_LEN);1674 1675 tre->dword[3] = u32_encode_bits(TRE_TYPE_GO, TRE_FLAGS_TYPE);1676 1677 if (i2c->multi_msg)1678 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK);1679 else1680 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);1681 }1682 1683 if (i2c->op == I2C_READ || i2c->multi_msg == false) {1684 /* create the DMA TRE */1685 tre = &desc->tre[tre_idx];1686 tre_idx++;1687 1688 address = sg_dma_address(sgl);1689 tre->dword[0] = lower_32_bits(address);1690 tre->dword[1] = upper_32_bits(address);1691 1692 tre->dword[2] = u32_encode_bits(sg_dma_len(sgl), TRE_DMA_LEN);1693 1694 tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE);1695 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT);1696 }1697 1698 for (i = 0; i < tre_idx; i++)1699 dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0],1700 desc->tre[i].dword[1], desc->tre[i].dword[2], desc->tre[i].dword[3]);1701 1702 return tre_idx;1703}1704 1705static int gpi_create_spi_tre(struct gchan *chan, struct gpi_desc *desc,1706 struct scatterlist *sgl, enum dma_transfer_direction direction)1707{1708 struct gpi_spi_config *spi = chan->config;1709 struct device *dev = chan->gpii->gpi_dev->dev;1710 unsigned int tre_idx = 0;1711 dma_addr_t address;1712 struct gpi_tre *tre;1713 unsigned int i;1714 1715 /* first create config tre if applicable */1716 if (direction == DMA_MEM_TO_DEV && spi->set_config) {1717 tre = &desc->tre[tre_idx];1718 tre_idx++;1719 1720 tre->dword[0] = u32_encode_bits(spi->word_len, TRE_SPI_C0_WORD_SZ);1721 tre->dword[0] |= u32_encode_bits(spi->loopback_en, TRE_SPI_C0_LOOPBACK);1722 tre->dword[0] |= u32_encode_bits(spi->clock_pol_high, TRE_SPI_C0_CPOL);1723 tre->dword[0] |= u32_encode_bits(spi->data_pol_high, TRE_SPI_C0_CPHA);1724 tre->dword[0] |= u32_encode_bits(spi->pack_en, TRE_SPI_C0_TX_PACK);1725 tre->dword[0] |= u32_encode_bits(spi->pack_en, TRE_SPI_C0_RX_PACK);1726 1727 tre->dword[1] = 0;1728 1729 tre->dword[2] = u32_encode_bits(spi->clk_div, TRE_C0_CLK_DIV);1730 tre->dword[2] |= u32_encode_bits(spi->clk_src, TRE_C0_CLK_SRC);1731 1732 tre->dword[3] = u32_encode_bits(TRE_TYPE_CONFIG0, TRE_FLAGS_TYPE);1733 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);1734 }1735 1736 /* create the GO tre for Tx */1737 if (direction == DMA_MEM_TO_DEV) {1738 tre = &desc->tre[tre_idx];1739 tre_idx++;1740 1741 tre->dword[0] = u32_encode_bits(spi->fragmentation, TRE_SPI_GO_FRAG);1742 tre->dword[0] |= u32_encode_bits(spi->cs, TRE_SPI_GO_CS);1743 tre->dword[0] |= u32_encode_bits(spi->cmd, TRE_SPI_GO_CMD);1744 1745 tre->dword[1] = 0;1746 1747 tre->dword[2] = u32_encode_bits(spi->rx_len, TRE_RX_LEN);1748 1749 tre->dword[3] = u32_encode_bits(TRE_TYPE_GO, TRE_FLAGS_TYPE);1750 if (spi->cmd == SPI_RX) {1751 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOB);1752 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK);1753 } else if (spi->cmd == SPI_TX) {1754 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);1755 } else { /* SPI_DUPLEX */1756 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);1757 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK);1758 }1759 }1760 1761 /* create the dma tre */1762 tre = &desc->tre[tre_idx];1763 tre_idx++;1764 1765 address = sg_dma_address(sgl);1766 tre->dword[0] = lower_32_bits(address);1767 tre->dword[1] = upper_32_bits(address);1768 1769 tre->dword[2] = u32_encode_bits(sg_dma_len(sgl), TRE_DMA_LEN);1770 1771 tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE);1772 if (direction == DMA_MEM_TO_DEV)1773 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT);1774 1775 for (i = 0; i < tre_idx; i++)1776 dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0],1777 desc->tre[i].dword[1], desc->tre[i].dword[2], desc->tre[i].dword[3]);1778 1779 return tre_idx;1780}1781 1782/* copy tre into transfer ring */1783static struct dma_async_tx_descriptor *1784gpi_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,1785 unsigned int sg_len, enum dma_transfer_direction direction,1786 unsigned long flags, void *context)1787{1788 struct gchan *gchan = to_gchan(chan);1789 struct gpii *gpii = gchan->gpii;1790 struct device *dev = gpii->gpi_dev->dev;1791 struct gpi_ring *ch_ring = &gchan->ch_ring;1792 struct gpi_desc *gpi_desc;1793 u32 nr, nr_tre = 0;1794 u8 set_config;1795 int i;1796 1797 gpii->ieob_set = false;1798 if (!is_slave_direction(direction)) {1799 dev_err(gpii->gpi_dev->dev, "invalid dma direction: %d\n", direction);1800 return NULL;1801 }1802 1803 if (sg_len > 1) {1804 dev_err(dev, "Multi sg sent, we support only one atm: %d\n", sg_len);1805 return NULL;1806 }1807 1808 nr_tre = 3;1809 set_config = *(u32 *)gchan->config;1810 if (!set_config)1811 nr_tre = 2;1812 if (direction == DMA_DEV_TO_MEM) /* rx */1813 nr_tre = 1;1814 1815 /* calculate # of elements required & available */1816 nr = gpi_ring_num_elements_avail(ch_ring);1817 if (nr < nr_tre) {1818 dev_err(dev, "not enough space in ring, avail:%u required:%u\n", nr, nr_tre);1819 return NULL;1820 }1821 1822 gpi_desc = kzalloc(sizeof(*gpi_desc), GFP_NOWAIT);1823 if (!gpi_desc)1824 return NULL;1825 1826 /* create TREs for xfer */1827 if (gchan->protocol == QCOM_GPI_SPI) {1828 i = gpi_create_spi_tre(gchan, gpi_desc, sgl, direction);1829 } else if (gchan->protocol == QCOM_GPI_I2C) {1830 i = gpi_create_i2c_tre(gchan, gpi_desc, sgl, direction);1831 } else {1832 dev_err(dev, "invalid peripheral: %d\n", gchan->protocol);1833 kfree(gpi_desc);1834 return NULL;1835 }1836 1837 /* set up the descriptor */1838 gpi_desc->gchan = gchan;1839 gpi_desc->len = sg_dma_len(sgl);1840 gpi_desc->num_tre = i;1841 1842 return vchan_tx_prep(&gchan->vc, &gpi_desc->vd, flags);1843}1844 1845/* rings transfer ring db to being transfer */1846static void gpi_issue_pending(struct dma_chan *chan)1847{1848 struct gchan *gchan = to_gchan(chan);1849 struct gpii *gpii = gchan->gpii;1850 unsigned long flags, pm_lock_flags;1851 struct virt_dma_desc *vd = NULL;1852 struct gpi_desc *gpi_desc;1853 struct gpi_ring *ch_ring = &gchan->ch_ring;1854 void *tre, *wp = NULL;1855 int i;1856 1857 read_lock_irqsave(&gpii->pm_lock, pm_lock_flags);1858 1859 /* move all submitted descriptors to issued list */1860 spin_lock_irqsave(&gchan->vc.lock, flags);1861 if (vchan_issue_pending(&gchan->vc))1862 vd = list_last_entry(&gchan->vc.desc_issued,1863 struct virt_dma_desc, node);1864 spin_unlock_irqrestore(&gchan->vc.lock, flags);1865 1866 /* nothing to do list is empty */1867 if (!vd) {1868 read_unlock_irqrestore(&gpii->pm_lock, pm_lock_flags);1869 return;1870 }1871 1872 gpi_desc = to_gpi_desc(vd);1873 for (i = 0; i < gpi_desc->num_tre; i++) {1874 tre = &gpi_desc->tre[i];1875 gpi_queue_xfer(gpii, gchan, tre, &wp);1876 }1877 1878 gpi_desc->db = ch_ring->wp;1879 gpi_write_ch_db(gchan, &gchan->ch_ring, gpi_desc->db);1880 read_unlock_irqrestore(&gpii->pm_lock, pm_lock_flags);1881}1882 1883static int gpi_ch_init(struct gchan *gchan)1884{1885 struct gpii *gpii = gchan->gpii;1886 const int ev_factor = gpii->gpi_dev->ev_factor;1887 u32 elements;1888 int i = 0, ret = 0;1889 1890 gchan->pm_state = CONFIG_STATE;1891 1892 /* check if both channels are configured before continue */1893 for (i = 0; i < MAX_CHANNELS_PER_GPII; i++)1894 if (gpii->gchan[i].pm_state != CONFIG_STATE)1895 goto exit_gpi_init;1896 1897 /* protocol must be same for both channels */1898 if (gpii->gchan[0].protocol != gpii->gchan[1].protocol) {1899 dev_err(gpii->gpi_dev->dev, "protocol did not match protocol %u != %u\n",1900 gpii->gchan[0].protocol, gpii->gchan[1].protocol);1901 ret = -EINVAL;1902 goto exit_gpi_init;1903 }1904 1905 /* allocate memory for event ring */1906 elements = CHAN_TRES << ev_factor;1907 ret = gpi_alloc_ring(&gpii->ev_ring, elements,1908 sizeof(union gpi_event), gpii);1909 if (ret)1910 goto exit_gpi_init;1911 1912 /* configure interrupts */1913 write_lock_irq(&gpii->pm_lock);1914 gpii->pm_state = PREPARE_HARDWARE;1915 write_unlock_irq(&gpii->pm_lock);1916 ret = gpi_config_interrupts(gpii, DEFAULT_IRQ_SETTINGS, 0);1917 if (ret) {1918 dev_err(gpii->gpi_dev->dev, "error config. interrupts, ret:%d\n", ret);1919 goto error_config_int;1920 }1921 1922 /* allocate event rings */1923 ret = gpi_alloc_ev_chan(gpii);1924 if (ret) {1925 dev_err(gpii->gpi_dev->dev, "error alloc_ev_chan:%d\n", ret);1926 goto error_alloc_ev_ring;1927 }1928 1929 /* Allocate all channels */1930 for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) {1931 ret = gpi_alloc_chan(&gpii->gchan[i], true);1932 if (ret) {1933 dev_err(gpii->gpi_dev->dev, "Error allocating chan:%d\n", ret);1934 goto error_alloc_chan;1935 }1936 }1937 1938 /* start channels */1939 for (i = 0; i < MAX_CHANNELS_PER_GPII; i++) {1940 ret = gpi_start_chan(&gpii->gchan[i]);1941 if (ret) {1942 dev_err(gpii->gpi_dev->dev, "Error start chan:%d\n", ret);1943 goto error_start_chan;1944 }1945 }1946 return ret;1947 1948error_start_chan:1949 for (i = i - 1; i >= 0; i--) {1950 gpi_stop_chan(&gpii->gchan[i]);1951 gpi_send_cmd(gpii, gchan, GPI_CH_CMD_RESET);1952 }1953 i = 2;1954error_alloc_chan:1955 for (i = i - 1; i >= 0; i--)1956 gpi_reset_chan(gchan, GPI_CH_CMD_DE_ALLOC);1957error_alloc_ev_ring:1958 gpi_disable_interrupts(gpii);1959error_config_int:1960 gpi_free_ring(&gpii->ev_ring, gpii);1961exit_gpi_init:1962 return ret;1963}1964 1965/* release all channel resources */1966static void gpi_free_chan_resources(struct dma_chan *chan)1967{1968 struct gchan *gchan = to_gchan(chan);1969 struct gpii *gpii = gchan->gpii;1970 enum gpi_pm_state cur_state;1971 int ret, i;1972 1973 mutex_lock(&gpii->ctrl_lock);1974 1975 cur_state = gchan->pm_state;1976 1977 /* disable ch state so no more TRE processing for this channel */1978 write_lock_irq(&gpii->pm_lock);1979 gchan->pm_state = PREPARE_TERMINATE;1980 write_unlock_irq(&gpii->pm_lock);1981 1982 /* attempt to do graceful hardware shutdown */1983 if (cur_state == ACTIVE_STATE) {1984 gpi_stop_chan(gchan);1985 1986 ret = gpi_send_cmd(gpii, gchan, GPI_CH_CMD_RESET);1987 if (ret)1988 dev_err(gpii->gpi_dev->dev, "error resetting channel:%d\n", ret);1989 1990 gpi_reset_chan(gchan, GPI_CH_CMD_DE_ALLOC);1991 }1992 1993 /* free all allocated memory */1994 gpi_free_ring(&gchan->ch_ring, gpii);1995 vchan_free_chan_resources(&gchan->vc);1996 kfree(gchan->config);1997 1998 write_lock_irq(&gpii->pm_lock);1999 gchan->pm_state = DISABLE_STATE;2000 write_unlock_irq(&gpii->pm_lock);2001 2002 /* if other rings are still active exit */2003 for (i = 0; i < MAX_CHANNELS_PER_GPII; i++)2004 if (gpii->gchan[i].ch_ring.configured)2005 goto exit_free;2006 2007 /* deallocate EV Ring */2008 cur_state = gpii->pm_state;2009 write_lock_irq(&gpii->pm_lock);2010 gpii->pm_state = PREPARE_TERMINATE;2011 write_unlock_irq(&gpii->pm_lock);2012 2013 /* wait for threads to complete out */2014 tasklet_kill(&gpii->ev_task);2015 2016 /* send command to de allocate event ring */2017 if (cur_state == ACTIVE_STATE)2018 gpi_send_cmd(gpii, NULL, GPI_EV_CMD_DEALLOC);2019 2020 gpi_free_ring(&gpii->ev_ring, gpii);2021 2022 /* disable interrupts */2023 if (cur_state == ACTIVE_STATE)2024 gpi_disable_interrupts(gpii);2025 2026 /* set final state to disable */2027 write_lock_irq(&gpii->pm_lock);2028 gpii->pm_state = DISABLE_STATE;2029 write_unlock_irq(&gpii->pm_lock);2030 2031exit_free:2032 mutex_unlock(&gpii->ctrl_lock);2033}2034 2035/* allocate channel resources */2036static int gpi_alloc_chan_resources(struct dma_chan *chan)2037{2038 struct gchan *gchan = to_gchan(chan);2039 struct gpii *gpii = gchan->gpii;2040 int ret;2041 2042 mutex_lock(&gpii->ctrl_lock);2043 2044 /* allocate memory for transfer ring */2045 ret = gpi_alloc_ring(&gchan->ch_ring, CHAN_TRES,2046 sizeof(struct gpi_tre), gpii);2047 if (ret)2048 goto xfer_alloc_err;2049 2050 ret = gpi_ch_init(gchan);2051 2052 mutex_unlock(&gpii->ctrl_lock);2053 2054 return ret;2055xfer_alloc_err:2056 mutex_unlock(&gpii->ctrl_lock);2057 2058 return ret;2059}2060 2061static int gpi_find_avail_gpii(struct gpi_dev *gpi_dev, u32 seid)2062{2063 struct gchan *tx_chan, *rx_chan;2064 unsigned int gpii;2065 2066 /* check if same seid is already configured for another chid */2067 for (gpii = 0; gpii < gpi_dev->max_gpii; gpii++) {2068 if (!((1 << gpii) & gpi_dev->gpii_mask))2069 continue;2070 2071 tx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_TX_CHAN];2072 rx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_RX_CHAN];2073 2074 if (rx_chan->vc.chan.client_count && rx_chan->seid == seid)2075 return gpii;2076 if (tx_chan->vc.chan.client_count && tx_chan->seid == seid)2077 return gpii;2078 }2079 2080 /* no channels configured with same seid, return next avail gpii */2081 for (gpii = 0; gpii < gpi_dev->max_gpii; gpii++) {2082 if (!((1 << gpii) & gpi_dev->gpii_mask))2083 continue;2084 2085 tx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_TX_CHAN];2086 rx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_RX_CHAN];2087 2088 /* check if gpii is configured */2089 if (tx_chan->vc.chan.client_count ||2090 rx_chan->vc.chan.client_count)2091 continue;2092 2093 /* found a free gpii */2094 return gpii;2095 }2096 2097 /* no gpii instance available to use */2098 return -EIO;2099}2100 2101/* gpi_of_dma_xlate: open client requested channel */2102static struct dma_chan *gpi_of_dma_xlate(struct of_phandle_args *args,2103 struct of_dma *of_dma)2104{2105 struct gpi_dev *gpi_dev = (struct gpi_dev *)of_dma->of_dma_data;2106 u32 seid, chid;2107 int gpii;2108 struct gchan *gchan;2109 2110 if (args->args_count < 3) {2111 dev_err(gpi_dev->dev, "gpii require minimum 2 args, client passed:%d args\n",2112 args->args_count);2113 return NULL;2114 }2115 2116 chid = args->args[0];2117 if (chid >= MAX_CHANNELS_PER_GPII) {2118 dev_err(gpi_dev->dev, "gpii channel:%d not valid\n", chid);2119 return NULL;2120 }2121 2122 seid = args->args[1];2123 2124 /* find next available gpii to use */2125 gpii = gpi_find_avail_gpii(gpi_dev, seid);2126 if (gpii < 0) {2127 dev_err(gpi_dev->dev, "no available gpii instances\n");2128 return NULL;2129 }2130 2131 gchan = &gpi_dev->gpiis[gpii].gchan[chid];2132 if (gchan->vc.chan.client_count) {2133 dev_err(gpi_dev->dev, "gpii:%d chid:%d seid:%d already configured\n",2134 gpii, chid, gchan->seid);2135 return NULL;2136 }2137 2138 gchan->seid = seid;2139 gchan->protocol = args->args[2];2140 2141 return dma_get_slave_channel(&gchan->vc.chan);2142}2143 2144static int gpi_probe(struct platform_device *pdev)2145{2146 struct gpi_dev *gpi_dev;2147 unsigned int i;2148 u32 ee_offset;2149 int ret;2150 2151 gpi_dev = devm_kzalloc(&pdev->dev, sizeof(*gpi_dev), GFP_KERNEL);2152 if (!gpi_dev)2153 return -ENOMEM;2154 2155 gpi_dev->dev = &pdev->dev;2156 gpi_dev->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &gpi_dev->res);2157 if (IS_ERR(gpi_dev->regs))2158 return PTR_ERR(gpi_dev->regs);2159 gpi_dev->ee_base = gpi_dev->regs;2160 2161 ret = of_property_read_u32(gpi_dev->dev->of_node, "dma-channels",2162 &gpi_dev->max_gpii);2163 if (ret) {2164 dev_err(gpi_dev->dev, "missing 'max-no-gpii' DT node\n");2165 return ret;2166 }2167 2168 ret = of_property_read_u32(gpi_dev->dev->of_node, "dma-channel-mask",2169 &gpi_dev->gpii_mask);2170 if (ret) {2171 dev_err(gpi_dev->dev, "missing 'gpii-mask' DT node\n");2172 return ret;2173 }2174 2175 ee_offset = (uintptr_t)device_get_match_data(gpi_dev->dev);2176 gpi_dev->ee_base = gpi_dev->ee_base - ee_offset;2177 2178 gpi_dev->ev_factor = EV_FACTOR;2179 2180 ret = dma_set_mask(gpi_dev->dev, DMA_BIT_MASK(64));2181 if (ret) {2182 dev_err(gpi_dev->dev, "Error setting dma_mask to 64, ret:%d\n", ret);2183 return ret;2184 }2185 2186 gpi_dev->gpiis = devm_kzalloc(gpi_dev->dev, sizeof(*gpi_dev->gpiis) *2187 gpi_dev->max_gpii, GFP_KERNEL);2188 if (!gpi_dev->gpiis)2189 return -ENOMEM;2190 2191 /* setup all the supported gpii */2192 INIT_LIST_HEAD(&gpi_dev->dma_device.channels);2193 for (i = 0; i < gpi_dev->max_gpii; i++) {2194 struct gpii *gpii = &gpi_dev->gpiis[i];2195 int chan;2196 2197 if (!((1 << i) & gpi_dev->gpii_mask))2198 continue;2199 2200 /* set up ev cntxt register map */2201 gpii->ev_cntxt_base_reg = gpi_dev->ee_base + GPII_n_EV_CH_k_CNTXT_0_OFFS(i, 0);2202 gpii->ev_cntxt_db_reg = gpi_dev->ee_base + GPII_n_EV_CH_k_DOORBELL_0_OFFS(i, 0);2203 gpii->ev_ring_rp_lsb_reg = gpii->ev_cntxt_base_reg + CNTXT_4_RING_RP_LSB;2204 gpii->ev_cmd_reg = gpi_dev->ee_base + GPII_n_EV_CH_CMD_OFFS(i);2205 gpii->ieob_clr_reg = gpi_dev->ee_base + GPII_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(i);2206 2207 /* set up irq */2208 ret = platform_get_irq(pdev, i);2209 if (ret < 0)2210 return ret;2211 gpii->irq = ret;2212 2213 /* set up channel specific register info */2214 for (chan = 0; chan < MAX_CHANNELS_PER_GPII; chan++) {2215 struct gchan *gchan = &gpii->gchan[chan];2216 2217 /* set up ch cntxt register map */2218 gchan->ch_cntxt_base_reg = gpi_dev->ee_base +2219 GPII_n_CH_k_CNTXT_0_OFFS(i, chan);2220 gchan->ch_cntxt_db_reg = gpi_dev->ee_base +2221 GPII_n_CH_k_DOORBELL_0_OFFS(i, chan);2222 gchan->ch_cmd_reg = gpi_dev->ee_base + GPII_n_CH_CMD_OFFS(i);2223 2224 /* vchan setup */2225 vchan_init(&gchan->vc, &gpi_dev->dma_device);2226 gchan->vc.desc_free = gpi_desc_free;2227 gchan->chid = chan;2228 gchan->gpii = gpii;2229 gchan->dir = GPII_CHAN_DIR[chan];2230 }2231 mutex_init(&gpii->ctrl_lock);2232 rwlock_init(&gpii->pm_lock);2233 tasklet_init(&gpii->ev_task, gpi_ev_tasklet,2234 (unsigned long)gpii);2235 init_completion(&gpii->cmd_completion);2236 gpii->gpii_id = i;2237 gpii->regs = gpi_dev->ee_base;2238 gpii->gpi_dev = gpi_dev;2239 }2240 2241 platform_set_drvdata(pdev, gpi_dev);2242 2243 /* clear and Set capabilities */2244 dma_cap_zero(gpi_dev->dma_device.cap_mask);2245 dma_cap_set(DMA_SLAVE, gpi_dev->dma_device.cap_mask);2246 2247 /* configure dmaengine apis */2248 gpi_dev->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);2249 gpi_dev->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;2250 gpi_dev->dma_device.src_addr_widths = DMA_SLAVE_BUSWIDTH_8_BYTES;2251 gpi_dev->dma_device.dst_addr_widths = DMA_SLAVE_BUSWIDTH_8_BYTES;2252 gpi_dev->dma_device.device_alloc_chan_resources = gpi_alloc_chan_resources;2253 gpi_dev->dma_device.device_free_chan_resources = gpi_free_chan_resources;2254 gpi_dev->dma_device.device_tx_status = dma_cookie_status;2255 gpi_dev->dma_device.device_issue_pending = gpi_issue_pending;2256 gpi_dev->dma_device.device_prep_slave_sg = gpi_prep_slave_sg;2257 gpi_dev->dma_device.device_config = gpi_peripheral_config;2258 gpi_dev->dma_device.device_terminate_all = gpi_terminate_all;2259 gpi_dev->dma_device.dev = gpi_dev->dev;2260 gpi_dev->dma_device.device_pause = gpi_pause;2261 gpi_dev->dma_device.device_resume = gpi_resume;2262 2263 /* register with dmaengine framework */2264 ret = dma_async_device_register(&gpi_dev->dma_device);2265 if (ret) {2266 dev_err(gpi_dev->dev, "async_device_register failed ret:%d", ret);2267 return ret;2268 }2269 2270 ret = of_dma_controller_register(gpi_dev->dev->of_node,2271 gpi_of_dma_xlate, gpi_dev);2272 if (ret) {2273 dev_err(gpi_dev->dev, "of_dma_controller_reg failed ret:%d", ret);2274 return ret;2275 }2276 2277 return ret;2278}2279 2280static const struct of_device_id gpi_of_match[] = {2281 { .compatible = "qcom,sdm845-gpi-dma", .data = (void *)0x0 },2282 { .compatible = "qcom,sm6350-gpi-dma", .data = (void *)0x10000 },2283 /*2284 * Do not grow the list for compatible devices. Instead use2285 * qcom,sdm845-gpi-dma (for ee_offset = 0x0) or qcom,sm6350-gpi-dma2286 * (for ee_offset = 0x10000).2287 */2288 { .compatible = "qcom,sc7280-gpi-dma", .data = (void *)0x10000 },2289 { .compatible = "qcom,sm8150-gpi-dma", .data = (void *)0x0 },2290 { .compatible = "qcom,sm8250-gpi-dma", .data = (void *)0x0 },2291 { .compatible = "qcom,sm8350-gpi-dma", .data = (void *)0x10000 },2292 { .compatible = "qcom,sm8450-gpi-dma", .data = (void *)0x10000 },2293 { },2294};2295MODULE_DEVICE_TABLE(of, gpi_of_match);2296 2297static struct platform_driver gpi_driver = {2298 .probe = gpi_probe,2299 .driver = {2300 .name = KBUILD_MODNAME,2301 .of_match_table = gpi_of_match,2302 },2303};2304 2305static int __init gpi_init(void)2306{2307 return platform_driver_register(&gpi_driver);2308}2309subsys_initcall(gpi_init)2310 2311MODULE_DESCRIPTION("QCOM GPI DMA engine driver");2312MODULE_LICENSE("GPL v2");2313