1792 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Driver for Atmel AT32 and AT91 SPI Controllers4 *5 * Copyright (C) 2006 Atmel Corporation6 */7 8#include <linux/kernel.h>9#include <linux/clk.h>10#include <linux/module.h>11#include <linux/platform_device.h>12#include <linux/delay.h>13#include <linux/dma-mapping.h>14#include <linux/dmaengine.h>15#include <linux/err.h>16#include <linux/interrupt.h>17#include <linux/spi/spi.h>18#include <linux/slab.h>19#include <linux/of.h>20 21#include <linux/io.h>22#include <linux/gpio/consumer.h>23#include <linux/pinctrl/consumer.h>24#include <linux/pm_runtime.h>25#include <linux/iopoll.h>26#include <trace/events/spi.h>27 28/* SPI register offsets */29#define SPI_CR 0x000030#define SPI_MR 0x000431#define SPI_RDR 0x000832#define SPI_TDR 0x000c33#define SPI_SR 0x001034#define SPI_IER 0x001435#define SPI_IDR 0x001836#define SPI_IMR 0x001c37#define SPI_CSR0 0x003038#define SPI_CSR1 0x003439#define SPI_CSR2 0x003840#define SPI_CSR3 0x003c41#define SPI_FMR 0x004042#define SPI_FLR 0x004443#define SPI_VERSION 0x00fc44#define SPI_RPR 0x010045#define SPI_RCR 0x010446#define SPI_TPR 0x010847#define SPI_TCR 0x010c48#define SPI_RNPR 0x011049#define SPI_RNCR 0x011450#define SPI_TNPR 0x011851#define SPI_TNCR 0x011c52#define SPI_PTCR 0x012053#define SPI_PTSR 0x012454 55/* Bitfields in CR */56#define SPI_SPIEN_OFFSET 057#define SPI_SPIEN_SIZE 158#define SPI_SPIDIS_OFFSET 159#define SPI_SPIDIS_SIZE 160#define SPI_SWRST_OFFSET 761#define SPI_SWRST_SIZE 162#define SPI_LASTXFER_OFFSET 2463#define SPI_LASTXFER_SIZE 164#define SPI_TXFCLR_OFFSET 1665#define SPI_TXFCLR_SIZE 166#define SPI_RXFCLR_OFFSET 1767#define SPI_RXFCLR_SIZE 168#define SPI_FIFOEN_OFFSET 3069#define SPI_FIFOEN_SIZE 170#define SPI_FIFODIS_OFFSET 3171#define SPI_FIFODIS_SIZE 172 73/* Bitfields in MR */74#define SPI_MSTR_OFFSET 075#define SPI_MSTR_SIZE 176#define SPI_PS_OFFSET 177#define SPI_PS_SIZE 178#define SPI_PCSDEC_OFFSET 279#define SPI_PCSDEC_SIZE 180#define SPI_FDIV_OFFSET 381#define SPI_FDIV_SIZE 182#define SPI_MODFDIS_OFFSET 483#define SPI_MODFDIS_SIZE 184#define SPI_WDRBT_OFFSET 585#define SPI_WDRBT_SIZE 186#define SPI_LLB_OFFSET 787#define SPI_LLB_SIZE 188#define SPI_PCS_OFFSET 1689#define SPI_PCS_SIZE 490#define SPI_DLYBCS_OFFSET 2491#define SPI_DLYBCS_SIZE 892 93/* Bitfields in RDR */94#define SPI_RD_OFFSET 095#define SPI_RD_SIZE 1696 97/* Bitfields in TDR */98#define SPI_TD_OFFSET 099#define SPI_TD_SIZE 16100 101/* Bitfields in SR */102#define SPI_RDRF_OFFSET 0103#define SPI_RDRF_SIZE 1104#define SPI_TDRE_OFFSET 1105#define SPI_TDRE_SIZE 1106#define SPI_MODF_OFFSET 2107#define SPI_MODF_SIZE 1108#define SPI_OVRES_OFFSET 3109#define SPI_OVRES_SIZE 1110#define SPI_ENDRX_OFFSET 4111#define SPI_ENDRX_SIZE 1112#define SPI_ENDTX_OFFSET 5113#define SPI_ENDTX_SIZE 1114#define SPI_RXBUFF_OFFSET 6115#define SPI_RXBUFF_SIZE 1116#define SPI_TXBUFE_OFFSET 7117#define SPI_TXBUFE_SIZE 1118#define SPI_NSSR_OFFSET 8119#define SPI_NSSR_SIZE 1120#define SPI_TXEMPTY_OFFSET 9121#define SPI_TXEMPTY_SIZE 1122#define SPI_SPIENS_OFFSET 16123#define SPI_SPIENS_SIZE 1124#define SPI_TXFEF_OFFSET 24125#define SPI_TXFEF_SIZE 1126#define SPI_TXFFF_OFFSET 25127#define SPI_TXFFF_SIZE 1128#define SPI_TXFTHF_OFFSET 26129#define SPI_TXFTHF_SIZE 1130#define SPI_RXFEF_OFFSET 27131#define SPI_RXFEF_SIZE 1132#define SPI_RXFFF_OFFSET 28133#define SPI_RXFFF_SIZE 1134#define SPI_RXFTHF_OFFSET 29135#define SPI_RXFTHF_SIZE 1136#define SPI_TXFPTEF_OFFSET 30137#define SPI_TXFPTEF_SIZE 1138#define SPI_RXFPTEF_OFFSET 31139#define SPI_RXFPTEF_SIZE 1140 141/* Bitfields in CSR0 */142#define SPI_CPOL_OFFSET 0143#define SPI_CPOL_SIZE 1144#define SPI_NCPHA_OFFSET 1145#define SPI_NCPHA_SIZE 1146#define SPI_CSAAT_OFFSET 3147#define SPI_CSAAT_SIZE 1148#define SPI_BITS_OFFSET 4149#define SPI_BITS_SIZE 4150#define SPI_SCBR_OFFSET 8151#define SPI_SCBR_SIZE 8152#define SPI_DLYBS_OFFSET 16153#define SPI_DLYBS_SIZE 8154#define SPI_DLYBCT_OFFSET 24155#define SPI_DLYBCT_SIZE 8156 157/* Bitfields in RCR */158#define SPI_RXCTR_OFFSET 0159#define SPI_RXCTR_SIZE 16160 161/* Bitfields in TCR */162#define SPI_TXCTR_OFFSET 0163#define SPI_TXCTR_SIZE 16164 165/* Bitfields in RNCR */166#define SPI_RXNCR_OFFSET 0167#define SPI_RXNCR_SIZE 16168 169/* Bitfields in TNCR */170#define SPI_TXNCR_OFFSET 0171#define SPI_TXNCR_SIZE 16172 173/* Bitfields in PTCR */174#define SPI_RXTEN_OFFSET 0175#define SPI_RXTEN_SIZE 1176#define SPI_RXTDIS_OFFSET 1177#define SPI_RXTDIS_SIZE 1178#define SPI_TXTEN_OFFSET 8179#define SPI_TXTEN_SIZE 1180#define SPI_TXTDIS_OFFSET 9181#define SPI_TXTDIS_SIZE 1182 183/* Bitfields in FMR */184#define SPI_TXRDYM_OFFSET 0185#define SPI_TXRDYM_SIZE 2186#define SPI_RXRDYM_OFFSET 4187#define SPI_RXRDYM_SIZE 2188#define SPI_TXFTHRES_OFFSET 16189#define SPI_TXFTHRES_SIZE 6190#define SPI_RXFTHRES_OFFSET 24191#define SPI_RXFTHRES_SIZE 6192 193/* Bitfields in FLR */194#define SPI_TXFL_OFFSET 0195#define SPI_TXFL_SIZE 6196#define SPI_RXFL_OFFSET 16197#define SPI_RXFL_SIZE 6198 199/* Constants for BITS */200#define SPI_BITS_8_BPT 0201#define SPI_BITS_9_BPT 1202#define SPI_BITS_10_BPT 2203#define SPI_BITS_11_BPT 3204#define SPI_BITS_12_BPT 4205#define SPI_BITS_13_BPT 5206#define SPI_BITS_14_BPT 6207#define SPI_BITS_15_BPT 7208#define SPI_BITS_16_BPT 8209#define SPI_ONE_DATA 0210#define SPI_TWO_DATA 1211#define SPI_FOUR_DATA 2212 213/* Bit manipulation macros */214#define SPI_BIT(name) \215 (1 << SPI_##name##_OFFSET)216#define SPI_BF(name, value) \217 (((value) & ((1 << SPI_##name##_SIZE) - 1)) << SPI_##name##_OFFSET)218#define SPI_BFEXT(name, value) \219 (((value) >> SPI_##name##_OFFSET) & ((1 << SPI_##name##_SIZE) - 1))220#define SPI_BFINS(name, value, old) \221 (((old) & ~(((1 << SPI_##name##_SIZE) - 1) << SPI_##name##_OFFSET)) \222 | SPI_BF(name, value))223 224/* Register access macros */225#define spi_readl(port, reg) \226 readl_relaxed((port)->regs + SPI_##reg)227#define spi_writel(port, reg, value) \228 writel_relaxed((value), (port)->regs + SPI_##reg)229#define spi_writew(port, reg, value) \230 writew_relaxed((value), (port)->regs + SPI_##reg)231 232/* use PIO for small transfers, avoiding DMA setup/teardown overhead and233 * cache operations; better heuristics consider wordsize and bitrate.234 */235#define DMA_MIN_BYTES 16236 237#define AUTOSUSPEND_TIMEOUT 2000238 239struct atmel_spi_caps {240 bool is_spi2;241 bool has_wdrbt;242 bool has_dma_support;243 bool has_pdc_support;244};245 246/*247 * The core SPI transfer engine just talks to a register bank to set up248 * DMA transfers; transfer queue progress is driven by IRQs. The clock249 * framework provides the base clock, subdivided for each spi_device.250 */251struct atmel_spi {252 spinlock_t lock;253 unsigned long flags;254 255 phys_addr_t phybase;256 void __iomem *regs;257 int irq;258 struct clk *clk;259 struct platform_device *pdev;260 unsigned long spi_clk;261 262 struct spi_transfer *current_transfer;263 int current_remaining_bytes;264 int done_status;265 dma_addr_t dma_addr_rx_bbuf;266 dma_addr_t dma_addr_tx_bbuf;267 void *addr_rx_bbuf;268 void *addr_tx_bbuf;269 270 struct completion xfer_completion;271 272 struct atmel_spi_caps caps;273 274 bool use_dma;275 bool use_pdc;276 277 bool keep_cs;278 279 u32 fifo_size;280 bool last_polarity;281 u8 native_cs_free;282 u8 native_cs_for_gpio;283};284 285/* Controller-specific per-slave state */286struct atmel_spi_device {287 u32 csr;288};289 290#define SPI_MAX_DMA_XFER 65535 /* true for both PDC and DMA */291#define INVALID_DMA_ADDRESS 0xffffffff292 293/*294 * This frequency can be anything supported by the controller, but to avoid295 * unnecessary delay, the highest possible frequency is chosen.296 *297 * This frequency is the highest possible which is not interfering with other298 * chip select registers (see Note for Serial Clock Bit Rate configuration in299 * Atmel-11121F-ATARM-SAMA5D3-Series-Datasheet_02-Feb-16, page 1283)300 */301#define DUMMY_MSG_FREQUENCY 0x02302/*303 * 8 bits is the minimum data the controller is capable of sending.304 *305 * This message can be anything as it should not be treated by any SPI device.306 */307#define DUMMY_MSG 0xAA308 309/*310 * Version 2 of the SPI controller has311 * - CR.LASTXFER312 * - SPI_MR.DIV32 may become FDIV or must-be-zero (here: always zero)313 * - SPI_SR.TXEMPTY, SPI_SR.NSSR (and corresponding irqs)314 * - SPI_CSRx.CSAAT315 * - SPI_CSRx.SBCR allows faster clocking316 */317static bool atmel_spi_is_v2(struct atmel_spi *as)318{319 return as->caps.is_spi2;320}321 322/*323 * Send a dummy message.324 *325 * This is sometimes needed when using a CS GPIO to force clock transition when326 * switching between devices with different polarities.327 */328static void atmel_spi_send_dummy(struct atmel_spi *as, struct spi_device *spi, int chip_select)329{330 u32 status;331 u32 csr;332 333 /*334 * Set a clock frequency to allow sending message on SPI bus.335 * The frequency here can be anything, but is needed for336 * the controller to send the data.337 */338 csr = spi_readl(as, CSR0 + 4 * chip_select);339 csr = SPI_BFINS(SCBR, DUMMY_MSG_FREQUENCY, csr);340 spi_writel(as, CSR0 + 4 * chip_select, csr);341 342 /*343 * Read all data coming from SPI bus, needed to be able to send344 * the message.345 */346 spi_readl(as, RDR);347 while (spi_readl(as, SR) & SPI_BIT(RDRF)) {348 spi_readl(as, RDR);349 cpu_relax();350 }351 352 spi_writel(as, TDR, DUMMY_MSG);353 354 readl_poll_timeout_atomic(as->regs + SPI_SR, status,355 (status & SPI_BIT(TXEMPTY)), 1, 1000);356}357 358 359/*360 * Earlier SPI controllers (e.g. on at91rm9200) have a design bug whereby361 * they assume that spi slave device state will not change on deselect, so362 * that automagic deselection is OK. ("NPCSx rises if no data is to be363 * transmitted") Not so! Workaround uses nCSx pins as GPIOs; or newer364 * controllers have CSAAT and friends.365 *366 * Even controller newer than ar91rm9200, using GPIOs can make sens as367 * it lets us support active-high chipselects despite the controller's368 * belief that only active-low devices/systems exists.369 *370 * However, at91rm9200 has a second erratum whereby nCS0 doesn't work371 * right when driven with GPIO. ("Mode Fault does not allow more than one372 * Master on Chip Select 0.") No workaround exists for that ... so for373 * nCS0 on that chip, we (a) don't use the GPIO, (b) can't support CS_HIGH,374 * and (c) will trigger that first erratum in some cases.375 *376 * When changing the clock polarity, the SPI controller waits for the next377 * transmission to enforce the default clock state. This may be an issue when378 * using a GPIO as Chip Select: the clock level is applied only when the first379 * packet is sent, once the CS has already been asserted. The workaround is to380 * avoid this by sending a first (dummy) message before toggling the CS state.381 */382static void cs_activate(struct atmel_spi *as, struct spi_device *spi)383{384 struct atmel_spi_device *asd = spi->controller_state;385 bool new_polarity;386 int chip_select;387 u32 mr;388 389 if (spi_get_csgpiod(spi, 0))390 chip_select = as->native_cs_for_gpio;391 else392 chip_select = spi_get_chipselect(spi, 0);393 394 if (atmel_spi_is_v2(as)) {395 spi_writel(as, CSR0 + 4 * chip_select, asd->csr);396 /* For the low SPI version, there is a issue that PDC transfer397 * on CS1,2,3 needs SPI_CSR0.BITS config as SPI_CSR1,2,3.BITS398 */399 spi_writel(as, CSR0, asd->csr);400 if (as->caps.has_wdrbt) {401 spi_writel(as, MR,402 SPI_BF(PCS, ~(0x01 << chip_select))403 | SPI_BIT(WDRBT)404 | SPI_BIT(MODFDIS)405 | SPI_BIT(MSTR));406 } else {407 spi_writel(as, MR,408 SPI_BF(PCS, ~(0x01 << chip_select))409 | SPI_BIT(MODFDIS)410 | SPI_BIT(MSTR));411 }412 413 mr = spi_readl(as, MR);414 415 /*416 * Ensures the clock polarity is valid before we actually417 * assert the CS to avoid spurious clock edges to be418 * processed by the spi devices.419 */420 if (spi_get_csgpiod(spi, 0)) {421 new_polarity = (asd->csr & SPI_BIT(CPOL)) != 0;422 if (new_polarity != as->last_polarity) {423 /*424 * Need to disable the GPIO before sending the dummy425 * message because it is already set by the spi core.426 */427 gpiod_set_value_cansleep(spi_get_csgpiod(spi, 0), 0);428 atmel_spi_send_dummy(as, spi, chip_select);429 as->last_polarity = new_polarity;430 gpiod_set_value_cansleep(spi_get_csgpiod(spi, 0), 1);431 }432 }433 } else {434 u32 cpol = (spi->mode & SPI_CPOL) ? SPI_BIT(CPOL) : 0;435 int i;436 u32 csr;437 438 /* Make sure clock polarity is correct */439 for (i = 0; i < spi->controller->num_chipselect; i++) {440 csr = spi_readl(as, CSR0 + 4 * i);441 if ((csr ^ cpol) & SPI_BIT(CPOL))442 spi_writel(as, CSR0 + 4 * i,443 csr ^ SPI_BIT(CPOL));444 }445 446 mr = spi_readl(as, MR);447 mr = SPI_BFINS(PCS, ~(1 << chip_select), mr);448 spi_writel(as, MR, mr);449 }450 451 dev_dbg(&spi->dev, "activate NPCS, mr %08x\n", mr);452}453 454static void cs_deactivate(struct atmel_spi *as, struct spi_device *spi)455{456 int chip_select;457 u32 mr;458 459 if (spi_get_csgpiod(spi, 0))460 chip_select = as->native_cs_for_gpio;461 else462 chip_select = spi_get_chipselect(spi, 0);463 464 /* only deactivate *this* device; sometimes transfers to465 * another device may be active when this routine is called.466 */467 mr = spi_readl(as, MR);468 if (~SPI_BFEXT(PCS, mr) & (1 << chip_select)) {469 mr = SPI_BFINS(PCS, 0xf, mr);470 spi_writel(as, MR, mr);471 }472 473 dev_dbg(&spi->dev, "DEactivate NPCS, mr %08x\n", mr);474 475 if (!spi_get_csgpiod(spi, 0))476 spi_writel(as, CR, SPI_BIT(LASTXFER));477}478 479static void atmel_spi_lock(struct atmel_spi *as) __acquires(&as->lock)480{481 spin_lock_irqsave(&as->lock, as->flags);482}483 484static void atmel_spi_unlock(struct atmel_spi *as) __releases(&as->lock)485{486 spin_unlock_irqrestore(&as->lock, as->flags);487}488 489static inline bool atmel_spi_is_vmalloc_xfer(struct spi_transfer *xfer)490{491 return is_vmalloc_addr(xfer->tx_buf) || is_vmalloc_addr(xfer->rx_buf);492}493 494static inline bool atmel_spi_use_dma(struct atmel_spi *as,495 struct spi_transfer *xfer)496{497 return as->use_dma && xfer->len >= DMA_MIN_BYTES;498}499 500static bool atmel_spi_can_dma(struct spi_controller *host,501 struct spi_device *spi,502 struct spi_transfer *xfer)503{504 struct atmel_spi *as = spi_controller_get_devdata(host);505 506 if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5))507 return atmel_spi_use_dma(as, xfer) &&508 !atmel_spi_is_vmalloc_xfer(xfer);509 else510 return atmel_spi_use_dma(as, xfer);511 512}513 514static int atmel_spi_dma_slave_config(struct atmel_spi *as, u8 bits_per_word)515{516 struct spi_controller *host = platform_get_drvdata(as->pdev);517 struct dma_slave_config slave_config;518 int err = 0;519 520 if (bits_per_word > 8) {521 slave_config.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;522 slave_config.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;523 } else {524 slave_config.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;525 slave_config.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;526 }527 528 slave_config.dst_addr = (dma_addr_t)as->phybase + SPI_TDR;529 slave_config.src_addr = (dma_addr_t)as->phybase + SPI_RDR;530 slave_config.src_maxburst = 1;531 slave_config.dst_maxburst = 1;532 slave_config.device_fc = false;533 534 /*535 * This driver uses fixed peripheral select mode (PS bit set to '0' in536 * the Mode Register).537 * So according to the datasheet, when FIFOs are available (and538 * enabled), the Transmit FIFO operates in Multiple Data Mode.539 * In this mode, up to 2 data, not 4, can be written into the Transmit540 * Data Register in a single access.541 * However, the first data has to be written into the lowest 16 bits and542 * the second data into the highest 16 bits of the Transmit543 * Data Register. For 8bit data (the most frequent case), it would544 * require to rework tx_buf so each data would actually fit 16 bits.545 * So we'd rather write only one data at the time. Hence the transmit546 * path works the same whether FIFOs are available (and enabled) or not.547 */548 if (dmaengine_slave_config(host->dma_tx, &slave_config)) {549 dev_err(&as->pdev->dev,550 "failed to configure tx dma channel\n");551 err = -EINVAL;552 }553 554 /*555 * This driver configures the spi controller for host mode (MSTR bit556 * set to '1' in the Mode Register).557 * So according to the datasheet, when FIFOs are available (and558 * enabled), the Receive FIFO operates in Single Data Mode.559 * So the receive path works the same whether FIFOs are available (and560 * enabled) or not.561 */562 if (dmaengine_slave_config(host->dma_rx, &slave_config)) {563 dev_err(&as->pdev->dev,564 "failed to configure rx dma channel\n");565 err = -EINVAL;566 }567 568 return err;569}570 571static int atmel_spi_configure_dma(struct spi_controller *host,572 struct atmel_spi *as)573{574 struct device *dev = &as->pdev->dev;575 int err;576 577 host->dma_tx = dma_request_chan(dev, "tx");578 if (IS_ERR(host->dma_tx)) {579 err = PTR_ERR(host->dma_tx);580 dev_dbg(dev, "No TX DMA channel, DMA is disabled\n");581 goto error_clear;582 }583 584 host->dma_rx = dma_request_chan(dev, "rx");585 if (IS_ERR(host->dma_rx)) {586 err = PTR_ERR(host->dma_rx);587 /*588 * No reason to check EPROBE_DEFER here since we have already589 * requested tx channel.590 */591 dev_dbg(dev, "No RX DMA channel, DMA is disabled\n");592 goto error;593 }594 595 err = atmel_spi_dma_slave_config(as, 8);596 if (err)597 goto error;598 599 dev_info(&as->pdev->dev,600 "Using %s (tx) and %s (rx) for DMA transfers\n",601 dma_chan_name(host->dma_tx),602 dma_chan_name(host->dma_rx));603 604 return 0;605error:606 if (!IS_ERR(host->dma_rx))607 dma_release_channel(host->dma_rx);608 if (!IS_ERR(host->dma_tx))609 dma_release_channel(host->dma_tx);610error_clear:611 host->dma_tx = host->dma_rx = NULL;612 return err;613}614 615static void atmel_spi_stop_dma(struct spi_controller *host)616{617 if (host->dma_rx)618 dmaengine_terminate_all(host->dma_rx);619 if (host->dma_tx)620 dmaengine_terminate_all(host->dma_tx);621}622 623static void atmel_spi_release_dma(struct spi_controller *host)624{625 if (host->dma_rx) {626 dma_release_channel(host->dma_rx);627 host->dma_rx = NULL;628 }629 if (host->dma_tx) {630 dma_release_channel(host->dma_tx);631 host->dma_tx = NULL;632 }633}634 635/* This function is called by the DMA driver from tasklet context */636static void dma_callback(void *data)637{638 struct spi_controller *host = data;639 struct atmel_spi *as = spi_controller_get_devdata(host);640 641 if (is_vmalloc_addr(as->current_transfer->rx_buf) &&642 IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {643 memcpy(as->current_transfer->rx_buf, as->addr_rx_bbuf,644 as->current_transfer->len);645 }646 complete(&as->xfer_completion);647}648 649/*650 * Next transfer using PIO without FIFO.651 */652static void atmel_spi_next_xfer_single(struct spi_controller *host,653 struct spi_transfer *xfer)654{655 struct atmel_spi *as = spi_controller_get_devdata(host);656 unsigned long xfer_pos = xfer->len - as->current_remaining_bytes;657 658 dev_vdbg(host->dev.parent, "atmel_spi_next_xfer_pio\n");659 660 /* Make sure data is not remaining in RDR */661 spi_readl(as, RDR);662 while (spi_readl(as, SR) & SPI_BIT(RDRF)) {663 spi_readl(as, RDR);664 cpu_relax();665 }666 667 if (xfer->bits_per_word > 8)668 spi_writel(as, TDR, *(u16 *)(xfer->tx_buf + xfer_pos));669 else670 spi_writel(as, TDR, *(u8 *)(xfer->tx_buf + xfer_pos));671 672 dev_dbg(host->dev.parent,673 " start pio xfer %p: len %u tx %p rx %p bitpw %d\n",674 xfer, xfer->len, xfer->tx_buf, xfer->rx_buf,675 xfer->bits_per_word);676 677 /* Enable relevant interrupts */678 spi_writel(as, IER, SPI_BIT(RDRF) | SPI_BIT(OVRES));679}680 681/*682 * Next transfer using PIO with FIFO.683 */684static void atmel_spi_next_xfer_fifo(struct spi_controller *host,685 struct spi_transfer *xfer)686{687 struct atmel_spi *as = spi_controller_get_devdata(host);688 u32 current_remaining_data, num_data;689 u32 offset = xfer->len - as->current_remaining_bytes;690 const u16 *words = (const u16 *)((u8 *)xfer->tx_buf + offset);691 const u8 *bytes = (const u8 *)((u8 *)xfer->tx_buf + offset);692 u16 td0, td1;693 u32 fifomr;694 695 dev_vdbg(host->dev.parent, "atmel_spi_next_xfer_fifo\n");696 697 /* Compute the number of data to transfer in the current iteration */698 current_remaining_data = ((xfer->bits_per_word > 8) ?699 ((u32)as->current_remaining_bytes >> 1) :700 (u32)as->current_remaining_bytes);701 num_data = min(current_remaining_data, as->fifo_size);702 703 /* Flush RX and TX FIFOs */704 spi_writel(as, CR, SPI_BIT(RXFCLR) | SPI_BIT(TXFCLR));705 while (spi_readl(as, FLR))706 cpu_relax();707 708 /* Set RX FIFO Threshold to the number of data to transfer */709 fifomr = spi_readl(as, FMR);710 spi_writel(as, FMR, SPI_BFINS(RXFTHRES, num_data, fifomr));711 712 /* Clear FIFO flags in the Status Register, especially RXFTHF */713 (void)spi_readl(as, SR);714 715 /* Fill TX FIFO */716 while (num_data >= 2) {717 if (xfer->bits_per_word > 8) {718 td0 = *words++;719 td1 = *words++;720 } else {721 td0 = *bytes++;722 td1 = *bytes++;723 }724 725 spi_writel(as, TDR, (td1 << 16) | td0);726 num_data -= 2;727 }728 729 if (num_data) {730 if (xfer->bits_per_word > 8)731 td0 = *words++;732 else733 td0 = *bytes++;734 735 spi_writew(as, TDR, td0);736 num_data--;737 }738 739 dev_dbg(host->dev.parent,740 " start fifo xfer %p: len %u tx %p rx %p bitpw %d\n",741 xfer, xfer->len, xfer->tx_buf, xfer->rx_buf,742 xfer->bits_per_word);743 744 /*745 * Enable RX FIFO Threshold Flag interrupt to be notified about746 * transfer completion.747 */748 spi_writel(as, IER, SPI_BIT(RXFTHF) | SPI_BIT(OVRES));749}750 751/*752 * Next transfer using PIO.753 */754static void atmel_spi_next_xfer_pio(struct spi_controller *host,755 struct spi_transfer *xfer)756{757 struct atmel_spi *as = spi_controller_get_devdata(host);758 759 if (as->fifo_size)760 atmel_spi_next_xfer_fifo(host, xfer);761 else762 atmel_spi_next_xfer_single(host, xfer);763}764 765/*766 * Submit next transfer for DMA.767 */768static int atmel_spi_next_xfer_dma_submit(struct spi_controller *host,769 struct spi_transfer *xfer,770 u32 *plen)771{772 struct atmel_spi *as = spi_controller_get_devdata(host);773 struct dma_chan *rxchan = host->dma_rx;774 struct dma_chan *txchan = host->dma_tx;775 struct dma_async_tx_descriptor *rxdesc;776 struct dma_async_tx_descriptor *txdesc;777 dma_cookie_t cookie;778 779 dev_vdbg(host->dev.parent, "atmel_spi_next_xfer_dma_submit\n");780 781 /* Check that the channels are available */782 if (!rxchan || !txchan)783 return -ENODEV;784 785 786 *plen = xfer->len;787 788 if (atmel_spi_dma_slave_config(as, xfer->bits_per_word))789 goto err_exit;790 791 /* Send both scatterlists */792 if (atmel_spi_is_vmalloc_xfer(xfer) &&793 IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {794 rxdesc = dmaengine_prep_slave_single(rxchan,795 as->dma_addr_rx_bbuf,796 xfer->len,797 DMA_DEV_TO_MEM,798 DMA_PREP_INTERRUPT |799 DMA_CTRL_ACK);800 } else {801 rxdesc = dmaengine_prep_slave_sg(rxchan,802 xfer->rx_sg.sgl,803 xfer->rx_sg.nents,804 DMA_DEV_TO_MEM,805 DMA_PREP_INTERRUPT |806 DMA_CTRL_ACK);807 }808 if (!rxdesc)809 goto err_dma;810 811 if (atmel_spi_is_vmalloc_xfer(xfer) &&812 IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {813 memcpy(as->addr_tx_bbuf, xfer->tx_buf, xfer->len);814 txdesc = dmaengine_prep_slave_single(txchan,815 as->dma_addr_tx_bbuf,816 xfer->len, DMA_MEM_TO_DEV,817 DMA_PREP_INTERRUPT |818 DMA_CTRL_ACK);819 } else {820 txdesc = dmaengine_prep_slave_sg(txchan,821 xfer->tx_sg.sgl,822 xfer->tx_sg.nents,823 DMA_MEM_TO_DEV,824 DMA_PREP_INTERRUPT |825 DMA_CTRL_ACK);826 }827 if (!txdesc)828 goto err_dma;829 830 dev_dbg(host->dev.parent,831 " start dma xfer %p: len %u tx %p/%08llx rx %p/%08llx\n",832 xfer, xfer->len, xfer->tx_buf, (unsigned long long)xfer->tx_dma,833 xfer->rx_buf, (unsigned long long)xfer->rx_dma);834 835 /* Enable relevant interrupts */836 spi_writel(as, IER, SPI_BIT(OVRES));837 838 /* Put the callback on the RX transfer only, that should finish last */839 rxdesc->callback = dma_callback;840 rxdesc->callback_param = host;841 842 /* Submit and fire RX and TX with TX last so we're ready to read! */843 cookie = rxdesc->tx_submit(rxdesc);844 if (dma_submit_error(cookie))845 goto err_dma;846 cookie = txdesc->tx_submit(txdesc);847 if (dma_submit_error(cookie))848 goto err_dma;849 rxchan->device->device_issue_pending(rxchan);850 txchan->device->device_issue_pending(txchan);851 852 return 0;853 854err_dma:855 spi_writel(as, IDR, SPI_BIT(OVRES));856 atmel_spi_stop_dma(host);857err_exit:858 return -ENOMEM;859}860 861static void atmel_spi_next_xfer_data(struct spi_controller *host,862 struct spi_transfer *xfer,863 dma_addr_t *tx_dma,864 dma_addr_t *rx_dma,865 u32 *plen)866{867 *rx_dma = xfer->rx_dma + xfer->len - *plen;868 *tx_dma = xfer->tx_dma + xfer->len - *plen;869 if (*plen > host->max_dma_len)870 *plen = host->max_dma_len;871}872 873static int atmel_spi_set_xfer_speed(struct atmel_spi *as,874 struct spi_device *spi,875 struct spi_transfer *xfer)876{877 u32 scbr, csr;878 unsigned long bus_hz;879 int chip_select;880 881 if (spi_get_csgpiod(spi, 0))882 chip_select = as->native_cs_for_gpio;883 else884 chip_select = spi_get_chipselect(spi, 0);885 886 /* v1 chips start out at half the peripheral bus speed. */887 bus_hz = as->spi_clk;888 if (!atmel_spi_is_v2(as))889 bus_hz /= 2;890 891 /*892 * Calculate the lowest divider that satisfies the893 * constraint, assuming div32/fdiv/mbz == 0.894 */895 scbr = DIV_ROUND_UP(bus_hz, xfer->speed_hz);896 897 /*898 * If the resulting divider doesn't fit into the899 * register bitfield, we can't satisfy the constraint.900 */901 if (scbr >= (1 << SPI_SCBR_SIZE)) {902 dev_err(&spi->dev,903 "setup: %d Hz too slow, scbr %u; min %ld Hz\n",904 xfer->speed_hz, scbr, bus_hz/255);905 return -EINVAL;906 }907 if (scbr == 0) {908 dev_err(&spi->dev,909 "setup: %d Hz too high, scbr %u; max %ld Hz\n",910 xfer->speed_hz, scbr, bus_hz);911 return -EINVAL;912 }913 csr = spi_readl(as, CSR0 + 4 * chip_select);914 csr = SPI_BFINS(SCBR, scbr, csr);915 spi_writel(as, CSR0 + 4 * chip_select, csr);916 xfer->effective_speed_hz = bus_hz / scbr;917 918 return 0;919}920 921/*922 * Submit next transfer for PDC.923 * lock is held, spi irq is blocked924 */925static void atmel_spi_pdc_next_xfer(struct spi_controller *host,926 struct spi_transfer *xfer)927{928 struct atmel_spi *as = spi_controller_get_devdata(host);929 u32 len;930 dma_addr_t tx_dma, rx_dma;931 932 spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS));933 934 len = as->current_remaining_bytes;935 atmel_spi_next_xfer_data(host, xfer, &tx_dma, &rx_dma, &len);936 as->current_remaining_bytes -= len;937 938 spi_writel(as, RPR, rx_dma);939 spi_writel(as, TPR, tx_dma);940 941 if (xfer->bits_per_word > 8)942 len >>= 1;943 spi_writel(as, RCR, len);944 spi_writel(as, TCR, len);945 946 dev_dbg(&host->dev,947 " start xfer %p: len %u tx %p/%08llx rx %p/%08llx\n",948 xfer, xfer->len, xfer->tx_buf,949 (unsigned long long)xfer->tx_dma, xfer->rx_buf,950 (unsigned long long)xfer->rx_dma);951 952 if (as->current_remaining_bytes) {953 len = as->current_remaining_bytes;954 atmel_spi_next_xfer_data(host, xfer, &tx_dma, &rx_dma, &len);955 as->current_remaining_bytes -= len;956 957 spi_writel(as, RNPR, rx_dma);958 spi_writel(as, TNPR, tx_dma);959 960 if (xfer->bits_per_word > 8)961 len >>= 1;962 spi_writel(as, RNCR, len);963 spi_writel(as, TNCR, len);964 965 dev_dbg(&host->dev,966 " next xfer %p: len %u tx %p/%08llx rx %p/%08llx\n",967 xfer, xfer->len, xfer->tx_buf,968 (unsigned long long)xfer->tx_dma, xfer->rx_buf,969 (unsigned long long)xfer->rx_dma);970 }971 972 /* REVISIT: We're waiting for RXBUFF before we start the next973 * transfer because we need to handle some difficult timing974 * issues otherwise. If we wait for TXBUFE in one transfer and975 * then starts waiting for RXBUFF in the next, it's difficult976 * to tell the difference between the RXBUFF interrupt we're977 * actually waiting for and the RXBUFF interrupt of the978 * previous transfer.979 *980 * It should be doable, though. Just not now...981 */982 spi_writel(as, IER, SPI_BIT(RXBUFF) | SPI_BIT(OVRES));983 spi_writel(as, PTCR, SPI_BIT(TXTEN) | SPI_BIT(RXTEN));984}985 986/*987 * For DMA, tx_buf/tx_dma have the same relationship as rx_buf/rx_dma:988 * - The buffer is either valid for CPU access, else NULL989 * - If the buffer is valid, so is its DMA address990 */991static int992atmel_spi_dma_map_xfer(struct atmel_spi *as, struct spi_transfer *xfer)993{994 struct device *dev = &as->pdev->dev;995 996 xfer->tx_dma = xfer->rx_dma = INVALID_DMA_ADDRESS;997 if (xfer->tx_buf) {998 /* tx_buf is a const void* where we need a void * for the dma999 * mapping */1000 void *nonconst_tx = (void *)xfer->tx_buf;1001 1002 xfer->tx_dma = dma_map_single(dev,1003 nonconst_tx, xfer->len,1004 DMA_TO_DEVICE);1005 if (dma_mapping_error(dev, xfer->tx_dma))1006 return -ENOMEM;1007 }1008 if (xfer->rx_buf) {1009 xfer->rx_dma = dma_map_single(dev,1010 xfer->rx_buf, xfer->len,1011 DMA_FROM_DEVICE);1012 if (dma_mapping_error(dev, xfer->rx_dma)) {1013 if (xfer->tx_buf)1014 dma_unmap_single(dev,1015 xfer->tx_dma, xfer->len,1016 DMA_TO_DEVICE);1017 return -ENOMEM;1018 }1019 }1020 return 0;1021}1022 1023static void atmel_spi_dma_unmap_xfer(struct spi_controller *host,1024 struct spi_transfer *xfer)1025{1026 if (xfer->tx_dma != INVALID_DMA_ADDRESS)1027 dma_unmap_single(host->dev.parent, xfer->tx_dma,1028 xfer->len, DMA_TO_DEVICE);1029 if (xfer->rx_dma != INVALID_DMA_ADDRESS)1030 dma_unmap_single(host->dev.parent, xfer->rx_dma,1031 xfer->len, DMA_FROM_DEVICE);1032}1033 1034static void atmel_spi_disable_pdc_transfer(struct atmel_spi *as)1035{1036 spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS));1037}1038 1039static void1040atmel_spi_pump_single_data(struct atmel_spi *as, struct spi_transfer *xfer)1041{1042 u8 *rxp;1043 u16 *rxp16;1044 unsigned long xfer_pos = xfer->len - as->current_remaining_bytes;1045 1046 if (xfer->bits_per_word > 8) {1047 rxp16 = (u16 *)(((u8 *)xfer->rx_buf) + xfer_pos);1048 *rxp16 = spi_readl(as, RDR);1049 } else {1050 rxp = ((u8 *)xfer->rx_buf) + xfer_pos;1051 *rxp = spi_readl(as, RDR);1052 }1053 if (xfer->bits_per_word > 8) {1054 if (as->current_remaining_bytes > 2)1055 as->current_remaining_bytes -= 2;1056 else1057 as->current_remaining_bytes = 0;1058 } else {1059 as->current_remaining_bytes--;1060 }1061}1062 1063static void1064atmel_spi_pump_fifo_data(struct atmel_spi *as, struct spi_transfer *xfer)1065{1066 u32 fifolr = spi_readl(as, FLR);1067 u32 num_bytes, num_data = SPI_BFEXT(RXFL, fifolr);1068 u32 offset = xfer->len - as->current_remaining_bytes;1069 u16 *words = (u16 *)((u8 *)xfer->rx_buf + offset);1070 u8 *bytes = (u8 *)((u8 *)xfer->rx_buf + offset);1071 u16 rd; /* RD field is the lowest 16 bits of RDR */1072 1073 /* Update the number of remaining bytes to transfer */1074 num_bytes = ((xfer->bits_per_word > 8) ?1075 (num_data << 1) :1076 num_data);1077 1078 if (as->current_remaining_bytes > num_bytes)1079 as->current_remaining_bytes -= num_bytes;1080 else1081 as->current_remaining_bytes = 0;1082 1083 /* Handle odd number of bytes when data are more than 8bit width */1084 if (xfer->bits_per_word > 8)1085 as->current_remaining_bytes &= ~0x1;1086 1087 /* Read data */1088 while (num_data) {1089 rd = spi_readl(as, RDR);1090 if (xfer->bits_per_word > 8)1091 *words++ = rd;1092 else1093 *bytes++ = rd;1094 num_data--;1095 }1096}1097 1098/* Called from IRQ1099 *1100 * Must update "current_remaining_bytes" to keep track of data1101 * to transfer.1102 */1103static void1104atmel_spi_pump_pio_data(struct atmel_spi *as, struct spi_transfer *xfer)1105{1106 if (as->fifo_size)1107 atmel_spi_pump_fifo_data(as, xfer);1108 else1109 atmel_spi_pump_single_data(as, xfer);1110}1111 1112/* Interrupt1113 *1114 */1115static irqreturn_t1116atmel_spi_pio_interrupt(int irq, void *dev_id)1117{1118 struct spi_controller *host = dev_id;1119 struct atmel_spi *as = spi_controller_get_devdata(host);1120 u32 status, pending, imr;1121 struct spi_transfer *xfer;1122 int ret = IRQ_NONE;1123 1124 imr = spi_readl(as, IMR);1125 status = spi_readl(as, SR);1126 pending = status & imr;1127 1128 if (pending & SPI_BIT(OVRES)) {1129 ret = IRQ_HANDLED;1130 spi_writel(as, IDR, SPI_BIT(OVRES));1131 dev_warn(host->dev.parent, "overrun\n");1132 1133 /*1134 * When we get an overrun, we disregard the current1135 * transfer. Data will not be copied back from any1136 * bounce buffer and msg->actual_len will not be1137 * updated with the last xfer.1138 *1139 * We will also not process any remaning transfers in1140 * the message.1141 */1142 as->done_status = -EIO;1143 smp_wmb();1144 1145 /* Clear any overrun happening while cleaning up */1146 spi_readl(as, SR);1147 1148 complete(&as->xfer_completion);1149 1150 } else if (pending & (SPI_BIT(RDRF) | SPI_BIT(RXFTHF))) {1151 atmel_spi_lock(as);1152 1153 if (as->current_remaining_bytes) {1154 ret = IRQ_HANDLED;1155 xfer = as->current_transfer;1156 atmel_spi_pump_pio_data(as, xfer);1157 if (!as->current_remaining_bytes)1158 spi_writel(as, IDR, pending);1159 1160 complete(&as->xfer_completion);1161 }1162 1163 atmel_spi_unlock(as);1164 } else {1165 WARN_ONCE(pending, "IRQ not handled, pending = %x\n", pending);1166 ret = IRQ_HANDLED;1167 spi_writel(as, IDR, pending);1168 }1169 1170 return ret;1171}1172 1173static irqreturn_t1174atmel_spi_pdc_interrupt(int irq, void *dev_id)1175{1176 struct spi_controller *host = dev_id;1177 struct atmel_spi *as = spi_controller_get_devdata(host);1178 u32 status, pending, imr;1179 int ret = IRQ_NONE;1180 1181 imr = spi_readl(as, IMR);1182 status = spi_readl(as, SR);1183 pending = status & imr;1184 1185 if (pending & SPI_BIT(OVRES)) {1186 1187 ret = IRQ_HANDLED;1188 1189 spi_writel(as, IDR, (SPI_BIT(RXBUFF) | SPI_BIT(ENDRX)1190 | SPI_BIT(OVRES)));1191 1192 /* Clear any overrun happening while cleaning up */1193 spi_readl(as, SR);1194 1195 as->done_status = -EIO;1196 1197 complete(&as->xfer_completion);1198 1199 } else if (pending & (SPI_BIT(RXBUFF) | SPI_BIT(ENDRX))) {1200 ret = IRQ_HANDLED;1201 1202 spi_writel(as, IDR, pending);1203 1204 complete(&as->xfer_completion);1205 }1206 1207 return ret;1208}1209 1210static int atmel_word_delay_csr(struct spi_device *spi, struct atmel_spi *as)1211{1212 struct spi_delay *delay = &spi->word_delay;1213 u32 value = delay->value;1214 1215 switch (delay->unit) {1216 case SPI_DELAY_UNIT_NSECS:1217 value /= 1000;1218 break;1219 case SPI_DELAY_UNIT_USECS:1220 break;1221 default:1222 return -EINVAL;1223 }1224 1225 return (as->spi_clk / 1000000 * value) >> 5;1226}1227 1228static void initialize_native_cs_for_gpio(struct atmel_spi *as)1229{1230 int i;1231 struct spi_controller *host = platform_get_drvdata(as->pdev);1232 1233 if (!as->native_cs_free)1234 return; /* already initialized */1235 1236 if (!host->cs_gpiods)1237 return; /* No CS GPIO */1238 1239 /*1240 * On the first version of the controller (AT91RM9200), CS01241 * can't be used associated with GPIO1242 */1243 if (atmel_spi_is_v2(as))1244 i = 0;1245 else1246 i = 1;1247 1248 for (; i < 4; i++)1249 if (host->cs_gpiods[i])1250 as->native_cs_free |= BIT(i);1251 1252 if (as->native_cs_free)1253 as->native_cs_for_gpio = ffs(as->native_cs_free);1254}1255 1256static int atmel_spi_setup(struct spi_device *spi)1257{1258 struct atmel_spi *as;1259 struct atmel_spi_device *asd;1260 u32 csr;1261 unsigned int bits = spi->bits_per_word;1262 int chip_select;1263 int word_delay_csr;1264 1265 as = spi_controller_get_devdata(spi->controller);1266 1267 /* see notes above re chipselect */1268 if (!spi_get_csgpiod(spi, 0) && (spi->mode & SPI_CS_HIGH)) {1269 dev_warn(&spi->dev, "setup: non GPIO CS can't be active-high\n");1270 return -EINVAL;1271 }1272 1273 /* Setup() is called during spi_register_controller(aka1274 * spi_register_master) but after all membmers of the cs_gpiod1275 * array have been filled, so we can looked for which native1276 * CS will be free for using with GPIO1277 */1278 initialize_native_cs_for_gpio(as);1279 1280 if (spi_get_csgpiod(spi, 0) && as->native_cs_free) {1281 dev_err(&spi->dev,1282 "No native CS available to support this GPIO CS\n");1283 return -EBUSY;1284 }1285 1286 if (spi_get_csgpiod(spi, 0))1287 chip_select = as->native_cs_for_gpio;1288 else1289 chip_select = spi_get_chipselect(spi, 0);1290 1291 csr = SPI_BF(BITS, bits - 8);1292 if (spi->mode & SPI_CPOL)1293 csr |= SPI_BIT(CPOL);1294 if (!(spi->mode & SPI_CPHA))1295 csr |= SPI_BIT(NCPHA);1296 1297 if (!spi_get_csgpiod(spi, 0))1298 csr |= SPI_BIT(CSAAT);1299 csr |= SPI_BF(DLYBS, 0);1300 1301 word_delay_csr = atmel_word_delay_csr(spi, as);1302 if (word_delay_csr < 0)1303 return word_delay_csr;1304 1305 /* DLYBCT adds delays between words. This is useful for slow devices1306 * that need a bit of time to setup the next transfer.1307 */1308 csr |= SPI_BF(DLYBCT, word_delay_csr);1309 1310 asd = spi->controller_state;1311 if (!asd) {1312 asd = kzalloc(sizeof(struct atmel_spi_device), GFP_KERNEL);1313 if (!asd)1314 return -ENOMEM;1315 1316 spi->controller_state = asd;1317 }1318 1319 asd->csr = csr;1320 1321 dev_dbg(&spi->dev,1322 "setup: bpw %u mode 0x%x -> csr%d %08x\n",1323 bits, spi->mode, spi_get_chipselect(spi, 0), csr);1324 1325 if (!atmel_spi_is_v2(as))1326 spi_writel(as, CSR0 + 4 * chip_select, csr);1327 1328 return 0;1329}1330 1331static void atmel_spi_set_cs(struct spi_device *spi, bool enable)1332{1333 struct atmel_spi *as = spi_controller_get_devdata(spi->controller);1334 /* the core doesn't really pass us enable/disable, but CS HIGH vs CS LOW1335 * since we already have routines for activate/deactivate translate1336 * high/low to active/inactive1337 */1338 enable = (!!(spi->mode & SPI_CS_HIGH) == enable);1339 1340 if (enable) {1341 cs_activate(as, spi);1342 } else {1343 cs_deactivate(as, spi);1344 }1345 1346}1347 1348static int atmel_spi_one_transfer(struct spi_controller *host,1349 struct spi_device *spi,1350 struct spi_transfer *xfer)1351{1352 struct atmel_spi *as;1353 u8 bits;1354 u32 len;1355 struct atmel_spi_device *asd;1356 int timeout;1357 int ret;1358 unsigned int dma_timeout;1359 long ret_timeout;1360 1361 as = spi_controller_get_devdata(host);1362 1363 asd = spi->controller_state;1364 bits = (asd->csr >> 4) & 0xf;1365 if (bits != xfer->bits_per_word - 8) {1366 dev_dbg(&spi->dev,1367 "you can't yet change bits_per_word in transfers\n");1368 return -ENOPROTOOPT;1369 }1370 1371 /*1372 * DMA map early, for performance (empties dcache ASAP) and1373 * better fault reporting.1374 */1375 if (as->use_pdc) {1376 if (atmel_spi_dma_map_xfer(as, xfer) < 0)1377 return -ENOMEM;1378 }1379 1380 atmel_spi_set_xfer_speed(as, spi, xfer);1381 1382 as->done_status = 0;1383 as->current_transfer = xfer;1384 as->current_remaining_bytes = xfer->len;1385 while (as->current_remaining_bytes) {1386 reinit_completion(&as->xfer_completion);1387 1388 if (as->use_pdc) {1389 atmel_spi_lock(as);1390 atmel_spi_pdc_next_xfer(host, xfer);1391 atmel_spi_unlock(as);1392 } else if (atmel_spi_use_dma(as, xfer)) {1393 len = as->current_remaining_bytes;1394 ret = atmel_spi_next_xfer_dma_submit(host,1395 xfer, &len);1396 if (ret) {1397 dev_err(&spi->dev,1398 "unable to use DMA, fallback to PIO\n");1399 as->done_status = ret;1400 break;1401 } else {1402 as->current_remaining_bytes -= len;1403 if (as->current_remaining_bytes < 0)1404 as->current_remaining_bytes = 0;1405 }1406 } else {1407 atmel_spi_lock(as);1408 atmel_spi_next_xfer_pio(host, xfer);1409 atmel_spi_unlock(as);1410 }1411 1412 dma_timeout = msecs_to_jiffies(spi_controller_xfer_timeout(host, xfer));1413 ret_timeout = wait_for_completion_timeout(&as->xfer_completion, dma_timeout);1414 if (!ret_timeout) {1415 dev_err(&spi->dev, "spi transfer timeout\n");1416 as->done_status = -EIO;1417 }1418 1419 if (as->done_status)1420 break;1421 }1422 1423 if (as->done_status) {1424 if (as->use_pdc) {1425 dev_warn(host->dev.parent,1426 "overrun (%u/%u remaining)\n",1427 spi_readl(as, TCR), spi_readl(as, RCR));1428 1429 /*1430 * Clean up DMA registers and make sure the data1431 * registers are empty.1432 */1433 spi_writel(as, RNCR, 0);1434 spi_writel(as, TNCR, 0);1435 spi_writel(as, RCR, 0);1436 spi_writel(as, TCR, 0);1437 for (timeout = 1000; timeout; timeout--)1438 if (spi_readl(as, SR) & SPI_BIT(TXEMPTY))1439 break;1440 if (!timeout)1441 dev_warn(host->dev.parent,1442 "timeout waiting for TXEMPTY");1443 while (spi_readl(as, SR) & SPI_BIT(RDRF))1444 spi_readl(as, RDR);1445 1446 /* Clear any overrun happening while cleaning up */1447 spi_readl(as, SR);1448 1449 } else if (atmel_spi_use_dma(as, xfer)) {1450 atmel_spi_stop_dma(host);1451 }1452 }1453 1454 if (as->use_pdc)1455 atmel_spi_dma_unmap_xfer(host, xfer);1456 1457 if (as->use_pdc)1458 atmel_spi_disable_pdc_transfer(as);1459 1460 return as->done_status;1461}1462 1463static void atmel_spi_cleanup(struct spi_device *spi)1464{1465 struct atmel_spi_device *asd = spi->controller_state;1466 1467 if (!asd)1468 return;1469 1470 spi->controller_state = NULL;1471 kfree(asd);1472}1473 1474static inline unsigned int atmel_get_version(struct atmel_spi *as)1475{1476 return spi_readl(as, VERSION) & 0x00000fff;1477}1478 1479static void atmel_get_caps(struct atmel_spi *as)1480{1481 unsigned int version;1482 1483 version = atmel_get_version(as);1484 1485 as->caps.is_spi2 = version > 0x121;1486 as->caps.has_wdrbt = version >= 0x210;1487 as->caps.has_dma_support = version >= 0x212;1488 as->caps.has_pdc_support = version < 0x212;1489}1490 1491static void atmel_spi_init(struct atmel_spi *as)1492{1493 spi_writel(as, CR, SPI_BIT(SWRST));1494 spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */1495 1496 /* It is recommended to enable FIFOs first thing after reset */1497 if (as->fifo_size)1498 spi_writel(as, CR, SPI_BIT(FIFOEN));1499 1500 if (as->caps.has_wdrbt) {1501 spi_writel(as, MR, SPI_BIT(WDRBT) | SPI_BIT(MODFDIS)1502 | SPI_BIT(MSTR));1503 } else {1504 spi_writel(as, MR, SPI_BIT(MSTR) | SPI_BIT(MODFDIS));1505 }1506 1507 if (as->use_pdc)1508 spi_writel(as, PTCR, SPI_BIT(RXTDIS) | SPI_BIT(TXTDIS));1509 spi_writel(as, CR, SPI_BIT(SPIEN));1510}1511 1512static int atmel_spi_probe(struct platform_device *pdev)1513{1514 struct resource *regs;1515 int irq;1516 struct clk *clk;1517 int ret;1518 struct spi_controller *host;1519 struct atmel_spi *as;1520 1521 /* Select default pin state */1522 pinctrl_pm_select_default_state(&pdev->dev);1523 1524 irq = platform_get_irq(pdev, 0);1525 if (irq < 0)1526 return irq;1527 1528 clk = devm_clk_get(&pdev->dev, "spi_clk");1529 if (IS_ERR(clk))1530 return PTR_ERR(clk);1531 1532 /* setup spi core then atmel-specific driver state */1533 host = spi_alloc_host(&pdev->dev, sizeof(*as));1534 if (!host)1535 return -ENOMEM;1536 1537 /* the spi->mode bits understood by this driver: */1538 host->use_gpio_descriptors = true;1539 host->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;1540 host->bits_per_word_mask = SPI_BPW_RANGE_MASK(8, 16);1541 host->dev.of_node = pdev->dev.of_node;1542 host->bus_num = pdev->id;1543 host->num_chipselect = 4;1544 host->setup = atmel_spi_setup;1545 host->flags = (SPI_CONTROLLER_MUST_RX | SPI_CONTROLLER_MUST_TX |1546 SPI_CONTROLLER_GPIO_SS);1547 host->transfer_one = atmel_spi_one_transfer;1548 host->set_cs = atmel_spi_set_cs;1549 host->cleanup = atmel_spi_cleanup;1550 host->auto_runtime_pm = true;1551 host->max_dma_len = SPI_MAX_DMA_XFER;1552 host->can_dma = atmel_spi_can_dma;1553 platform_set_drvdata(pdev, host);1554 1555 as = spi_controller_get_devdata(host);1556 1557 spin_lock_init(&as->lock);1558 1559 as->pdev = pdev;1560 as->regs = devm_platform_get_and_ioremap_resource(pdev, 0, ®s);1561 if (IS_ERR(as->regs)) {1562 ret = PTR_ERR(as->regs);1563 goto out_unmap_regs;1564 }1565 as->phybase = regs->start;1566 as->irq = irq;1567 as->clk = clk;1568 1569 init_completion(&as->xfer_completion);1570 1571 atmel_get_caps(as);1572 1573 as->use_dma = false;1574 as->use_pdc = false;1575 if (as->caps.has_dma_support) {1576 ret = atmel_spi_configure_dma(host, as);1577 if (ret == 0) {1578 as->use_dma = true;1579 } else if (ret == -EPROBE_DEFER) {1580 goto out_unmap_regs;1581 }1582 } else if (as->caps.has_pdc_support) {1583 as->use_pdc = true;1584 }1585 1586 if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {1587 as->addr_rx_bbuf = dma_alloc_coherent(&pdev->dev,1588 SPI_MAX_DMA_XFER,1589 &as->dma_addr_rx_bbuf,1590 GFP_KERNEL | GFP_DMA);1591 if (!as->addr_rx_bbuf) {1592 as->use_dma = false;1593 } else {1594 as->addr_tx_bbuf = dma_alloc_coherent(&pdev->dev,1595 SPI_MAX_DMA_XFER,1596 &as->dma_addr_tx_bbuf,1597 GFP_KERNEL | GFP_DMA);1598 if (!as->addr_tx_bbuf) {1599 as->use_dma = false;1600 dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER,1601 as->addr_rx_bbuf,1602 as->dma_addr_rx_bbuf);1603 }1604 }1605 if (!as->use_dma)1606 dev_info(host->dev.parent,1607 " can not allocate dma coherent memory\n");1608 }1609 1610 if (as->caps.has_dma_support && !as->use_dma)1611 dev_info(&pdev->dev, "Atmel SPI Controller using PIO only\n");1612 1613 if (as->use_pdc) {1614 ret = devm_request_irq(&pdev->dev, irq, atmel_spi_pdc_interrupt,1615 0, dev_name(&pdev->dev), host);1616 } else {1617 ret = devm_request_irq(&pdev->dev, irq, atmel_spi_pio_interrupt,1618 0, dev_name(&pdev->dev), host);1619 }1620 if (ret)1621 goto out_unmap_regs;1622 1623 /* Initialize the hardware */1624 ret = clk_prepare_enable(clk);1625 if (ret)1626 goto out_free_irq;1627 1628 as->spi_clk = clk_get_rate(clk);1629 1630 as->fifo_size = 0;1631 if (!of_property_read_u32(pdev->dev.of_node, "atmel,fifo-size",1632 &as->fifo_size)) {1633 dev_info(&pdev->dev, "Using FIFO (%u data)\n", as->fifo_size);1634 }1635 1636 atmel_spi_init(as);1637 1638 pm_runtime_set_autosuspend_delay(&pdev->dev, AUTOSUSPEND_TIMEOUT);1639 pm_runtime_use_autosuspend(&pdev->dev);1640 pm_runtime_set_active(&pdev->dev);1641 pm_runtime_enable(&pdev->dev);1642 1643 ret = devm_spi_register_controller(&pdev->dev, host);1644 if (ret)1645 goto out_free_dma;1646 1647 /* go! */1648 dev_info(&pdev->dev, "Atmel SPI Controller version 0x%x at 0x%08lx (irq %d)\n",1649 atmel_get_version(as), (unsigned long)regs->start,1650 irq);1651 1652 return 0;1653 1654out_free_dma:1655 pm_runtime_disable(&pdev->dev);1656 pm_runtime_set_suspended(&pdev->dev);1657 1658 if (as->use_dma)1659 atmel_spi_release_dma(host);1660 1661 spi_writel(as, CR, SPI_BIT(SWRST));1662 spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */1663 clk_disable_unprepare(clk);1664out_free_irq:1665out_unmap_regs:1666 spi_controller_put(host);1667 return ret;1668}1669 1670static void atmel_spi_remove(struct platform_device *pdev)1671{1672 struct spi_controller *host = platform_get_drvdata(pdev);1673 struct atmel_spi *as = spi_controller_get_devdata(host);1674 1675 pm_runtime_get_sync(&pdev->dev);1676 1677 /* reset the hardware and block queue progress */1678 if (as->use_dma) {1679 atmel_spi_stop_dma(host);1680 atmel_spi_release_dma(host);1681 if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {1682 dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER,1683 as->addr_tx_bbuf,1684 as->dma_addr_tx_bbuf);1685 dma_free_coherent(&pdev->dev, SPI_MAX_DMA_XFER,1686 as->addr_rx_bbuf,1687 as->dma_addr_rx_bbuf);1688 }1689 }1690 1691 spin_lock_irq(&as->lock);1692 spi_writel(as, CR, SPI_BIT(SWRST));1693 spi_writel(as, CR, SPI_BIT(SWRST)); /* AT91SAM9263 Rev B workaround */1694 spi_readl(as, SR);1695 spin_unlock_irq(&as->lock);1696 1697 clk_disable_unprepare(as->clk);1698 1699 pm_runtime_put_noidle(&pdev->dev);1700 pm_runtime_disable(&pdev->dev);1701}1702 1703static int atmel_spi_runtime_suspend(struct device *dev)1704{1705 struct spi_controller *host = dev_get_drvdata(dev);1706 struct atmel_spi *as = spi_controller_get_devdata(host);1707 1708 clk_disable_unprepare(as->clk);1709 pinctrl_pm_select_sleep_state(dev);1710 1711 return 0;1712}1713 1714static int atmel_spi_runtime_resume(struct device *dev)1715{1716 struct spi_controller *host = dev_get_drvdata(dev);1717 struct atmel_spi *as = spi_controller_get_devdata(host);1718 1719 pinctrl_pm_select_default_state(dev);1720 1721 return clk_prepare_enable(as->clk);1722}1723 1724static int atmel_spi_suspend(struct device *dev)1725{1726 struct spi_controller *host = dev_get_drvdata(dev);1727 int ret;1728 1729 /* Stop the queue running */1730 ret = spi_controller_suspend(host);1731 if (ret)1732 return ret;1733 1734 if (!pm_runtime_suspended(dev))1735 atmel_spi_runtime_suspend(dev);1736 1737 return 0;1738}1739 1740static int atmel_spi_resume(struct device *dev)1741{1742 struct spi_controller *host = dev_get_drvdata(dev);1743 struct atmel_spi *as = spi_controller_get_devdata(host);1744 int ret;1745 1746 ret = clk_prepare_enable(as->clk);1747 if (ret)1748 return ret;1749 1750 atmel_spi_init(as);1751 1752 clk_disable_unprepare(as->clk);1753 1754 if (!pm_runtime_suspended(dev)) {1755 ret = atmel_spi_runtime_resume(dev);1756 if (ret)1757 return ret;1758 }1759 1760 /* Start the queue running */1761 return spi_controller_resume(host);1762}1763 1764static const struct dev_pm_ops atmel_spi_pm_ops = {1765 SYSTEM_SLEEP_PM_OPS(atmel_spi_suspend, atmel_spi_resume)1766 RUNTIME_PM_OPS(atmel_spi_runtime_suspend,1767 atmel_spi_runtime_resume, NULL)1768};1769 1770static const struct of_device_id atmel_spi_dt_ids[] = {1771 { .compatible = "atmel,at91rm9200-spi" },1772 { /* sentinel */ }1773};1774 1775MODULE_DEVICE_TABLE(of, atmel_spi_dt_ids);1776 1777static struct platform_driver atmel_spi_driver = {1778 .driver = {1779 .name = "atmel_spi",1780 .pm = pm_ptr(&atmel_spi_pm_ops),1781 .of_match_table = atmel_spi_dt_ids,1782 },1783 .probe = atmel_spi_probe,1784 .remove_new = atmel_spi_remove,1785};1786module_platform_driver(atmel_spi_driver);1787 1788MODULE_DESCRIPTION("Atmel AT32/AT91 SPI Controller driver");1789MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");1790MODULE_LICENSE("GPL");1791MODULE_ALIAS("platform:atmel_spi");1792