528 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Altera TSE SGDMA and MSGDMA Linux driver3 * Copyright (C) 2014 Altera Corporation. All rights reserved4 */5 6#include <linux/list.h>7#include "altera_utils.h"8#include "altera_tse.h"9#include "altera_sgdmahw.h"10#include "altera_sgdma.h"11 12static void sgdma_setup_descrip(struct sgdma_descrip __iomem *desc,13 struct sgdma_descrip __iomem *ndesc,14 dma_addr_t ndesc_phys,15 dma_addr_t raddr,16 dma_addr_t waddr,17 u16 length,18 int generate_eop,19 int rfixed,20 int wfixed);21 22static int sgdma_async_write(struct altera_tse_private *priv,23 struct sgdma_descrip __iomem *desc);24 25static int sgdma_async_read(struct altera_tse_private *priv);26 27static dma_addr_t28sgdma_txphysaddr(struct altera_tse_private *priv,29 struct sgdma_descrip __iomem *desc);30 31static dma_addr_t32sgdma_rxphysaddr(struct altera_tse_private *priv,33 struct sgdma_descrip __iomem *desc);34 35static int sgdma_txbusy(struct altera_tse_private *priv);36 37static int sgdma_rxbusy(struct altera_tse_private *priv);38 39static void40queue_tx(struct altera_tse_private *priv, struct tse_buffer *buffer);41 42static void43queue_rx(struct altera_tse_private *priv, struct tse_buffer *buffer);44 45static struct tse_buffer *46dequeue_tx(struct altera_tse_private *priv);47 48static struct tse_buffer *49dequeue_rx(struct altera_tse_private *priv);50 51static struct tse_buffer *52queue_rx_peekhead(struct altera_tse_private *priv);53 54int sgdma_initialize(struct altera_tse_private *priv)55{56 priv->txctrlreg = SGDMA_CTRLREG_ILASTD |57 SGDMA_CTRLREG_INTEN;58 59 priv->rxctrlreg = SGDMA_CTRLREG_IDESCRIP |60 SGDMA_CTRLREG_INTEN |61 SGDMA_CTRLREG_ILASTD;62 63 INIT_LIST_HEAD(&priv->txlisthd);64 INIT_LIST_HEAD(&priv->rxlisthd);65 66 priv->rxdescphys = (dma_addr_t) 0;67 priv->txdescphys = (dma_addr_t) 0;68 69 priv->rxdescphys = dma_map_single(priv->device,70 (void __force *)priv->rx_dma_desc,71 priv->rxdescmem, DMA_BIDIRECTIONAL);72 73 if (dma_mapping_error(priv->device, priv->rxdescphys)) {74 sgdma_uninitialize(priv);75 netdev_err(priv->dev, "error mapping rx descriptor memory\n");76 return -EINVAL;77 }78 79 priv->txdescphys = dma_map_single(priv->device,80 (void __force *)priv->tx_dma_desc,81 priv->txdescmem, DMA_TO_DEVICE);82 83 if (dma_mapping_error(priv->device, priv->txdescphys)) {84 sgdma_uninitialize(priv);85 netdev_err(priv->dev, "error mapping tx descriptor memory\n");86 return -EINVAL;87 }88 89 /* Initialize descriptor memory to all 0's, sync memory to cache */90 memset_io(priv->tx_dma_desc, 0, priv->txdescmem);91 memset_io(priv->rx_dma_desc, 0, priv->rxdescmem);92 93 dma_sync_single_for_device(priv->device, priv->txdescphys,94 priv->txdescmem, DMA_TO_DEVICE);95 96 dma_sync_single_for_device(priv->device, priv->rxdescphys,97 priv->rxdescmem, DMA_TO_DEVICE);98 99 return 0;100}101 102void sgdma_uninitialize(struct altera_tse_private *priv)103{104 if (priv->rxdescphys)105 dma_unmap_single(priv->device, priv->rxdescphys,106 priv->rxdescmem, DMA_BIDIRECTIONAL);107 108 if (priv->txdescphys)109 dma_unmap_single(priv->device, priv->txdescphys,110 priv->txdescmem, DMA_TO_DEVICE);111}112 113/* This function resets the SGDMA controller and clears the114 * descriptor memory used for transmits and receives.115 */116void sgdma_reset(struct altera_tse_private *priv)117{118 /* Initialize descriptor memory to 0 */119 memset_io(priv->tx_dma_desc, 0, priv->txdescmem);120 memset_io(priv->rx_dma_desc, 0, priv->rxdescmem);121 122 csrwr32(SGDMA_CTRLREG_RESET, priv->tx_dma_csr, sgdma_csroffs(control));123 csrwr32(0, priv->tx_dma_csr, sgdma_csroffs(control));124 125 csrwr32(SGDMA_CTRLREG_RESET, priv->rx_dma_csr, sgdma_csroffs(control));126 csrwr32(0, priv->rx_dma_csr, sgdma_csroffs(control));127}128 129/* For SGDMA, interrupts remain enabled after initially enabling,130 * so no need to provide implementations for abstract enable131 * and disable132 */133 134void sgdma_enable_rxirq(struct altera_tse_private *priv)135{136}137 138void sgdma_enable_txirq(struct altera_tse_private *priv)139{140}141 142void sgdma_disable_rxirq(struct altera_tse_private *priv)143{144}145 146void sgdma_disable_txirq(struct altera_tse_private *priv)147{148}149 150void sgdma_clear_rxirq(struct altera_tse_private *priv)151{152 tse_set_bit(priv->rx_dma_csr, sgdma_csroffs(control),153 SGDMA_CTRLREG_CLRINT);154}155 156void sgdma_clear_txirq(struct altera_tse_private *priv)157{158 tse_set_bit(priv->tx_dma_csr, sgdma_csroffs(control),159 SGDMA_CTRLREG_CLRINT);160}161 162/* transmits buffer through SGDMA. Returns number of buffers163 * transmitted, 0 if not possible.164 *165 * tx_lock is held by the caller166 */167int sgdma_tx_buffer(struct altera_tse_private *priv, struct tse_buffer *buffer)168{169 struct sgdma_descrip __iomem *descbase =170 (struct sgdma_descrip __iomem *)priv->tx_dma_desc;171 172 struct sgdma_descrip __iomem *cdesc = &descbase[0];173 struct sgdma_descrip __iomem *ndesc = &descbase[1];174 175 /* wait 'til the tx sgdma is ready for the next transmit request */176 if (sgdma_txbusy(priv))177 return 0;178 179 sgdma_setup_descrip(cdesc, /* current descriptor */180 ndesc, /* next descriptor */181 sgdma_txphysaddr(priv, ndesc),182 buffer->dma_addr, /* address of packet to xmit */183 0, /* write addr 0 for tx dma */184 buffer->len, /* length of packet */185 SGDMA_CONTROL_EOP, /* Generate EOP */186 0, /* read fixed */187 SGDMA_CONTROL_WR_FIXED); /* Generate SOP */188 189 sgdma_async_write(priv, cdesc);190 191 /* enqueue the request to the pending transmit queue */192 queue_tx(priv, buffer);193 194 return 1;195}196 197 198/* tx_lock held to protect access to queued tx list199 */200u32 sgdma_tx_completions(struct altera_tse_private *priv)201{202 u32 ready = 0;203 204 if (!sgdma_txbusy(priv) &&205 ((csrrd8(priv->tx_dma_desc, sgdma_descroffs(control))206 & SGDMA_CONTROL_HW_OWNED) == 0) &&207 (dequeue_tx(priv))) {208 ready = 1;209 }210 211 return ready;212}213 214void sgdma_start_rxdma(struct altera_tse_private *priv)215{216 sgdma_async_read(priv);217}218 219void sgdma_add_rx_desc(struct altera_tse_private *priv,220 struct tse_buffer *rxbuffer)221{222 queue_rx(priv, rxbuffer);223}224 225/* status is returned on upper 16 bits,226 * length is returned in lower 16 bits227 */228u32 sgdma_rx_status(struct altera_tse_private *priv)229{230 struct sgdma_descrip __iomem *base =231 (struct sgdma_descrip __iomem *)priv->rx_dma_desc;232 struct sgdma_descrip __iomem *desc = NULL;233 struct tse_buffer *rxbuffer = NULL;234 unsigned int rxstatus = 0;235 236 u32 sts = csrrd32(priv->rx_dma_csr, sgdma_csroffs(status));237 238 desc = &base[0];239 if (sts & SGDMA_STSREG_EOP) {240 unsigned int pktlength = 0;241 unsigned int pktstatus = 0;242 dma_sync_single_for_cpu(priv->device,243 priv->rxdescphys,244 SGDMA_DESC_LEN,245 DMA_FROM_DEVICE);246 247 pktlength = csrrd16(desc, sgdma_descroffs(bytes_xferred));248 pktstatus = csrrd8(desc, sgdma_descroffs(status));249 rxstatus = pktstatus & ~SGDMA_STATUS_EOP;250 rxstatus = rxstatus << 16;251 rxstatus |= (pktlength & 0xffff);252 253 if (rxstatus) {254 csrwr8(0, desc, sgdma_descroffs(status));255 256 rxbuffer = dequeue_rx(priv);257 if (rxbuffer == NULL)258 netdev_info(priv->dev,259 "sgdma rx and rx queue empty!\n");260 261 /* Clear control */262 csrwr32(0, priv->rx_dma_csr, sgdma_csroffs(control));263 /* clear status */264 csrwr32(0xf, priv->rx_dma_csr, sgdma_csroffs(status));265 266 /* kick the rx sgdma after reaping this descriptor */267 sgdma_async_read(priv);268 269 } else {270 /* If the SGDMA indicated an end of packet on recv,271 * then it's expected that the rxstatus from the272 * descriptor is non-zero - meaning a valid packet273 * with a nonzero length, or an error has been274 * indicated. if not, then all we can do is signal275 * an error and return no packet received. Most likely276 * there is a system design error, or an error in the277 * underlying kernel (cache or cache management problem)278 */279 netdev_err(priv->dev,280 "SGDMA RX Error Info: %x, %x, %x\n",281 sts, csrrd8(desc, sgdma_descroffs(status)),282 rxstatus);283 }284 } else if (sts == 0) {285 sgdma_async_read(priv);286 }287 288 return rxstatus;289}290 291 292/* Private functions */293static void sgdma_setup_descrip(struct sgdma_descrip __iomem *desc,294 struct sgdma_descrip __iomem *ndesc,295 dma_addr_t ndesc_phys,296 dma_addr_t raddr,297 dma_addr_t waddr,298 u16 length,299 int generate_eop,300 int rfixed,301 int wfixed)302{303 /* Clear the next descriptor as not owned by hardware */304 305 u32 ctrl = csrrd8(ndesc, sgdma_descroffs(control));306 ctrl &= ~SGDMA_CONTROL_HW_OWNED;307 csrwr8(ctrl, ndesc, sgdma_descroffs(control));308 309 ctrl = SGDMA_CONTROL_HW_OWNED;310 ctrl |= generate_eop;311 ctrl |= rfixed;312 ctrl |= wfixed;313 314 /* Channel is implicitly zero, initialized to 0 by default */315 csrwr32(lower_32_bits(raddr), desc, sgdma_descroffs(raddr));316 csrwr32(lower_32_bits(waddr), desc, sgdma_descroffs(waddr));317 318 csrwr32(0, desc, sgdma_descroffs(pad1));319 csrwr32(0, desc, sgdma_descroffs(pad2));320 csrwr32(lower_32_bits(ndesc_phys), desc, sgdma_descroffs(next));321 322 csrwr8(ctrl, desc, sgdma_descroffs(control));323 csrwr8(0, desc, sgdma_descroffs(status));324 csrwr8(0, desc, sgdma_descroffs(wburst));325 csrwr8(0, desc, sgdma_descroffs(rburst));326 csrwr16(length, desc, sgdma_descroffs(bytes));327 csrwr16(0, desc, sgdma_descroffs(bytes_xferred));328}329 330/* If hardware is busy, don't restart async read.331 * if status register is 0 - meaning initial state, restart async read,332 * probably for the first time when populating a receive buffer.333 * If read status indicate not busy and a status, restart the async334 * DMA read.335 */336static int sgdma_async_read(struct altera_tse_private *priv)337{338 struct sgdma_descrip __iomem *descbase =339 (struct sgdma_descrip __iomem *)priv->rx_dma_desc;340 341 struct sgdma_descrip __iomem *cdesc = &descbase[0];342 struct sgdma_descrip __iomem *ndesc = &descbase[1];343 struct tse_buffer *rxbuffer = NULL;344 345 if (!sgdma_rxbusy(priv)) {346 rxbuffer = queue_rx_peekhead(priv);347 if (rxbuffer == NULL) {348 netdev_err(priv->dev, "no rx buffers available\n");349 return 0;350 }351 352 sgdma_setup_descrip(cdesc, /* current descriptor */353 ndesc, /* next descriptor */354 sgdma_rxphysaddr(priv, ndesc),355 0, /* read addr 0 for rx dma */356 rxbuffer->dma_addr, /* write addr for rx dma */357 0, /* read 'til EOP */358 0, /* EOP: NA for rx dma */359 0, /* read fixed: NA for rx dma */360 0); /* SOP: NA for rx DMA */361 362 dma_sync_single_for_device(priv->device,363 priv->rxdescphys,364 SGDMA_DESC_LEN,365 DMA_TO_DEVICE);366 367 csrwr32(lower_32_bits(sgdma_rxphysaddr(priv, cdesc)),368 priv->rx_dma_csr,369 sgdma_csroffs(next_descrip));370 371 csrwr32((priv->rxctrlreg | SGDMA_CTRLREG_START),372 priv->rx_dma_csr,373 sgdma_csroffs(control));374 375 return 1;376 }377 378 return 0;379}380 381static int sgdma_async_write(struct altera_tse_private *priv,382 struct sgdma_descrip __iomem *desc)383{384 if (sgdma_txbusy(priv))385 return 0;386 387 /* clear control and status */388 csrwr32(0, priv->tx_dma_csr, sgdma_csroffs(control));389 csrwr32(0x1f, priv->tx_dma_csr, sgdma_csroffs(status));390 391 dma_sync_single_for_device(priv->device, priv->txdescphys,392 SGDMA_DESC_LEN, DMA_TO_DEVICE);393 394 csrwr32(lower_32_bits(sgdma_txphysaddr(priv, desc)),395 priv->tx_dma_csr,396 sgdma_csroffs(next_descrip));397 398 csrwr32((priv->txctrlreg | SGDMA_CTRLREG_START),399 priv->tx_dma_csr,400 sgdma_csroffs(control));401 402 return 1;403}404 405static dma_addr_t406sgdma_txphysaddr(struct altera_tse_private *priv,407 struct sgdma_descrip __iomem *desc)408{409 dma_addr_t paddr = priv->txdescmem_busaddr;410 uintptr_t offs = (uintptr_t)desc - (uintptr_t)priv->tx_dma_desc;411 return (dma_addr_t)((uintptr_t)paddr + offs);412}413 414static dma_addr_t415sgdma_rxphysaddr(struct altera_tse_private *priv,416 struct sgdma_descrip __iomem *desc)417{418 dma_addr_t paddr = priv->rxdescmem_busaddr;419 uintptr_t offs = (uintptr_t)desc - (uintptr_t)priv->rx_dma_desc;420 return (dma_addr_t)((uintptr_t)paddr + offs);421}422 423#define list_remove_head(list, entry, type, member) \424 do { \425 entry = NULL; \426 if (!list_empty(list)) { \427 entry = list_entry((list)->next, type, member); \428 list_del_init(&entry->member); \429 } \430 } while (0)431 432#define list_peek_head(list, entry, type, member) \433 do { \434 entry = NULL; \435 if (!list_empty(list)) { \436 entry = list_entry((list)->next, type, member); \437 } \438 } while (0)439 440/* adds a tse_buffer to the tail of a tx buffer list.441 * assumes the caller is managing and holding a mutual exclusion442 * primitive to avoid simultaneous pushes/pops to the list.443 */444static void445queue_tx(struct altera_tse_private *priv, struct tse_buffer *buffer)446{447 list_add_tail(&buffer->lh, &priv->txlisthd);448}449 450 451/* adds a tse_buffer to the tail of a rx buffer list452 * assumes the caller is managing and holding a mutual exclusion453 * primitive to avoid simultaneous pushes/pops to the list.454 */455static void456queue_rx(struct altera_tse_private *priv, struct tse_buffer *buffer)457{458 list_add_tail(&buffer->lh, &priv->rxlisthd);459}460 461/* dequeues a tse_buffer from the transmit buffer list, otherwise462 * returns NULL if empty.463 * assumes the caller is managing and holding a mutual exclusion464 * primitive to avoid simultaneous pushes/pops to the list.465 */466static struct tse_buffer *467dequeue_tx(struct altera_tse_private *priv)468{469 struct tse_buffer *buffer = NULL;470 list_remove_head(&priv->txlisthd, buffer, struct tse_buffer, lh);471 return buffer;472}473 474/* dequeues a tse_buffer from the receive buffer list, otherwise475 * returns NULL if empty476 * assumes the caller is managing and holding a mutual exclusion477 * primitive to avoid simultaneous pushes/pops to the list.478 */479static struct tse_buffer *480dequeue_rx(struct altera_tse_private *priv)481{482 struct tse_buffer *buffer = NULL;483 list_remove_head(&priv->rxlisthd, buffer, struct tse_buffer, lh);484 return buffer;485}486 487/* dequeues a tse_buffer from the receive buffer list, otherwise488 * returns NULL if empty489 * assumes the caller is managing and holding a mutual exclusion490 * primitive to avoid simultaneous pushes/pops to the list while the491 * head is being examined.492 */493static struct tse_buffer *494queue_rx_peekhead(struct altera_tse_private *priv)495{496 struct tse_buffer *buffer = NULL;497 list_peek_head(&priv->rxlisthd, buffer, struct tse_buffer, lh);498 return buffer;499}500 501/* check and return rx sgdma status without polling502 */503static int sgdma_rxbusy(struct altera_tse_private *priv)504{505 return csrrd32(priv->rx_dma_csr, sgdma_csroffs(status))506 & SGDMA_STSREG_BUSY;507}508 509/* waits for the tx sgdma to finish it's current operation, returns 0510 * when it transitions to nonbusy, returns 1 if the operation times out511 */512static int sgdma_txbusy(struct altera_tse_private *priv)513{514 int delay = 0;515 516 /* if DMA is busy, wait for current transaction to finish */517 while ((csrrd32(priv->tx_dma_csr, sgdma_csroffs(status))518 & SGDMA_STSREG_BUSY) && (delay++ < 100))519 udelay(1);520 521 if (csrrd32(priv->tx_dma_csr, sgdma_csroffs(status))522 & SGDMA_STSREG_BUSY) {523 netdev_err(priv->dev, "timeout waiting for tx dma\n");524 return 1;525 }526 return 0;527}528