645 lines · c
1/* Synopsys DesignWare Core Enterprise Ethernet (XLGMAC) Driver2 *3 * Copyright (c) 2017 Synopsys, Inc. (www.synopsys.com)4 *5 * This program is dual-licensed; you may select either version 2 of6 * the GNU General Public License ("GPL") or BSD license ("BSD").7 *8 * This Synopsys DWC XLGMAC software driver and associated documentation9 * (hereinafter the "Software") is an unsupported proprietary work of10 * Synopsys, Inc. unless otherwise expressly agreed to in writing between11 * Synopsys and you. The Software IS NOT an item of Licensed Software or a12 * Licensed Product under any End User Software License Agreement or13 * Agreement for Licensed Products with Synopsys or any supplement thereto.14 * Synopsys is a registered trademark of Synopsys, Inc. Other names included15 * in the SOFTWARE may be the trademarks of their respective owners.16 */17 18#include "dwc-xlgmac.h"19#include "dwc-xlgmac-reg.h"20 21static void xlgmac_unmap_desc_data(struct xlgmac_pdata *pdata,22 struct xlgmac_desc_data *desc_data)23{24 if (desc_data->skb_dma) {25 if (desc_data->mapped_as_page) {26 dma_unmap_page(pdata->dev, desc_data->skb_dma,27 desc_data->skb_dma_len, DMA_TO_DEVICE);28 } else {29 dma_unmap_single(pdata->dev, desc_data->skb_dma,30 desc_data->skb_dma_len, DMA_TO_DEVICE);31 }32 desc_data->skb_dma = 0;33 desc_data->skb_dma_len = 0;34 }35 36 if (desc_data->skb) {37 dev_kfree_skb_any(desc_data->skb);38 desc_data->skb = NULL;39 }40 41 if (desc_data->rx.hdr.pa.pages)42 put_page(desc_data->rx.hdr.pa.pages);43 44 if (desc_data->rx.hdr.pa_unmap.pages) {45 dma_unmap_page(pdata->dev, desc_data->rx.hdr.pa_unmap.pages_dma,46 desc_data->rx.hdr.pa_unmap.pages_len,47 DMA_FROM_DEVICE);48 put_page(desc_data->rx.hdr.pa_unmap.pages);49 }50 51 if (desc_data->rx.buf.pa.pages)52 put_page(desc_data->rx.buf.pa.pages);53 54 if (desc_data->rx.buf.pa_unmap.pages) {55 dma_unmap_page(pdata->dev, desc_data->rx.buf.pa_unmap.pages_dma,56 desc_data->rx.buf.pa_unmap.pages_len,57 DMA_FROM_DEVICE);58 put_page(desc_data->rx.buf.pa_unmap.pages);59 }60 61 memset(&desc_data->tx, 0, sizeof(desc_data->tx));62 memset(&desc_data->rx, 0, sizeof(desc_data->rx));63 64 desc_data->mapped_as_page = 0;65 66 if (desc_data->state_saved) {67 desc_data->state_saved = 0;68 desc_data->state.skb = NULL;69 desc_data->state.len = 0;70 desc_data->state.error = 0;71 }72}73 74static void xlgmac_free_ring(struct xlgmac_pdata *pdata,75 struct xlgmac_ring *ring)76{77 struct xlgmac_desc_data *desc_data;78 unsigned int i;79 80 if (!ring)81 return;82 83 if (ring->desc_data_head) {84 for (i = 0; i < ring->dma_desc_count; i++) {85 desc_data = XLGMAC_GET_DESC_DATA(ring, i);86 xlgmac_unmap_desc_data(pdata, desc_data);87 }88 89 kfree(ring->desc_data_head);90 ring->desc_data_head = NULL;91 }92 93 if (ring->rx_hdr_pa.pages) {94 dma_unmap_page(pdata->dev, ring->rx_hdr_pa.pages_dma,95 ring->rx_hdr_pa.pages_len, DMA_FROM_DEVICE);96 put_page(ring->rx_hdr_pa.pages);97 98 ring->rx_hdr_pa.pages = NULL;99 ring->rx_hdr_pa.pages_len = 0;100 ring->rx_hdr_pa.pages_offset = 0;101 ring->rx_hdr_pa.pages_dma = 0;102 }103 104 if (ring->rx_buf_pa.pages) {105 dma_unmap_page(pdata->dev, ring->rx_buf_pa.pages_dma,106 ring->rx_buf_pa.pages_len, DMA_FROM_DEVICE);107 put_page(ring->rx_buf_pa.pages);108 109 ring->rx_buf_pa.pages = NULL;110 ring->rx_buf_pa.pages_len = 0;111 ring->rx_buf_pa.pages_offset = 0;112 ring->rx_buf_pa.pages_dma = 0;113 }114 115 if (ring->dma_desc_head) {116 dma_free_coherent(pdata->dev,117 (sizeof(struct xlgmac_dma_desc) *118 ring->dma_desc_count),119 ring->dma_desc_head,120 ring->dma_desc_head_addr);121 ring->dma_desc_head = NULL;122 }123}124 125static int xlgmac_init_ring(struct xlgmac_pdata *pdata,126 struct xlgmac_ring *ring,127 unsigned int dma_desc_count)128{129 if (!ring)130 return 0;131 132 /* Descriptors */133 ring->dma_desc_count = dma_desc_count;134 ring->dma_desc_head = dma_alloc_coherent(pdata->dev,135 (sizeof(struct xlgmac_dma_desc) *136 dma_desc_count),137 &ring->dma_desc_head_addr,138 GFP_KERNEL);139 if (!ring->dma_desc_head)140 return -ENOMEM;141 142 /* Array of descriptor data */143 ring->desc_data_head = kcalloc(dma_desc_count,144 sizeof(struct xlgmac_desc_data),145 GFP_KERNEL);146 if (!ring->desc_data_head)147 return -ENOMEM;148 149 netif_dbg(pdata, drv, pdata->netdev,150 "dma_desc_head=%p, dma_desc_head_addr=%pad, desc_data_head=%p\n",151 ring->dma_desc_head,152 &ring->dma_desc_head_addr,153 ring->desc_data_head);154 155 return 0;156}157 158static void xlgmac_free_rings(struct xlgmac_pdata *pdata)159{160 struct xlgmac_channel *channel;161 unsigned int i;162 163 if (!pdata->channel_head)164 return;165 166 channel = pdata->channel_head;167 for (i = 0; i < pdata->channel_count; i++, channel++) {168 xlgmac_free_ring(pdata, channel->tx_ring);169 xlgmac_free_ring(pdata, channel->rx_ring);170 }171}172 173static int xlgmac_alloc_rings(struct xlgmac_pdata *pdata)174{175 struct xlgmac_channel *channel;176 unsigned int i;177 int ret;178 179 channel = pdata->channel_head;180 for (i = 0; i < pdata->channel_count; i++, channel++) {181 netif_dbg(pdata, drv, pdata->netdev, "%s - Tx ring:\n",182 channel->name);183 184 ret = xlgmac_init_ring(pdata, channel->tx_ring,185 pdata->tx_desc_count);186 187 if (ret) {188 netdev_alert(pdata->netdev,189 "error initializing Tx ring");190 goto err_init_ring;191 }192 193 netif_dbg(pdata, drv, pdata->netdev, "%s - Rx ring:\n",194 channel->name);195 196 ret = xlgmac_init_ring(pdata, channel->rx_ring,197 pdata->rx_desc_count);198 if (ret) {199 netdev_alert(pdata->netdev,200 "error initializing Rx ring\n");201 goto err_init_ring;202 }203 }204 205 return 0;206 207err_init_ring:208 xlgmac_free_rings(pdata);209 210 return ret;211}212 213static void xlgmac_free_channels(struct xlgmac_pdata *pdata)214{215 if (!pdata->channel_head)216 return;217 218 kfree(pdata->channel_head->tx_ring);219 pdata->channel_head->tx_ring = NULL;220 221 kfree(pdata->channel_head->rx_ring);222 pdata->channel_head->rx_ring = NULL;223 224 kfree(pdata->channel_head);225 226 pdata->channel_head = NULL;227 pdata->channel_count = 0;228}229 230static int xlgmac_alloc_channels(struct xlgmac_pdata *pdata)231{232 struct xlgmac_channel *channel_head, *channel;233 struct xlgmac_ring *tx_ring, *rx_ring;234 int ret = -ENOMEM;235 unsigned int i;236 237 channel_head = kcalloc(pdata->channel_count,238 sizeof(struct xlgmac_channel), GFP_KERNEL);239 if (!channel_head)240 return ret;241 242 netif_dbg(pdata, drv, pdata->netdev,243 "channel_head=%p\n", channel_head);244 245 tx_ring = kcalloc(pdata->tx_ring_count, sizeof(struct xlgmac_ring),246 GFP_KERNEL);247 if (!tx_ring)248 goto err_tx_ring;249 250 rx_ring = kcalloc(pdata->rx_ring_count, sizeof(struct xlgmac_ring),251 GFP_KERNEL);252 if (!rx_ring)253 goto err_rx_ring;254 255 for (i = 0, channel = channel_head; i < pdata->channel_count;256 i++, channel++) {257 snprintf(channel->name, sizeof(channel->name), "channel-%u", i);258 channel->pdata = pdata;259 channel->queue_index = i;260 channel->dma_regs = pdata->mac_regs + DMA_CH_BASE +261 (DMA_CH_INC * i);262 263 if (pdata->per_channel_irq) {264 /* Get the per DMA interrupt */265 ret = pdata->channel_irq[i];266 if (ret < 0) {267 netdev_err(pdata->netdev,268 "get_irq %u failed\n",269 i + 1);270 goto err_irq;271 }272 channel->dma_irq = ret;273 }274 275 if (i < pdata->tx_ring_count)276 channel->tx_ring = tx_ring++;277 278 if (i < pdata->rx_ring_count)279 channel->rx_ring = rx_ring++;280 281 netif_dbg(pdata, drv, pdata->netdev,282 "%s: dma_regs=%p, tx_ring=%p, rx_ring=%p\n",283 channel->name, channel->dma_regs,284 channel->tx_ring, channel->rx_ring);285 }286 287 pdata->channel_head = channel_head;288 289 return 0;290 291err_irq:292 kfree(rx_ring);293 294err_rx_ring:295 kfree(tx_ring);296 297err_tx_ring:298 kfree(channel_head);299 300 return ret;301}302 303static void xlgmac_free_channels_and_rings(struct xlgmac_pdata *pdata)304{305 xlgmac_free_rings(pdata);306 307 xlgmac_free_channels(pdata);308}309 310static int xlgmac_alloc_channels_and_rings(struct xlgmac_pdata *pdata)311{312 int ret;313 314 ret = xlgmac_alloc_channels(pdata);315 if (ret)316 goto err_alloc;317 318 ret = xlgmac_alloc_rings(pdata);319 if (ret)320 goto err_alloc;321 322 return 0;323 324err_alloc:325 xlgmac_free_channels_and_rings(pdata);326 327 return ret;328}329 330static int xlgmac_alloc_pages(struct xlgmac_pdata *pdata,331 struct xlgmac_page_alloc *pa,332 gfp_t gfp, int order)333{334 struct page *pages = NULL;335 dma_addr_t pages_dma;336 337 /* Try to obtain pages, decreasing order if necessary */338 gfp |= __GFP_COMP | __GFP_NOWARN;339 while (order >= 0) {340 pages = alloc_pages(gfp, order);341 if (pages)342 break;343 344 order--;345 }346 if (!pages)347 return -ENOMEM;348 349 /* Map the pages */350 pages_dma = dma_map_page(pdata->dev, pages, 0,351 PAGE_SIZE << order, DMA_FROM_DEVICE);352 if (dma_mapping_error(pdata->dev, pages_dma)) {353 put_page(pages);354 return -ENOMEM;355 }356 357 pa->pages = pages;358 pa->pages_len = PAGE_SIZE << order;359 pa->pages_offset = 0;360 pa->pages_dma = pages_dma;361 362 return 0;363}364 365static void xlgmac_set_buffer_data(struct xlgmac_buffer_data *bd,366 struct xlgmac_page_alloc *pa,367 unsigned int len)368{369 get_page(pa->pages);370 bd->pa = *pa;371 372 bd->dma_base = pa->pages_dma;373 bd->dma_off = pa->pages_offset;374 bd->dma_len = len;375 376 pa->pages_offset += len;377 if ((pa->pages_offset + len) > pa->pages_len) {378 /* This data descriptor is responsible for unmapping page(s) */379 bd->pa_unmap = *pa;380 381 /* Get a new allocation next time */382 pa->pages = NULL;383 pa->pages_len = 0;384 pa->pages_offset = 0;385 pa->pages_dma = 0;386 }387}388 389static int xlgmac_map_rx_buffer(struct xlgmac_pdata *pdata,390 struct xlgmac_ring *ring,391 struct xlgmac_desc_data *desc_data)392{393 int order, ret;394 395 if (!ring->rx_hdr_pa.pages) {396 ret = xlgmac_alloc_pages(pdata, &ring->rx_hdr_pa,397 GFP_ATOMIC, 0);398 if (ret)399 return ret;400 }401 402 if (!ring->rx_buf_pa.pages) {403 order = max_t(int, PAGE_ALLOC_COSTLY_ORDER - 1, 0);404 ret = xlgmac_alloc_pages(pdata, &ring->rx_buf_pa,405 GFP_ATOMIC, order);406 if (ret)407 return ret;408 }409 410 /* Set up the header page info */411 xlgmac_set_buffer_data(&desc_data->rx.hdr, &ring->rx_hdr_pa,412 XLGMAC_SKB_ALLOC_SIZE);413 414 /* Set up the buffer page info */415 xlgmac_set_buffer_data(&desc_data->rx.buf, &ring->rx_buf_pa,416 pdata->rx_buf_size);417 418 return 0;419}420 421static void xlgmac_tx_desc_init(struct xlgmac_pdata *pdata)422{423 struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;424 struct xlgmac_desc_data *desc_data;425 struct xlgmac_dma_desc *dma_desc;426 struct xlgmac_channel *channel;427 struct xlgmac_ring *ring;428 dma_addr_t dma_desc_addr;429 unsigned int i, j;430 431 channel = pdata->channel_head;432 for (i = 0; i < pdata->channel_count; i++, channel++) {433 ring = channel->tx_ring;434 if (!ring)435 break;436 437 dma_desc = ring->dma_desc_head;438 dma_desc_addr = ring->dma_desc_head_addr;439 440 for (j = 0; j < ring->dma_desc_count; j++) {441 desc_data = XLGMAC_GET_DESC_DATA(ring, j);442 443 desc_data->dma_desc = dma_desc;444 desc_data->dma_desc_addr = dma_desc_addr;445 446 dma_desc++;447 dma_desc_addr += sizeof(struct xlgmac_dma_desc);448 }449 450 ring->cur = 0;451 ring->dirty = 0;452 memset(&ring->tx, 0, sizeof(ring->tx));453 454 hw_ops->tx_desc_init(channel);455 }456}457 458static void xlgmac_rx_desc_init(struct xlgmac_pdata *pdata)459{460 struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;461 struct xlgmac_desc_data *desc_data;462 struct xlgmac_dma_desc *dma_desc;463 struct xlgmac_channel *channel;464 struct xlgmac_ring *ring;465 dma_addr_t dma_desc_addr;466 unsigned int i, j;467 468 channel = pdata->channel_head;469 for (i = 0; i < pdata->channel_count; i++, channel++) {470 ring = channel->rx_ring;471 if (!ring)472 break;473 474 dma_desc = ring->dma_desc_head;475 dma_desc_addr = ring->dma_desc_head_addr;476 477 for (j = 0; j < ring->dma_desc_count; j++) {478 desc_data = XLGMAC_GET_DESC_DATA(ring, j);479 480 desc_data->dma_desc = dma_desc;481 desc_data->dma_desc_addr = dma_desc_addr;482 483 if (xlgmac_map_rx_buffer(pdata, ring, desc_data))484 break;485 486 dma_desc++;487 dma_desc_addr += sizeof(struct xlgmac_dma_desc);488 }489 490 ring->cur = 0;491 ring->dirty = 0;492 493 hw_ops->rx_desc_init(channel);494 }495}496 497static int xlgmac_map_tx_skb(struct xlgmac_channel *channel,498 struct sk_buff *skb)499{500 struct xlgmac_pdata *pdata = channel->pdata;501 struct xlgmac_ring *ring = channel->tx_ring;502 unsigned int start_index, cur_index;503 struct xlgmac_desc_data *desc_data;504 unsigned int offset, datalen, len;505 struct xlgmac_pkt_info *pkt_info;506 skb_frag_t *frag;507 unsigned int tso, vlan;508 dma_addr_t skb_dma;509 unsigned int i;510 511 offset = 0;512 start_index = ring->cur;513 cur_index = ring->cur;514 515 pkt_info = &ring->pkt_info;516 pkt_info->desc_count = 0;517 pkt_info->length = 0;518 519 tso = XLGMAC_GET_REG_BITS(pkt_info->attributes,520 TX_PACKET_ATTRIBUTES_TSO_ENABLE_POS,521 TX_PACKET_ATTRIBUTES_TSO_ENABLE_LEN);522 vlan = XLGMAC_GET_REG_BITS(pkt_info->attributes,523 TX_PACKET_ATTRIBUTES_VLAN_CTAG_POS,524 TX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN);525 526 /* Save space for a context descriptor if needed */527 if ((tso && (pkt_info->mss != ring->tx.cur_mss)) ||528 (vlan && (pkt_info->vlan_ctag != ring->tx.cur_vlan_ctag)))529 cur_index++;530 desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);531 532 if (tso) {533 /* Map the TSO header */534 skb_dma = dma_map_single(pdata->dev, skb->data,535 pkt_info->header_len, DMA_TO_DEVICE);536 if (dma_mapping_error(pdata->dev, skb_dma)) {537 netdev_alert(pdata->netdev, "dma_map_single failed\n");538 goto err_out;539 }540 desc_data->skb_dma = skb_dma;541 desc_data->skb_dma_len = pkt_info->header_len;542 netif_dbg(pdata, tx_queued, pdata->netdev,543 "skb header: index=%u, dma=%pad, len=%u\n",544 cur_index, &skb_dma, pkt_info->header_len);545 546 offset = pkt_info->header_len;547 548 pkt_info->length += pkt_info->header_len;549 550 cur_index++;551 desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);552 }553 554 /* Map the (remainder of the) packet */555 for (datalen = skb_headlen(skb) - offset; datalen; ) {556 len = min_t(unsigned int, datalen, XLGMAC_TX_MAX_BUF_SIZE);557 558 skb_dma = dma_map_single(pdata->dev, skb->data + offset, len,559 DMA_TO_DEVICE);560 if (dma_mapping_error(pdata->dev, skb_dma)) {561 netdev_alert(pdata->netdev, "dma_map_single failed\n");562 goto err_out;563 }564 desc_data->skb_dma = skb_dma;565 desc_data->skb_dma_len = len;566 netif_dbg(pdata, tx_queued, pdata->netdev,567 "skb data: index=%u, dma=%pad, len=%u\n",568 cur_index, &skb_dma, len);569 570 datalen -= len;571 offset += len;572 573 pkt_info->length += len;574 575 cur_index++;576 desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);577 }578 579 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {580 netif_dbg(pdata, tx_queued, pdata->netdev,581 "mapping frag %u\n", i);582 583 frag = &skb_shinfo(skb)->frags[i];584 offset = 0;585 586 for (datalen = skb_frag_size(frag); datalen; ) {587 len = min_t(unsigned int, datalen,588 XLGMAC_TX_MAX_BUF_SIZE);589 590 skb_dma = skb_frag_dma_map(pdata->dev, frag, offset,591 len, DMA_TO_DEVICE);592 if (dma_mapping_error(pdata->dev, skb_dma)) {593 netdev_alert(pdata->netdev,594 "skb_frag_dma_map failed\n");595 goto err_out;596 }597 desc_data->skb_dma = skb_dma;598 desc_data->skb_dma_len = len;599 desc_data->mapped_as_page = 1;600 netif_dbg(pdata, tx_queued, pdata->netdev,601 "skb frag: index=%u, dma=%pad, len=%u\n",602 cur_index, &skb_dma, len);603 604 datalen -= len;605 offset += len;606 607 pkt_info->length += len;608 609 cur_index++;610 desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index);611 }612 }613 614 /* Save the skb address in the last entry. We always have some data615 * that has been mapped so desc_data is always advanced past the last616 * piece of mapped data - use the entry pointed to by cur_index - 1.617 */618 desc_data = XLGMAC_GET_DESC_DATA(ring, cur_index - 1);619 desc_data->skb = skb;620 621 /* Save the number of descriptor entries used */622 pkt_info->desc_count = cur_index - start_index;623 624 return pkt_info->desc_count;625 626err_out:627 while (start_index < cur_index) {628 desc_data = XLGMAC_GET_DESC_DATA(ring, start_index++);629 xlgmac_unmap_desc_data(pdata, desc_data);630 }631 632 return 0;633}634 635void xlgmac_init_desc_ops(struct xlgmac_desc_ops *desc_ops)636{637 desc_ops->alloc_channels_and_rings = xlgmac_alloc_channels_and_rings;638 desc_ops->free_channels_and_rings = xlgmac_free_channels_and_rings;639 desc_ops->map_tx_skb = xlgmac_map_tx_skb;640 desc_ops->map_rx_buffer = xlgmac_map_rx_buffer;641 desc_ops->unmap_desc_data = xlgmac_unmap_desc_data;642 desc_ops->tx_desc_init = xlgmac_tx_desc_init;643 desc_ops->rx_desc_init = xlgmac_rx_desc_init;644}645