1566 lines · c
1/*2 * Copyright (c) 2010 Broadcom Corporation3 *4 * Permission to use, copy, modify, and/or distribute this software for any5 * purpose with or without fee is hereby granted, provided that the above6 * copyright notice and this permission notice appear in all copies.7 *8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY11 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION13 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN14 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.15 */16 17#include <linux/slab.h>18#include <linux/delay.h>19#include <linux/pci.h>20#include <net/cfg80211.h>21#include <net/mac80211.h>22 23#include <brcmu_utils.h>24#include <aiutils.h>25#include "types.h"26#include "main.h"27#include "dma.h"28#include "soc.h"29#include "scb.h"30#include "ampdu.h"31#include "debug.h"32#include "brcms_trace_events.h"33 34/*35 * dma register field offset calculation36 */37#define DMA64REGOFFS(field) offsetof(struct dma64regs, field)38#define DMA64TXREGOFFS(di, field) (di->d64txregbase + DMA64REGOFFS(field))39#define DMA64RXREGOFFS(di, field) (di->d64rxregbase + DMA64REGOFFS(field))40 41/*42 * DMA hardware requires each descriptor ring to be 8kB aligned, and fit within43 * a contiguous 8kB physical address.44 */45#define D64RINGALIGN_BITS 1346#define D64MAXRINGSZ (1 << D64RINGALIGN_BITS)47#define D64RINGALIGN (1 << D64RINGALIGN_BITS)48 49#define D64MAXDD (D64MAXRINGSZ / sizeof(struct dma64desc))50 51/* transmit channel control */52#define D64_XC_XE 0x00000001 /* transmit enable */53#define D64_XC_SE 0x00000002 /* transmit suspend request */54#define D64_XC_LE 0x00000004 /* loopback enable */55#define D64_XC_FL 0x00000010 /* flush request */56#define D64_XC_PD 0x00000800 /* parity check disable */57#define D64_XC_AE 0x00030000 /* address extension bits */58#define D64_XC_AE_SHIFT 1659 60/* transmit descriptor table pointer */61#define D64_XP_LD_MASK 0x00000fff /* last valid descriptor */62 63/* transmit channel status */64#define D64_XS0_CD_MASK 0x00001fff /* current descriptor pointer */65#define D64_XS0_XS_MASK 0xf0000000 /* transmit state */66#define D64_XS0_XS_SHIFT 2867#define D64_XS0_XS_DISABLED 0x00000000 /* disabled */68#define D64_XS0_XS_ACTIVE 0x10000000 /* active */69#define D64_XS0_XS_IDLE 0x20000000 /* idle wait */70#define D64_XS0_XS_STOPPED 0x30000000 /* stopped */71#define D64_XS0_XS_SUSP 0x40000000 /* suspend pending */72 73#define D64_XS1_AD_MASK 0x00001fff /* active descriptor */74#define D64_XS1_XE_MASK 0xf0000000 /* transmit errors */75#define D64_XS1_XE_SHIFT 2876#define D64_XS1_XE_NOERR 0x00000000 /* no error */77#define D64_XS1_XE_DPE 0x10000000 /* descriptor protocol error */78#define D64_XS1_XE_DFU 0x20000000 /* data fifo underrun */79#define D64_XS1_XE_DTE 0x30000000 /* data transfer error */80#define D64_XS1_XE_DESRE 0x40000000 /* descriptor read error */81#define D64_XS1_XE_COREE 0x50000000 /* core error */82 83/* receive channel control */84/* receive enable */85#define D64_RC_RE 0x0000000186/* receive frame offset */87#define D64_RC_RO_MASK 0x000000fe88#define D64_RC_RO_SHIFT 189/* direct fifo receive (pio) mode */90#define D64_RC_FM 0x0000010091/* separate rx header descriptor enable */92#define D64_RC_SH 0x0000020093/* overflow continue */94#define D64_RC_OC 0x0000040095/* parity check disable */96#define D64_RC_PD 0x0000080097/* address extension bits */98#define D64_RC_AE 0x0003000099#define D64_RC_AE_SHIFT 16100 101/* flags for dma controller */102/* partity enable */103#define DMA_CTRL_PEN (1 << 0)104/* rx overflow continue */105#define DMA_CTRL_ROC (1 << 1)106/* allow rx scatter to multiple descriptors */107#define DMA_CTRL_RXMULTI (1 << 2)108/* Unframed Rx/Tx data */109#define DMA_CTRL_UNFRAMED (1 << 3)110 111/* receive descriptor table pointer */112#define D64_RP_LD_MASK 0x00000fff /* last valid descriptor */113 114/* receive channel status */115#define D64_RS0_CD_MASK 0x00001fff /* current descriptor pointer */116#define D64_RS0_RS_MASK 0xf0000000 /* receive state */117#define D64_RS0_RS_SHIFT 28118#define D64_RS0_RS_DISABLED 0x00000000 /* disabled */119#define D64_RS0_RS_ACTIVE 0x10000000 /* active */120#define D64_RS0_RS_IDLE 0x20000000 /* idle wait */121#define D64_RS0_RS_STOPPED 0x30000000 /* stopped */122#define D64_RS0_RS_SUSP 0x40000000 /* suspend pending */123 124#define D64_RS1_AD_MASK 0x0001ffff /* active descriptor */125#define D64_RS1_RE_MASK 0xf0000000 /* receive errors */126#define D64_RS1_RE_SHIFT 28127#define D64_RS1_RE_NOERR 0x00000000 /* no error */128#define D64_RS1_RE_DPO 0x10000000 /* descriptor protocol error */129#define D64_RS1_RE_DFU 0x20000000 /* data fifo overflow */130#define D64_RS1_RE_DTE 0x30000000 /* data transfer error */131#define D64_RS1_RE_DESRE 0x40000000 /* descriptor read error */132#define D64_RS1_RE_COREE 0x50000000 /* core error */133 134/* fifoaddr */135#define D64_FA_OFF_MASK 0xffff /* offset */136#define D64_FA_SEL_MASK 0xf0000 /* select */137#define D64_FA_SEL_SHIFT 16138#define D64_FA_SEL_XDD 0x00000 /* transmit dma data */139#define D64_FA_SEL_XDP 0x10000 /* transmit dma pointers */140#define D64_FA_SEL_RDD 0x40000 /* receive dma data */141#define D64_FA_SEL_RDP 0x50000 /* receive dma pointers */142#define D64_FA_SEL_XFD 0x80000 /* transmit fifo data */143#define D64_FA_SEL_XFP 0x90000 /* transmit fifo pointers */144#define D64_FA_SEL_RFD 0xc0000 /* receive fifo data */145#define D64_FA_SEL_RFP 0xd0000 /* receive fifo pointers */146#define D64_FA_SEL_RSD 0xe0000 /* receive frame status data */147#define D64_FA_SEL_RSP 0xf0000 /* receive frame status pointers */148 149/* descriptor control flags 1 */150#define D64_CTRL_COREFLAGS 0x0ff00000 /* core specific flags */151#define D64_CTRL1_EOT ((u32)1 << 28) /* end of descriptor table */152#define D64_CTRL1_IOC ((u32)1 << 29) /* interrupt on completion */153#define D64_CTRL1_EOF ((u32)1 << 30) /* end of frame */154#define D64_CTRL1_SOF ((u32)1 << 31) /* start of frame */155 156/* descriptor control flags 2 */157/* buffer byte count. real data len must <= 16KB */158#define D64_CTRL2_BC_MASK 0x00007fff159/* address extension bits */160#define D64_CTRL2_AE 0x00030000161#define D64_CTRL2_AE_SHIFT 16162/* parity bit */163#define D64_CTRL2_PARITY 0x00040000164 165/* control flags in the range [27:20] are core-specific and not defined here */166#define D64_CTRL_CORE_MASK 0x0ff00000167 168#define D64_RX_FRM_STS_LEN 0x0000ffff /* frame length mask */169#define D64_RX_FRM_STS_OVFL 0x00800000 /* RxOverFlow */170#define D64_RX_FRM_STS_DSCRCNT 0x0f000000 /* no. of descriptors used - 1 */171#define D64_RX_FRM_STS_DATATYPE 0xf0000000 /* core-dependent data type */172 173/*174 * packet headroom necessary to accommodate the largest header175 * in the system, (i.e TXOFF). By doing, we avoid the need to176 * allocate an extra buffer for the header when bridging to WL.177 * There is a compile time check in wlc.c which ensure that this178 * value is at least as big as TXOFF. This value is used in179 * dma_rxfill().180 */181 182#define BCMEXTRAHDROOM 172183 184#define MAXNAMEL 8 /* 8 char names */185 186/* macros to convert between byte offsets and indexes */187#define B2I(bytes, type) ((bytes) / sizeof(type))188#define I2B(index, type) ((index) * sizeof(type))189 190#define PCI32ADDR_HIGH 0xc0000000 /* address[31:30] */191#define PCI32ADDR_HIGH_SHIFT 30 /* address[31:30] */192 193#define PCI64ADDR_HIGH 0x80000000 /* address[63] */194#define PCI64ADDR_HIGH_SHIFT 31 /* address[63] */195 196/*197 * DMA Descriptor198 * Descriptors are only read by the hardware, never written back.199 */200struct dma64desc {201 __le32 ctrl1; /* misc control bits & bufcount */202 __le32 ctrl2; /* buffer count and address extension */203 __le32 addrlow; /* memory address of the date buffer, bits 31:0 */204 __le32 addrhigh; /* memory address of the date buffer, bits 63:32 */205};206 207/* dma engine software state */208struct dma_info {209 struct dma_pub dma; /* exported structure */210 char name[MAXNAMEL]; /* callers name for diag msgs */211 212 struct bcma_device *core;213 struct device *dmadev;214 215 /* session information for AMPDU */216 struct brcms_ampdu_session ampdu_session;217 218 bool dma64; /* this dma engine is operating in 64-bit mode */219 bool addrext; /* this dma engine supports DmaExtendedAddrChanges */220 221 /* 64-bit dma tx engine registers */222 uint d64txregbase;223 /* 64-bit dma rx engine registers */224 uint d64rxregbase;225 /* pointer to dma64 tx descriptor ring */226 struct dma64desc *txd64;227 /* pointer to dma64 rx descriptor ring */228 struct dma64desc *rxd64;229 230 u16 dmadesc_align; /* alignment requirement for dma descriptors */231 232 u16 ntxd; /* # tx descriptors tunable */233 u16 txin; /* index of next descriptor to reclaim */234 u16 txout; /* index of next descriptor to post */235 /* pointer to parallel array of pointers to packets */236 struct sk_buff **txp;237 /* Aligned physical address of descriptor ring */238 dma_addr_t txdpa;239 /* Original physical address of descriptor ring */240 dma_addr_t txdpaorig;241 u16 txdalign; /* #bytes added to alloc'd mem to align txd */242 u32 txdalloc; /* #bytes allocated for the ring */243 u32 xmtptrbase; /* When using unaligned descriptors, the ptr register244 * is not just an index, it needs all 13 bits to be245 * an offset from the addr register.246 */247 248 u16 nrxd; /* # rx descriptors tunable */249 u16 rxin; /* index of next descriptor to reclaim */250 u16 rxout; /* index of next descriptor to post */251 /* pointer to parallel array of pointers to packets */252 struct sk_buff **rxp;253 /* Aligned physical address of descriptor ring */254 dma_addr_t rxdpa;255 /* Original physical address of descriptor ring */256 dma_addr_t rxdpaorig;257 u16 rxdalign; /* #bytes added to alloc'd mem to align rxd */258 u32 rxdalloc; /* #bytes allocated for the ring */259 u32 rcvptrbase; /* Base for ptr reg when using unaligned descriptors */260 261 /* tunables */262 unsigned int rxbufsize; /* rx buffer size in bytes, not including263 * the extra headroom264 */265 uint rxextrahdrroom; /* extra rx headroom, reverseved to assist upper266 * stack, e.g. some rx pkt buffers will be267 * bridged to tx side without byte copying.268 * The extra headroom needs to be large enough269 * to fit txheader needs. Some dongle driver may270 * not need it.271 */272 uint nrxpost; /* # rx buffers to keep posted */273 unsigned int rxoffset; /* rxcontrol offset */274 /* add to get dma address of descriptor ring, low 32 bits */275 uint ddoffsetlow;276 /* high 32 bits */277 uint ddoffsethigh;278 /* add to get dma address of data buffer, low 32 bits */279 uint dataoffsetlow;280 /* high 32 bits */281 uint dataoffsethigh;282 /* descriptor base need to be aligned or not */283 bool aligndesc_4k;284};285 286/* Check for odd number of 1's */287static u32 parity32(__le32 data)288{289 /* no swap needed for counting 1's */290 u32 par_data = *(u32 *)&data;291 292 par_data ^= par_data >> 16;293 par_data ^= par_data >> 8;294 par_data ^= par_data >> 4;295 par_data ^= par_data >> 2;296 par_data ^= par_data >> 1;297 298 return par_data & 1;299}300 301static bool dma64_dd_parity(struct dma64desc *dd)302{303 return parity32(dd->addrlow ^ dd->addrhigh ^ dd->ctrl1 ^ dd->ctrl2);304}305 306/* descriptor bumping functions */307 308static uint xxd(uint x, uint n)309{310 return x & (n - 1); /* faster than %, but n must be power of 2 */311}312 313static uint txd(struct dma_info *di, uint x)314{315 return xxd(x, di->ntxd);316}317 318static uint rxd(struct dma_info *di, uint x)319{320 return xxd(x, di->nrxd);321}322 323static uint nexttxd(struct dma_info *di, uint i)324{325 return txd(di, i + 1);326}327 328static uint prevtxd(struct dma_info *di, uint i)329{330 return txd(di, i - 1);331}332 333static uint nextrxd(struct dma_info *di, uint i)334{335 return rxd(di, i + 1);336}337 338static uint ntxdactive(struct dma_info *di, uint h, uint t)339{340 return txd(di, t-h);341}342 343static uint nrxdactive(struct dma_info *di, uint h, uint t)344{345 return rxd(di, t-h);346}347 348static uint _dma_ctrlflags(struct dma_info *di, uint mask, uint flags)349{350 uint dmactrlflags;351 352 if (di == NULL)353 return 0;354 355 dmactrlflags = di->dma.dmactrlflags;356 dmactrlflags &= ~mask;357 dmactrlflags |= flags;358 359 /* If trying to enable parity, check if parity is actually supported */360 if (dmactrlflags & DMA_CTRL_PEN) {361 u32 control;362 363 control = bcma_read32(di->core, DMA64TXREGOFFS(di, control));364 bcma_write32(di->core, DMA64TXREGOFFS(di, control),365 control | D64_XC_PD);366 if (bcma_read32(di->core, DMA64TXREGOFFS(di, control)) &367 D64_XC_PD)368 /* We *can* disable it so it is supported,369 * restore control register370 */371 bcma_write32(di->core, DMA64TXREGOFFS(di, control),372 control);373 else374 /* Not supported, don't allow it to be enabled */375 dmactrlflags &= ~DMA_CTRL_PEN;376 }377 378 di->dma.dmactrlflags = dmactrlflags;379 380 return dmactrlflags;381}382 383static bool _dma64_addrext(struct dma_info *di, uint ctrl_offset)384{385 u32 w;386 bcma_set32(di->core, ctrl_offset, D64_XC_AE);387 w = bcma_read32(di->core, ctrl_offset);388 bcma_mask32(di->core, ctrl_offset, ~D64_XC_AE);389 return (w & D64_XC_AE) == D64_XC_AE;390}391 392/*393 * return true if this dma engine supports DmaExtendedAddrChanges,394 * otherwise false395 */396static bool _dma_isaddrext(struct dma_info *di)397{398 /* DMA64 supports full 32- or 64-bit operation. AE is always valid */399 400 /* not all tx or rx channel are available */401 if (di->d64txregbase != 0) {402 if (!_dma64_addrext(di, DMA64TXREGOFFS(di, control)))403 brcms_dbg_dma(di->core,404 "%s: DMA64 tx doesn't have AE set\n",405 di->name);406 return true;407 } else if (di->d64rxregbase != 0) {408 if (!_dma64_addrext(di, DMA64RXREGOFFS(di, control)))409 brcms_dbg_dma(di->core,410 "%s: DMA64 rx doesn't have AE set\n",411 di->name);412 return true;413 }414 415 return false;416}417 418static bool _dma_descriptor_align(struct dma_info *di)419{420 u32 addrl;421 422 /* Check to see if the descriptors need to be aligned on 4K/8K or not */423 if (di->d64txregbase != 0) {424 bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow), 0xff0);425 addrl = bcma_read32(di->core, DMA64TXREGOFFS(di, addrlow));426 if (addrl != 0)427 return false;428 } else if (di->d64rxregbase != 0) {429 bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow), 0xff0);430 addrl = bcma_read32(di->core, DMA64RXREGOFFS(di, addrlow));431 if (addrl != 0)432 return false;433 }434 return true;435}436 437/*438 * Descriptor table must start at the DMA hardware dictated alignment, so439 * allocated memory must be large enough to support this requirement.440 */441static void *dma_alloc_consistent(struct dma_info *di, uint size,442 u16 align_bits, uint *alloced,443 dma_addr_t *pap)444{445 if (align_bits) {446 u16 align = (1 << align_bits);447 if (!IS_ALIGNED(PAGE_SIZE, align))448 size += align;449 *alloced = size;450 }451 return dma_alloc_coherent(di->dmadev, size, pap, GFP_ATOMIC);452}453 454static455u8 dma_align_sizetobits(uint size)456{457 u8 bitpos = 0;458 while (size >>= 1)459 bitpos++;460 return bitpos;461}462 463/* This function ensures that the DMA descriptor ring will not get allocated464 * across Page boundary. If the allocation is done across the page boundary465 * at the first time, then it is freed and the allocation is done at466 * descriptor ring size aligned location. This will ensure that the ring will467 * not cross page boundary468 */469static void *dma_ringalloc(struct dma_info *di, u32 boundary, uint size,470 u16 *alignbits, uint *alloced,471 dma_addr_t *descpa)472{473 void *va;474 u32 desc_strtaddr;475 u32 alignbytes = 1 << *alignbits;476 477 va = dma_alloc_consistent(di, size, *alignbits, alloced, descpa);478 479 if (NULL == va)480 return NULL;481 482 desc_strtaddr = (u32) roundup((unsigned long)va, alignbytes);483 if (((desc_strtaddr + size - 1) & boundary) != (desc_strtaddr484 & boundary)) {485 *alignbits = dma_align_sizetobits(size);486 dma_free_coherent(di->dmadev, size, va, *descpa);487 va = dma_alloc_consistent(di, size, *alignbits,488 alloced, descpa);489 }490 return va;491}492 493static bool dma64_alloc(struct dma_info *di, uint direction)494{495 u16 size;496 uint ddlen;497 void *va;498 uint alloced = 0;499 u16 align;500 u16 align_bits;501 502 ddlen = sizeof(struct dma64desc);503 504 size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);505 align_bits = di->dmadesc_align;506 align = (1 << align_bits);507 508 if (direction == DMA_TX) {509 va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,510 &alloced, &di->txdpaorig);511 if (va == NULL) {512 brcms_dbg_dma(di->core,513 "%s: DMA_ALLOC_CONSISTENT(ntxd) failed\n",514 di->name);515 return false;516 }517 align = (1 << align_bits);518 di->txd64 = (struct dma64desc *)519 roundup((unsigned long)va, align);520 di->txdalign = (uint) ((s8 *)di->txd64 - (s8 *) va);521 di->txdpa = di->txdpaorig + di->txdalign;522 di->txdalloc = alloced;523 } else {524 va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,525 &alloced, &di->rxdpaorig);526 if (va == NULL) {527 brcms_dbg_dma(di->core,528 "%s: DMA_ALLOC_CONSISTENT(nrxd) failed\n",529 di->name);530 return false;531 }532 align = (1 << align_bits);533 di->rxd64 = (struct dma64desc *)534 roundup((unsigned long)va, align);535 di->rxdalign = (uint) ((s8 *)di->rxd64 - (s8 *) va);536 di->rxdpa = di->rxdpaorig + di->rxdalign;537 di->rxdalloc = alloced;538 }539 540 return true;541}542 543static bool _dma_alloc(struct dma_info *di, uint direction)544{545 return dma64_alloc(di, direction);546}547 548struct dma_pub *dma_attach(char *name, struct brcms_c_info *wlc,549 uint txregbase, uint rxregbase, uint ntxd, uint nrxd,550 uint rxbufsize, int rxextheadroom,551 uint nrxpost, uint rxoffset)552{553 struct si_pub *sih = wlc->hw->sih;554 struct bcma_device *core = wlc->hw->d11core;555 struct dma_info *di;556 u8 rev = core->id.rev;557 uint size;558 struct si_info *sii = container_of(sih, struct si_info, pub);559 560 /* allocate private info structure */561 di = kzalloc(sizeof(*di), GFP_ATOMIC);562 if (di == NULL)563 return NULL;564 565 di->dma64 =566 ((bcma_aread32(core, BCMA_IOST) & SISF_DMA64) == SISF_DMA64);567 568 /* init dma reg info */569 di->core = core;570 di->d64txregbase = txregbase;571 di->d64rxregbase = rxregbase;572 573 /*574 * Default flags (which can be changed by the driver calling575 * dma_ctrlflags before enable): For backwards compatibility576 * both Rx Overflow Continue and Parity are DISABLED.577 */578 _dma_ctrlflags(di, DMA_CTRL_ROC | DMA_CTRL_PEN, 0);579 580 brcms_dbg_dma(di->core, "%s: %s flags 0x%x ntxd %d nrxd %d "581 "rxbufsize %d rxextheadroom %d nrxpost %d rxoffset %d "582 "txregbase %u rxregbase %u\n", name, "DMA64",583 di->dma.dmactrlflags, ntxd, nrxd, rxbufsize,584 rxextheadroom, nrxpost, rxoffset, txregbase, rxregbase);585 586 /* make a private copy of our callers name */587 strscpy(di->name, name, sizeof(di->name));588 589 di->dmadev = core->dma_dev;590 591 /* save tunables */592 di->ntxd = (u16) ntxd;593 di->nrxd = (u16) nrxd;594 595 /* the actual dma size doesn't include the extra headroom */596 di->rxextrahdrroom =597 (rxextheadroom == -1) ? BCMEXTRAHDROOM : rxextheadroom;598 if (rxbufsize > BCMEXTRAHDROOM)599 di->rxbufsize = (u16) (rxbufsize - di->rxextrahdrroom);600 else601 di->rxbufsize = (u16) rxbufsize;602 603 di->nrxpost = (u16) nrxpost;604 di->rxoffset = (u8) rxoffset;605 606 /*607 * figure out the DMA physical address offset for dd and data608 * PCI/PCIE: they map silicon backplace address to zero609 * based memory, need offset610 * Other bus: use zero SI_BUS BIGENDIAN kludge: use sdram611 * swapped region for data buffer, not descriptor612 */613 di->ddoffsetlow = 0;614 di->dataoffsetlow = 0;615 /* for pci bus, add offset */616 if (sii->icbus->hosttype == BCMA_HOSTTYPE_PCI) {617 /* add offset for pcie with DMA64 bus */618 di->ddoffsetlow = 0;619 di->ddoffsethigh = SI_PCIE_DMA_H32;620 }621 di->dataoffsetlow = di->ddoffsetlow;622 di->dataoffsethigh = di->ddoffsethigh;623 624 /* WAR64450 : DMACtl.Addr ext fields are not supported in SDIOD core. */625 if ((core->id.id == BCMA_CORE_SDIO_DEV)626 && ((rev > 0) && (rev <= 2)))627 di->addrext = false;628 else if ((core->id.id == BCMA_CORE_I2S) &&629 ((rev == 0) || (rev == 1)))630 di->addrext = false;631 else632 di->addrext = _dma_isaddrext(di);633 634 /* does the descriptor need to be aligned and if yes, on 4K/8K or not */635 di->aligndesc_4k = _dma_descriptor_align(di);636 if (di->aligndesc_4k) {637 di->dmadesc_align = D64RINGALIGN_BITS;638 if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2))639 /* for smaller dd table, HW relax alignment reqmnt */640 di->dmadesc_align = D64RINGALIGN_BITS - 1;641 } else {642 di->dmadesc_align = 4; /* 16 byte alignment */643 }644 645 brcms_dbg_dma(di->core, "DMA descriptor align_needed %d, align %d\n",646 di->aligndesc_4k, di->dmadesc_align);647 648 /* allocate tx packet pointer vector */649 if (ntxd) {650 size = ntxd * sizeof(void *);651 di->txp = kzalloc(size, GFP_ATOMIC);652 if (di->txp == NULL)653 goto fail;654 }655 656 /* allocate rx packet pointer vector */657 if (nrxd) {658 size = nrxd * sizeof(void *);659 di->rxp = kzalloc(size, GFP_ATOMIC);660 if (di->rxp == NULL)661 goto fail;662 }663 664 /*665 * allocate transmit descriptor ring, only need ntxd descriptors666 * but it must be aligned667 */668 if (ntxd) {669 if (!_dma_alloc(di, DMA_TX))670 goto fail;671 }672 673 /*674 * allocate receive descriptor ring, only need nrxd descriptors675 * but it must be aligned676 */677 if (nrxd) {678 if (!_dma_alloc(di, DMA_RX))679 goto fail;680 }681 682 if ((di->ddoffsetlow != 0) && !di->addrext) {683 if (di->txdpa > SI_PCI_DMA_SZ) {684 brcms_dbg_dma(di->core,685 "%s: txdpa 0x%x: addrext not supported\n",686 di->name, (u32)di->txdpa);687 goto fail;688 }689 if (di->rxdpa > SI_PCI_DMA_SZ) {690 brcms_dbg_dma(di->core,691 "%s: rxdpa 0x%x: addrext not supported\n",692 di->name, (u32)di->rxdpa);693 goto fail;694 }695 }696 697 /* Initialize AMPDU session */698 brcms_c_ampdu_reset_session(&di->ampdu_session, wlc);699 700 brcms_dbg_dma(di->core,701 "ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh 0x%x addrext %d\n",702 di->ddoffsetlow, di->ddoffsethigh,703 di->dataoffsetlow, di->dataoffsethigh,704 di->addrext);705 706 return (struct dma_pub *) di;707 708 fail:709 dma_detach((struct dma_pub *)di);710 return NULL;711}712 713static inline void714dma64_dd_upd(struct dma_info *di, struct dma64desc *ddring,715 dma_addr_t pa, uint outidx, u32 *flags, u32 bufcount)716{717 u32 ctrl2 = bufcount & D64_CTRL2_BC_MASK;718 719 /* PCI bus with big(>1G) physical address, use address extension */720 if ((di->dataoffsetlow == 0) || !(pa & PCI32ADDR_HIGH)) {721 ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow);722 ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh);723 ddring[outidx].ctrl1 = cpu_to_le32(*flags);724 ddring[outidx].ctrl2 = cpu_to_le32(ctrl2);725 } else {726 /* address extension for 32-bit PCI */727 u32 ae;728 729 ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;730 pa &= ~PCI32ADDR_HIGH;731 732 ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE;733 ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow);734 ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh);735 ddring[outidx].ctrl1 = cpu_to_le32(*flags);736 ddring[outidx].ctrl2 = cpu_to_le32(ctrl2);737 }738 if (di->dma.dmactrlflags & DMA_CTRL_PEN) {739 if (dma64_dd_parity(&ddring[outidx]))740 ddring[outidx].ctrl2 =741 cpu_to_le32(ctrl2 | D64_CTRL2_PARITY);742 }743}744 745/* !! may be called with core in reset */746void dma_detach(struct dma_pub *pub)747{748 struct dma_info *di = container_of(pub, struct dma_info, dma);749 750 brcms_dbg_dma(di->core, "%s:\n", di->name);751 752 /* free dma descriptor rings */753 if (di->txd64)754 dma_free_coherent(di->dmadev, di->txdalloc,755 ((s8 *)di->txd64 - di->txdalign),756 (di->txdpaorig));757 if (di->rxd64)758 dma_free_coherent(di->dmadev, di->rxdalloc,759 ((s8 *)di->rxd64 - di->rxdalign),760 (di->rxdpaorig));761 762 /* free packet pointer vectors */763 kfree(di->txp);764 kfree(di->rxp);765 766 /* free our private info structure */767 kfree(di);768 769}770 771/* initialize descriptor table base address */772static void773_dma_ddtable_init(struct dma_info *di, uint direction, dma_addr_t pa)774{775 if (!di->aligndesc_4k) {776 if (direction == DMA_TX)777 di->xmtptrbase = pa;778 else779 di->rcvptrbase = pa;780 }781 782 if ((di->ddoffsetlow == 0)783 || !(pa & PCI32ADDR_HIGH)) {784 if (direction == DMA_TX) {785 bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow),786 pa + di->ddoffsetlow);787 bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh),788 di->ddoffsethigh);789 } else {790 bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow),791 pa + di->ddoffsetlow);792 bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh),793 di->ddoffsethigh);794 }795 } else {796 /* DMA64 32bits address extension */797 u32 ae;798 799 /* shift the high bit(s) from pa to ae */800 ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;801 pa &= ~PCI32ADDR_HIGH;802 803 if (direction == DMA_TX) {804 bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow),805 pa + di->ddoffsetlow);806 bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh),807 di->ddoffsethigh);808 bcma_maskset32(di->core, DMA64TXREGOFFS(di, control),809 D64_XC_AE, (ae << D64_XC_AE_SHIFT));810 } else {811 bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow),812 pa + di->ddoffsetlow);813 bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh),814 di->ddoffsethigh);815 bcma_maskset32(di->core, DMA64RXREGOFFS(di, control),816 D64_RC_AE, (ae << D64_RC_AE_SHIFT));817 }818 }819}820 821static void _dma_rxenable(struct dma_info *di)822{823 uint dmactrlflags = di->dma.dmactrlflags;824 u32 control;825 826 brcms_dbg_dma(di->core, "%s:\n", di->name);827 828 control = D64_RC_RE | (bcma_read32(di->core,829 DMA64RXREGOFFS(di, control)) &830 D64_RC_AE);831 832 if ((dmactrlflags & DMA_CTRL_PEN) == 0)833 control |= D64_RC_PD;834 835 if (dmactrlflags & DMA_CTRL_ROC)836 control |= D64_RC_OC;837 838 bcma_write32(di->core, DMA64RXREGOFFS(di, control),839 ((di->rxoffset << D64_RC_RO_SHIFT) | control));840}841 842void dma_rxinit(struct dma_pub *pub)843{844 struct dma_info *di = container_of(pub, struct dma_info, dma);845 846 brcms_dbg_dma(di->core, "%s:\n", di->name);847 848 if (di->nrxd == 0)849 return;850 851 di->rxin = di->rxout = 0;852 853 /* clear rx descriptor ring */854 memset(di->rxd64, '\0', di->nrxd * sizeof(struct dma64desc));855 856 /* DMA engine with out alignment requirement requires table to be inited857 * before enabling the engine858 */859 if (!di->aligndesc_4k)860 _dma_ddtable_init(di, DMA_RX, di->rxdpa);861 862 _dma_rxenable(di);863 864 if (di->aligndesc_4k)865 _dma_ddtable_init(di, DMA_RX, di->rxdpa);866}867 868static struct sk_buff *dma64_getnextrxp(struct dma_info *di, bool forceall)869{870 uint i, curr;871 struct sk_buff *rxp;872 dma_addr_t pa;873 874 i = di->rxin;875 876 /* return if no packets posted */877 if (i == di->rxout)878 return NULL;879 880 curr =881 B2I(((bcma_read32(di->core,882 DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) -883 di->rcvptrbase) & D64_RS0_CD_MASK, struct dma64desc);884 885 /* ignore curr if forceall */886 if (!forceall && (i == curr))887 return NULL;888 889 /* get the packet pointer that corresponds to the rx descriptor */890 rxp = di->rxp[i];891 di->rxp[i] = NULL;892 893 pa = le32_to_cpu(di->rxd64[i].addrlow) - di->dataoffsetlow;894 895 /* clear this packet from the descriptor ring */896 dma_unmap_single(di->dmadev, pa, di->rxbufsize, DMA_FROM_DEVICE);897 898 di->rxd64[i].addrlow = cpu_to_le32(0xdeadbeef);899 di->rxd64[i].addrhigh = cpu_to_le32(0xdeadbeef);900 901 di->rxin = nextrxd(di, i);902 903 return rxp;904}905 906static struct sk_buff *_dma_getnextrxp(struct dma_info *di, bool forceall)907{908 if (di->nrxd == 0)909 return NULL;910 911 return dma64_getnextrxp(di, forceall);912}913 914/*915 * !! rx entry routine916 * returns the number packages in the next frame, or 0 if there are no more917 * if DMA_CTRL_RXMULTI is defined, DMA scattering(multiple buffers) is918 * supported with pkts chain919 * otherwise, it's treated as giant pkt and will be tossed.920 * The DMA scattering starts with normal DMA header, followed by first921 * buffer data. After it reaches the max size of buffer, the data continues922 * in next DMA descriptor buffer WITHOUT DMA header923 */924int dma_rx(struct dma_pub *pub, struct sk_buff_head *skb_list)925{926 struct dma_info *di = container_of(pub, struct dma_info, dma);927 struct sk_buff_head dma_frames;928 struct sk_buff *p, *next;929 uint len;930 uint pkt_len;931 int resid = 0;932 int pktcnt = 1;933 934 skb_queue_head_init(&dma_frames);935 next_frame:936 p = _dma_getnextrxp(di, false);937 if (p == NULL)938 return 0;939 940 len = le16_to_cpu(*(__le16 *) (p->data));941 brcms_dbg_dma(di->core, "%s: dma_rx len %d\n", di->name, len);942 dma_spin_for_len(len, p);943 944 /* set actual length */945 pkt_len = min((di->rxoffset + len), di->rxbufsize);946 __skb_trim(p, pkt_len);947 skb_queue_tail(&dma_frames, p);948 resid = len - (di->rxbufsize - di->rxoffset);949 950 /* check for single or multi-buffer rx */951 if (resid > 0) {952 while ((resid > 0) && (p = _dma_getnextrxp(di, false))) {953 pkt_len = min_t(uint, resid, di->rxbufsize);954 __skb_trim(p, pkt_len);955 skb_queue_tail(&dma_frames, p);956 resid -= di->rxbufsize;957 pktcnt++;958 }959 960#ifdef DEBUG961 if (resid > 0) {962 uint cur;963 cur =964 B2I(((bcma_read32(di->core,965 DMA64RXREGOFFS(di, status0)) &966 D64_RS0_CD_MASK) - di->rcvptrbase) &967 D64_RS0_CD_MASK, struct dma64desc);968 brcms_dbg_dma(di->core,969 "rxin %d rxout %d, hw_curr %d\n",970 di->rxin, di->rxout, cur);971 }972#endif /* DEBUG */973 974 if ((di->dma.dmactrlflags & DMA_CTRL_RXMULTI) == 0) {975 brcms_dbg_dma(di->core, "%s: bad frame length (%d)\n",976 di->name, len);977 skb_queue_walk_safe(&dma_frames, p, next) {978 skb_unlink(p, &dma_frames);979 brcmu_pkt_buf_free_skb(p);980 }981 di->dma.rxgiants++;982 pktcnt = 1;983 goto next_frame;984 }985 }986 987 skb_queue_splice_tail(&dma_frames, skb_list);988 return pktcnt;989}990 991static bool dma64_rxidle(struct dma_info *di)992{993 brcms_dbg_dma(di->core, "%s:\n", di->name);994 995 if (di->nrxd == 0)996 return true;997 998 return ((bcma_read32(di->core,999 DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) ==1000 (bcma_read32(di->core, DMA64RXREGOFFS(di, ptr)) &1001 D64_RS0_CD_MASK));1002}1003 1004static bool dma64_txidle(struct dma_info *di)1005{1006 if (di->ntxd == 0)1007 return true;1008 1009 return ((bcma_read32(di->core,1010 DMA64TXREGOFFS(di, status0)) & D64_XS0_CD_MASK) ==1011 (bcma_read32(di->core, DMA64TXREGOFFS(di, ptr)) &1012 D64_XS0_CD_MASK));1013}1014 1015/*1016 * post receive buffers1017 * Return false if refill failed completely or dma mapping failed. The ring1018 * is empty, which will stall the rx dma and user might want to call rxfill1019 * again asap. This is unlikely to happen on a memory-rich NIC, but often on1020 * memory-constrained dongle.1021 */1022bool dma_rxfill(struct dma_pub *pub)1023{1024 struct dma_info *di = container_of(pub, struct dma_info, dma);1025 struct sk_buff *p;1026 u16 rxin, rxout;1027 u32 flags = 0;1028 uint n;1029 uint i;1030 dma_addr_t pa;1031 uint extra_offset = 0;1032 bool ring_empty;1033 1034 ring_empty = false;1035 1036 /*1037 * Determine how many receive buffers we're lacking1038 * from the full complement, allocate, initialize,1039 * and post them, then update the chip rx lastdscr.1040 */1041 1042 rxin = di->rxin;1043 rxout = di->rxout;1044 1045 n = di->nrxpost - nrxdactive(di, rxin, rxout);1046 1047 brcms_dbg_dma(di->core, "%s: post %d\n", di->name, n);1048 1049 if (di->rxbufsize > BCMEXTRAHDROOM)1050 extra_offset = di->rxextrahdrroom;1051 1052 for (i = 0; i < n; i++) {1053 /*1054 * the di->rxbufsize doesn't include the extra headroom,1055 * we need to add it to the size to be allocated1056 */1057 p = brcmu_pkt_buf_get_skb(di->rxbufsize + extra_offset);1058 1059 if (p == NULL) {1060 brcms_dbg_dma(di->core, "%s: out of rxbufs\n",1061 di->name);1062 if (i == 0 && dma64_rxidle(di)) {1063 brcms_dbg_dma(di->core, "%s: ring is empty !\n",1064 di->name);1065 ring_empty = true;1066 }1067 di->dma.rxnobuf++;1068 break;1069 }1070 /* reserve an extra headroom, if applicable */1071 if (extra_offset)1072 skb_pull(p, extra_offset);1073 1074 /* Do a cached write instead of uncached write since DMA_MAP1075 * will flush the cache.1076 */1077 *(u32 *) (p->data) = 0;1078 1079 pa = dma_map_single(di->dmadev, p->data, di->rxbufsize,1080 DMA_FROM_DEVICE);1081 if (dma_mapping_error(di->dmadev, pa)) {1082 brcmu_pkt_buf_free_skb(p);1083 return false;1084 }1085 1086 /* save the free packet pointer */1087 di->rxp[rxout] = p;1088 1089 /* reset flags for each descriptor */1090 flags = 0;1091 if (rxout == (di->nrxd - 1))1092 flags = D64_CTRL1_EOT;1093 1094 dma64_dd_upd(di, di->rxd64, pa, rxout, &flags,1095 di->rxbufsize);1096 rxout = nextrxd(di, rxout);1097 }1098 1099 di->rxout = rxout;1100 1101 /* update the chip lastdscr pointer */1102 bcma_write32(di->core, DMA64RXREGOFFS(di, ptr),1103 di->rcvptrbase + I2B(rxout, struct dma64desc));1104 1105 return ring_empty;1106}1107 1108void dma_rxreclaim(struct dma_pub *pub)1109{1110 struct dma_info *di = container_of(pub, struct dma_info, dma);1111 struct sk_buff *p;1112 1113 brcms_dbg_dma(di->core, "%s:\n", di->name);1114 1115 while ((p = _dma_getnextrxp(di, true)))1116 brcmu_pkt_buf_free_skb(p);1117}1118 1119void dma_counterreset(struct dma_pub *pub)1120{1121 /* reset all software counters */1122 pub->rxgiants = 0;1123 pub->rxnobuf = 0;1124 pub->txnobuf = 0;1125}1126 1127/* get the address of the var in order to change later */1128unsigned long dma_getvar(struct dma_pub *pub, const char *name)1129{1130 struct dma_info *di = container_of(pub, struct dma_info, dma);1131 1132 if (!strcmp(name, "&txavail"))1133 return (unsigned long)&(di->dma.txavail);1134 return 0;1135}1136 1137/* 64-bit DMA functions */1138 1139void dma_txinit(struct dma_pub *pub)1140{1141 struct dma_info *di = container_of(pub, struct dma_info, dma);1142 u32 control = D64_XC_XE;1143 1144 brcms_dbg_dma(di->core, "%s:\n", di->name);1145 1146 if (di->ntxd == 0)1147 return;1148 1149 di->txin = di->txout = 0;1150 di->dma.txavail = di->ntxd - 1;1151 1152 /* clear tx descriptor ring */1153 memset(di->txd64, '\0', (di->ntxd * sizeof(struct dma64desc)));1154 1155 /* DMA engine with out alignment requirement requires table to be inited1156 * before enabling the engine1157 */1158 if (!di->aligndesc_4k)1159 _dma_ddtable_init(di, DMA_TX, di->txdpa);1160 1161 if ((di->dma.dmactrlflags & DMA_CTRL_PEN) == 0)1162 control |= D64_XC_PD;1163 bcma_set32(di->core, DMA64TXREGOFFS(di, control), control);1164 1165 /* DMA engine with alignment requirement requires table to be inited1166 * before enabling the engine1167 */1168 if (di->aligndesc_4k)1169 _dma_ddtable_init(di, DMA_TX, di->txdpa);1170}1171 1172void dma_txsuspend(struct dma_pub *pub)1173{1174 struct dma_info *di = container_of(pub, struct dma_info, dma);1175 1176 brcms_dbg_dma(di->core, "%s:\n", di->name);1177 1178 if (di->ntxd == 0)1179 return;1180 1181 bcma_set32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE);1182}1183 1184void dma_txresume(struct dma_pub *pub)1185{1186 struct dma_info *di = container_of(pub, struct dma_info, dma);1187 1188 brcms_dbg_dma(di->core, "%s:\n", di->name);1189 1190 if (di->ntxd == 0)1191 return;1192 1193 bcma_mask32(di->core, DMA64TXREGOFFS(di, control), ~D64_XC_SE);1194}1195 1196bool dma_txsuspended(struct dma_pub *pub)1197{1198 struct dma_info *di = container_of(pub, struct dma_info, dma);1199 1200 return (di->ntxd == 0) ||1201 ((bcma_read32(di->core,1202 DMA64TXREGOFFS(di, control)) & D64_XC_SE) ==1203 D64_XC_SE);1204}1205 1206void dma_txreclaim(struct dma_pub *pub, enum txd_range range)1207{1208 struct dma_info *di = container_of(pub, struct dma_info, dma);1209 struct sk_buff *p;1210 1211 brcms_dbg_dma(di->core, "%s: %s\n",1212 di->name,1213 range == DMA_RANGE_ALL ? "all" :1214 range == DMA_RANGE_TRANSMITTED ? "transmitted" :1215 "transferred");1216 1217 if (di->txin == di->txout)1218 return;1219 1220 while ((p = dma_getnexttxp(pub, range))) {1221 /* For unframed data, we don't have any packets to free */1222 if (!(di->dma.dmactrlflags & DMA_CTRL_UNFRAMED))1223 brcmu_pkt_buf_free_skb(p);1224 }1225}1226 1227bool dma_txreset(struct dma_pub *pub)1228{1229 struct dma_info *di = container_of(pub, struct dma_info, dma);1230 u32 status;1231 1232 if (di->ntxd == 0)1233 return true;1234 1235 /* suspend tx DMA first */1236 bcma_write32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE);1237 SPINWAIT(((status =1238 (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) &1239 D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED) &&1240 (status != D64_XS0_XS_IDLE) && (status != D64_XS0_XS_STOPPED),1241 10000);1242 1243 bcma_write32(di->core, DMA64TXREGOFFS(di, control), 0);1244 SPINWAIT(((status =1245 (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) &1246 D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED), 10000);1247 1248 /* wait for the last transaction to complete */1249 udelay(300);1250 1251 return status == D64_XS0_XS_DISABLED;1252}1253 1254bool dma_rxreset(struct dma_pub *pub)1255{1256 struct dma_info *di = container_of(pub, struct dma_info, dma);1257 u32 status;1258 1259 if (di->nrxd == 0)1260 return true;1261 1262 bcma_write32(di->core, DMA64RXREGOFFS(di, control), 0);1263 SPINWAIT(((status =1264 (bcma_read32(di->core, DMA64RXREGOFFS(di, status0)) &1265 D64_RS0_RS_MASK)) != D64_RS0_RS_DISABLED), 10000);1266 1267 return status == D64_RS0_RS_DISABLED;1268}1269 1270static void dma_txenq(struct dma_info *di, struct sk_buff *p)1271{1272 unsigned char *data;1273 uint len;1274 u16 txout;1275 u32 flags = 0;1276 dma_addr_t pa;1277 1278 txout = di->txout;1279 1280 if (WARN_ON(nexttxd(di, txout) == di->txin))1281 return;1282 1283 /*1284 * obtain and initialize transmit descriptor entry.1285 */1286 data = p->data;1287 len = p->len;1288 1289 /* get physical address of buffer start */1290 pa = dma_map_single(di->dmadev, data, len, DMA_TO_DEVICE);1291 /* if mapping failed, free skb */1292 if (dma_mapping_error(di->dmadev, pa)) {1293 brcmu_pkt_buf_free_skb(p);1294 return;1295 }1296 /* With a DMA segment list, Descriptor table is filled1297 * using the segment list instead of looping over1298 * buffers in multi-chain DMA. Therefore, EOF for SGLIST1299 * is when end of segment list is reached.1300 */1301 flags = D64_CTRL1_SOF | D64_CTRL1_IOC | D64_CTRL1_EOF;1302 if (txout == (di->ntxd - 1))1303 flags |= D64_CTRL1_EOT;1304 1305 dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);1306 1307 txout = nexttxd(di, txout);1308 1309 /* save the packet */1310 di->txp[prevtxd(di, txout)] = p;1311 1312 /* bump the tx descriptor index */1313 di->txout = txout;1314}1315 1316static void ampdu_finalize(struct dma_info *di)1317{1318 struct brcms_ampdu_session *session = &di->ampdu_session;1319 struct sk_buff *p;1320 1321 trace_brcms_ampdu_session(&session->wlc->hw->d11core->dev,1322 session->max_ampdu_len,1323 session->max_ampdu_frames,1324 session->ampdu_len,1325 skb_queue_len(&session->skb_list),1326 session->dma_len);1327 1328 if (WARN_ON(skb_queue_empty(&session->skb_list)))1329 return;1330 1331 brcms_c_ampdu_finalize(session);1332 1333 while (!skb_queue_empty(&session->skb_list)) {1334 p = skb_dequeue(&session->skb_list);1335 dma_txenq(di, p);1336 }1337 1338 bcma_write32(di->core, DMA64TXREGOFFS(di, ptr),1339 di->xmtptrbase + I2B(di->txout, struct dma64desc));1340 brcms_c_ampdu_reset_session(session, session->wlc);1341}1342 1343static void prep_ampdu_frame(struct dma_info *di, struct sk_buff *p)1344{1345 struct brcms_ampdu_session *session = &di->ampdu_session;1346 int ret;1347 1348 ret = brcms_c_ampdu_add_frame(session, p);1349 if (ret == -ENOSPC) {1350 /*1351 * AMPDU cannot accomodate this frame. Close out the in-1352 * progress AMPDU session and start a new one.1353 */1354 ampdu_finalize(di);1355 ret = brcms_c_ampdu_add_frame(session, p);1356 }1357 1358 WARN_ON(ret);1359}1360 1361/* Update count of available tx descriptors based on current DMA state */1362static void dma_update_txavail(struct dma_info *di)1363{1364 /*1365 * Available space is number of descriptors less the number of1366 * active descriptors and the number of queued AMPDU frames.1367 */1368 di->dma.txavail = di->ntxd - ntxdactive(di, di->txin, di->txout) -1369 skb_queue_len(&di->ampdu_session.skb_list) - 1;1370}1371 1372/*1373 * !! tx entry routine1374 * WARNING: call must check the return value for error.1375 * the error(toss frames) could be fatal and cause many subsequent hard1376 * to debug problems1377 */1378int dma_txfast(struct brcms_c_info *wlc, struct dma_pub *pub,1379 struct sk_buff *p)1380{1381 struct dma_info *di = container_of(pub, struct dma_info, dma);1382 struct brcms_ampdu_session *session = &di->ampdu_session;1383 struct ieee80211_tx_info *tx_info;1384 bool is_ampdu;1385 1386 /* no use to transmit a zero length packet */1387 if (p->len == 0)1388 return 0;1389 1390 /* return nonzero if out of tx descriptors */1391 if (di->dma.txavail == 0 || nexttxd(di, di->txout) == di->txin)1392 goto outoftxd;1393 1394 tx_info = IEEE80211_SKB_CB(p);1395 is_ampdu = tx_info->flags & IEEE80211_TX_CTL_AMPDU;1396 if (is_ampdu)1397 prep_ampdu_frame(di, p);1398 else1399 dma_txenq(di, p);1400 1401 /* tx flow control */1402 dma_update_txavail(di);1403 1404 /* kick the chip */1405 if (is_ampdu) {1406 /*1407 * Start sending data if we've got a full AMPDU, there's1408 * no more space in the DMA ring, or the ring isn't1409 * currently transmitting.1410 */1411 if (skb_queue_len(&session->skb_list) == session->max_ampdu_frames ||1412 di->dma.txavail == 0 || dma64_txidle(di))1413 ampdu_finalize(di);1414 } else {1415 bcma_write32(di->core, DMA64TXREGOFFS(di, ptr),1416 di->xmtptrbase + I2B(di->txout, struct dma64desc));1417 }1418 1419 return 0;1420 1421 outoftxd:1422 brcms_dbg_dma(di->core, "%s: out of txds !!!\n", di->name);1423 brcmu_pkt_buf_free_skb(p);1424 di->dma.txavail = 0;1425 di->dma.txnobuf++;1426 return -ENOSPC;1427}1428 1429void dma_txflush(struct dma_pub *pub)1430{1431 struct dma_info *di = container_of(pub, struct dma_info, dma);1432 struct brcms_ampdu_session *session = &di->ampdu_session;1433 1434 if (!skb_queue_empty(&session->skb_list))1435 ampdu_finalize(di);1436}1437 1438int dma_txpending(struct dma_pub *pub)1439{1440 struct dma_info *di = container_of(pub, struct dma_info, dma);1441 return ntxdactive(di, di->txin, di->txout);1442}1443 1444/*1445 * If we have an active AMPDU session and are not transmitting,1446 * this function will force tx to start.1447 */1448void dma_kick_tx(struct dma_pub *pub)1449{1450 struct dma_info *di = container_of(pub, struct dma_info, dma);1451 struct brcms_ampdu_session *session = &di->ampdu_session;1452 1453 if (!skb_queue_empty(&session->skb_list) && dma64_txidle(di))1454 ampdu_finalize(di);1455}1456 1457/*1458 * Reclaim next completed txd (txds if using chained buffers) in the range1459 * specified and return associated packet.1460 * If range is DMA_RANGE_TRANSMITTED, reclaim descriptors that have be1461 * transmitted as noted by the hardware "CurrDescr" pointer.1462 * If range is DMA_RANGE_TRANSFERED, reclaim descriptors that have be1463 * transferred by the DMA as noted by the hardware "ActiveDescr" pointer.1464 * If range is DMA_RANGE_ALL, reclaim all txd(s) posted to the ring and1465 * return associated packet regardless of the value of hardware pointers.1466 */1467struct sk_buff *dma_getnexttxp(struct dma_pub *pub, enum txd_range range)1468{1469 struct dma_info *di = container_of(pub, struct dma_info, dma);1470 u16 start, end, i;1471 u16 active_desc;1472 struct sk_buff *txp;1473 1474 brcms_dbg_dma(di->core, "%s: %s\n",1475 di->name,1476 range == DMA_RANGE_ALL ? "all" :1477 range == DMA_RANGE_TRANSMITTED ? "transmitted" :1478 "transferred");1479 1480 if (di->ntxd == 0)1481 return NULL;1482 1483 txp = NULL;1484 1485 start = di->txin;1486 if (range == DMA_RANGE_ALL)1487 end = di->txout;1488 else {1489 end = (u16) (B2I(((bcma_read32(di->core,1490 DMA64TXREGOFFS(di, status0)) &1491 D64_XS0_CD_MASK) - di->xmtptrbase) &1492 D64_XS0_CD_MASK, struct dma64desc));1493 1494 if (range == DMA_RANGE_TRANSFERED) {1495 active_desc =1496 (u16)(bcma_read32(di->core,1497 DMA64TXREGOFFS(di, status1)) &1498 D64_XS1_AD_MASK);1499 active_desc =1500 (active_desc - di->xmtptrbase) & D64_XS0_CD_MASK;1501 active_desc = B2I(active_desc, struct dma64desc);1502 if (end != active_desc)1503 end = prevtxd(di, active_desc);1504 }1505 }1506 1507 if ((start == 0) && (end > di->txout))1508 goto bogus;1509 1510 for (i = start; i != end && !txp; i = nexttxd(di, i)) {1511 dma_addr_t pa;1512 uint size;1513 1514 pa = le32_to_cpu(di->txd64[i].addrlow) - di->dataoffsetlow;1515 1516 size =1517 (le32_to_cpu(di->txd64[i].ctrl2) &1518 D64_CTRL2_BC_MASK);1519 1520 di->txd64[i].addrlow = cpu_to_le32(0xdeadbeef);1521 di->txd64[i].addrhigh = cpu_to_le32(0xdeadbeef);1522 1523 txp = di->txp[i];1524 di->txp[i] = NULL;1525 1526 dma_unmap_single(di->dmadev, pa, size, DMA_TO_DEVICE);1527 }1528 1529 di->txin = i;1530 1531 /* tx flow control */1532 dma_update_txavail(di);1533 1534 return txp;1535 1536 bogus:1537 brcms_dbg_dma(di->core, "bogus curr: start %d end %d txout %d\n",1538 start, end, di->txout);1539 return NULL;1540}1541 1542/*1543 * Mac80211 initiated actions sometimes require packets in the DMA queue to be1544 * modified. The modified portion of the packet is not under control of the DMA1545 * engine. This function calls a caller-supplied function for each packet in1546 * the caller specified dma chain.1547 */1548void dma_walk_packets(struct dma_pub *dmah, void (*callback_fnc)1549 (void *pkt, void *arg_a), void *arg_a)1550{1551 struct dma_info *di = container_of(dmah, struct dma_info, dma);1552 uint i = di->txin;1553 uint end = di->txout;1554 struct sk_buff *skb;1555 struct ieee80211_tx_info *tx_info;1556 1557 while (i != end) {1558 skb = di->txp[i];1559 if (skb != NULL) {1560 tx_info = (struct ieee80211_tx_info *)skb->cb;1561 (callback_fnc)(tx_info, arg_a);1562 }1563 i = nexttxd(di, i);1564 }1565}1566