1970 lines · c
1// SPDX-License-Identifier: ISC2/*3 * Copyright (c) 2005-2011 Atheros Communications Inc.4 * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.5 * Copyright (c) 2018 The Linux Foundation. All rights reserved.6 * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.7 */8 9#include "hif.h"10#include "ce.h"11#include "debug.h"12 13/*14 * Support for Copy Engine hardware, which is mainly used for15 * communication between Host and Target over a PCIe interconnect.16 */17 18/*19 * A single CopyEngine (CE) comprises two "rings":20 * a source ring21 * a destination ring22 *23 * Each ring consists of a number of descriptors which specify24 * an address, length, and meta-data.25 *26 * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)27 * controls one ring and the other side controls the other ring.28 * The source side chooses when to initiate a transfer and it29 * chooses what to send (buffer address, length). The destination30 * side keeps a supply of "anonymous receive buffers" available and31 * it handles incoming data as it arrives (when the destination32 * receives an interrupt).33 *34 * The sender may send a simple buffer (address/length) or it may35 * send a small list of buffers. When a small list is sent, hardware36 * "gathers" these and they end up in a single destination buffer37 * with a single interrupt.38 *39 * There are several "contexts" managed by this layer -- more, it40 * may seem -- than should be needed. These are provided mainly for41 * maximum flexibility and especially to facilitate a simpler HIF42 * implementation. There are per-CopyEngine recv, send, and watermark43 * contexts. These are supplied by the caller when a recv, send,44 * or watermark handler is established and they are echoed back to45 * the caller when the respective callbacks are invoked. There is46 * also a per-transfer context supplied by the caller when a buffer47 * (or sendlist) is sent and when a buffer is enqueued for recv.48 * These per-transfer contexts are echoed back to the caller when49 * the buffer is sent/received.50 */51 52static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,53 struct ath10k_ce_pipe *ce_state)54{55 u32 ce_id = ce_state->id;56 u32 addr = 0;57 58 switch (ce_id) {59 case 0:60 addr = 0x00032000;61 break;62 case 3:63 addr = 0x0003200C;64 break;65 case 4:66 addr = 0x00032010;67 break;68 case 5:69 addr = 0x00032014;70 break;71 case 7:72 addr = 0x0003201C;73 break;74 default:75 ath10k_warn(ar, "invalid CE id: %d", ce_id);76 break;77 }78 return addr;79}80 81static inline unsigned int82ath10k_set_ring_byte(unsigned int offset,83 struct ath10k_hw_ce_regs_addr_map *addr_map)84{85 return ((offset << addr_map->lsb) & addr_map->mask);86}87 88static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)89{90 struct ath10k_ce *ce = ath10k_ce_priv(ar);91 92 return ce->bus_ops->read32(ar, offset);93}94 95static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)96{97 struct ath10k_ce *ce = ath10k_ce_priv(ar);98 99 ce->bus_ops->write32(ar, offset, value);100}101 102static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,103 u32 ce_ctrl_addr,104 unsigned int n)105{106 ath10k_ce_write32(ar, ce_ctrl_addr +107 ar->hw_ce_regs->dst_wr_index_addr, n);108}109 110static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,111 u32 ce_ctrl_addr)112{113 return ath10k_ce_read32(ar, ce_ctrl_addr +114 ar->hw_ce_regs->dst_wr_index_addr);115}116 117static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,118 u32 ce_ctrl_addr,119 unsigned int n)120{121 ath10k_ce_write32(ar, ce_ctrl_addr +122 ar->hw_ce_regs->sr_wr_index_addr, n);123}124 125static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,126 u32 ce_ctrl_addr)127{128 return ath10k_ce_read32(ar, ce_ctrl_addr +129 ar->hw_ce_regs->sr_wr_index_addr);130}131 132static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,133 u32 ce_id)134{135 struct ath10k_ce *ce = ath10k_ce_priv(ar);136 137 return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;138}139 140static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,141 u32 ce_ctrl_addr)142{143 struct ath10k_ce *ce = ath10k_ce_priv(ar);144 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);145 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];146 u32 index;147 148 if (ar->hw_params.rri_on_ddr &&149 (ce_state->attr_flags & CE_ATTR_DIS_INTR))150 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);151 else152 index = ath10k_ce_read32(ar, ce_ctrl_addr +153 ar->hw_ce_regs->current_srri_addr);154 155 return index;156}157 158static inline void159ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,160 struct ath10k_ce_pipe *ce_state,161 unsigned int value)162{163 ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);164}165 166static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,167 u32 ce_id,168 u64 addr)169{170 struct ath10k_ce *ce = ath10k_ce_priv(ar);171 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];172 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);173 u32 addr_lo = lower_32_bits(addr);174 175 ath10k_ce_write32(ar, ce_ctrl_addr +176 ar->hw_ce_regs->sr_base_addr_lo, addr_lo);177 178 if (ce_state->ops->ce_set_src_ring_base_addr_hi) {179 ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr,180 addr);181 }182}183 184static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar,185 u32 ce_ctrl_addr,186 u64 addr)187{188 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;189 190 ath10k_ce_write32(ar, ce_ctrl_addr +191 ar->hw_ce_regs->sr_base_addr_hi, addr_hi);192}193 194static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,195 u32 ce_ctrl_addr,196 unsigned int n)197{198 ath10k_ce_write32(ar, ce_ctrl_addr +199 ar->hw_ce_regs->sr_size_addr, n);200}201 202static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,203 u32 ce_ctrl_addr,204 unsigned int n)205{206 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;207 208 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +209 ctrl_regs->addr);210 211 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,212 (ctrl1_addr & ~(ctrl_regs->dmax->mask)) |213 ath10k_set_ring_byte(n, ctrl_regs->dmax));214}215 216static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,217 u32 ce_ctrl_addr,218 unsigned int n)219{220 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;221 222 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +223 ctrl_regs->addr);224 225 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,226 (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |227 ath10k_set_ring_byte(n, ctrl_regs->src_ring));228}229 230static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,231 u32 ce_ctrl_addr,232 unsigned int n)233{234 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;235 236 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +237 ctrl_regs->addr);238 239 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,240 (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |241 ath10k_set_ring_byte(n, ctrl_regs->dst_ring));242}243 244static inline245 u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)246{247 struct ath10k_ce *ce = ath10k_ce_priv(ar);248 249 return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &250 CE_DDR_RRI_MASK;251}252 253static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,254 u32 ce_ctrl_addr)255{256 struct ath10k_ce *ce = ath10k_ce_priv(ar);257 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);258 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];259 u32 index;260 261 if (ar->hw_params.rri_on_ddr &&262 (ce_state->attr_flags & CE_ATTR_DIS_INTR))263 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);264 else265 index = ath10k_ce_read32(ar, ce_ctrl_addr +266 ar->hw_ce_regs->current_drri_addr);267 268 return index;269}270 271static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,272 u32 ce_id,273 u64 addr)274{275 struct ath10k_ce *ce = ath10k_ce_priv(ar);276 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];277 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);278 u32 addr_lo = lower_32_bits(addr);279 280 ath10k_ce_write32(ar, ce_ctrl_addr +281 ar->hw_ce_regs->dr_base_addr_lo, addr_lo);282 283 if (ce_state->ops->ce_set_dest_ring_base_addr_hi) {284 ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr,285 addr);286 }287}288 289static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar,290 u32 ce_ctrl_addr,291 u64 addr)292{293 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;294 u32 reg_value;295 296 reg_value = ath10k_ce_read32(ar, ce_ctrl_addr +297 ar->hw_ce_regs->dr_base_addr_hi);298 reg_value &= ~CE_DESC_ADDR_HI_MASK;299 reg_value |= addr_hi;300 ath10k_ce_write32(ar, ce_ctrl_addr +301 ar->hw_ce_regs->dr_base_addr_hi, reg_value);302}303 304static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,305 u32 ce_ctrl_addr,306 unsigned int n)307{308 ath10k_ce_write32(ar, ce_ctrl_addr +309 ar->hw_ce_regs->dr_size_addr, n);310}311 312static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,313 u32 ce_ctrl_addr,314 unsigned int n)315{316 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;317 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);318 319 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,320 (addr & ~(srcr_wm->wm_high->mask)) |321 (ath10k_set_ring_byte(n, srcr_wm->wm_high)));322}323 324static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,325 u32 ce_ctrl_addr,326 unsigned int n)327{328 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;329 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);330 331 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,332 (addr & ~(srcr_wm->wm_low->mask)) |333 (ath10k_set_ring_byte(n, srcr_wm->wm_low)));334}335 336static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,337 u32 ce_ctrl_addr,338 unsigned int n)339{340 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;341 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);342 343 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,344 (addr & ~(dstr_wm->wm_high->mask)) |345 (ath10k_set_ring_byte(n, dstr_wm->wm_high)));346}347 348static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,349 u32 ce_ctrl_addr,350 unsigned int n)351{352 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;353 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);354 355 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,356 (addr & ~(dstr_wm->wm_low->mask)) |357 (ath10k_set_ring_byte(n, dstr_wm->wm_low)));358}359 360static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,361 u32 ce_ctrl_addr)362{363 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;364 365 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +366 ar->hw_ce_regs->host_ie_addr);367 368 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,369 host_ie_addr | host_ie->copy_complete->mask);370}371 372static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,373 u32 ce_ctrl_addr)374{375 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;376 377 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +378 ar->hw_ce_regs->host_ie_addr);379 380 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,381 host_ie_addr & ~(host_ie->copy_complete->mask));382}383 384static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,385 u32 ce_ctrl_addr)386{387 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;388 389 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +390 ar->hw_ce_regs->host_ie_addr);391 392 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,393 host_ie_addr & ~(wm_regs->wm_mask));394}395 396static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,397 u32 ce_ctrl_addr)398{399 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;400 401 u32 misc_ie_addr = ath10k_ce_read32(ar,402 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);403 404 ath10k_ce_write32(ar,405 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,406 misc_ie_addr & ~(misc_regs->err_mask));407}408 409static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,410 u32 ce_ctrl_addr,411 unsigned int mask)412{413 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;414 415 ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);416}417 418/*419 * Guts of ath10k_ce_send.420 * The caller takes responsibility for any needed locking.421 */422static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,423 void *per_transfer_context,424 dma_addr_t buffer,425 unsigned int nbytes,426 unsigned int transfer_id,427 unsigned int flags)428{429 struct ath10k *ar = ce_state->ar;430 struct ath10k_ce_ring *src_ring = ce_state->src_ring;431 struct ce_desc *desc, sdesc;432 unsigned int nentries_mask = src_ring->nentries_mask;433 unsigned int sw_index = src_ring->sw_index;434 unsigned int write_index = src_ring->write_index;435 u32 ctrl_addr = ce_state->ctrl_addr;436 u32 desc_flags = 0;437 int ret = 0;438 439 if (nbytes > ce_state->src_sz_max)440 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",441 __func__, nbytes, ce_state->src_sz_max);442 443 if (unlikely(CE_RING_DELTA(nentries_mask,444 write_index, sw_index - 1) <= 0)) {445 ret = -ENOSR;446 goto exit;447 }448 449 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,450 write_index);451 452 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);453 454 if (flags & CE_SEND_FLAG_GATHER)455 desc_flags |= CE_DESC_FLAGS_GATHER;456 if (flags & CE_SEND_FLAG_BYTE_SWAP)457 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;458 459 sdesc.addr = __cpu_to_le32(buffer);460 sdesc.nbytes = __cpu_to_le16(nbytes);461 sdesc.flags = __cpu_to_le16(desc_flags);462 463 *desc = sdesc;464 465 src_ring->per_transfer_context[write_index] = per_transfer_context;466 467 /* Update Source Ring Write Index */468 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);469 470 /* WORKAROUND */471 if (!(flags & CE_SEND_FLAG_GATHER))472 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);473 474 src_ring->write_index = write_index;475exit:476 return ret;477}478 479static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,480 void *per_transfer_context,481 dma_addr_t buffer,482 unsigned int nbytes,483 unsigned int transfer_id,484 unsigned int flags)485{486 struct ath10k *ar = ce_state->ar;487 struct ath10k_ce_ring *src_ring = ce_state->src_ring;488 struct ce_desc_64 *desc, sdesc;489 unsigned int nentries_mask = src_ring->nentries_mask;490 unsigned int sw_index;491 unsigned int write_index = src_ring->write_index;492 u32 ctrl_addr = ce_state->ctrl_addr;493 __le32 *addr;494 u32 desc_flags = 0;495 int ret = 0;496 497 if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))498 return -ESHUTDOWN;499 500 if (nbytes > ce_state->src_sz_max)501 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",502 __func__, nbytes, ce_state->src_sz_max);503 504 if (ar->hw_params.rri_on_ddr)505 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);506 else507 sw_index = src_ring->sw_index;508 509 if (unlikely(CE_RING_DELTA(nentries_mask,510 write_index, sw_index - 1) <= 0)) {511 ret = -ENOSR;512 goto exit;513 }514 515 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,516 write_index);517 518 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);519 520 if (flags & CE_SEND_FLAG_GATHER)521 desc_flags |= CE_DESC_FLAGS_GATHER;522 523 if (flags & CE_SEND_FLAG_BYTE_SWAP)524 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;525 526 addr = (__le32 *)&sdesc.addr;527 528 flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK;529 addr[0] = __cpu_to_le32(buffer);530 addr[1] = __cpu_to_le32(flags);531 if (flags & CE_SEND_FLAG_GATHER)532 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);533 else534 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));535 536 sdesc.nbytes = __cpu_to_le16(nbytes);537 sdesc.flags = __cpu_to_le16(desc_flags);538 539 *desc = sdesc;540 541 src_ring->per_transfer_context[write_index] = per_transfer_context;542 543 /* Update Source Ring Write Index */544 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);545 546 if (!(flags & CE_SEND_FLAG_GATHER)) {547 if (ar->hw_params.shadow_reg_support)548 ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,549 write_index);550 else551 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr,552 write_index);553 }554 555 src_ring->write_index = write_index;556exit:557 return ret;558}559 560int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,561 void *per_transfer_context,562 dma_addr_t buffer,563 unsigned int nbytes,564 unsigned int transfer_id,565 unsigned int flags)566{567 return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,568 buffer, nbytes, transfer_id, flags);569}570EXPORT_SYMBOL(ath10k_ce_send_nolock);571 572void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)573{574 struct ath10k *ar = pipe->ar;575 struct ath10k_ce *ce = ath10k_ce_priv(ar);576 struct ath10k_ce_ring *src_ring = pipe->src_ring;577 u32 ctrl_addr = pipe->ctrl_addr;578 579 lockdep_assert_held(&ce->ce_lock);580 581 /*582 * This function must be called only if there is an incomplete583 * scatter-gather transfer (before index register is updated)584 * that needs to be cleaned up.585 */586 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))587 return;588 589 if (WARN_ON_ONCE(src_ring->write_index ==590 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))591 return;592 593 src_ring->write_index--;594 src_ring->write_index &= src_ring->nentries_mask;595 596 src_ring->per_transfer_context[src_ring->write_index] = NULL;597}598EXPORT_SYMBOL(__ath10k_ce_send_revert);599 600int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,601 void *per_transfer_context,602 dma_addr_t buffer,603 unsigned int nbytes,604 unsigned int transfer_id,605 unsigned int flags)606{607 struct ath10k *ar = ce_state->ar;608 struct ath10k_ce *ce = ath10k_ce_priv(ar);609 int ret;610 611 spin_lock_bh(&ce->ce_lock);612 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,613 buffer, nbytes, transfer_id, flags);614 spin_unlock_bh(&ce->ce_lock);615 616 return ret;617}618EXPORT_SYMBOL(ath10k_ce_send);619 620int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)621{622 struct ath10k *ar = pipe->ar;623 struct ath10k_ce *ce = ath10k_ce_priv(ar);624 int delta;625 626 spin_lock_bh(&ce->ce_lock);627 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,628 pipe->src_ring->write_index,629 pipe->src_ring->sw_index - 1);630 spin_unlock_bh(&ce->ce_lock);631 632 return delta;633}634EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);635 636int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)637{638 struct ath10k *ar = pipe->ar;639 struct ath10k_ce *ce = ath10k_ce_priv(ar);640 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;641 unsigned int nentries_mask = dest_ring->nentries_mask;642 unsigned int write_index = dest_ring->write_index;643 unsigned int sw_index = dest_ring->sw_index;644 645 lockdep_assert_held(&ce->ce_lock);646 647 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);648}649EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);650 651static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,652 dma_addr_t paddr)653{654 struct ath10k *ar = pipe->ar;655 struct ath10k_ce *ce = ath10k_ce_priv(ar);656 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;657 unsigned int nentries_mask = dest_ring->nentries_mask;658 unsigned int write_index = dest_ring->write_index;659 unsigned int sw_index = dest_ring->sw_index;660 struct ce_desc *base = dest_ring->base_addr_owner_space;661 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);662 u32 ctrl_addr = pipe->ctrl_addr;663 664 lockdep_assert_held(&ce->ce_lock);665 666 if ((pipe->id != 5) &&667 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)668 return -ENOSPC;669 670 desc->addr = __cpu_to_le32(paddr);671 desc->nbytes = 0;672 673 dest_ring->per_transfer_context[write_index] = ctx;674 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);675 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);676 dest_ring->write_index = write_index;677 678 return 0;679}680 681static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,682 void *ctx,683 dma_addr_t paddr)684{685 struct ath10k *ar = pipe->ar;686 struct ath10k_ce *ce = ath10k_ce_priv(ar);687 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;688 unsigned int nentries_mask = dest_ring->nentries_mask;689 unsigned int write_index = dest_ring->write_index;690 unsigned int sw_index = dest_ring->sw_index;691 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;692 struct ce_desc_64 *desc =693 CE_DEST_RING_TO_DESC_64(base, write_index);694 u32 ctrl_addr = pipe->ctrl_addr;695 696 lockdep_assert_held(&ce->ce_lock);697 698 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)699 return -ENOSPC;700 701 desc->addr = __cpu_to_le64(paddr);702 desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK);703 704 desc->nbytes = 0;705 706 dest_ring->per_transfer_context[write_index] = ctx;707 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);708 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);709 dest_ring->write_index = write_index;710 711 return 0;712}713 714void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)715{716 struct ath10k *ar = pipe->ar;717 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;718 unsigned int nentries_mask = dest_ring->nentries_mask;719 unsigned int write_index = dest_ring->write_index;720 u32 ctrl_addr = pipe->ctrl_addr;721 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);722 723 /* Prevent CE ring stuck issue that will occur when ring is full.724 * Make sure that write index is 1 less than read index.725 */726 if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)727 nentries -= 1;728 729 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);730 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);731 dest_ring->write_index = write_index;732}733EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);734 735int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,736 dma_addr_t paddr)737{738 struct ath10k *ar = pipe->ar;739 struct ath10k_ce *ce = ath10k_ce_priv(ar);740 int ret;741 742 spin_lock_bh(&ce->ce_lock);743 ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);744 spin_unlock_bh(&ce->ce_lock);745 746 return ret;747}748EXPORT_SYMBOL(ath10k_ce_rx_post_buf);749 750/*751 * Guts of ath10k_ce_completed_recv_next.752 * The caller takes responsibility for any necessary locking.753 */754static int755 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,756 void **per_transfer_contextp,757 unsigned int *nbytesp)758{759 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;760 unsigned int nentries_mask = dest_ring->nentries_mask;761 unsigned int sw_index = dest_ring->sw_index;762 763 struct ce_desc *base = dest_ring->base_addr_owner_space;764 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);765 struct ce_desc sdesc;766 u16 nbytes;767 768 /* Copy in one go for performance reasons */769 sdesc = *desc;770 771 nbytes = __le16_to_cpu(sdesc.nbytes);772 if (nbytes == 0) {773 /*774 * This closes a relatively unusual race where the Host775 * sees the updated DRRI before the update to the776 * corresponding descriptor has completed. We treat this777 * as a descriptor that is not yet done.778 */779 return -EIO;780 }781 782 desc->nbytes = 0;783 784 /* Return data from completed destination descriptor */785 *nbytesp = nbytes;786 787 if (per_transfer_contextp)788 *per_transfer_contextp =789 dest_ring->per_transfer_context[sw_index];790 791 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.792 * So update transfer context all CEs except CE5.793 */794 if (ce_state->id != 5)795 dest_ring->per_transfer_context[sw_index] = NULL;796 797 /* Update sw_index */798 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);799 dest_ring->sw_index = sw_index;800 801 return 0;802}803 804static int805_ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,806 void **per_transfer_contextp,807 unsigned int *nbytesp)808{809 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;810 unsigned int nentries_mask = dest_ring->nentries_mask;811 unsigned int sw_index = dest_ring->sw_index;812 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;813 struct ce_desc_64 *desc =814 CE_DEST_RING_TO_DESC_64(base, sw_index);815 struct ce_desc_64 sdesc;816 u16 nbytes;817 818 /* Copy in one go for performance reasons */819 sdesc = *desc;820 821 nbytes = __le16_to_cpu(sdesc.nbytes);822 if (nbytes == 0) {823 /* This closes a relatively unusual race where the Host824 * sees the updated DRRI before the update to the825 * corresponding descriptor has completed. We treat this826 * as a descriptor that is not yet done.827 */828 return -EIO;829 }830 831 desc->nbytes = 0;832 833 /* Return data from completed destination descriptor */834 *nbytesp = nbytes;835 836 if (per_transfer_contextp)837 *per_transfer_contextp =838 dest_ring->per_transfer_context[sw_index];839 840 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.841 * So update transfer context all CEs except CE5.842 */843 if (ce_state->id != 5)844 dest_ring->per_transfer_context[sw_index] = NULL;845 846 /* Update sw_index */847 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);848 dest_ring->sw_index = sw_index;849 850 return 0;851}852 853int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,854 void **per_transfer_ctx,855 unsigned int *nbytesp)856{857 return ce_state->ops->ce_completed_recv_next_nolock(ce_state,858 per_transfer_ctx,859 nbytesp);860}861EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);862 863int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,864 void **per_transfer_contextp,865 unsigned int *nbytesp)866{867 struct ath10k *ar = ce_state->ar;868 struct ath10k_ce *ce = ath10k_ce_priv(ar);869 int ret;870 871 spin_lock_bh(&ce->ce_lock);872 ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,873 per_transfer_contextp,874 nbytesp);875 876 spin_unlock_bh(&ce->ce_lock);877 878 return ret;879}880EXPORT_SYMBOL(ath10k_ce_completed_recv_next);881 882static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,883 void **per_transfer_contextp,884 dma_addr_t *bufferp)885{886 struct ath10k_ce_ring *dest_ring;887 unsigned int nentries_mask;888 unsigned int sw_index;889 unsigned int write_index;890 int ret;891 struct ath10k *ar;892 struct ath10k_ce *ce;893 894 dest_ring = ce_state->dest_ring;895 896 if (!dest_ring)897 return -EIO;898 899 ar = ce_state->ar;900 ce = ath10k_ce_priv(ar);901 902 spin_lock_bh(&ce->ce_lock);903 904 nentries_mask = dest_ring->nentries_mask;905 sw_index = dest_ring->sw_index;906 write_index = dest_ring->write_index;907 if (write_index != sw_index) {908 struct ce_desc *base = dest_ring->base_addr_owner_space;909 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);910 911 /* Return data from completed destination descriptor */912 *bufferp = __le32_to_cpu(desc->addr);913 914 if (per_transfer_contextp)915 *per_transfer_contextp =916 dest_ring->per_transfer_context[sw_index];917 918 /* sanity */919 dest_ring->per_transfer_context[sw_index] = NULL;920 desc->nbytes = 0;921 922 /* Update sw_index */923 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);924 dest_ring->sw_index = sw_index;925 ret = 0;926 } else {927 ret = -EIO;928 }929 930 spin_unlock_bh(&ce->ce_lock);931 932 return ret;933}934 935static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,936 void **per_transfer_contextp,937 dma_addr_t *bufferp)938{939 struct ath10k_ce_ring *dest_ring;940 unsigned int nentries_mask;941 unsigned int sw_index;942 unsigned int write_index;943 int ret;944 struct ath10k *ar;945 struct ath10k_ce *ce;946 947 dest_ring = ce_state->dest_ring;948 949 if (!dest_ring)950 return -EIO;951 952 ar = ce_state->ar;953 ce = ath10k_ce_priv(ar);954 955 spin_lock_bh(&ce->ce_lock);956 957 nentries_mask = dest_ring->nentries_mask;958 sw_index = dest_ring->sw_index;959 write_index = dest_ring->write_index;960 if (write_index != sw_index) {961 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;962 struct ce_desc_64 *desc =963 CE_DEST_RING_TO_DESC_64(base, sw_index);964 965 /* Return data from completed destination descriptor */966 *bufferp = __le64_to_cpu(desc->addr);967 968 if (per_transfer_contextp)969 *per_transfer_contextp =970 dest_ring->per_transfer_context[sw_index];971 972 /* sanity */973 dest_ring->per_transfer_context[sw_index] = NULL;974 desc->nbytes = 0;975 976 /* Update sw_index */977 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);978 dest_ring->sw_index = sw_index;979 ret = 0;980 } else {981 ret = -EIO;982 }983 984 spin_unlock_bh(&ce->ce_lock);985 986 return ret;987}988 989int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,990 void **per_transfer_contextp,991 dma_addr_t *bufferp)992{993 return ce_state->ops->ce_revoke_recv_next(ce_state,994 per_transfer_contextp,995 bufferp);996}997EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);998 999/*1000 * Guts of ath10k_ce_completed_send_next.1001 * The caller takes responsibility for any necessary locking.1002 */1003static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,1004 void **per_transfer_contextp)1005{1006 struct ath10k_ce_ring *src_ring = ce_state->src_ring;1007 u32 ctrl_addr = ce_state->ctrl_addr;1008 struct ath10k *ar = ce_state->ar;1009 unsigned int nentries_mask = src_ring->nentries_mask;1010 unsigned int sw_index = src_ring->sw_index;1011 unsigned int read_index;1012 struct ce_desc *desc;1013 1014 if (src_ring->hw_index == sw_index) {1015 /*1016 * The SW completion index has caught up with the cached1017 * version of the HW completion index.1018 * Update the cached HW completion index to see whether1019 * the SW has really caught up to the HW, or if the cached1020 * value of the HW index has become stale.1021 */1022 1023 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);1024 if (read_index == 0xffffffff)1025 return -ENODEV;1026 1027 read_index &= nentries_mask;1028 src_ring->hw_index = read_index;1029 }1030 1031 if (ar->hw_params.rri_on_ddr)1032 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);1033 else1034 read_index = src_ring->hw_index;1035 1036 if (read_index == sw_index)1037 return -EIO;1038 1039 if (per_transfer_contextp)1040 *per_transfer_contextp =1041 src_ring->per_transfer_context[sw_index];1042 1043 /* sanity */1044 src_ring->per_transfer_context[sw_index] = NULL;1045 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,1046 sw_index);1047 desc->nbytes = 0;1048 1049 /* Update sw_index */1050 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);1051 src_ring->sw_index = sw_index;1052 1053 return 0;1054}1055 1056static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state,1057 void **per_transfer_contextp)1058{1059 struct ath10k_ce_ring *src_ring = ce_state->src_ring;1060 u32 ctrl_addr = ce_state->ctrl_addr;1061 struct ath10k *ar = ce_state->ar;1062 unsigned int nentries_mask = src_ring->nentries_mask;1063 unsigned int sw_index = src_ring->sw_index;1064 unsigned int read_index;1065 struct ce_desc_64 *desc;1066 1067 if (src_ring->hw_index == sw_index) {1068 /*1069 * The SW completion index has caught up with the cached1070 * version of the HW completion index.1071 * Update the cached HW completion index to see whether1072 * the SW has really caught up to the HW, or if the cached1073 * value of the HW index has become stale.1074 */1075 1076 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);1077 if (read_index == 0xffffffff)1078 return -ENODEV;1079 1080 read_index &= nentries_mask;1081 src_ring->hw_index = read_index;1082 }1083 1084 if (ar->hw_params.rri_on_ddr)1085 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);1086 else1087 read_index = src_ring->hw_index;1088 1089 if (read_index == sw_index)1090 return -EIO;1091 1092 if (per_transfer_contextp)1093 *per_transfer_contextp =1094 src_ring->per_transfer_context[sw_index];1095 1096 /* sanity */1097 src_ring->per_transfer_context[sw_index] = NULL;1098 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,1099 sw_index);1100 desc->nbytes = 0;1101 1102 /* Update sw_index */1103 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);1104 src_ring->sw_index = sw_index;1105 1106 return 0;1107}1108 1109int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,1110 void **per_transfer_contextp)1111{1112 return ce_state->ops->ce_completed_send_next_nolock(ce_state,1113 per_transfer_contextp);1114}1115EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);1116 1117static void ath10k_ce_extract_desc_data(struct ath10k *ar,1118 struct ath10k_ce_ring *src_ring,1119 u32 sw_index,1120 dma_addr_t *bufferp,1121 u32 *nbytesp,1122 u32 *transfer_idp)1123{1124 struct ce_desc *base = src_ring->base_addr_owner_space;1125 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);1126 1127 /* Return data from completed source descriptor */1128 *bufferp = __le32_to_cpu(desc->addr);1129 *nbytesp = __le16_to_cpu(desc->nbytes);1130 *transfer_idp = MS(__le16_to_cpu(desc->flags),1131 CE_DESC_FLAGS_META_DATA);1132}1133 1134static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,1135 struct ath10k_ce_ring *src_ring,1136 u32 sw_index,1137 dma_addr_t *bufferp,1138 u32 *nbytesp,1139 u32 *transfer_idp)1140{1141 struct ce_desc_64 *base = src_ring->base_addr_owner_space;1142 struct ce_desc_64 *desc =1143 CE_SRC_RING_TO_DESC_64(base, sw_index);1144 1145 /* Return data from completed source descriptor */1146 *bufferp = __le64_to_cpu(desc->addr);1147 *nbytesp = __le16_to_cpu(desc->nbytes);1148 *transfer_idp = MS(__le16_to_cpu(desc->flags),1149 CE_DESC_FLAGS_META_DATA);1150}1151 1152/* NB: Modeled after ath10k_ce_completed_send_next */1153int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,1154 void **per_transfer_contextp,1155 dma_addr_t *bufferp,1156 unsigned int *nbytesp,1157 unsigned int *transfer_idp)1158{1159 struct ath10k_ce_ring *src_ring;1160 unsigned int nentries_mask;1161 unsigned int sw_index;1162 unsigned int write_index;1163 int ret;1164 struct ath10k *ar;1165 struct ath10k_ce *ce;1166 1167 src_ring = ce_state->src_ring;1168 1169 if (!src_ring)1170 return -EIO;1171 1172 ar = ce_state->ar;1173 ce = ath10k_ce_priv(ar);1174 1175 spin_lock_bh(&ce->ce_lock);1176 1177 nentries_mask = src_ring->nentries_mask;1178 sw_index = src_ring->sw_index;1179 write_index = src_ring->write_index;1180 1181 if (write_index != sw_index) {1182 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,1183 bufferp, nbytesp,1184 transfer_idp);1185 1186 if (per_transfer_contextp)1187 *per_transfer_contextp =1188 src_ring->per_transfer_context[sw_index];1189 1190 /* sanity */1191 src_ring->per_transfer_context[sw_index] = NULL;1192 1193 /* Update sw_index */1194 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);1195 src_ring->sw_index = sw_index;1196 ret = 0;1197 } else {1198 ret = -EIO;1199 }1200 1201 spin_unlock_bh(&ce->ce_lock);1202 1203 return ret;1204}1205EXPORT_SYMBOL(ath10k_ce_cancel_send_next);1206 1207int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,1208 void **per_transfer_contextp)1209{1210 struct ath10k *ar = ce_state->ar;1211 struct ath10k_ce *ce = ath10k_ce_priv(ar);1212 int ret;1213 1214 spin_lock_bh(&ce->ce_lock);1215 ret = ath10k_ce_completed_send_next_nolock(ce_state,1216 per_transfer_contextp);1217 spin_unlock_bh(&ce->ce_lock);1218 1219 return ret;1220}1221EXPORT_SYMBOL(ath10k_ce_completed_send_next);1222 1223/*1224 * Guts of interrupt handler for per-engine interrupts on a particular CE.1225 *1226 * Invokes registered callbacks for recv_complete,1227 * send_complete, and watermarks.1228 */1229void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)1230{1231 struct ath10k_ce *ce = ath10k_ce_priv(ar);1232 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];1233 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;1234 u32 ctrl_addr = ce_state->ctrl_addr;1235 1236 /*1237 * Clear before handling1238 *1239 * Misc CE interrupts are not being handled, but still need1240 * to be cleared.1241 *1242 * NOTE: When the last copy engine interrupt is cleared the1243 * hardware will go to sleep. Once this happens any access to1244 * the CE registers can cause a hardware fault.1245 */1246 ath10k_ce_engine_int_status_clear(ar, ctrl_addr,1247 wm_regs->cc_mask | wm_regs->wm_mask);1248 1249 if (ce_state->recv_cb)1250 ce_state->recv_cb(ce_state);1251 1252 if (ce_state->send_cb)1253 ce_state->send_cb(ce_state);1254}1255EXPORT_SYMBOL(ath10k_ce_per_engine_service);1256 1257/*1258 * Handler for per-engine interrupts on ALL active CEs.1259 * This is used in cases where the system is sharing a1260 * single interrupt for all CEs1261 */1262 1263void ath10k_ce_per_engine_service_any(struct ath10k *ar)1264{1265 int ce_id;1266 u32 intr_summary;1267 1268 intr_summary = ath10k_ce_interrupt_summary(ar);1269 1270 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {1271 if (intr_summary & (1 << ce_id))1272 intr_summary &= ~(1 << ce_id);1273 else1274 /* no intr pending on this CE */1275 continue;1276 1277 ath10k_ce_per_engine_service(ar, ce_id);1278 }1279}1280EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);1281 1282/*1283 * Adjust interrupts for the copy complete handler.1284 * If it's needed for either send or recv, then unmask1285 * this interrupt; otherwise, mask it.1286 *1287 * Called with ce_lock held.1288 */1289static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)1290{1291 u32 ctrl_addr = ce_state->ctrl_addr;1292 struct ath10k *ar = ce_state->ar;1293 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;1294 1295 if ((!disable_copy_compl_intr) &&1296 (ce_state->send_cb || ce_state->recv_cb))1297 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);1298 else1299 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);1300 1301 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);1302}1303 1304void ath10k_ce_disable_interrupt(struct ath10k *ar, int ce_id)1305{1306 struct ath10k_ce *ce = ath10k_ce_priv(ar);1307 struct ath10k_ce_pipe *ce_state;1308 u32 ctrl_addr;1309 1310 ce_state = &ce->ce_states[ce_id];1311 if (ce_state->attr_flags & CE_ATTR_POLL)1312 return;1313 1314 ctrl_addr = ath10k_ce_base_address(ar, ce_id);1315 1316 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);1317 ath10k_ce_error_intr_disable(ar, ctrl_addr);1318 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);1319}1320EXPORT_SYMBOL(ath10k_ce_disable_interrupt);1321 1322void ath10k_ce_disable_interrupts(struct ath10k *ar)1323{1324 int ce_id;1325 1326 for (ce_id = 0; ce_id < CE_COUNT; ce_id++)1327 ath10k_ce_disable_interrupt(ar, ce_id);1328}1329EXPORT_SYMBOL(ath10k_ce_disable_interrupts);1330 1331void ath10k_ce_enable_interrupt(struct ath10k *ar, int ce_id)1332{1333 struct ath10k_ce *ce = ath10k_ce_priv(ar);1334 struct ath10k_ce_pipe *ce_state;1335 1336 ce_state = &ce->ce_states[ce_id];1337 if (ce_state->attr_flags & CE_ATTR_POLL)1338 return;1339 1340 ath10k_ce_per_engine_handler_adjust(ce_state);1341}1342EXPORT_SYMBOL(ath10k_ce_enable_interrupt);1343 1344void ath10k_ce_enable_interrupts(struct ath10k *ar)1345{1346 int ce_id;1347 1348 /* Enable interrupts for copy engine that1349 * are not using polling mode.1350 */1351 for (ce_id = 0; ce_id < CE_COUNT; ce_id++)1352 ath10k_ce_enable_interrupt(ar, ce_id);1353}1354EXPORT_SYMBOL(ath10k_ce_enable_interrupts);1355 1356static int ath10k_ce_init_src_ring(struct ath10k *ar,1357 unsigned int ce_id,1358 const struct ce_attr *attr)1359{1360 struct ath10k_ce *ce = ath10k_ce_priv(ar);1361 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];1362 struct ath10k_ce_ring *src_ring = ce_state->src_ring;1363 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);1364 1365 nentries = roundup_pow_of_two(attr->src_nentries);1366 1367 if (ar->hw_params.target_64bit)1368 memset(src_ring->base_addr_owner_space, 0,1369 nentries * sizeof(struct ce_desc_64));1370 else1371 memset(src_ring->base_addr_owner_space, 0,1372 nentries * sizeof(struct ce_desc));1373 1374 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);1375 src_ring->sw_index &= src_ring->nentries_mask;1376 src_ring->hw_index = src_ring->sw_index;1377 1378 src_ring->write_index =1379 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);1380 src_ring->write_index &= src_ring->nentries_mask;1381 1382 ath10k_ce_src_ring_base_addr_set(ar, ce_id,1383 src_ring->base_addr_ce_space);1384 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);1385 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);1386 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);1387 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);1388 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);1389 1390 ath10k_dbg(ar, ATH10K_DBG_BOOT,1391 "boot init ce src ring id %d entries %d base_addr %pK\n",1392 ce_id, nentries, src_ring->base_addr_owner_space);1393 1394 return 0;1395}1396 1397static int ath10k_ce_init_dest_ring(struct ath10k *ar,1398 unsigned int ce_id,1399 const struct ce_attr *attr)1400{1401 struct ath10k_ce *ce = ath10k_ce_priv(ar);1402 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];1403 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;1404 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);1405 1406 nentries = roundup_pow_of_two(attr->dest_nentries);1407 1408 if (ar->hw_params.target_64bit)1409 memset(dest_ring->base_addr_owner_space, 0,1410 nentries * sizeof(struct ce_desc_64));1411 else1412 memset(dest_ring->base_addr_owner_space, 0,1413 nentries * sizeof(struct ce_desc));1414 1415 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);1416 dest_ring->sw_index &= dest_ring->nentries_mask;1417 dest_ring->write_index =1418 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);1419 dest_ring->write_index &= dest_ring->nentries_mask;1420 1421 ath10k_ce_dest_ring_base_addr_set(ar, ce_id,1422 dest_ring->base_addr_ce_space);1423 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);1424 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);1425 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);1426 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);1427 1428 ath10k_dbg(ar, ATH10K_DBG_BOOT,1429 "boot ce dest ring id %d entries %d base_addr %pK\n",1430 ce_id, nentries, dest_ring->base_addr_owner_space);1431 1432 return 0;1433}1434 1435static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,1436 struct ath10k_ce_ring *src_ring,1437 u32 nentries)1438{1439 src_ring->shadow_base_unaligned = kcalloc(nentries,1440 sizeof(struct ce_desc_64),1441 GFP_KERNEL);1442 if (!src_ring->shadow_base_unaligned)1443 return -ENOMEM;1444 1445 src_ring->shadow_base = (struct ce_desc_64 *)1446 PTR_ALIGN(src_ring->shadow_base_unaligned,1447 CE_DESC_RING_ALIGN);1448 return 0;1449}1450 1451static struct ath10k_ce_ring *1452ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,1453 const struct ce_attr *attr)1454{1455 struct ath10k_ce_ring *src_ring;1456 u32 nentries = attr->src_nentries;1457 dma_addr_t base_addr;1458 int ret;1459 1460 nentries = roundup_pow_of_two(nentries);1461 1462 src_ring = kzalloc(struct_size(src_ring, per_transfer_context,1463 nentries), GFP_KERNEL);1464 if (src_ring == NULL)1465 return ERR_PTR(-ENOMEM);1466 1467 src_ring->nentries = nentries;1468 src_ring->nentries_mask = nentries - 1;1469 1470 /*1471 * Legacy platforms that do not support cache1472 * coherent DMA are unsupported1473 */1474 src_ring->base_addr_owner_space_unaligned =1475 dma_alloc_coherent(ar->dev,1476 (nentries * sizeof(struct ce_desc) +1477 CE_DESC_RING_ALIGN),1478 &base_addr, GFP_KERNEL);1479 if (!src_ring->base_addr_owner_space_unaligned) {1480 kfree(src_ring);1481 return ERR_PTR(-ENOMEM);1482 }1483 1484 src_ring->base_addr_ce_space_unaligned = base_addr;1485 1486 src_ring->base_addr_owner_space =1487 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,1488 CE_DESC_RING_ALIGN);1489 src_ring->base_addr_ce_space =1490 ALIGN(src_ring->base_addr_ce_space_unaligned,1491 CE_DESC_RING_ALIGN);1492 1493 if (ar->hw_params.shadow_reg_support) {1494 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);1495 if (ret) {1496 dma_free_coherent(ar->dev,1497 (nentries * sizeof(struct ce_desc) +1498 CE_DESC_RING_ALIGN),1499 src_ring->base_addr_owner_space_unaligned,1500 base_addr);1501 kfree(src_ring);1502 return ERR_PTR(ret);1503 }1504 }1505 1506 return src_ring;1507}1508 1509static struct ath10k_ce_ring *1510ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,1511 const struct ce_attr *attr)1512{1513 struct ath10k_ce_ring *src_ring;1514 u32 nentries = attr->src_nentries;1515 dma_addr_t base_addr;1516 int ret;1517 1518 nentries = roundup_pow_of_two(nentries);1519 1520 src_ring = kzalloc(struct_size(src_ring, per_transfer_context,1521 nentries), GFP_KERNEL);1522 if (!src_ring)1523 return ERR_PTR(-ENOMEM);1524 1525 src_ring->nentries = nentries;1526 src_ring->nentries_mask = nentries - 1;1527 1528 /* Legacy platforms that do not support cache1529 * coherent DMA are unsupported1530 */1531 src_ring->base_addr_owner_space_unaligned =1532 dma_alloc_coherent(ar->dev,1533 (nentries * sizeof(struct ce_desc_64) +1534 CE_DESC_RING_ALIGN),1535 &base_addr, GFP_KERNEL);1536 if (!src_ring->base_addr_owner_space_unaligned) {1537 kfree(src_ring);1538 return ERR_PTR(-ENOMEM);1539 }1540 1541 src_ring->base_addr_ce_space_unaligned = base_addr;1542 1543 src_ring->base_addr_owner_space =1544 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,1545 CE_DESC_RING_ALIGN);1546 src_ring->base_addr_ce_space =1547 ALIGN(src_ring->base_addr_ce_space_unaligned,1548 CE_DESC_RING_ALIGN);1549 1550 if (ar->hw_params.shadow_reg_support) {1551 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);1552 if (ret) {1553 dma_free_coherent(ar->dev,1554 (nentries * sizeof(struct ce_desc_64) +1555 CE_DESC_RING_ALIGN),1556 src_ring->base_addr_owner_space_unaligned,1557 base_addr);1558 kfree(src_ring);1559 return ERR_PTR(ret);1560 }1561 }1562 1563 return src_ring;1564}1565 1566static struct ath10k_ce_ring *1567ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,1568 const struct ce_attr *attr)1569{1570 struct ath10k_ce_ring *dest_ring;1571 u32 nentries;1572 dma_addr_t base_addr;1573 1574 nentries = roundup_pow_of_two(attr->dest_nentries);1575 1576 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,1577 nentries), GFP_KERNEL);1578 if (dest_ring == NULL)1579 return ERR_PTR(-ENOMEM);1580 1581 dest_ring->nentries = nentries;1582 dest_ring->nentries_mask = nentries - 1;1583 1584 /*1585 * Legacy platforms that do not support cache1586 * coherent DMA are unsupported1587 */1588 dest_ring->base_addr_owner_space_unaligned =1589 dma_alloc_coherent(ar->dev,1590 (nentries * sizeof(struct ce_desc) +1591 CE_DESC_RING_ALIGN),1592 &base_addr, GFP_KERNEL);1593 if (!dest_ring->base_addr_owner_space_unaligned) {1594 kfree(dest_ring);1595 return ERR_PTR(-ENOMEM);1596 }1597 1598 dest_ring->base_addr_ce_space_unaligned = base_addr;1599 1600 dest_ring->base_addr_owner_space =1601 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,1602 CE_DESC_RING_ALIGN);1603 dest_ring->base_addr_ce_space =1604 ALIGN(dest_ring->base_addr_ce_space_unaligned,1605 CE_DESC_RING_ALIGN);1606 1607 return dest_ring;1608}1609 1610static struct ath10k_ce_ring *1611ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,1612 const struct ce_attr *attr)1613{1614 struct ath10k_ce_ring *dest_ring;1615 u32 nentries;1616 dma_addr_t base_addr;1617 1618 nentries = roundup_pow_of_two(attr->dest_nentries);1619 1620 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,1621 nentries), GFP_KERNEL);1622 if (!dest_ring)1623 return ERR_PTR(-ENOMEM);1624 1625 dest_ring->nentries = nentries;1626 dest_ring->nentries_mask = nentries - 1;1627 1628 /* Legacy platforms that do not support cache1629 * coherent DMA are unsupported1630 */1631 dest_ring->base_addr_owner_space_unaligned =1632 dma_alloc_coherent(ar->dev,1633 (nentries * sizeof(struct ce_desc_64) +1634 CE_DESC_RING_ALIGN),1635 &base_addr, GFP_KERNEL);1636 if (!dest_ring->base_addr_owner_space_unaligned) {1637 kfree(dest_ring);1638 return ERR_PTR(-ENOMEM);1639 }1640 1641 dest_ring->base_addr_ce_space_unaligned = base_addr;1642 1643 /* Correctly initialize memory to 0 to prevent garbage1644 * data crashing system when download firmware1645 */1646 dest_ring->base_addr_owner_space =1647 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,1648 CE_DESC_RING_ALIGN);1649 dest_ring->base_addr_ce_space =1650 ALIGN(dest_ring->base_addr_ce_space_unaligned,1651 CE_DESC_RING_ALIGN);1652 1653 return dest_ring;1654}1655 1656/*1657 * Initialize a Copy Engine based on caller-supplied attributes.1658 * This may be called once to initialize both source and destination1659 * rings or it may be called twice for separate source and destination1660 * initialization. It may be that only one side or the other is1661 * initialized by software/firmware.1662 */1663int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,1664 const struct ce_attr *attr)1665{1666 int ret;1667 1668 if (attr->src_nentries) {1669 ret = ath10k_ce_init_src_ring(ar, ce_id, attr);1670 if (ret) {1671 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",1672 ce_id, ret);1673 return ret;1674 }1675 }1676 1677 if (attr->dest_nentries) {1678 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);1679 if (ret) {1680 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",1681 ce_id, ret);1682 return ret;1683 }1684 }1685 1686 return 0;1687}1688EXPORT_SYMBOL(ath10k_ce_init_pipe);1689 1690static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)1691{1692 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);1693 1694 ath10k_ce_src_ring_base_addr_set(ar, ce_id, 0);1695 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);1696 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);1697 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);1698}1699 1700static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)1701{1702 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);1703 1704 ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 0);1705 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);1706 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);1707}1708 1709void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)1710{1711 ath10k_ce_deinit_src_ring(ar, ce_id);1712 ath10k_ce_deinit_dest_ring(ar, ce_id);1713}1714EXPORT_SYMBOL(ath10k_ce_deinit_pipe);1715 1716static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)1717{1718 struct ath10k_ce *ce = ath10k_ce_priv(ar);1719 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];1720 1721 if (ce_state->src_ring) {1722 if (ar->hw_params.shadow_reg_support)1723 kfree(ce_state->src_ring->shadow_base_unaligned);1724 dma_free_coherent(ar->dev,1725 (ce_state->src_ring->nentries *1726 sizeof(struct ce_desc) +1727 CE_DESC_RING_ALIGN),1728 ce_state->src_ring->base_addr_owner_space,1729 ce_state->src_ring->base_addr_ce_space);1730 kfree(ce_state->src_ring);1731 }1732 1733 if (ce_state->dest_ring) {1734 dma_free_coherent(ar->dev,1735 (ce_state->dest_ring->nentries *1736 sizeof(struct ce_desc) +1737 CE_DESC_RING_ALIGN),1738 ce_state->dest_ring->base_addr_owner_space,1739 ce_state->dest_ring->base_addr_ce_space);1740 kfree(ce_state->dest_ring);1741 }1742 1743 ce_state->src_ring = NULL;1744 ce_state->dest_ring = NULL;1745}1746 1747static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)1748{1749 struct ath10k_ce *ce = ath10k_ce_priv(ar);1750 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];1751 1752 if (ce_state->src_ring) {1753 if (ar->hw_params.shadow_reg_support)1754 kfree(ce_state->src_ring->shadow_base_unaligned);1755 dma_free_coherent(ar->dev,1756 (ce_state->src_ring->nentries *1757 sizeof(struct ce_desc_64) +1758 CE_DESC_RING_ALIGN),1759 ce_state->src_ring->base_addr_owner_space,1760 ce_state->src_ring->base_addr_ce_space);1761 kfree(ce_state->src_ring);1762 }1763 1764 if (ce_state->dest_ring) {1765 dma_free_coherent(ar->dev,1766 (ce_state->dest_ring->nentries *1767 sizeof(struct ce_desc_64) +1768 CE_DESC_RING_ALIGN),1769 ce_state->dest_ring->base_addr_owner_space,1770 ce_state->dest_ring->base_addr_ce_space);1771 kfree(ce_state->dest_ring);1772 }1773 1774 ce_state->src_ring = NULL;1775 ce_state->dest_ring = NULL;1776}1777 1778void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)1779{1780 struct ath10k_ce *ce = ath10k_ce_priv(ar);1781 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];1782 1783 ce_state->ops->ce_free_pipe(ar, ce_id);1784}1785EXPORT_SYMBOL(ath10k_ce_free_pipe);1786 1787void ath10k_ce_dump_registers(struct ath10k *ar,1788 struct ath10k_fw_crash_data *crash_data)1789{1790 struct ath10k_ce *ce = ath10k_ce_priv(ar);1791 struct ath10k_ce_crash_data ce_data;1792 u32 addr, id;1793 1794 lockdep_assert_held(&ar->dump_mutex);1795 1796 ath10k_err(ar, "Copy Engine register dump:\n");1797 1798 spin_lock_bh(&ce->ce_lock);1799 for (id = 0; id < CE_COUNT; id++) {1800 addr = ath10k_ce_base_address(ar, id);1801 ce_data.base_addr = cpu_to_le32(addr);1802 1803 ce_data.src_wr_idx =1804 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));1805 ce_data.src_r_idx =1806 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));1807 ce_data.dst_wr_idx =1808 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));1809 ce_data.dst_r_idx =1810 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));1811 1812 if (crash_data)1813 crash_data->ce_crash_data[id] = ce_data;1814 1815 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,1816 le32_to_cpu(ce_data.base_addr),1817 le32_to_cpu(ce_data.src_wr_idx),1818 le32_to_cpu(ce_data.src_r_idx),1819 le32_to_cpu(ce_data.dst_wr_idx),1820 le32_to_cpu(ce_data.dst_r_idx));1821 }1822 1823 spin_unlock_bh(&ce->ce_lock);1824}1825EXPORT_SYMBOL(ath10k_ce_dump_registers);1826 1827static const struct ath10k_ce_ops ce_ops = {1828 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring,1829 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,1830 .ce_rx_post_buf = __ath10k_ce_rx_post_buf,1831 .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,1832 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,1833 .ce_extract_desc_data = ath10k_ce_extract_desc_data,1834 .ce_free_pipe = _ath10k_ce_free_pipe,1835 .ce_send_nolock = _ath10k_ce_send_nolock,1836 .ce_set_src_ring_base_addr_hi = NULL,1837 .ce_set_dest_ring_base_addr_hi = NULL,1838 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock,1839};1840 1841static const struct ath10k_ce_ops ce_64_ops = {1842 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,1843 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,1844 .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,1845 .ce_completed_recv_next_nolock =1846 _ath10k_ce_completed_recv_next_nolock_64,1847 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,1848 .ce_extract_desc_data = ath10k_ce_extract_desc_data_64,1849 .ce_free_pipe = _ath10k_ce_free_pipe_64,1850 .ce_send_nolock = _ath10k_ce_send_nolock_64,1851 .ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi,1852 .ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi,1853 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock_64,1854};1855 1856static void ath10k_ce_set_ops(struct ath10k *ar,1857 struct ath10k_ce_pipe *ce_state)1858{1859 switch (ar->hw_rev) {1860 case ATH10K_HW_WCN3990:1861 ce_state->ops = &ce_64_ops;1862 break;1863 default:1864 ce_state->ops = &ce_ops;1865 break;1866 }1867}1868 1869int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,1870 const struct ce_attr *attr)1871{1872 struct ath10k_ce *ce = ath10k_ce_priv(ar);1873 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];1874 int ret;1875 1876 ath10k_ce_set_ops(ar, ce_state);1877 /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid1878 * additional TX locking checks.1879 *1880 * For the lack of a better place do the check here.1881 */1882 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >1883 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));1884 BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >1885 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));1886 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >1887 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));1888 1889 ce_state->ar = ar;1890 ce_state->id = ce_id;1891 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);1892 ce_state->attr_flags = attr->flags;1893 ce_state->src_sz_max = attr->src_sz_max;1894 1895 if (attr->src_nentries)1896 ce_state->send_cb = attr->send_cb;1897 1898 if (attr->dest_nentries)1899 ce_state->recv_cb = attr->recv_cb;1900 1901 if (attr->src_nentries) {1902 ce_state->src_ring =1903 ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);1904 if (IS_ERR(ce_state->src_ring)) {1905 ret = PTR_ERR(ce_state->src_ring);1906 ath10k_err(ar, "failed to alloc CE src ring %d: %d\n",1907 ce_id, ret);1908 ce_state->src_ring = NULL;1909 return ret;1910 }1911 }1912 1913 if (attr->dest_nentries) {1914 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,1915 ce_id,1916 attr);1917 if (IS_ERR(ce_state->dest_ring)) {1918 ret = PTR_ERR(ce_state->dest_ring);1919 ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n",1920 ce_id, ret);1921 ce_state->dest_ring = NULL;1922 return ret;1923 }1924 }1925 1926 return 0;1927}1928EXPORT_SYMBOL(ath10k_ce_alloc_pipe);1929 1930void ath10k_ce_alloc_rri(struct ath10k *ar)1931{1932 int i;1933 u32 value;1934 u32 ctrl1_regs;1935 u32 ce_base_addr;1936 struct ath10k_ce *ce = ath10k_ce_priv(ar);1937 1938 ce->vaddr_rri = dma_alloc_coherent(ar->dev,1939 (CE_COUNT * sizeof(u32)),1940 &ce->paddr_rri, GFP_KERNEL);1941 1942 if (!ce->vaddr_rri)1943 return;1944 1945 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low,1946 lower_32_bits(ce->paddr_rri));1947 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high,1948 (upper_32_bits(ce->paddr_rri) &1949 CE_DESC_ADDR_HI_MASK));1950 1951 for (i = 0; i < CE_COUNT; i++) {1952 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;1953 ce_base_addr = ath10k_ce_base_address(ar, i);1954 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs);1955 value |= ar->hw_ce_regs->upd->mask;1956 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value);1957 }1958}1959EXPORT_SYMBOL(ath10k_ce_alloc_rri);1960 1961void ath10k_ce_free_rri(struct ath10k *ar)1962{1963 struct ath10k_ce *ce = ath10k_ce_priv(ar);1964 1965 dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)),1966 ce->vaddr_rri,1967 ce->paddr_rri);1968}1969EXPORT_SYMBOL(ath10k_ce_free_rri);1970