1408 lines · c
1// SPDX-License-Identifier: BSD-3-Clause-Clear2/*3 * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.4 * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved.5 */6#include <linux/dma-mapping.h>7#include "hal_tx.h"8#include "debug.h"9#include "hal_desc.h"10#include "hif.h"11 12static const struct hal_srng_config hw_srng_config_template[] = {13 /* TODO: max_rings can populated by querying HW capabilities */14 { /* REO_DST */15 .start_ring_id = HAL_SRNG_RING_ID_REO2SW1,16 .max_rings = 4,17 .entry_size = sizeof(struct hal_reo_dest_ring) >> 2,18 .lmac_ring = false,19 .ring_dir = HAL_SRNG_DIR_DST,20 .max_size = HAL_REO_REO2SW1_RING_BASE_MSB_RING_SIZE,21 },22 { /* REO_EXCEPTION */23 /* Designating REO2TCL ring as exception ring. This ring is24 * similar to other REO2SW rings though it is named as REO2TCL.25 * Any of theREO2SW rings can be used as exception ring.26 */27 .start_ring_id = HAL_SRNG_RING_ID_REO2TCL,28 .max_rings = 1,29 .entry_size = sizeof(struct hal_reo_dest_ring) >> 2,30 .lmac_ring = false,31 .ring_dir = HAL_SRNG_DIR_DST,32 .max_size = HAL_REO_REO2TCL_RING_BASE_MSB_RING_SIZE,33 },34 { /* REO_REINJECT */35 .start_ring_id = HAL_SRNG_RING_ID_SW2REO,36 .max_rings = 1,37 .entry_size = sizeof(struct hal_reo_entrance_ring) >> 2,38 .lmac_ring = false,39 .ring_dir = HAL_SRNG_DIR_SRC,40 .max_size = HAL_REO_SW2REO_RING_BASE_MSB_RING_SIZE,41 },42 { /* REO_CMD */43 .start_ring_id = HAL_SRNG_RING_ID_REO_CMD,44 .max_rings = 1,45 .entry_size = (sizeof(struct hal_tlv_hdr) +46 sizeof(struct hal_reo_get_queue_stats)) >> 2,47 .lmac_ring = false,48 .ring_dir = HAL_SRNG_DIR_SRC,49 .max_size = HAL_REO_CMD_RING_BASE_MSB_RING_SIZE,50 },51 { /* REO_STATUS */52 .start_ring_id = HAL_SRNG_RING_ID_REO_STATUS,53 .max_rings = 1,54 .entry_size = (sizeof(struct hal_tlv_hdr) +55 sizeof(struct hal_reo_get_queue_stats_status)) >> 2,56 .lmac_ring = false,57 .ring_dir = HAL_SRNG_DIR_DST,58 .max_size = HAL_REO_STATUS_RING_BASE_MSB_RING_SIZE,59 },60 { /* TCL_DATA */61 .start_ring_id = HAL_SRNG_RING_ID_SW2TCL1,62 .max_rings = 3,63 .entry_size = (sizeof(struct hal_tlv_hdr) +64 sizeof(struct hal_tcl_data_cmd)) >> 2,65 .lmac_ring = false,66 .ring_dir = HAL_SRNG_DIR_SRC,67 .max_size = HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE,68 },69 { /* TCL_CMD */70 .start_ring_id = HAL_SRNG_RING_ID_SW2TCL_CMD,71 .max_rings = 1,72 .entry_size = (sizeof(struct hal_tlv_hdr) +73 sizeof(struct hal_tcl_gse_cmd)) >> 2,74 .lmac_ring = false,75 .ring_dir = HAL_SRNG_DIR_SRC,76 .max_size = HAL_SW2TCL1_CMD_RING_BASE_MSB_RING_SIZE,77 },78 { /* TCL_STATUS */79 .start_ring_id = HAL_SRNG_RING_ID_TCL_STATUS,80 .max_rings = 1,81 .entry_size = (sizeof(struct hal_tlv_hdr) +82 sizeof(struct hal_tcl_status_ring)) >> 2,83 .lmac_ring = false,84 .ring_dir = HAL_SRNG_DIR_DST,85 .max_size = HAL_TCL_STATUS_RING_BASE_MSB_RING_SIZE,86 },87 { /* CE_SRC */88 .start_ring_id = HAL_SRNG_RING_ID_CE0_SRC,89 .max_rings = 12,90 .entry_size = sizeof(struct hal_ce_srng_src_desc) >> 2,91 .lmac_ring = false,92 .ring_dir = HAL_SRNG_DIR_SRC,93 .max_size = HAL_CE_SRC_RING_BASE_MSB_RING_SIZE,94 },95 { /* CE_DST */96 .start_ring_id = HAL_SRNG_RING_ID_CE0_DST,97 .max_rings = 12,98 .entry_size = sizeof(struct hal_ce_srng_dest_desc) >> 2,99 .lmac_ring = false,100 .ring_dir = HAL_SRNG_DIR_SRC,101 .max_size = HAL_CE_DST_RING_BASE_MSB_RING_SIZE,102 },103 { /* CE_DST_STATUS */104 .start_ring_id = HAL_SRNG_RING_ID_CE0_DST_STATUS,105 .max_rings = 12,106 .entry_size = sizeof(struct hal_ce_srng_dst_status_desc) >> 2,107 .lmac_ring = false,108 .ring_dir = HAL_SRNG_DIR_DST,109 .max_size = HAL_CE_DST_STATUS_RING_BASE_MSB_RING_SIZE,110 },111 { /* WBM_IDLE_LINK */112 .start_ring_id = HAL_SRNG_RING_ID_WBM_IDLE_LINK,113 .max_rings = 1,114 .entry_size = sizeof(struct hal_wbm_link_desc) >> 2,115 .lmac_ring = false,116 .ring_dir = HAL_SRNG_DIR_SRC,117 .max_size = HAL_WBM_IDLE_LINK_RING_BASE_MSB_RING_SIZE,118 },119 { /* SW2WBM_RELEASE */120 .start_ring_id = HAL_SRNG_RING_ID_WBM_SW_RELEASE,121 .max_rings = 1,122 .entry_size = sizeof(struct hal_wbm_release_ring) >> 2,123 .lmac_ring = false,124 .ring_dir = HAL_SRNG_DIR_SRC,125 .max_size = HAL_SW2WBM_RELEASE_RING_BASE_MSB_RING_SIZE,126 },127 { /* WBM2SW_RELEASE */128 .start_ring_id = HAL_SRNG_RING_ID_WBM2SW0_RELEASE,129 .max_rings = 5,130 .entry_size = sizeof(struct hal_wbm_release_ring) >> 2,131 .lmac_ring = false,132 .ring_dir = HAL_SRNG_DIR_DST,133 .max_size = HAL_WBM2SW_RELEASE_RING_BASE_MSB_RING_SIZE,134 },135 { /* RXDMA_BUF */136 .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA0_BUF,137 .max_rings = 2,138 .entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,139 .lmac_ring = true,140 .ring_dir = HAL_SRNG_DIR_SRC,141 .max_size = HAL_RXDMA_RING_MAX_SIZE,142 },143 { /* RXDMA_DST */144 .start_ring_id = HAL_SRNG_RING_ID_WMAC1_RXDMA2SW0,145 .max_rings = 1,146 .entry_size = sizeof(struct hal_reo_entrance_ring) >> 2,147 .lmac_ring = true,148 .ring_dir = HAL_SRNG_DIR_DST,149 .max_size = HAL_RXDMA_RING_MAX_SIZE,150 },151 { /* RXDMA_MONITOR_BUF */152 .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA2_BUF,153 .max_rings = 1,154 .entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,155 .lmac_ring = true,156 .ring_dir = HAL_SRNG_DIR_SRC,157 .max_size = HAL_RXDMA_RING_MAX_SIZE,158 },159 { /* RXDMA_MONITOR_STATUS */160 .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA1_STATBUF,161 .max_rings = 1,162 .entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,163 .lmac_ring = true,164 .ring_dir = HAL_SRNG_DIR_SRC,165 .max_size = HAL_RXDMA_RING_MAX_SIZE,166 },167 { /* RXDMA_MONITOR_DST */168 .start_ring_id = HAL_SRNG_RING_ID_WMAC1_RXDMA2SW1,169 .max_rings = 1,170 .entry_size = sizeof(struct hal_reo_entrance_ring) >> 2,171 .lmac_ring = true,172 .ring_dir = HAL_SRNG_DIR_DST,173 .max_size = HAL_RXDMA_RING_MAX_SIZE,174 },175 { /* RXDMA_MONITOR_DESC */176 .start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA1_DESC,177 .max_rings = 1,178 .entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,179 .lmac_ring = true,180 .ring_dir = HAL_SRNG_DIR_SRC,181 .max_size = HAL_RXDMA_RING_MAX_SIZE,182 },183 { /* RXDMA DIR BUF */184 .start_ring_id = HAL_SRNG_RING_ID_RXDMA_DIR_BUF,185 .max_rings = 1,186 .entry_size = 8 >> 2, /* TODO: Define the struct */187 .lmac_ring = true,188 .ring_dir = HAL_SRNG_DIR_SRC,189 .max_size = HAL_RXDMA_RING_MAX_SIZE,190 },191};192 193static int ath11k_hal_alloc_cont_rdp(struct ath11k_base *ab)194{195 struct ath11k_hal *hal = &ab->hal;196 size_t size;197 198 size = sizeof(u32) * HAL_SRNG_RING_ID_MAX;199 hal->rdp.vaddr = dma_alloc_coherent(ab->dev, size, &hal->rdp.paddr,200 GFP_KERNEL);201 if (!hal->rdp.vaddr)202 return -ENOMEM;203 204 return 0;205}206 207static void ath11k_hal_free_cont_rdp(struct ath11k_base *ab)208{209 struct ath11k_hal *hal = &ab->hal;210 size_t size;211 212 if (!hal->rdp.vaddr)213 return;214 215 size = sizeof(u32) * HAL_SRNG_RING_ID_MAX;216 dma_free_coherent(ab->dev, size,217 hal->rdp.vaddr, hal->rdp.paddr);218 hal->rdp.vaddr = NULL;219}220 221static int ath11k_hal_alloc_cont_wrp(struct ath11k_base *ab)222{223 struct ath11k_hal *hal = &ab->hal;224 size_t size;225 226 size = sizeof(u32) * HAL_SRNG_NUM_LMAC_RINGS;227 hal->wrp.vaddr = dma_alloc_coherent(ab->dev, size, &hal->wrp.paddr,228 GFP_KERNEL);229 if (!hal->wrp.vaddr)230 return -ENOMEM;231 232 return 0;233}234 235static void ath11k_hal_free_cont_wrp(struct ath11k_base *ab)236{237 struct ath11k_hal *hal = &ab->hal;238 size_t size;239 240 if (!hal->wrp.vaddr)241 return;242 243 size = sizeof(u32) * HAL_SRNG_NUM_LMAC_RINGS;244 dma_free_coherent(ab->dev, size,245 hal->wrp.vaddr, hal->wrp.paddr);246 hal->wrp.vaddr = NULL;247}248 249static void ath11k_hal_ce_dst_setup(struct ath11k_base *ab,250 struct hal_srng *srng, int ring_num)251{252 struct hal_srng_config *srng_config = &ab->hal.srng_config[HAL_CE_DST];253 u32 addr;254 u32 val;255 256 addr = HAL_CE_DST_RING_CTRL +257 srng_config->reg_start[HAL_SRNG_REG_GRP_R0] +258 ring_num * srng_config->reg_size[HAL_SRNG_REG_GRP_R0];259 260 val = ath11k_hif_read32(ab, addr);261 val &= ~HAL_CE_DST_R0_DEST_CTRL_MAX_LEN;262 val |= FIELD_PREP(HAL_CE_DST_R0_DEST_CTRL_MAX_LEN,263 srng->u.dst_ring.max_buffer_length);264 ath11k_hif_write32(ab, addr, val);265}266 267static void ath11k_hal_srng_dst_hw_init(struct ath11k_base *ab,268 struct hal_srng *srng)269{270 struct ath11k_hal *hal = &ab->hal;271 u32 val;272 u64 hp_addr;273 u32 reg_base;274 275 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];276 277 if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) {278 ath11k_hif_write32(ab, reg_base +279 HAL_REO1_RING_MSI1_BASE_LSB_OFFSET(ab),280 srng->msi_addr);281 282 val = FIELD_PREP(HAL_REO1_RING_MSI1_BASE_MSB_ADDR,283 ((u64)srng->msi_addr >>284 HAL_ADDR_MSB_REG_SHIFT)) |285 HAL_REO1_RING_MSI1_BASE_MSB_MSI1_ENABLE;286 ath11k_hif_write32(ab, reg_base +287 HAL_REO1_RING_MSI1_BASE_MSB_OFFSET(ab), val);288 289 ath11k_hif_write32(ab,290 reg_base + HAL_REO1_RING_MSI1_DATA_OFFSET(ab),291 srng->msi_data);292 }293 294 ath11k_hif_write32(ab, reg_base, srng->ring_base_paddr);295 296 val = FIELD_PREP(HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB,297 ((u64)srng->ring_base_paddr >>298 HAL_ADDR_MSB_REG_SHIFT)) |299 FIELD_PREP(HAL_REO1_RING_BASE_MSB_RING_SIZE,300 (srng->entry_size * srng->num_entries));301 ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_BASE_MSB_OFFSET(ab), val);302 303 val = FIELD_PREP(HAL_REO1_RING_ID_RING_ID, srng->ring_id) |304 FIELD_PREP(HAL_REO1_RING_ID_ENTRY_SIZE, srng->entry_size);305 ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_ID_OFFSET(ab), val);306 307 /* interrupt setup */308 val = FIELD_PREP(HAL_REO1_RING_PRDR_INT_SETUP_INTR_TMR_THOLD,309 (srng->intr_timer_thres_us >> 3));310 311 val |= FIELD_PREP(HAL_REO1_RING_PRDR_INT_SETUP_BATCH_COUNTER_THOLD,312 (srng->intr_batch_cntr_thres_entries *313 srng->entry_size));314 315 ath11k_hif_write32(ab,316 reg_base + HAL_REO1_RING_PRODUCER_INT_SETUP_OFFSET(ab),317 val);318 319 hp_addr = hal->rdp.paddr +320 ((unsigned long)srng->u.dst_ring.hp_addr -321 (unsigned long)hal->rdp.vaddr);322 ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_HP_ADDR_LSB_OFFSET(ab),323 hp_addr & HAL_ADDR_LSB_REG_MASK);324 ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_HP_ADDR_MSB_OFFSET(ab),325 hp_addr >> HAL_ADDR_MSB_REG_SHIFT);326 327 /* Initialize head and tail pointers to indicate ring is empty */328 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];329 ath11k_hif_write32(ab, reg_base, 0);330 ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_TP_OFFSET(ab), 0);331 *srng->u.dst_ring.hp_addr = 0;332 333 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];334 val = 0;335 if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP)336 val |= HAL_REO1_RING_MISC_DATA_TLV_SWAP;337 if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP)338 val |= HAL_REO1_RING_MISC_HOST_FW_SWAP;339 if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP)340 val |= HAL_REO1_RING_MISC_MSI_SWAP;341 val |= HAL_REO1_RING_MISC_SRNG_ENABLE;342 343 ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_MISC_OFFSET(ab), val);344}345 346static void ath11k_hal_srng_src_hw_init(struct ath11k_base *ab,347 struct hal_srng *srng)348{349 struct ath11k_hal *hal = &ab->hal;350 u32 val;351 u64 tp_addr;352 u32 reg_base;353 354 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];355 356 if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) {357 ath11k_hif_write32(ab, reg_base +358 HAL_TCL1_RING_MSI1_BASE_LSB_OFFSET(ab),359 srng->msi_addr);360 361 val = FIELD_PREP(HAL_TCL1_RING_MSI1_BASE_MSB_ADDR,362 ((u64)srng->msi_addr >>363 HAL_ADDR_MSB_REG_SHIFT)) |364 HAL_TCL1_RING_MSI1_BASE_MSB_MSI1_ENABLE;365 ath11k_hif_write32(ab, reg_base +366 HAL_TCL1_RING_MSI1_BASE_MSB_OFFSET(ab),367 val);368 369 ath11k_hif_write32(ab, reg_base +370 HAL_TCL1_RING_MSI1_DATA_OFFSET(ab),371 srng->msi_data);372 }373 374 ath11k_hif_write32(ab, reg_base, srng->ring_base_paddr);375 376 val = FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB,377 ((u64)srng->ring_base_paddr >>378 HAL_ADDR_MSB_REG_SHIFT)) |379 FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_SIZE,380 (srng->entry_size * srng->num_entries));381 ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_BASE_MSB_OFFSET(ab), val);382 383 val = FIELD_PREP(HAL_REO1_RING_ID_ENTRY_SIZE, srng->entry_size);384 ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_ID_OFFSET(ab), val);385 386 if (srng->ring_id == HAL_SRNG_RING_ID_WBM_IDLE_LINK) {387 ath11k_hif_write32(ab, reg_base, (u32)srng->ring_base_paddr);388 val = FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB,389 ((u64)srng->ring_base_paddr >>390 HAL_ADDR_MSB_REG_SHIFT)) |391 FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_SIZE,392 (srng->entry_size * srng->num_entries));393 ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_BASE_MSB_OFFSET(ab), val);394 }395 396 /* interrupt setup */397 /* NOTE: IPQ8074 v2 requires the interrupt timer threshold in the398 * unit of 8 usecs instead of 1 usec (as required by v1).399 */400 val = FIELD_PREP(HAL_TCL1_RING_CONSR_INT_SETUP_IX0_INTR_TMR_THOLD,401 srng->intr_timer_thres_us);402 403 val |= FIELD_PREP(HAL_TCL1_RING_CONSR_INT_SETUP_IX0_BATCH_COUNTER_THOLD,404 (srng->intr_batch_cntr_thres_entries *405 srng->entry_size));406 407 ath11k_hif_write32(ab,408 reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX0_OFFSET(ab),409 val);410 411 val = 0;412 if (srng->flags & HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN) {413 val |= FIELD_PREP(HAL_TCL1_RING_CONSR_INT_SETUP_IX1_LOW_THOLD,414 srng->u.src_ring.low_threshold);415 }416 ath11k_hif_write32(ab,417 reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX1_OFFSET(ab),418 val);419 420 if (srng->ring_id != HAL_SRNG_RING_ID_WBM_IDLE_LINK) {421 tp_addr = hal->rdp.paddr +422 ((unsigned long)srng->u.src_ring.tp_addr -423 (unsigned long)hal->rdp.vaddr);424 ath11k_hif_write32(ab,425 reg_base + HAL_TCL1_RING_TP_ADDR_LSB_OFFSET(ab),426 tp_addr & HAL_ADDR_LSB_REG_MASK);427 ath11k_hif_write32(ab,428 reg_base + HAL_TCL1_RING_TP_ADDR_MSB_OFFSET(ab),429 tp_addr >> HAL_ADDR_MSB_REG_SHIFT);430 }431 432 /* Initialize head and tail pointers to indicate ring is empty */433 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];434 ath11k_hif_write32(ab, reg_base, 0);435 ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_TP_OFFSET, 0);436 *srng->u.src_ring.tp_addr = 0;437 438 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];439 val = 0;440 if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP)441 val |= HAL_TCL1_RING_MISC_DATA_TLV_SWAP;442 if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP)443 val |= HAL_TCL1_RING_MISC_HOST_FW_SWAP;444 if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP)445 val |= HAL_TCL1_RING_MISC_MSI_SWAP;446 447 /* Loop count is not used for SRC rings */448 val |= HAL_TCL1_RING_MISC_MSI_LOOPCNT_DISABLE;449 450 val |= HAL_TCL1_RING_MISC_SRNG_ENABLE;451 452 ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_MISC_OFFSET(ab), val);453}454 455static void ath11k_hal_srng_hw_init(struct ath11k_base *ab,456 struct hal_srng *srng)457{458 if (srng->ring_dir == HAL_SRNG_DIR_SRC)459 ath11k_hal_srng_src_hw_init(ab, srng);460 else461 ath11k_hal_srng_dst_hw_init(ab, srng);462}463 464static int ath11k_hal_srng_get_ring_id(struct ath11k_base *ab,465 enum hal_ring_type type,466 int ring_num, int mac_id)467{468 struct hal_srng_config *srng_config = &ab->hal.srng_config[type];469 int ring_id;470 471 if (ring_num >= srng_config->max_rings) {472 ath11k_warn(ab, "invalid ring number :%d\n", ring_num);473 return -EINVAL;474 }475 476 ring_id = srng_config->start_ring_id + ring_num;477 if (srng_config->lmac_ring)478 ring_id += mac_id * HAL_SRNG_RINGS_PER_LMAC;479 480 if (WARN_ON(ring_id >= HAL_SRNG_RING_ID_MAX))481 return -EINVAL;482 483 return ring_id;484}485 486int ath11k_hal_srng_get_entrysize(struct ath11k_base *ab, u32 ring_type)487{488 struct hal_srng_config *srng_config;489 490 if (WARN_ON(ring_type >= HAL_MAX_RING_TYPES))491 return -EINVAL;492 493 srng_config = &ab->hal.srng_config[ring_type];494 495 return (srng_config->entry_size << 2);496}497 498int ath11k_hal_srng_get_max_entries(struct ath11k_base *ab, u32 ring_type)499{500 struct hal_srng_config *srng_config;501 502 if (WARN_ON(ring_type >= HAL_MAX_RING_TYPES))503 return -EINVAL;504 505 srng_config = &ab->hal.srng_config[ring_type];506 507 return (srng_config->max_size / srng_config->entry_size);508}509 510void ath11k_hal_srng_get_params(struct ath11k_base *ab, struct hal_srng *srng,511 struct hal_srng_params *params)512{513 params->ring_base_paddr = srng->ring_base_paddr;514 params->ring_base_vaddr = srng->ring_base_vaddr;515 params->num_entries = srng->num_entries;516 params->intr_timer_thres_us = srng->intr_timer_thres_us;517 params->intr_batch_cntr_thres_entries =518 srng->intr_batch_cntr_thres_entries;519 params->low_threshold = srng->u.src_ring.low_threshold;520 params->msi_addr = srng->msi_addr;521 params->msi_data = srng->msi_data;522 params->flags = srng->flags;523}524 525dma_addr_t ath11k_hal_srng_get_hp_addr(struct ath11k_base *ab,526 struct hal_srng *srng)527{528 if (!(srng->flags & HAL_SRNG_FLAGS_LMAC_RING))529 return 0;530 531 if (srng->ring_dir == HAL_SRNG_DIR_SRC)532 return ab->hal.wrp.paddr +533 ((unsigned long)srng->u.src_ring.hp_addr -534 (unsigned long)ab->hal.wrp.vaddr);535 else536 return ab->hal.rdp.paddr +537 ((unsigned long)srng->u.dst_ring.hp_addr -538 (unsigned long)ab->hal.rdp.vaddr);539}540 541dma_addr_t ath11k_hal_srng_get_tp_addr(struct ath11k_base *ab,542 struct hal_srng *srng)543{544 if (!(srng->flags & HAL_SRNG_FLAGS_LMAC_RING))545 return 0;546 547 if (srng->ring_dir == HAL_SRNG_DIR_SRC)548 return ab->hal.rdp.paddr +549 ((unsigned long)srng->u.src_ring.tp_addr -550 (unsigned long)ab->hal.rdp.vaddr);551 else552 return ab->hal.wrp.paddr +553 ((unsigned long)srng->u.dst_ring.tp_addr -554 (unsigned long)ab->hal.wrp.vaddr);555}556 557u32 ath11k_hal_ce_get_desc_size(enum hal_ce_desc type)558{559 switch (type) {560 case HAL_CE_DESC_SRC:561 return sizeof(struct hal_ce_srng_src_desc);562 case HAL_CE_DESC_DST:563 return sizeof(struct hal_ce_srng_dest_desc);564 case HAL_CE_DESC_DST_STATUS:565 return sizeof(struct hal_ce_srng_dst_status_desc);566 }567 568 return 0;569}570 571void ath11k_hal_ce_src_set_desc(void *buf, dma_addr_t paddr, u32 len, u32 id,572 u8 byte_swap_data)573{574 struct hal_ce_srng_src_desc *desc = buf;575 576 desc->buffer_addr_low = paddr & HAL_ADDR_LSB_REG_MASK;577 desc->buffer_addr_info =578 FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_ADDR_HI,579 ((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT)) |580 FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_BYTE_SWAP,581 byte_swap_data) |582 FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_GATHER, 0) |583 FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_LEN, len);584 desc->meta_info = FIELD_PREP(HAL_CE_SRC_DESC_META_INFO_DATA, id);585}586 587void ath11k_hal_ce_dst_set_desc(void *buf, dma_addr_t paddr)588{589 struct hal_ce_srng_dest_desc *desc = buf;590 591 desc->buffer_addr_low = paddr & HAL_ADDR_LSB_REG_MASK;592 desc->buffer_addr_info =593 FIELD_PREP(HAL_CE_DEST_DESC_ADDR_INFO_ADDR_HI,594 ((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT));595}596 597u32 ath11k_hal_ce_dst_status_get_length(void *buf)598{599 struct hal_ce_srng_dst_status_desc *desc = buf;600 u32 len;601 602 len = FIELD_GET(HAL_CE_DST_STATUS_DESC_FLAGS_LEN, desc->flags);603 desc->flags &= ~HAL_CE_DST_STATUS_DESC_FLAGS_LEN;604 605 return len;606}607 608void ath11k_hal_set_link_desc_addr(struct hal_wbm_link_desc *desc, u32 cookie,609 dma_addr_t paddr)610{611 desc->buf_addr_info.info0 = FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,612 (paddr & HAL_ADDR_LSB_REG_MASK));613 desc->buf_addr_info.info1 = FIELD_PREP(BUFFER_ADDR_INFO1_ADDR,614 ((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT)) |615 FIELD_PREP(BUFFER_ADDR_INFO1_RET_BUF_MGR, 1) |616 FIELD_PREP(BUFFER_ADDR_INFO1_SW_COOKIE, cookie);617}618 619u32 *ath11k_hal_srng_dst_peek(struct ath11k_base *ab, struct hal_srng *srng)620{621 lockdep_assert_held(&srng->lock);622 623 if (srng->u.dst_ring.tp != srng->u.dst_ring.cached_hp)624 return (srng->ring_base_vaddr + srng->u.dst_ring.tp);625 626 return NULL;627}628 629static u32 *ath11k_hal_srng_dst_peek_with_dma(struct ath11k_base *ab,630 struct hal_srng *srng, dma_addr_t *paddr)631{632 lockdep_assert_held(&srng->lock);633 634 if (srng->u.dst_ring.tp != srng->u.dst_ring.cached_hp) {635 *paddr = srng->ring_base_paddr +636 sizeof(*srng->ring_base_vaddr) * srng->u.dst_ring.tp;637 return srng->ring_base_vaddr + srng->u.dst_ring.tp;638 }639 640 return NULL;641}642 643static void ath11k_hal_srng_prefetch_desc(struct ath11k_base *ab,644 struct hal_srng *srng)645{646 dma_addr_t desc_paddr;647 u32 *desc;648 649 /* prefetch only if desc is available */650 desc = ath11k_hal_srng_dst_peek_with_dma(ab, srng, &desc_paddr);651 if (likely(desc)) {652 dma_sync_single_for_cpu(ab->dev, desc_paddr,653 (srng->entry_size * sizeof(u32)),654 DMA_FROM_DEVICE);655 prefetch(desc);656 }657}658 659u32 *ath11k_hal_srng_dst_get_next_entry(struct ath11k_base *ab,660 struct hal_srng *srng)661{662 u32 *desc;663 664 lockdep_assert_held(&srng->lock);665 666 if (srng->u.dst_ring.tp == srng->u.dst_ring.cached_hp)667 return NULL;668 669 desc = srng->ring_base_vaddr + srng->u.dst_ring.tp;670 671 srng->u.dst_ring.tp += srng->entry_size;672 673 /* wrap around to start of ring*/674 if (srng->u.dst_ring.tp == srng->ring_size)675 srng->u.dst_ring.tp = 0;676 677 /* Try to prefetch the next descriptor in the ring */678 if (srng->flags & HAL_SRNG_FLAGS_CACHED)679 ath11k_hal_srng_prefetch_desc(ab, srng);680 681 return desc;682}683 684int ath11k_hal_srng_dst_num_free(struct ath11k_base *ab, struct hal_srng *srng,685 bool sync_hw_ptr)686{687 u32 tp, hp;688 689 lockdep_assert_held(&srng->lock);690 691 tp = srng->u.dst_ring.tp;692 693 if (sync_hw_ptr) {694 hp = *srng->u.dst_ring.hp_addr;695 srng->u.dst_ring.cached_hp = hp;696 } else {697 hp = srng->u.dst_ring.cached_hp;698 }699 700 if (hp >= tp)701 return (hp - tp) / srng->entry_size;702 else703 return (srng->ring_size - tp + hp) / srng->entry_size;704}705 706/* Returns number of available entries in src ring */707int ath11k_hal_srng_src_num_free(struct ath11k_base *ab, struct hal_srng *srng,708 bool sync_hw_ptr)709{710 u32 tp, hp;711 712 lockdep_assert_held(&srng->lock);713 714 hp = srng->u.src_ring.hp;715 716 if (sync_hw_ptr) {717 tp = *srng->u.src_ring.tp_addr;718 srng->u.src_ring.cached_tp = tp;719 } else {720 tp = srng->u.src_ring.cached_tp;721 }722 723 if (tp > hp)724 return ((tp - hp) / srng->entry_size) - 1;725 else726 return ((srng->ring_size - hp + tp) / srng->entry_size) - 1;727}728 729u32 *ath11k_hal_srng_src_get_next_entry(struct ath11k_base *ab,730 struct hal_srng *srng)731{732 u32 *desc;733 u32 next_hp;734 735 lockdep_assert_held(&srng->lock);736 737 /* TODO: Using % is expensive, but we have to do this since size of some738 * SRNG rings is not power of 2 (due to descriptor sizes). Need to see739 * if separate function is defined for rings having power of 2 ring size740 * (TCL2SW, REO2SW, SW2RXDMA and CE rings) so that we can avoid the741 * overhead of % by using mask (with &).742 */743 next_hp = (srng->u.src_ring.hp + srng->entry_size) % srng->ring_size;744 745 if (next_hp == srng->u.src_ring.cached_tp)746 return NULL;747 748 desc = srng->ring_base_vaddr + srng->u.src_ring.hp;749 srng->u.src_ring.hp = next_hp;750 751 /* TODO: Reap functionality is not used by all rings. If particular752 * ring does not use reap functionality, we need not update reap_hp753 * with next_hp pointer. Need to make sure a separate function is used754 * before doing any optimization by removing below code updating755 * reap_hp.756 */757 srng->u.src_ring.reap_hp = next_hp;758 759 return desc;760}761 762u32 *ath11k_hal_srng_src_reap_next(struct ath11k_base *ab,763 struct hal_srng *srng)764{765 u32 *desc;766 u32 next_reap_hp;767 768 lockdep_assert_held(&srng->lock);769 770 next_reap_hp = (srng->u.src_ring.reap_hp + srng->entry_size) %771 srng->ring_size;772 773 if (next_reap_hp == srng->u.src_ring.cached_tp)774 return NULL;775 776 desc = srng->ring_base_vaddr + next_reap_hp;777 srng->u.src_ring.reap_hp = next_reap_hp;778 779 return desc;780}781 782u32 *ath11k_hal_srng_src_get_next_reaped(struct ath11k_base *ab,783 struct hal_srng *srng)784{785 u32 *desc;786 787 lockdep_assert_held(&srng->lock);788 789 if (srng->u.src_ring.hp == srng->u.src_ring.reap_hp)790 return NULL;791 792 desc = srng->ring_base_vaddr + srng->u.src_ring.hp;793 srng->u.src_ring.hp = (srng->u.src_ring.hp + srng->entry_size) %794 srng->ring_size;795 796 return desc;797}798 799u32 *ath11k_hal_srng_src_next_peek(struct ath11k_base *ab, struct hal_srng *srng)800{801 u32 next_hp;802 803 lockdep_assert_held(&srng->lock);804 805 next_hp = (srng->u.src_ring.hp + srng->entry_size) % srng->ring_size;806 807 if (next_hp != srng->u.src_ring.cached_tp)808 return srng->ring_base_vaddr + next_hp;809 810 return NULL;811}812 813u32 *ath11k_hal_srng_src_peek(struct ath11k_base *ab, struct hal_srng *srng)814{815 lockdep_assert_held(&srng->lock);816 817 if (((srng->u.src_ring.hp + srng->entry_size) % srng->ring_size) ==818 srng->u.src_ring.cached_tp)819 return NULL;820 821 return srng->ring_base_vaddr + srng->u.src_ring.hp;822}823 824void ath11k_hal_srng_access_begin(struct ath11k_base *ab, struct hal_srng *srng)825{826 lockdep_assert_held(&srng->lock);827 828 if (srng->ring_dir == HAL_SRNG_DIR_SRC) {829 srng->u.src_ring.cached_tp =830 *(volatile u32 *)srng->u.src_ring.tp_addr;831 } else {832 srng->u.dst_ring.cached_hp = *srng->u.dst_ring.hp_addr;833 834 /* Try to prefetch the next descriptor in the ring */835 if (srng->flags & HAL_SRNG_FLAGS_CACHED)836 ath11k_hal_srng_prefetch_desc(ab, srng);837 }838}839 840/* Update cached ring head/tail pointers to HW. ath11k_hal_srng_access_begin()841 * should have been called before this.842 */843void ath11k_hal_srng_access_end(struct ath11k_base *ab, struct hal_srng *srng)844{845 lockdep_assert_held(&srng->lock);846 847 /* TODO: See if we need a write memory barrier here */848 if (srng->flags & HAL_SRNG_FLAGS_LMAC_RING) {849 /* For LMAC rings, ring pointer updates are done through FW and850 * hence written to a shared memory location that is read by FW851 */852 if (srng->ring_dir == HAL_SRNG_DIR_SRC) {853 srng->u.src_ring.last_tp =854 *(volatile u32 *)srng->u.src_ring.tp_addr;855 *srng->u.src_ring.hp_addr = srng->u.src_ring.hp;856 } else {857 srng->u.dst_ring.last_hp = *srng->u.dst_ring.hp_addr;858 *srng->u.dst_ring.tp_addr = srng->u.dst_ring.tp;859 }860 } else {861 if (srng->ring_dir == HAL_SRNG_DIR_SRC) {862 srng->u.src_ring.last_tp =863 *(volatile u32 *)srng->u.src_ring.tp_addr;864 ath11k_hif_write32(ab,865 (unsigned long)srng->u.src_ring.hp_addr -866 (unsigned long)ab->mem,867 srng->u.src_ring.hp);868 } else {869 srng->u.dst_ring.last_hp = *srng->u.dst_ring.hp_addr;870 ath11k_hif_write32(ab,871 (unsigned long)srng->u.dst_ring.tp_addr -872 (unsigned long)ab->mem,873 srng->u.dst_ring.tp);874 }875 }876 877 srng->timestamp = jiffies;878}879 880void ath11k_hal_setup_link_idle_list(struct ath11k_base *ab,881 struct hal_wbm_idle_scatter_list *sbuf,882 u32 nsbufs, u32 tot_link_desc,883 u32 end_offset)884{885 struct ath11k_buffer_addr *link_addr;886 int i;887 u32 reg_scatter_buf_sz = HAL_WBM_IDLE_SCATTER_BUF_SIZE / 64;888 889 link_addr = (void *)sbuf[0].vaddr + HAL_WBM_IDLE_SCATTER_BUF_SIZE;890 891 for (i = 1; i < nsbufs; i++) {892 link_addr->info0 = sbuf[i].paddr & HAL_ADDR_LSB_REG_MASK;893 link_addr->info1 = FIELD_PREP(894 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32,895 (u64)sbuf[i].paddr >> HAL_ADDR_MSB_REG_SHIFT) |896 FIELD_PREP(897 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG,898 BASE_ADDR_MATCH_TAG_VAL);899 900 link_addr = (void *)sbuf[i].vaddr +901 HAL_WBM_IDLE_SCATTER_BUF_SIZE;902 }903 904 ath11k_hif_write32(ab,905 HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_R0_IDLE_LIST_CONTROL_ADDR,906 FIELD_PREP(HAL_WBM_SCATTER_BUFFER_SIZE, reg_scatter_buf_sz) |907 FIELD_PREP(HAL_WBM_LINK_DESC_IDLE_LIST_MODE, 0x1));908 ath11k_hif_write32(ab,909 HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_R0_IDLE_LIST_SIZE_ADDR,910 FIELD_PREP(HAL_WBM_SCATTER_RING_SIZE_OF_IDLE_LINK_DESC_LIST,911 reg_scatter_buf_sz * nsbufs));912 ath11k_hif_write32(ab,913 HAL_SEQ_WCSS_UMAC_WBM_REG +914 HAL_WBM_SCATTERED_RING_BASE_LSB,915 FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,916 sbuf[0].paddr & HAL_ADDR_LSB_REG_MASK));917 ath11k_hif_write32(ab,918 HAL_SEQ_WCSS_UMAC_WBM_REG +919 HAL_WBM_SCATTERED_RING_BASE_MSB,920 FIELD_PREP(921 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32,922 (u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT) |923 FIELD_PREP(924 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG,925 BASE_ADDR_MATCH_TAG_VAL));926 927 /* Setup head and tail pointers for the idle list */928 ath11k_hif_write32(ab,929 HAL_SEQ_WCSS_UMAC_WBM_REG +930 HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0,931 FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,932 sbuf[nsbufs - 1].paddr));933 ath11k_hif_write32(ab,934 HAL_SEQ_WCSS_UMAC_WBM_REG +935 HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX1,936 FIELD_PREP(937 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32,938 ((u64)sbuf[nsbufs - 1].paddr >>939 HAL_ADDR_MSB_REG_SHIFT)) |940 FIELD_PREP(HAL_WBM_SCATTERED_DESC_HEAD_P_OFFSET_IX1,941 (end_offset >> 2)));942 ath11k_hif_write32(ab,943 HAL_SEQ_WCSS_UMAC_WBM_REG +944 HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0,945 FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,946 sbuf[0].paddr));947 948 ath11k_hif_write32(ab,949 HAL_SEQ_WCSS_UMAC_WBM_REG +950 HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX0,951 FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,952 sbuf[0].paddr));953 ath11k_hif_write32(ab,954 HAL_SEQ_WCSS_UMAC_WBM_REG +955 HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX1,956 FIELD_PREP(957 HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32,958 ((u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT)) |959 FIELD_PREP(HAL_WBM_SCATTERED_DESC_TAIL_P_OFFSET_IX1,960 0));961 ath11k_hif_write32(ab,962 HAL_SEQ_WCSS_UMAC_WBM_REG +963 HAL_WBM_SCATTERED_DESC_PTR_HP_ADDR,964 2 * tot_link_desc);965 966 /* Enable the SRNG */967 ath11k_hif_write32(ab,968 HAL_SEQ_WCSS_UMAC_WBM_REG +969 HAL_WBM_IDLE_LINK_RING_MISC_ADDR(ab), 0x40);970}971 972int ath11k_hal_srng_setup(struct ath11k_base *ab, enum hal_ring_type type,973 int ring_num, int mac_id,974 struct hal_srng_params *params)975{976 struct ath11k_hal *hal = &ab->hal;977 struct hal_srng_config *srng_config = &ab->hal.srng_config[type];978 struct hal_srng *srng;979 int ring_id;980 u32 lmac_idx;981 int i;982 u32 reg_base;983 984 ring_id = ath11k_hal_srng_get_ring_id(ab, type, ring_num, mac_id);985 if (ring_id < 0)986 return ring_id;987 988 srng = &hal->srng_list[ring_id];989 990 srng->ring_id = ring_id;991 srng->ring_dir = srng_config->ring_dir;992 srng->ring_base_paddr = params->ring_base_paddr;993 srng->ring_base_vaddr = params->ring_base_vaddr;994 srng->entry_size = srng_config->entry_size;995 srng->num_entries = params->num_entries;996 srng->ring_size = srng->entry_size * srng->num_entries;997 srng->intr_batch_cntr_thres_entries =998 params->intr_batch_cntr_thres_entries;999 srng->intr_timer_thres_us = params->intr_timer_thres_us;1000 srng->flags = params->flags;1001 srng->msi_addr = params->msi_addr;1002 srng->msi_data = params->msi_data;1003 srng->initialized = 1;1004 spin_lock_init(&srng->lock);1005 lockdep_set_class(&srng->lock, hal->srng_key + ring_id);1006 1007 for (i = 0; i < HAL_SRNG_NUM_REG_GRP; i++) {1008 srng->hwreg_base[i] = srng_config->reg_start[i] +1009 (ring_num * srng_config->reg_size[i]);1010 }1011 1012 memset(srng->ring_base_vaddr, 0,1013 (srng->entry_size * srng->num_entries) << 2);1014 1015 /* TODO: Add comments on these swap configurations */1016 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))1017 srng->flags |= HAL_SRNG_FLAGS_MSI_SWAP | HAL_SRNG_FLAGS_DATA_TLV_SWAP |1018 HAL_SRNG_FLAGS_RING_PTR_SWAP;1019 1020 reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];1021 1022 if (srng->ring_dir == HAL_SRNG_DIR_SRC) {1023 srng->u.src_ring.hp = 0;1024 srng->u.src_ring.cached_tp = 0;1025 srng->u.src_ring.reap_hp = srng->ring_size - srng->entry_size;1026 srng->u.src_ring.tp_addr = (void *)(hal->rdp.vaddr + ring_id);1027 srng->u.src_ring.low_threshold = params->low_threshold *1028 srng->entry_size;1029 if (srng_config->lmac_ring) {1030 lmac_idx = ring_id - HAL_SRNG_RING_ID_LMAC1_ID_START;1031 srng->u.src_ring.hp_addr = (void *)(hal->wrp.vaddr +1032 lmac_idx);1033 srng->flags |= HAL_SRNG_FLAGS_LMAC_RING;1034 } else {1035 if (!ab->hw_params.supports_shadow_regs)1036 srng->u.src_ring.hp_addr =1037 (u32 *)((unsigned long)ab->mem + reg_base);1038 else1039 ath11k_dbg(ab, ATH11K_DBG_HAL,1040 "type %d ring_num %d reg_base 0x%x shadow 0x%lx\n",1041 type, ring_num,1042 reg_base,1043 (unsigned long)srng->u.src_ring.hp_addr -1044 (unsigned long)ab->mem);1045 }1046 } else {1047 /* During initialization loop count in all the descriptors1048 * will be set to zero, and HW will set it to 1 on completing1049 * descriptor update in first loop, and increments it by 1 on1050 * subsequent loops (loop count wraps around after reaching1051 * 0xffff). The 'loop_cnt' in SW ring state is the expected1052 * loop count in descriptors updated by HW (to be processed1053 * by SW).1054 */1055 srng->u.dst_ring.loop_cnt = 1;1056 srng->u.dst_ring.tp = 0;1057 srng->u.dst_ring.cached_hp = 0;1058 srng->u.dst_ring.hp_addr = (void *)(hal->rdp.vaddr + ring_id);1059 if (srng_config->lmac_ring) {1060 /* For LMAC rings, tail pointer updates will be done1061 * through FW by writing to a shared memory location1062 */1063 lmac_idx = ring_id - HAL_SRNG_RING_ID_LMAC1_ID_START;1064 srng->u.dst_ring.tp_addr = (void *)(hal->wrp.vaddr +1065 lmac_idx);1066 srng->flags |= HAL_SRNG_FLAGS_LMAC_RING;1067 } else {1068 if (!ab->hw_params.supports_shadow_regs)1069 srng->u.dst_ring.tp_addr =1070 (u32 *)((unsigned long)ab->mem + reg_base +1071 (HAL_REO1_RING_TP(ab) - HAL_REO1_RING_HP(ab)));1072 else1073 ath11k_dbg(ab, ATH11K_DBG_HAL,1074 "type %d ring_num %d target_reg 0x%x shadow 0x%lx\n",1075 type, ring_num,1076 reg_base + (HAL_REO1_RING_TP(ab) -1077 HAL_REO1_RING_HP(ab)),1078 (unsigned long)srng->u.dst_ring.tp_addr -1079 (unsigned long)ab->mem);1080 }1081 }1082 1083 if (srng_config->lmac_ring)1084 return ring_id;1085 1086 ath11k_hal_srng_hw_init(ab, srng);1087 1088 if (type == HAL_CE_DST) {1089 srng->u.dst_ring.max_buffer_length = params->max_buffer_len;1090 ath11k_hal_ce_dst_setup(ab, srng, ring_num);1091 }1092 1093 return ring_id;1094}1095 1096static void ath11k_hal_srng_update_hp_tp_addr(struct ath11k_base *ab,1097 int shadow_cfg_idx,1098 enum hal_ring_type ring_type,1099 int ring_num)1100{1101 struct hal_srng *srng;1102 struct ath11k_hal *hal = &ab->hal;1103 int ring_id;1104 struct hal_srng_config *srng_config = &hal->srng_config[ring_type];1105 1106 ring_id = ath11k_hal_srng_get_ring_id(ab, ring_type, ring_num, 0);1107 if (ring_id < 0)1108 return;1109 1110 srng = &hal->srng_list[ring_id];1111 1112 if (srng_config->ring_dir == HAL_SRNG_DIR_DST)1113 srng->u.dst_ring.tp_addr = (u32 *)(HAL_SHADOW_REG(ab, shadow_cfg_idx) +1114 (unsigned long)ab->mem);1115 else1116 srng->u.src_ring.hp_addr = (u32 *)(HAL_SHADOW_REG(ab, shadow_cfg_idx) +1117 (unsigned long)ab->mem);1118}1119 1120int ath11k_hal_srng_update_shadow_config(struct ath11k_base *ab,1121 enum hal_ring_type ring_type,1122 int ring_num)1123{1124 struct ath11k_hal *hal = &ab->hal;1125 struct hal_srng_config *srng_config = &hal->srng_config[ring_type];1126 int shadow_cfg_idx = hal->num_shadow_reg_configured;1127 u32 target_reg;1128 1129 if (shadow_cfg_idx >= HAL_SHADOW_NUM_REGS)1130 return -EINVAL;1131 1132 hal->num_shadow_reg_configured++;1133 1134 target_reg = srng_config->reg_start[HAL_HP_OFFSET_IN_REG_START];1135 target_reg += srng_config->reg_size[HAL_HP_OFFSET_IN_REG_START] *1136 ring_num;1137 1138 /* For destination ring, shadow the TP */1139 if (srng_config->ring_dir == HAL_SRNG_DIR_DST)1140 target_reg += HAL_OFFSET_FROM_HP_TO_TP;1141 1142 hal->shadow_reg_addr[shadow_cfg_idx] = target_reg;1143 1144 /* update hp/tp addr to hal structure*/1145 ath11k_hal_srng_update_hp_tp_addr(ab, shadow_cfg_idx, ring_type,1146 ring_num);1147 1148 ath11k_dbg(ab, ATH11K_DBG_HAL,1149 "update shadow config target_reg %x shadow reg 0x%x shadow_idx 0x%x ring_type %d ring num %d",1150 target_reg,1151 HAL_SHADOW_REG(ab, shadow_cfg_idx),1152 shadow_cfg_idx,1153 ring_type, ring_num);1154 1155 return 0;1156}1157 1158void ath11k_hal_srng_shadow_config(struct ath11k_base *ab)1159{1160 struct ath11k_hal *hal = &ab->hal;1161 int ring_type, ring_num;1162 1163 /* update all the non-CE srngs. */1164 for (ring_type = 0; ring_type < HAL_MAX_RING_TYPES; ring_type++) {1165 struct hal_srng_config *srng_config = &hal->srng_config[ring_type];1166 1167 if (ring_type == HAL_CE_SRC ||1168 ring_type == HAL_CE_DST ||1169 ring_type == HAL_CE_DST_STATUS)1170 continue;1171 1172 if (srng_config->lmac_ring)1173 continue;1174 1175 for (ring_num = 0; ring_num < srng_config->max_rings; ring_num++)1176 ath11k_hal_srng_update_shadow_config(ab, ring_type, ring_num);1177 }1178}1179 1180void ath11k_hal_srng_get_shadow_config(struct ath11k_base *ab,1181 u32 **cfg, u32 *len)1182{1183 struct ath11k_hal *hal = &ab->hal;1184 1185 *len = hal->num_shadow_reg_configured;1186 *cfg = hal->shadow_reg_addr;1187}1188 1189void ath11k_hal_srng_shadow_update_hp_tp(struct ath11k_base *ab,1190 struct hal_srng *srng)1191{1192 lockdep_assert_held(&srng->lock);1193 1194 /* check whether the ring is empty. Update the shadow1195 * HP only when then ring isn't empty.1196 */1197 if (srng->ring_dir == HAL_SRNG_DIR_SRC &&1198 *srng->u.src_ring.tp_addr != srng->u.src_ring.hp)1199 ath11k_hal_srng_access_end(ab, srng);1200}1201 1202static int ath11k_hal_srng_create_config(struct ath11k_base *ab)1203{1204 struct ath11k_hal *hal = &ab->hal;1205 struct hal_srng_config *s;1206 1207 hal->srng_config = kmemdup(hw_srng_config_template,1208 sizeof(hw_srng_config_template),1209 GFP_KERNEL);1210 if (!hal->srng_config)1211 return -ENOMEM;1212 1213 s = &hal->srng_config[HAL_REO_DST];1214 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_BASE_LSB(ab);1215 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_HP(ab);1216 s->reg_size[0] = HAL_REO2_RING_BASE_LSB(ab) - HAL_REO1_RING_BASE_LSB(ab);1217 s->reg_size[1] = HAL_REO2_RING_HP(ab) - HAL_REO1_RING_HP(ab);1218 1219 s = &hal->srng_config[HAL_REO_EXCEPTION];1220 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_TCL_RING_BASE_LSB(ab);1221 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_TCL_RING_HP(ab);1222 1223 s = &hal->srng_config[HAL_REO_REINJECT];1224 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_BASE_LSB(ab);1225 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_HP(ab);1226 1227 s = &hal->srng_config[HAL_REO_CMD];1228 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_RING_BASE_LSB(ab);1229 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_HP(ab);1230 1231 s = &hal->srng_config[HAL_REO_STATUS];1232 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_RING_BASE_LSB(ab);1233 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_HP(ab);1234 1235 s = &hal->srng_config[HAL_TCL_DATA];1236 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_BASE_LSB(ab);1237 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_HP;1238 s->reg_size[0] = HAL_TCL2_RING_BASE_LSB(ab) - HAL_TCL1_RING_BASE_LSB(ab);1239 s->reg_size[1] = HAL_TCL2_RING_HP - HAL_TCL1_RING_HP;1240 1241 s = &hal->srng_config[HAL_TCL_CMD];1242 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_BASE_LSB(ab);1243 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_HP;1244 1245 s = &hal->srng_config[HAL_TCL_STATUS];1246 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_BASE_LSB(ab);1247 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_HP;1248 1249 s = &hal->srng_config[HAL_CE_SRC];1250 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_BASE_LSB +1251 ATH11K_CE_OFFSET(ab);1252 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_HP +1253 ATH11K_CE_OFFSET(ab);1254 s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) -1255 HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab);1256 s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) -1257 HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab);1258 1259 s = &hal->srng_config[HAL_CE_DST];1260 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_BASE_LSB +1261 ATH11K_CE_OFFSET(ab);1262 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_HP +1263 ATH11K_CE_OFFSET(ab);1264 s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -1265 HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);1266 s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -1267 HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);1268 1269 s = &hal->srng_config[HAL_CE_DST_STATUS];1270 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) +1271 HAL_CE_DST_STATUS_RING_BASE_LSB + ATH11K_CE_OFFSET(ab);1272 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_STATUS_RING_HP +1273 ATH11K_CE_OFFSET(ab);1274 s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -1275 HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);1276 s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -1277 HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);1278 1279 s = &hal->srng_config[HAL_WBM_IDLE_LINK];1280 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_BASE_LSB(ab);1281 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_HP;1282 1283 s = &hal->srng_config[HAL_SW2WBM_RELEASE];1284 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_RELEASE_RING_BASE_LSB(ab);1285 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_RELEASE_RING_HP;1286 1287 s = &hal->srng_config[HAL_WBM2SW_RELEASE];1288 s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_BASE_LSB(ab);1289 s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_HP;1290 s->reg_size[0] = HAL_WBM1_RELEASE_RING_BASE_LSB(ab) -1291 HAL_WBM0_RELEASE_RING_BASE_LSB(ab);1292 s->reg_size[1] = HAL_WBM1_RELEASE_RING_HP - HAL_WBM0_RELEASE_RING_HP;1293 1294 return 0;1295}1296 1297static void ath11k_hal_register_srng_key(struct ath11k_base *ab)1298{1299 struct ath11k_hal *hal = &ab->hal;1300 u32 ring_id;1301 1302 for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++)1303 lockdep_register_key(hal->srng_key + ring_id);1304}1305 1306static void ath11k_hal_unregister_srng_key(struct ath11k_base *ab)1307{1308 struct ath11k_hal *hal = &ab->hal;1309 u32 ring_id;1310 1311 for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++)1312 lockdep_unregister_key(hal->srng_key + ring_id);1313}1314 1315int ath11k_hal_srng_init(struct ath11k_base *ab)1316{1317 struct ath11k_hal *hal = &ab->hal;1318 int ret;1319 1320 memset(hal, 0, sizeof(*hal));1321 1322 ret = ath11k_hal_srng_create_config(ab);1323 if (ret)1324 goto err_hal;1325 1326 ret = ath11k_hal_alloc_cont_rdp(ab);1327 if (ret)1328 goto err_hal;1329 1330 ret = ath11k_hal_alloc_cont_wrp(ab);1331 if (ret)1332 goto err_free_cont_rdp;1333 1334 ath11k_hal_register_srng_key(ab);1335 1336 return 0;1337 1338err_free_cont_rdp:1339 ath11k_hal_free_cont_rdp(ab);1340 1341err_hal:1342 return ret;1343}1344EXPORT_SYMBOL(ath11k_hal_srng_init);1345 1346void ath11k_hal_srng_deinit(struct ath11k_base *ab)1347{1348 struct ath11k_hal *hal = &ab->hal;1349 1350 ath11k_hal_unregister_srng_key(ab);1351 ath11k_hal_free_cont_rdp(ab);1352 ath11k_hal_free_cont_wrp(ab);1353 kfree(hal->srng_config);1354}1355EXPORT_SYMBOL(ath11k_hal_srng_deinit);1356 1357void ath11k_hal_dump_srng_stats(struct ath11k_base *ab)1358{1359 struct hal_srng *srng;1360 struct ath11k_ext_irq_grp *irq_grp;1361 struct ath11k_ce_pipe *ce_pipe;1362 int i;1363 1364 ath11k_err(ab, "Last interrupt received for each CE:\n");1365 for (i = 0; i < ab->hw_params.ce_count; i++) {1366 ce_pipe = &ab->ce.ce_pipe[i];1367 1368 if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)1369 continue;1370 1371 ath11k_err(ab, "CE_id %d pipe_num %d %ums before\n",1372 i, ce_pipe->pipe_num,1373 jiffies_to_msecs(jiffies - ce_pipe->timestamp));1374 }1375 1376 ath11k_err(ab, "\nLast interrupt received for each group:\n");1377 for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {1378 irq_grp = &ab->ext_irq_grp[i];1379 ath11k_err(ab, "group_id %d %ums before\n",1380 irq_grp->grp_id,1381 jiffies_to_msecs(jiffies - irq_grp->timestamp));1382 }1383 1384 for (i = 0; i < HAL_SRNG_RING_ID_MAX; i++) {1385 srng = &ab->hal.srng_list[i];1386 1387 if (!srng->initialized)1388 continue;1389 1390 if (srng->ring_dir == HAL_SRNG_DIR_SRC)1391 ath11k_err(ab,1392 "src srng id %u hp %u, reap_hp %u, cur tp %u, cached tp %u last tp %u napi processed before %ums\n",1393 srng->ring_id, srng->u.src_ring.hp,1394 srng->u.src_ring.reap_hp,1395 *srng->u.src_ring.tp_addr, srng->u.src_ring.cached_tp,1396 srng->u.src_ring.last_tp,1397 jiffies_to_msecs(jiffies - srng->timestamp));1398 else if (srng->ring_dir == HAL_SRNG_DIR_DST)1399 ath11k_err(ab,1400 "dst srng id %u tp %u, cur hp %u, cached hp %u last hp %u napi processed before %ums\n",1401 srng->ring_id, srng->u.dst_ring.tp,1402 *srng->u.dst_ring.hp_addr,1403 srng->u.dst_ring.cached_hp,1404 srng->u.dst_ring.last_hp,1405 jiffies_to_msecs(jiffies - srng->timestamp));1406 }1407}1408