531 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause2/*3 * Copyright (C) 2003-2014, 2018-2022, 2024 Intel Corporation4 * Copyright (C) 2015-2016 Intel Deutschland GmbH5 */6#include <linux/delay.h>7#include <linux/device.h>8#include <linux/export.h>9 10#include "iwl-drv.h"11#include "iwl-io.h"12#include "iwl-csr.h"13#include "iwl-debug.h"14#include "iwl-prph.h"15#include "iwl-fh.h"16 17void iwl_write8(struct iwl_trans *trans, u32 ofs, u8 val)18{19 trace_iwlwifi_dev_iowrite8(trans->dev, ofs, val);20 iwl_trans_write8(trans, ofs, val);21}22IWL_EXPORT_SYMBOL(iwl_write8);23 24void iwl_write32(struct iwl_trans *trans, u32 ofs, u32 val)25{26 trace_iwlwifi_dev_iowrite32(trans->dev, ofs, val);27 iwl_trans_write32(trans, ofs, val);28}29IWL_EXPORT_SYMBOL(iwl_write32);30 31void iwl_write64(struct iwl_trans *trans, u64 ofs, u64 val)32{33 trace_iwlwifi_dev_iowrite64(trans->dev, ofs, val);34 iwl_trans_write32(trans, ofs, lower_32_bits(val));35 iwl_trans_write32(trans, ofs + 4, upper_32_bits(val));36}37IWL_EXPORT_SYMBOL(iwl_write64);38 39u32 iwl_read32(struct iwl_trans *trans, u32 ofs)40{41 u32 val = iwl_trans_read32(trans, ofs);42 43 trace_iwlwifi_dev_ioread32(trans->dev, ofs, val);44 return val;45}46IWL_EXPORT_SYMBOL(iwl_read32);47 48#define IWL_POLL_INTERVAL 10 /* microseconds */49 50int iwl_poll_bit(struct iwl_trans *trans, u32 addr,51 u32 bits, u32 mask, int timeout)52{53 int t = 0;54 55 do {56 if ((iwl_read32(trans, addr) & mask) == (bits & mask))57 return t;58 udelay(IWL_POLL_INTERVAL);59 t += IWL_POLL_INTERVAL;60 } while (t < timeout);61 62 return -ETIMEDOUT;63}64IWL_EXPORT_SYMBOL(iwl_poll_bit);65 66u32 iwl_read_direct32(struct iwl_trans *trans, u32 reg)67{68 if (iwl_trans_grab_nic_access(trans)) {69 u32 value = iwl_read32(trans, reg);70 71 iwl_trans_release_nic_access(trans);72 return value;73 }74 75 /* return as if we have a HW timeout/failure */76 return 0x5a5a5a5a;77}78IWL_EXPORT_SYMBOL(iwl_read_direct32);79 80void iwl_write_direct32(struct iwl_trans *trans, u32 reg, u32 value)81{82 if (iwl_trans_grab_nic_access(trans)) {83 iwl_write32(trans, reg, value);84 iwl_trans_release_nic_access(trans);85 }86}87IWL_EXPORT_SYMBOL(iwl_write_direct32);88 89void iwl_write_direct64(struct iwl_trans *trans, u64 reg, u64 value)90{91 if (iwl_trans_grab_nic_access(trans)) {92 iwl_write64(trans, reg, value);93 iwl_trans_release_nic_access(trans);94 }95}96IWL_EXPORT_SYMBOL(iwl_write_direct64);97 98int iwl_poll_direct_bit(struct iwl_trans *trans, u32 addr, u32 mask,99 int timeout)100{101 int t = 0;102 103 do {104 if ((iwl_read_direct32(trans, addr) & mask) == mask)105 return t;106 udelay(IWL_POLL_INTERVAL);107 t += IWL_POLL_INTERVAL;108 } while (t < timeout);109 110 return -ETIMEDOUT;111}112IWL_EXPORT_SYMBOL(iwl_poll_direct_bit);113 114u32 iwl_read_prph_no_grab(struct iwl_trans *trans, u32 ofs)115{116 u32 val = iwl_trans_read_prph(trans, ofs);117 trace_iwlwifi_dev_ioread_prph32(trans->dev, ofs, val);118 return val;119}120IWL_EXPORT_SYMBOL(iwl_read_prph_no_grab);121 122void iwl_write_prph_no_grab(struct iwl_trans *trans, u32 ofs, u32 val)123{124 trace_iwlwifi_dev_iowrite_prph32(trans->dev, ofs, val);125 iwl_trans_write_prph(trans, ofs, val);126}127IWL_EXPORT_SYMBOL(iwl_write_prph_no_grab);128 129void iwl_write_prph64_no_grab(struct iwl_trans *trans, u64 ofs, u64 val)130{131 trace_iwlwifi_dev_iowrite_prph64(trans->dev, ofs, val);132 iwl_write_prph_no_grab(trans, ofs, val & 0xffffffff);133 iwl_write_prph_no_grab(trans, ofs + 4, val >> 32);134}135IWL_EXPORT_SYMBOL(iwl_write_prph64_no_grab);136 137u32 iwl_read_prph(struct iwl_trans *trans, u32 ofs)138{139 if (iwl_trans_grab_nic_access(trans)) {140 u32 val = iwl_read_prph_no_grab(trans, ofs);141 142 iwl_trans_release_nic_access(trans);143 144 return val;145 }146 147 /* return as if we have a HW timeout/failure */148 return 0x5a5a5a5a;149}150IWL_EXPORT_SYMBOL(iwl_read_prph);151 152void iwl_write_prph_delay(struct iwl_trans *trans, u32 ofs, u32 val, u32 delay_ms)153{154 if (iwl_trans_grab_nic_access(trans)) {155 mdelay(delay_ms);156 iwl_write_prph_no_grab(trans, ofs, val);157 iwl_trans_release_nic_access(trans);158 }159}160IWL_EXPORT_SYMBOL(iwl_write_prph_delay);161 162int iwl_poll_prph_bit(struct iwl_trans *trans, u32 addr,163 u32 bits, u32 mask, int timeout)164{165 int t = 0;166 167 do {168 if ((iwl_read_prph(trans, addr) & mask) == (bits & mask))169 return t;170 udelay(IWL_POLL_INTERVAL);171 t += IWL_POLL_INTERVAL;172 } while (t < timeout);173 174 return -ETIMEDOUT;175}176 177void iwl_set_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask)178{179 if (iwl_trans_grab_nic_access(trans)) {180 iwl_write_prph_no_grab(trans, ofs,181 iwl_read_prph_no_grab(trans, ofs) |182 mask);183 iwl_trans_release_nic_access(trans);184 }185}186IWL_EXPORT_SYMBOL(iwl_set_bits_prph);187 188void iwl_set_bits_mask_prph(struct iwl_trans *trans, u32 ofs,189 u32 bits, u32 mask)190{191 if (iwl_trans_grab_nic_access(trans)) {192 iwl_write_prph_no_grab(trans, ofs,193 (iwl_read_prph_no_grab(trans, ofs) &194 mask) | bits);195 iwl_trans_release_nic_access(trans);196 }197}198IWL_EXPORT_SYMBOL(iwl_set_bits_mask_prph);199 200void iwl_clear_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask)201{202 u32 val;203 204 if (iwl_trans_grab_nic_access(trans)) {205 val = iwl_read_prph_no_grab(trans, ofs);206 iwl_write_prph_no_grab(trans, ofs, (val & ~mask));207 iwl_trans_release_nic_access(trans);208 }209}210IWL_EXPORT_SYMBOL(iwl_clear_bits_prph);211 212void iwl_force_nmi(struct iwl_trans *trans)213{214 if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_9000)215 iwl_write_prph_delay(trans, DEVICE_SET_NMI_REG,216 DEVICE_SET_NMI_VAL_DRV, 1);217 else if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210)218 iwl_write_umac_prph(trans, UREG_NIC_SET_NMI_DRIVER,219 UREG_NIC_SET_NMI_DRIVER_NMI_FROM_DRIVER);220 else if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_BZ)221 iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,222 UREG_DOORBELL_TO_ISR6_NMI_BIT);223 else224 iwl_write32(trans, CSR_DOORBELL_VECTOR,225 UREG_DOORBELL_TO_ISR6_NMI_BIT);226}227IWL_EXPORT_SYMBOL(iwl_force_nmi);228 229static const char *get_rfh_string(int cmd)230{231#define IWL_CMD(x) case x: return #x232#define IWL_CMD_MQ(arg, reg, q) { if (arg == reg(q)) return #reg; }233 234 int i;235 236 for (i = 0; i < IWL_MAX_RX_HW_QUEUES; i++) {237 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_BA_LSB, i);238 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_WIDX, i);239 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_RIDX, i);240 IWL_CMD_MQ(cmd, RFH_Q_URBD_STTS_WPTR_LSB, i);241 }242 243 switch (cmd) {244 IWL_CMD(RFH_RXF_DMA_CFG);245 IWL_CMD(RFH_GEN_CFG);246 IWL_CMD(RFH_GEN_STATUS);247 IWL_CMD(FH_TSSR_TX_STATUS_REG);248 IWL_CMD(FH_TSSR_TX_ERROR_REG);249 default:250 return "UNKNOWN";251 }252#undef IWL_CMD_MQ253}254 255struct reg {256 u32 addr;257 bool is64;258};259 260static int iwl_dump_rfh(struct iwl_trans *trans, char **buf)261{262 int i, q;263 int num_q = trans->num_rx_queues;264 static const u32 rfh_tbl[] = {265 RFH_RXF_DMA_CFG,266 RFH_GEN_CFG,267 RFH_GEN_STATUS,268 FH_TSSR_TX_STATUS_REG,269 FH_TSSR_TX_ERROR_REG,270 };271 static const struct reg rfh_mq_tbl[] = {272 { RFH_Q0_FRBDCB_BA_LSB, true },273 { RFH_Q0_FRBDCB_WIDX, false },274 { RFH_Q0_FRBDCB_RIDX, false },275 { RFH_Q0_URBD_STTS_WPTR_LSB, true },276 };277 278#ifdef CONFIG_IWLWIFI_DEBUGFS279 if (buf) {280 int pos = 0;281 /*282 * Register (up to 34 for name + 8 blank/q for MQ): 40 chars283 * Colon + space: 2 characters284 * 0X%08x: 10 characters285 * New line: 1 character286 * Total of 53 characters287 */288 size_t bufsz = ARRAY_SIZE(rfh_tbl) * 53 +289 ARRAY_SIZE(rfh_mq_tbl) * 53 * num_q + 40;290 291 *buf = kmalloc(bufsz, GFP_KERNEL);292 if (!*buf)293 return -ENOMEM;294 295 pos += scnprintf(*buf + pos, bufsz - pos,296 "RFH register values:\n");297 298 for (i = 0; i < ARRAY_SIZE(rfh_tbl); i++)299 pos += scnprintf(*buf + pos, bufsz - pos,300 "%40s: 0X%08x\n",301 get_rfh_string(rfh_tbl[i]),302 iwl_read_prph(trans, rfh_tbl[i]));303 304 for (i = 0; i < ARRAY_SIZE(rfh_mq_tbl); i++)305 for (q = 0; q < num_q; q++) {306 u32 addr = rfh_mq_tbl[i].addr;307 308 addr += q * (rfh_mq_tbl[i].is64 ? 8 : 4);309 pos += scnprintf(*buf + pos, bufsz - pos,310 "%34s(q %2d): 0X%08x\n",311 get_rfh_string(addr), q,312 iwl_read_prph(trans, addr));313 }314 315 return pos;316 }317#endif318 319 IWL_ERR(trans, "RFH register values:\n");320 for (i = 0; i < ARRAY_SIZE(rfh_tbl); i++)321 IWL_ERR(trans, " %34s: 0X%08x\n",322 get_rfh_string(rfh_tbl[i]),323 iwl_read_prph(trans, rfh_tbl[i]));324 325 for (i = 0; i < ARRAY_SIZE(rfh_mq_tbl); i++)326 for (q = 0; q < num_q; q++) {327 u32 addr = rfh_mq_tbl[i].addr;328 329 addr += q * (rfh_mq_tbl[i].is64 ? 8 : 4);330 IWL_ERR(trans, " %34s(q %d): 0X%08x\n",331 get_rfh_string(addr), q,332 iwl_read_prph(trans, addr));333 }334 335 return 0;336}337 338static const char *get_fh_string(int cmd)339{340 switch (cmd) {341 IWL_CMD(FH_RSCSR_CHNL0_STTS_WPTR_REG);342 IWL_CMD(FH_RSCSR_CHNL0_RBDCB_BASE_REG);343 IWL_CMD(FH_RSCSR_CHNL0_WPTR);344 IWL_CMD(FH_MEM_RCSR_CHNL0_CONFIG_REG);345 IWL_CMD(FH_MEM_RSSR_SHARED_CTRL_REG);346 IWL_CMD(FH_MEM_RSSR_RX_STATUS_REG);347 IWL_CMD(FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV);348 IWL_CMD(FH_TSSR_TX_STATUS_REG);349 IWL_CMD(FH_TSSR_TX_ERROR_REG);350 default:351 return "UNKNOWN";352 }353#undef IWL_CMD354}355 356int iwl_dump_fh(struct iwl_trans *trans, char **buf)357{358 int i;359 static const u32 fh_tbl[] = {360 FH_RSCSR_CHNL0_STTS_WPTR_REG,361 FH_RSCSR_CHNL0_RBDCB_BASE_REG,362 FH_RSCSR_CHNL0_WPTR,363 FH_MEM_RCSR_CHNL0_CONFIG_REG,364 FH_MEM_RSSR_SHARED_CTRL_REG,365 FH_MEM_RSSR_RX_STATUS_REG,366 FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV,367 FH_TSSR_TX_STATUS_REG,368 FH_TSSR_TX_ERROR_REG369 };370 371 if (trans->trans_cfg->mq_rx_supported)372 return iwl_dump_rfh(trans, buf);373 374#ifdef CONFIG_IWLWIFI_DEBUGFS375 if (buf) {376 int pos = 0;377 size_t bufsz = ARRAY_SIZE(fh_tbl) * 48 + 40;378 379 *buf = kmalloc(bufsz, GFP_KERNEL);380 if (!*buf)381 return -ENOMEM;382 383 pos += scnprintf(*buf + pos, bufsz - pos,384 "FH register values:\n");385 386 for (i = 0; i < ARRAY_SIZE(fh_tbl); i++)387 pos += scnprintf(*buf + pos, bufsz - pos,388 " %34s: 0X%08x\n",389 get_fh_string(fh_tbl[i]),390 iwl_read_direct32(trans, fh_tbl[i]));391 392 return pos;393 }394#endif395 396 IWL_ERR(trans, "FH register values:\n");397 for (i = 0; i < ARRAY_SIZE(fh_tbl); i++)398 IWL_ERR(trans, " %34s: 0X%08x\n",399 get_fh_string(fh_tbl[i]),400 iwl_read_direct32(trans, fh_tbl[i]));401 402 return 0;403}404 405#define IWL_HOST_MON_BLOCK_PEMON 0x00406#define IWL_HOST_MON_BLOCK_HIPM 0x22407 408#define IWL_HOST_MON_BLOCK_PEMON_VEC0 0x00409#define IWL_HOST_MON_BLOCK_PEMON_VEC1 0x01410#define IWL_HOST_MON_BLOCK_PEMON_WFPM 0x06411 412static void iwl_dump_host_monitor_block(struct iwl_trans *trans,413 u32 block, u32 vec, u32 iter)414{415 int i;416 417 IWL_ERR(trans, "Host monitor block 0x%x vector 0x%x\n", block, vec);418 iwl_write32(trans, CSR_MONITOR_CFG_REG, (block << 8) | vec);419 for (i = 0; i < iter; i++)420 IWL_ERR(trans, " value [iter %d]: 0x%08x\n",421 i, iwl_read32(trans, CSR_MONITOR_STATUS_REG));422}423 424static void iwl_dump_host_monitor(struct iwl_trans *trans)425{426 switch (trans->trans_cfg->device_family) {427 case IWL_DEVICE_FAMILY_22000:428 case IWL_DEVICE_FAMILY_AX210:429 IWL_ERR(trans, "CSR_RESET = 0x%x\n",430 iwl_read32(trans, CSR_RESET));431 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_PEMON,432 IWL_HOST_MON_BLOCK_PEMON_VEC0, 15);433 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_PEMON,434 IWL_HOST_MON_BLOCK_PEMON_VEC1, 15);435 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_PEMON,436 IWL_HOST_MON_BLOCK_PEMON_WFPM, 15);437 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_HIPM,438 IWL_HOST_MON_BLOCK_PEMON_VEC0, 1);439 break;440 default:441 /* not supported yet */442 return;443 }444}445 446int iwl_finish_nic_init(struct iwl_trans *trans)447{448 const struct iwl_cfg_trans_params *cfg_trans = trans->trans_cfg;449 u32 poll_ready;450 int err;451 452 if (cfg_trans->bisr_workaround) {453 /* ensure the TOP FSM isn't still in previous reset */454 mdelay(2);455 }456 457 /*458 * Set "initialization complete" bit to move adapter from459 * D0U* --> D0A* (powered-up active) state.460 */461 if (cfg_trans->device_family >= IWL_DEVICE_FAMILY_BZ) {462 iwl_set_bit(trans, CSR_GP_CNTRL,463 CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ |464 CSR_GP_CNTRL_REG_FLAG_MAC_INIT);465 poll_ready = CSR_GP_CNTRL_REG_FLAG_MAC_STATUS;466 } else {467 iwl_set_bit(trans, CSR_GP_CNTRL,468 CSR_GP_CNTRL_REG_FLAG_INIT_DONE);469 poll_ready = CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY;470 }471 472 if (cfg_trans->device_family == IWL_DEVICE_FAMILY_8000)473 udelay(2);474 475 /*476 * Wait for clock stabilization; once stabilized, access to477 * device-internal resources is supported, e.g. iwl_write_prph()478 * and accesses to uCode SRAM.479 */480 err = iwl_poll_bit(trans, CSR_GP_CNTRL, poll_ready, poll_ready, 25000);481 if (err < 0) {482 IWL_DEBUG_INFO(trans, "Failed to wake NIC\n");483 484 iwl_dump_host_monitor(trans);485 }486 487 if (cfg_trans->bisr_workaround) {488 /* ensure BISR shift has finished */489 udelay(200);490 }491 492 return err < 0 ? err : 0;493}494IWL_EXPORT_SYMBOL(iwl_finish_nic_init);495 496void iwl_trans_sync_nmi_with_addr(struct iwl_trans *trans, u32 inta_addr,497 u32 sw_err_bit)498{499 unsigned long timeout = jiffies + IWL_TRANS_NMI_TIMEOUT;500 bool interrupts_enabled = test_bit(STATUS_INT_ENABLED, &trans->status);501 502 /* if the interrupts were already disabled, there is no point in503 * calling iwl_disable_interrupts504 */505 if (interrupts_enabled)506 iwl_trans_interrupts(trans, false);507 508 iwl_force_nmi(trans);509 while (time_after(timeout, jiffies)) {510 u32 inta_hw = iwl_read32(trans, inta_addr);511 512 /* Error detected by uCode */513 if (inta_hw & sw_err_bit) {514 /* Clear causes register */515 iwl_write32(trans, inta_addr, inta_hw & sw_err_bit);516 break;517 }518 519 mdelay(1);520 }521 522 /* enable interrupts only if there were already enabled before this523 * function to avoid a case were the driver enable interrupts before524 * proper configurations were made525 */526 if (interrupts_enabled)527 iwl_trans_interrupts(trans, true);528 529 iwl_trans_fw_error(trans, false);530}531