1720 lines · c
1/*2 * This file is part of the Chelsio FCoE driver for Linux.3 *4 * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved.5 *6 * This software is available to you under a choice of one of two7 * licenses. You may choose to be licensed under the terms of the GNU8 * General Public License (GPL) Version 2, available from the file9 * COPYING in the main directory of this source tree, or the10 * OpenIB.org BSD license below:11 *12 * Redistribution and use in source and binary forms, with or13 * without modification, are permitted provided that the following14 * conditions are met:15 *16 * - Redistributions of source code must retain the above17 * copyright notice, this list of conditions and the following18 * disclaimer.19 *20 * - Redistributions in binary form must reproduce the above21 * copyright notice, this list of conditions and the following22 * disclaimer in the documentation and/or other materials23 * provided with the distribution.24 *25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE32 * SOFTWARE.33 */34 35#include <linux/kernel.h>36#include <linux/string.h>37#include <linux/compiler.h>38#include <linux/slab.h>39#include <asm/page.h>40#include <linux/cache.h>41 42#include "t4_values.h"43#include "csio_hw.h"44#include "csio_wr.h"45#include "csio_mb.h"46#include "csio_defs.h"47 48int csio_intr_coalesce_cnt; /* value:SGE_INGRESS_RX_THRESHOLD[0] */49static int csio_sge_thresh_reg; /* SGE_INGRESS_RX_THRESHOLD[0] */50 51int csio_intr_coalesce_time = 10; /* value:SGE_TIMER_VALUE_1 */52static int csio_sge_timer_reg = 1;53 54#define CSIO_SET_FLBUF_SIZE(_hw, _reg, _val) \55 csio_wr_reg32((_hw), (_val), SGE_FL_BUFFER_SIZE##_reg##_A)56 57static void58csio_get_flbuf_size(struct csio_hw *hw, struct csio_sge *sge, uint32_t reg)59{60 sge->sge_fl_buf_size[reg] = csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE0_A +61 reg * sizeof(uint32_t));62}63 64/* Free list buffer size */65static inline uint32_t66csio_wr_fl_bufsz(struct csio_sge *sge, struct csio_dma_buf *buf)67{68 return sge->sge_fl_buf_size[buf->paddr & 0xF];69}70 71/* Size of the egress queue status page */72static inline uint32_t73csio_wr_qstat_pgsz(struct csio_hw *hw)74{75 return (hw->wrm.sge.sge_control & EGRSTATUSPAGESIZE_F) ? 128 : 64;76}77 78/* Ring freelist doorbell */79static inline void80csio_wr_ring_fldb(struct csio_hw *hw, struct csio_q *flq)81{82 /*83 * Ring the doorbell only when we have atleast CSIO_QCREDIT_SZ84 * number of bytes in the freelist queue. This translates to atleast85 * 8 freelist buffer pointers (since each pointer is 8 bytes).86 */87 if (flq->inc_idx >= 8) {88 csio_wr_reg32(hw, DBPRIO_F | QID_V(flq->un.fl.flid) |89 PIDX_T5_V(flq->inc_idx / 8) | DBTYPE_F,90 MYPF_REG(SGE_PF_KDOORBELL_A));91 flq->inc_idx &= 7;92 }93}94 95/* Write a 0 cidx increment value to enable SGE interrupts for this queue */96static void97csio_wr_sge_intr_enable(struct csio_hw *hw, uint16_t iqid)98{99 csio_wr_reg32(hw, CIDXINC_V(0) |100 INGRESSQID_V(iqid) |101 TIMERREG_V(X_TIMERREG_RESTART_COUNTER),102 MYPF_REG(SGE_PF_GTS_A));103}104 105/*106 * csio_wr_fill_fl - Populate the FL buffers of a FL queue.107 * @hw: HW module.108 * @flq: Freelist queue.109 *110 * Fill up freelist buffer entries with buffers of size specified111 * in the size register.112 *113 */114static int115csio_wr_fill_fl(struct csio_hw *hw, struct csio_q *flq)116{117 struct csio_wrm *wrm = csio_hw_to_wrm(hw);118 struct csio_sge *sge = &wrm->sge;119 __be64 *d = (__be64 *)(flq->vstart);120 struct csio_dma_buf *buf = &flq->un.fl.bufs[0];121 uint64_t paddr;122 int sreg = flq->un.fl.sreg;123 int n = flq->credits;124 125 while (n--) {126 buf->len = sge->sge_fl_buf_size[sreg];127 buf->vaddr = dma_alloc_coherent(&hw->pdev->dev, buf->len,128 &buf->paddr, GFP_KERNEL);129 if (!buf->vaddr) {130 csio_err(hw, "Could only fill %d buffers!\n", n + 1);131 return -ENOMEM;132 }133 134 paddr = buf->paddr | (sreg & 0xF);135 136 *d++ = cpu_to_be64(paddr);137 buf++;138 }139 140 return 0;141}142 143/*144 * csio_wr_update_fl -145 * @hw: HW module.146 * @flq: Freelist queue.147 *148 *149 */150static inline void151csio_wr_update_fl(struct csio_hw *hw, struct csio_q *flq, uint16_t n)152{153 154 flq->inc_idx += n;155 flq->pidx += n;156 if (unlikely(flq->pidx >= flq->credits))157 flq->pidx -= (uint16_t)flq->credits;158 159 CSIO_INC_STATS(flq, n_flq_refill);160}161 162/*163 * csio_wr_alloc_q - Allocate a WR queue and initialize it.164 * @hw: HW module165 * @qsize: Size of the queue in bytes166 * @wrsize: Since of WR in this queue, if fixed.167 * @type: Type of queue (Ingress/Egress/Freelist)168 * @owner: Module that owns this queue.169 * @nflb: Number of freelist buffers for FL.170 * @sreg: What is the FL buffer size register?171 * @iq_int_handler: Ingress queue handler in INTx mode.172 *173 * This function allocates and sets up a queue for the caller174 * of size qsize, aligned at the required boundary. This is subject to175 * be free entries being available in the queue array. If one is found,176 * it is initialized with the allocated queue, marked as being used (owner),177 * and a handle returned to the caller in form of the queue's index178 * into the q_arr array.179 * If user has indicated a freelist (by specifying nflb > 0), create180 * another queue (with its own index into q_arr) for the freelist. Allocate181 * memory for DMA buffer metadata (vaddr, len etc). Save off the freelist182 * idx in the ingress queue's flq.idx. This is how a Freelist is associated183 * with its owning ingress queue.184 */185int186csio_wr_alloc_q(struct csio_hw *hw, uint32_t qsize, uint32_t wrsize,187 uint16_t type, void *owner, uint32_t nflb, int sreg,188 iq_handler_t iq_intx_handler)189{190 struct csio_wrm *wrm = csio_hw_to_wrm(hw);191 struct csio_q *q, *flq;192 int free_idx = wrm->free_qidx;193 int ret_idx = free_idx;194 uint32_t qsz;195 int flq_idx;196 197 if (free_idx >= wrm->num_q) {198 csio_err(hw, "No more free queues.\n");199 return -1;200 }201 202 switch (type) {203 case CSIO_EGRESS:204 qsz = ALIGN(qsize, CSIO_QCREDIT_SZ) + csio_wr_qstat_pgsz(hw);205 break;206 case CSIO_INGRESS:207 switch (wrsize) {208 case 16:209 case 32:210 case 64:211 case 128:212 break;213 default:214 csio_err(hw, "Invalid Ingress queue WR size:%d\n",215 wrsize);216 return -1;217 }218 219 /*220 * Number of elements must be a multiple of 16221 * So this includes status page size222 */223 qsz = ALIGN(qsize/wrsize, 16) * wrsize;224 225 break;226 case CSIO_FREELIST:227 qsz = ALIGN(qsize/wrsize, 8) * wrsize + csio_wr_qstat_pgsz(hw);228 break;229 default:230 csio_err(hw, "Invalid queue type: 0x%x\n", type);231 return -1;232 }233 234 q = wrm->q_arr[free_idx];235 236 q->vstart = dma_alloc_coherent(&hw->pdev->dev, qsz, &q->pstart,237 GFP_KERNEL);238 if (!q->vstart) {239 csio_err(hw,240 "Failed to allocate DMA memory for "241 "queue at id: %d size: %d\n", free_idx, qsize);242 return -1;243 }244 245 q->type = type;246 q->owner = owner;247 q->pidx = q->cidx = q->inc_idx = 0;248 q->size = qsz;249 q->wr_sz = wrsize; /* If using fixed size WRs */250 251 wrm->free_qidx++;252 253 if (type == CSIO_INGRESS) {254 /* Since queue area is set to zero */255 q->un.iq.genbit = 1;256 257 /*258 * Ingress queue status page size is always the size of259 * the ingress queue entry.260 */261 q->credits = (qsz - q->wr_sz) / q->wr_sz;262 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz263 - q->wr_sz);264 265 /* Allocate memory for FL if requested */266 if (nflb > 0) {267 flq_idx = csio_wr_alloc_q(hw, nflb * sizeof(__be64),268 sizeof(__be64), CSIO_FREELIST,269 owner, 0, sreg, NULL);270 if (flq_idx == -1) {271 csio_err(hw,272 "Failed to allocate FL queue"273 " for IQ idx:%d\n", free_idx);274 return -1;275 }276 277 /* Associate the new FL with the Ingress quue */278 q->un.iq.flq_idx = flq_idx;279 280 flq = wrm->q_arr[q->un.iq.flq_idx];281 flq->un.fl.bufs = kcalloc(flq->credits,282 sizeof(struct csio_dma_buf),283 GFP_KERNEL);284 if (!flq->un.fl.bufs) {285 csio_err(hw,286 "Failed to allocate FL queue bufs"287 " for IQ idx:%d\n", free_idx);288 return -1;289 }290 291 flq->un.fl.packen = 0;292 flq->un.fl.offset = 0;293 flq->un.fl.sreg = sreg;294 295 /* Fill up the free list buffers */296 if (csio_wr_fill_fl(hw, flq))297 return -1;298 299 /*300 * Make sure in a FLQ, atleast 1 credit (8 FL buffers)301 * remains unpopulated,otherwise HW thinks302 * FLQ is empty.303 */304 flq->pidx = flq->inc_idx = flq->credits - 8;305 } else {306 q->un.iq.flq_idx = -1;307 }308 309 /* Associate the IQ INTx handler. */310 q->un.iq.iq_intx_handler = iq_intx_handler;311 312 csio_q_iqid(hw, ret_idx) = CSIO_MAX_QID;313 314 } else if (type == CSIO_EGRESS) {315 q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / CSIO_QCREDIT_SZ;316 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz317 - csio_wr_qstat_pgsz(hw));318 csio_q_eqid(hw, ret_idx) = CSIO_MAX_QID;319 } else { /* Freelist */320 q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / sizeof(__be64);321 q->vwrap = (void *)((uintptr_t)(q->vstart) + qsz322 - csio_wr_qstat_pgsz(hw));323 csio_q_flid(hw, ret_idx) = CSIO_MAX_QID;324 }325 326 return ret_idx;327}328 329/*330 * csio_wr_iq_create_rsp - Response handler for IQ creation.331 * @hw: The HW module.332 * @mbp: Mailbox.333 * @iq_idx: Ingress queue that got created.334 *335 * Handle FW_IQ_CMD mailbox completion. Save off the assigned IQ/FL ids.336 */337static int338csio_wr_iq_create_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)339{340 struct csio_iq_params iqp;341 enum fw_retval retval;342 uint32_t iq_id;343 int flq_idx;344 345 memset(&iqp, 0, sizeof(struct csio_iq_params));346 347 csio_mb_iq_alloc_write_rsp(hw, mbp, &retval, &iqp);348 349 if (retval != FW_SUCCESS) {350 csio_err(hw, "IQ cmd returned 0x%x!\n", retval);351 mempool_free(mbp, hw->mb_mempool);352 return -EINVAL;353 }354 355 csio_q_iqid(hw, iq_idx) = iqp.iqid;356 csio_q_physiqid(hw, iq_idx) = iqp.physiqid;357 csio_q_pidx(hw, iq_idx) = csio_q_cidx(hw, iq_idx) = 0;358 csio_q_inc_idx(hw, iq_idx) = 0;359 360 /* Actual iq-id. */361 iq_id = iqp.iqid - hw->wrm.fw_iq_start;362 363 /* Set the iq-id to iq map table. */364 if (iq_id >= CSIO_MAX_IQ) {365 csio_err(hw,366 "Exceeding MAX_IQ(%d) supported!"367 " iqid:%d rel_iqid:%d FW iq_start:%d\n",368 CSIO_MAX_IQ, iq_id, iqp.iqid, hw->wrm.fw_iq_start);369 mempool_free(mbp, hw->mb_mempool);370 return -EINVAL;371 }372 csio_q_set_intr_map(hw, iq_idx, iq_id);373 374 /*375 * During FW_IQ_CMD, FW sets interrupt_sent bit to 1 in the SGE376 * ingress context of this queue. This will block interrupts to377 * this queue until the next GTS write. Therefore, we do a378 * 0-cidx increment GTS write for this queue just to clear the379 * interrupt_sent bit. This will re-enable interrupts to this380 * queue.381 */382 csio_wr_sge_intr_enable(hw, iqp.physiqid);383 384 flq_idx = csio_q_iq_flq_idx(hw, iq_idx);385 if (flq_idx != -1) {386 struct csio_q *flq = hw->wrm.q_arr[flq_idx];387 388 csio_q_flid(hw, flq_idx) = iqp.fl0id;389 csio_q_cidx(hw, flq_idx) = 0;390 csio_q_pidx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8;391 csio_q_inc_idx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8;392 393 /* Now update SGE about the buffers allocated during init */394 csio_wr_ring_fldb(hw, flq);395 }396 397 mempool_free(mbp, hw->mb_mempool);398 399 return 0;400}401 402/*403 * csio_wr_iq_create - Configure an Ingress queue with FW.404 * @hw: The HW module.405 * @priv: Private data object.406 * @iq_idx: Ingress queue index in the WR module.407 * @vec: MSIX vector.408 * @portid: PCIE Channel to be associated with this queue.409 * @async: Is this a FW asynchronous message handling queue?410 * @cbfn: Completion callback.411 *412 * This API configures an ingress queue with FW by issuing a FW_IQ_CMD mailbox413 * with alloc/write bits set.414 */415int416csio_wr_iq_create(struct csio_hw *hw, void *priv, int iq_idx,417 uint32_t vec, uint8_t portid, bool async,418 void (*cbfn) (struct csio_hw *, struct csio_mb *))419{420 struct csio_mb *mbp;421 struct csio_iq_params iqp;422 int flq_idx;423 424 memset(&iqp, 0, sizeof(struct csio_iq_params));425 csio_q_portid(hw, iq_idx) = portid;426 427 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);428 if (!mbp) {429 csio_err(hw, "IQ command out of memory!\n");430 return -ENOMEM;431 }432 433 switch (hw->intr_mode) {434 case CSIO_IM_INTX:435 case CSIO_IM_MSI:436 /* For interrupt forwarding queue only */437 if (hw->intr_iq_idx == iq_idx)438 iqp.iqandst = X_INTERRUPTDESTINATION_PCIE;439 else440 iqp.iqandst = X_INTERRUPTDESTINATION_IQ;441 iqp.iqandstindex =442 csio_q_physiqid(hw, hw->intr_iq_idx);443 break;444 case CSIO_IM_MSIX:445 iqp.iqandst = X_INTERRUPTDESTINATION_PCIE;446 iqp.iqandstindex = (uint16_t)vec;447 break;448 case CSIO_IM_NONE:449 mempool_free(mbp, hw->mb_mempool);450 return -EINVAL;451 }452 453 /* Pass in the ingress queue cmd parameters */454 iqp.pfn = hw->pfn;455 iqp.vfn = 0;456 iqp.iq_start = 1;457 iqp.viid = 0;458 iqp.type = FW_IQ_TYPE_FL_INT_CAP;459 iqp.iqasynch = async;460 if (csio_intr_coalesce_cnt)461 iqp.iqanus = X_UPDATESCHEDULING_COUNTER_OPTTIMER;462 else463 iqp.iqanus = X_UPDATESCHEDULING_TIMER;464 iqp.iqanud = X_UPDATEDELIVERY_INTERRUPT;465 iqp.iqpciech = portid;466 iqp.iqintcntthresh = (uint8_t)csio_sge_thresh_reg;467 468 switch (csio_q_wr_sz(hw, iq_idx)) {469 case 16:470 iqp.iqesize = 0; break;471 case 32:472 iqp.iqesize = 1; break;473 case 64:474 iqp.iqesize = 2; break;475 case 128:476 iqp.iqesize = 3; break;477 }478 479 iqp.iqsize = csio_q_size(hw, iq_idx) /480 csio_q_wr_sz(hw, iq_idx);481 iqp.iqaddr = csio_q_pstart(hw, iq_idx);482 483 flq_idx = csio_q_iq_flq_idx(hw, iq_idx);484 if (flq_idx != -1) {485 enum chip_type chip = CHELSIO_CHIP_VERSION(hw->chip_id);486 struct csio_q *flq = hw->wrm.q_arr[flq_idx];487 488 iqp.fl0paden = 1;489 iqp.fl0packen = flq->un.fl.packen ? 1 : 0;490 iqp.fl0fbmin = X_FETCHBURSTMIN_64B;491 iqp.fl0fbmax = ((chip == CHELSIO_T5) ?492 X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B);493 iqp.fl0size = csio_q_size(hw, flq_idx) / CSIO_QCREDIT_SZ;494 iqp.fl0addr = csio_q_pstart(hw, flq_idx);495 }496 497 csio_mb_iq_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);498 499 if (csio_mb_issue(hw, mbp)) {500 csio_err(hw, "Issue of IQ cmd failed!\n");501 mempool_free(mbp, hw->mb_mempool);502 return -EINVAL;503 }504 505 if (cbfn != NULL)506 return 0;507 508 return csio_wr_iq_create_rsp(hw, mbp, iq_idx);509}510 511/*512 * csio_wr_eq_create_rsp - Response handler for EQ creation.513 * @hw: The HW module.514 * @mbp: Mailbox.515 * @eq_idx: Egress queue that got created.516 *517 * Handle FW_EQ_OFLD_CMD mailbox completion. Save off the assigned EQ ids.518 */519static int520csio_wr_eq_cfg_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)521{522 struct csio_eq_params eqp;523 enum fw_retval retval;524 525 memset(&eqp, 0, sizeof(struct csio_eq_params));526 527 csio_mb_eq_ofld_alloc_write_rsp(hw, mbp, &retval, &eqp);528 529 if (retval != FW_SUCCESS) {530 csio_err(hw, "EQ OFLD cmd returned 0x%x!\n", retval);531 mempool_free(mbp, hw->mb_mempool);532 return -EINVAL;533 }534 535 csio_q_eqid(hw, eq_idx) = (uint16_t)eqp.eqid;536 csio_q_physeqid(hw, eq_idx) = (uint16_t)eqp.physeqid;537 csio_q_pidx(hw, eq_idx) = csio_q_cidx(hw, eq_idx) = 0;538 csio_q_inc_idx(hw, eq_idx) = 0;539 540 mempool_free(mbp, hw->mb_mempool);541 542 return 0;543}544 545/*546 * csio_wr_eq_create - Configure an Egress queue with FW.547 * @hw: HW module.548 * @priv: Private data.549 * @eq_idx: Egress queue index in the WR module.550 * @iq_idx: Associated ingress queue index.551 * @cbfn: Completion callback.552 *553 * This API configures a offload egress queue with FW by issuing a554 * FW_EQ_OFLD_CMD (with alloc + write ) mailbox.555 */556int557csio_wr_eq_create(struct csio_hw *hw, void *priv, int eq_idx,558 int iq_idx, uint8_t portid,559 void (*cbfn) (struct csio_hw *, struct csio_mb *))560{561 struct csio_mb *mbp;562 struct csio_eq_params eqp;563 564 memset(&eqp, 0, sizeof(struct csio_eq_params));565 566 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);567 if (!mbp) {568 csio_err(hw, "EQ command out of memory!\n");569 return -ENOMEM;570 }571 572 eqp.pfn = hw->pfn;573 eqp.vfn = 0;574 eqp.eqstart = 1;575 eqp.hostfcmode = X_HOSTFCMODE_STATUS_PAGE;576 eqp.iqid = csio_q_iqid(hw, iq_idx);577 eqp.fbmin = X_FETCHBURSTMIN_64B;578 eqp.fbmax = X_FETCHBURSTMAX_512B;579 eqp.cidxfthresh = 0;580 eqp.pciechn = portid;581 eqp.eqsize = csio_q_size(hw, eq_idx) / CSIO_QCREDIT_SZ;582 eqp.eqaddr = csio_q_pstart(hw, eq_idx);583 584 csio_mb_eq_ofld_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO,585 &eqp, cbfn);586 587 if (csio_mb_issue(hw, mbp)) {588 csio_err(hw, "Issue of EQ OFLD cmd failed!\n");589 mempool_free(mbp, hw->mb_mempool);590 return -EINVAL;591 }592 593 if (cbfn != NULL)594 return 0;595 596 return csio_wr_eq_cfg_rsp(hw, mbp, eq_idx);597}598 599/*600 * csio_wr_iq_destroy_rsp - Response handler for IQ removal.601 * @hw: The HW module.602 * @mbp: Mailbox.603 * @iq_idx: Ingress queue that was freed.604 *605 * Handle FW_IQ_CMD (free) mailbox completion.606 */607static int608csio_wr_iq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)609{610 enum fw_retval retval = csio_mb_fw_retval(mbp);611 int rv = 0;612 613 if (retval != FW_SUCCESS)614 rv = -EINVAL;615 616 mempool_free(mbp, hw->mb_mempool);617 618 return rv;619}620 621/*622 * csio_wr_iq_destroy - Free an ingress queue.623 * @hw: The HW module.624 * @priv: Private data object.625 * @iq_idx: Ingress queue index to destroy626 * @cbfn: Completion callback.627 *628 * This API frees an ingress queue by issuing the FW_IQ_CMD629 * with the free bit set.630 */631static int632csio_wr_iq_destroy(struct csio_hw *hw, void *priv, int iq_idx,633 void (*cbfn)(struct csio_hw *, struct csio_mb *))634{635 int rv = 0;636 struct csio_mb *mbp;637 struct csio_iq_params iqp;638 int flq_idx;639 640 memset(&iqp, 0, sizeof(struct csio_iq_params));641 642 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);643 if (!mbp)644 return -ENOMEM;645 646 iqp.pfn = hw->pfn;647 iqp.vfn = 0;648 iqp.iqid = csio_q_iqid(hw, iq_idx);649 iqp.type = FW_IQ_TYPE_FL_INT_CAP;650 651 flq_idx = csio_q_iq_flq_idx(hw, iq_idx);652 if (flq_idx != -1)653 iqp.fl0id = csio_q_flid(hw, flq_idx);654 else655 iqp.fl0id = 0xFFFF;656 657 iqp.fl1id = 0xFFFF;658 659 csio_mb_iq_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);660 661 rv = csio_mb_issue(hw, mbp);662 if (rv != 0) {663 mempool_free(mbp, hw->mb_mempool);664 return rv;665 }666 667 if (cbfn != NULL)668 return 0;669 670 return csio_wr_iq_destroy_rsp(hw, mbp, iq_idx);671}672 673/*674 * csio_wr_eq_destroy_rsp - Response handler for OFLD EQ creation.675 * @hw: The HW module.676 * @mbp: Mailbox.677 * @eq_idx: Egress queue that was freed.678 *679 * Handle FW_OFLD_EQ_CMD (free) mailbox completion.680 */681static int682csio_wr_eq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)683{684 enum fw_retval retval = csio_mb_fw_retval(mbp);685 int rv = 0;686 687 if (retval != FW_SUCCESS)688 rv = -EINVAL;689 690 mempool_free(mbp, hw->mb_mempool);691 692 return rv;693}694 695/*696 * csio_wr_eq_destroy - Free an Egress queue.697 * @hw: The HW module.698 * @priv: Private data object.699 * @eq_idx: Egress queue index to destroy700 * @cbfn: Completion callback.701 *702 * This API frees an Egress queue by issuing the FW_EQ_OFLD_CMD703 * with the free bit set.704 */705static int706csio_wr_eq_destroy(struct csio_hw *hw, void *priv, int eq_idx,707 void (*cbfn) (struct csio_hw *, struct csio_mb *))708{709 int rv = 0;710 struct csio_mb *mbp;711 struct csio_eq_params eqp;712 713 memset(&eqp, 0, sizeof(struct csio_eq_params));714 715 mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);716 if (!mbp)717 return -ENOMEM;718 719 eqp.pfn = hw->pfn;720 eqp.vfn = 0;721 eqp.eqid = csio_q_eqid(hw, eq_idx);722 723 csio_mb_eq_ofld_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &eqp, cbfn);724 725 rv = csio_mb_issue(hw, mbp);726 if (rv != 0) {727 mempool_free(mbp, hw->mb_mempool);728 return rv;729 }730 731 if (cbfn != NULL)732 return 0;733 734 return csio_wr_eq_destroy_rsp(hw, mbp, eq_idx);735}736 737/*738 * csio_wr_cleanup_eq_stpg - Cleanup Egress queue status page739 * @hw: HW module740 * @qidx: Egress queue index741 *742 * Cleanup the Egress queue status page.743 */744static void745csio_wr_cleanup_eq_stpg(struct csio_hw *hw, int qidx)746{747 struct csio_q *q = csio_hw_to_wrm(hw)->q_arr[qidx];748 struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;749 750 memset(stp, 0, sizeof(*stp));751}752 753/*754 * csio_wr_cleanup_iq_ftr - Cleanup Footer entries in IQ755 * @hw: HW module756 * @qidx: Ingress queue index757 *758 * Cleanup the footer entries in the given ingress queue,759 * set to 1 the internal copy of genbit.760 */761static void762csio_wr_cleanup_iq_ftr(struct csio_hw *hw, int qidx)763{764 struct csio_wrm *wrm = csio_hw_to_wrm(hw);765 struct csio_q *q = wrm->q_arr[qidx];766 void *wr;767 struct csio_iqwr_footer *ftr;768 uint32_t i = 0;769 770 /* set to 1 since we are just about zero out genbit */771 q->un.iq.genbit = 1;772 773 for (i = 0; i < q->credits; i++) {774 /* Get the WR */775 wr = (void *)((uintptr_t)q->vstart +776 (i * q->wr_sz));777 /* Get the footer */778 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +779 (q->wr_sz - sizeof(*ftr)));780 /* Zero out footer */781 memset(ftr, 0, sizeof(*ftr));782 }783}784 785int786csio_wr_destroy_queues(struct csio_hw *hw, bool cmd)787{788 int i, flq_idx;789 struct csio_q *q;790 struct csio_wrm *wrm = csio_hw_to_wrm(hw);791 int rv;792 793 for (i = 0; i < wrm->free_qidx; i++) {794 q = wrm->q_arr[i];795 796 switch (q->type) {797 case CSIO_EGRESS:798 if (csio_q_eqid(hw, i) != CSIO_MAX_QID) {799 csio_wr_cleanup_eq_stpg(hw, i);800 if (!cmd) {801 csio_q_eqid(hw, i) = CSIO_MAX_QID;802 continue;803 }804 805 rv = csio_wr_eq_destroy(hw, NULL, i, NULL);806 if ((rv == -EBUSY) || (rv == -ETIMEDOUT))807 cmd = false;808 809 csio_q_eqid(hw, i) = CSIO_MAX_QID;810 }811 fallthrough;812 case CSIO_INGRESS:813 if (csio_q_iqid(hw, i) != CSIO_MAX_QID) {814 csio_wr_cleanup_iq_ftr(hw, i);815 if (!cmd) {816 csio_q_iqid(hw, i) = CSIO_MAX_QID;817 flq_idx = csio_q_iq_flq_idx(hw, i);818 if (flq_idx != -1)819 csio_q_flid(hw, flq_idx) =820 CSIO_MAX_QID;821 continue;822 }823 824 rv = csio_wr_iq_destroy(hw, NULL, i, NULL);825 if ((rv == -EBUSY) || (rv == -ETIMEDOUT))826 cmd = false;827 828 csio_q_iqid(hw, i) = CSIO_MAX_QID;829 flq_idx = csio_q_iq_flq_idx(hw, i);830 if (flq_idx != -1)831 csio_q_flid(hw, flq_idx) = CSIO_MAX_QID;832 }833 break;834 default:835 break;836 }837 }838 839 hw->flags &= ~CSIO_HWF_Q_FW_ALLOCED;840 841 return 0;842}843 844/*845 * csio_wr_get - Get requested size of WR entry/entries from queue.846 * @hw: HW module.847 * @qidx: Index of queue.848 * @size: Cumulative size of Work request(s).849 * @wrp: Work request pair.850 *851 * If requested credits are available, return the start address of the852 * work request in the work request pair. Set pidx accordingly and853 * return.854 *855 * NOTE about WR pair:856 * ==================857 * A WR can start towards the end of a queue, and then continue at the858 * beginning, since the queue is considered to be circular. This will859 * require a pair of address/size to be passed back to the caller -860 * hence Work request pair format.861 */862int863csio_wr_get(struct csio_hw *hw, int qidx, uint32_t size,864 struct csio_wr_pair *wrp)865{866 struct csio_wrm *wrm = csio_hw_to_wrm(hw);867 struct csio_q *q = wrm->q_arr[qidx];868 void *cwr = (void *)((uintptr_t)(q->vstart) +869 (q->pidx * CSIO_QCREDIT_SZ));870 struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;871 uint16_t cidx = q->cidx = ntohs(stp->cidx);872 uint16_t pidx = q->pidx;873 uint32_t req_sz = ALIGN(size, CSIO_QCREDIT_SZ);874 int req_credits = req_sz / CSIO_QCREDIT_SZ;875 int credits;876 877 CSIO_DB_ASSERT(q->owner != NULL);878 CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));879 CSIO_DB_ASSERT(cidx <= q->credits);880 881 /* Calculate credits */882 if (pidx > cidx) {883 credits = q->credits - (pidx - cidx) - 1;884 } else if (cidx > pidx) {885 credits = cidx - pidx - 1;886 } else {887 /* cidx == pidx, empty queue */888 credits = q->credits;889 CSIO_INC_STATS(q, n_qempty);890 }891 892 /*893 * Check if we have enough credits.894 * credits = 1 implies queue is full.895 */896 if (!credits || (req_credits > credits)) {897 CSIO_INC_STATS(q, n_qfull);898 return -EBUSY;899 }900 901 /*902 * If we are here, we have enough credits to satisfy the903 * request. Check if we are near the end of q, and if WR spills over.904 * If it does, use the first addr/size to cover the queue until905 * the end. Fit the remainder portion of the request at the top906 * of queue and return it in the second addr/len. Set pidx907 * accordingly.908 */909 if (unlikely(((uintptr_t)cwr + req_sz) > (uintptr_t)(q->vwrap))) {910 wrp->addr1 = cwr;911 wrp->size1 = (uint32_t)((uintptr_t)q->vwrap - (uintptr_t)cwr);912 wrp->addr2 = q->vstart;913 wrp->size2 = req_sz - wrp->size1;914 q->pidx = (uint16_t)(ALIGN(wrp->size2, CSIO_QCREDIT_SZ) /915 CSIO_QCREDIT_SZ);916 CSIO_INC_STATS(q, n_qwrap);917 CSIO_INC_STATS(q, n_eq_wr_split);918 } else {919 wrp->addr1 = cwr;920 wrp->size1 = req_sz;921 wrp->addr2 = NULL;922 wrp->size2 = 0;923 q->pidx += (uint16_t)req_credits;924 925 /* We are the end of queue, roll back pidx to top of queue */926 if (unlikely(q->pidx == q->credits)) {927 q->pidx = 0;928 CSIO_INC_STATS(q, n_qwrap);929 }930 }931 932 q->inc_idx = (uint16_t)req_credits;933 934 CSIO_INC_STATS(q, n_tot_reqs);935 936 return 0;937}938 939/*940 * csio_wr_copy_to_wrp - Copies given data into WR.941 * @data_buf - Data buffer942 * @wrp - Work request pair.943 * @wr_off - Work request offset.944 * @data_len - Data length.945 *946 * Copies the given data in Work Request. Work request pair(wrp) specifies947 * address information of Work request.948 * Returns: none949 */950void951csio_wr_copy_to_wrp(void *data_buf, struct csio_wr_pair *wrp,952 uint32_t wr_off, uint32_t data_len)953{954 uint32_t nbytes;955 956 /* Number of space available in buffer addr1 of WRP */957 nbytes = ((wrp->size1 - wr_off) >= data_len) ?958 data_len : (wrp->size1 - wr_off);959 960 memcpy((uint8_t *) wrp->addr1 + wr_off, data_buf, nbytes);961 data_len -= nbytes;962 963 /* Write the remaining data from the begining of circular buffer */964 if (data_len) {965 CSIO_DB_ASSERT(data_len <= wrp->size2);966 CSIO_DB_ASSERT(wrp->addr2 != NULL);967 memcpy(wrp->addr2, (uint8_t *) data_buf + nbytes, data_len);968 }969}970 971/*972 * csio_wr_issue - Notify chip of Work request.973 * @hw: HW module.974 * @qidx: Index of queue.975 * @prio: 0: Low priority, 1: High priority976 *977 * Rings the SGE Doorbell by writing the current producer index of the passed978 * in queue into the register.979 *980 */981int982csio_wr_issue(struct csio_hw *hw, int qidx, bool prio)983{984 struct csio_wrm *wrm = csio_hw_to_wrm(hw);985 struct csio_q *q = wrm->q_arr[qidx];986 987 CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));988 989 wmb();990 /* Ring SGE Doorbell writing q->pidx into it */991 csio_wr_reg32(hw, DBPRIO_V(prio) | QID_V(q->un.eq.physeqid) |992 PIDX_T5_V(q->inc_idx) | DBTYPE_F,993 MYPF_REG(SGE_PF_KDOORBELL_A));994 q->inc_idx = 0;995 996 return 0;997}998 999static inline uint32_t1000csio_wr_avail_qcredits(struct csio_q *q)1001{1002 if (q->pidx > q->cidx)1003 return q->pidx - q->cidx;1004 else if (q->cidx > q->pidx)1005 return q->credits - (q->cidx - q->pidx);1006 else1007 return 0; /* cidx == pidx, empty queue */1008}1009 1010/*1011 * csio_wr_inval_flq_buf - Invalidate a free list buffer entry.1012 * @hw: HW module.1013 * @flq: The freelist queue.1014 *1015 * Invalidate the driver's version of a freelist buffer entry,1016 * without freeing the associated the DMA memory. The entry1017 * to be invalidated is picked up from the current Free list1018 * queue cidx.1019 *1020 */1021static inline void1022csio_wr_inval_flq_buf(struct csio_hw *hw, struct csio_q *flq)1023{1024 flq->cidx++;1025 if (flq->cidx == flq->credits) {1026 flq->cidx = 0;1027 CSIO_INC_STATS(flq, n_qwrap);1028 }1029}1030 1031/*1032 * csio_wr_process_fl - Process a freelist completion.1033 * @hw: HW module.1034 * @q: The ingress queue attached to the Freelist.1035 * @wr: The freelist completion WR in the ingress queue.1036 * @len_to_qid: The lower 32-bits of the first flit of the RSP footer1037 * @iq_handler: Caller's handler for this completion.1038 * @priv: Private pointer of caller1039 *1040 */1041static inline void1042csio_wr_process_fl(struct csio_hw *hw, struct csio_q *q,1043 void *wr, uint32_t len_to_qid,1044 void (*iq_handler)(struct csio_hw *, void *,1045 uint32_t, struct csio_fl_dma_buf *,1046 void *),1047 void *priv)1048{1049 struct csio_wrm *wrm = csio_hw_to_wrm(hw);1050 struct csio_sge *sge = &wrm->sge;1051 struct csio_fl_dma_buf flb;1052 struct csio_dma_buf *buf, *fbuf;1053 uint32_t bufsz, len, lastlen = 0;1054 struct csio_q *flq = hw->wrm.q_arr[q->un.iq.flq_idx];1055 1056 CSIO_DB_ASSERT(flq != NULL);1057 1058 len = len_to_qid;1059 1060 if (len & IQWRF_NEWBUF) {1061 if (flq->un.fl.offset > 0) {1062 csio_wr_inval_flq_buf(hw, flq);1063 flq->un.fl.offset = 0;1064 }1065 len = IQWRF_LEN_GET(len);1066 }1067 1068 CSIO_DB_ASSERT(len != 0);1069 1070 flb.totlen = len;1071 1072 /* Consume all freelist buffers used for len bytes */1073 for (fbuf = flb.flbufs; ; fbuf++) {1074 buf = &flq->un.fl.bufs[flq->cidx];1075 bufsz = csio_wr_fl_bufsz(sge, buf);1076 1077 fbuf->paddr = buf->paddr;1078 fbuf->vaddr = buf->vaddr;1079 1080 flb.offset = flq->un.fl.offset;1081 lastlen = min(bufsz, len);1082 fbuf->len = lastlen;1083 1084 len -= lastlen;1085 if (!len)1086 break;1087 csio_wr_inval_flq_buf(hw, flq);1088 }1089 1090 flb.defer_free = flq->un.fl.packen ? 0 : 1;1091 1092 iq_handler(hw, wr, q->wr_sz - sizeof(struct csio_iqwr_footer),1093 &flb, priv);1094 1095 if (flq->un.fl.packen)1096 flq->un.fl.offset += ALIGN(lastlen, sge->csio_fl_align);1097 else1098 csio_wr_inval_flq_buf(hw, flq);1099 1100}1101 1102/*1103 * csio_is_new_iqwr - Is this a new Ingress queue entry ?1104 * @q: Ingress quueue.1105 * @ftr: Ingress queue WR SGE footer.1106 *1107 * The entry is new if our generation bit matches the corresponding1108 * bit in the footer of the current WR.1109 */1110static inline bool1111csio_is_new_iqwr(struct csio_q *q, struct csio_iqwr_footer *ftr)1112{1113 return (q->un.iq.genbit == (ftr->u.type_gen >> IQWRF_GEN_SHIFT));1114}1115 1116/*1117 * csio_wr_process_iq - Process elements in Ingress queue.1118 * @hw: HW pointer1119 * @qidx: Index of queue1120 * @iq_handler: Handler for this queue1121 * @priv: Caller's private pointer1122 *1123 * This routine walks through every entry of the ingress queue, calling1124 * the provided iq_handler with the entry, until the generation bit1125 * flips.1126 */1127int1128csio_wr_process_iq(struct csio_hw *hw, struct csio_q *q,1129 void (*iq_handler)(struct csio_hw *, void *,1130 uint32_t, struct csio_fl_dma_buf *,1131 void *),1132 void *priv)1133{1134 struct csio_wrm *wrm = csio_hw_to_wrm(hw);1135 void *wr = (void *)((uintptr_t)q->vstart + (q->cidx * q->wr_sz));1136 struct csio_iqwr_footer *ftr;1137 uint32_t wr_type, fw_qid, qid;1138 struct csio_q *q_completed;1139 struct csio_q *flq = csio_iq_has_fl(q) ?1140 wrm->q_arr[q->un.iq.flq_idx] : NULL;1141 int rv = 0;1142 1143 /* Get the footer */1144 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +1145 (q->wr_sz - sizeof(*ftr)));1146 1147 /*1148 * When q wrapped around last time, driver should have inverted1149 * ic.genbit as well.1150 */1151 while (csio_is_new_iqwr(q, ftr)) {1152 1153 CSIO_DB_ASSERT(((uintptr_t)wr + q->wr_sz) <=1154 (uintptr_t)q->vwrap);1155 rmb();1156 wr_type = IQWRF_TYPE_GET(ftr->u.type_gen);1157 1158 switch (wr_type) {1159 case X_RSPD_TYPE_CPL:1160 /* Subtract footer from WR len */1161 iq_handler(hw, wr, q->wr_sz - sizeof(*ftr), NULL, priv);1162 break;1163 case X_RSPD_TYPE_FLBUF:1164 csio_wr_process_fl(hw, q, wr,1165 ntohl(ftr->pldbuflen_qid),1166 iq_handler, priv);1167 break;1168 case X_RSPD_TYPE_INTR:1169 fw_qid = ntohl(ftr->pldbuflen_qid);1170 qid = fw_qid - wrm->fw_iq_start;1171 q_completed = hw->wrm.intr_map[qid];1172 1173 if (unlikely(qid ==1174 csio_q_physiqid(hw, hw->intr_iq_idx))) {1175 /*1176 * We are already in the Forward Interrupt1177 * Interrupt Queue Service! Do-not service1178 * again!1179 *1180 */1181 } else {1182 CSIO_DB_ASSERT(q_completed);1183 CSIO_DB_ASSERT(1184 q_completed->un.iq.iq_intx_handler);1185 1186 /* Call the queue handler. */1187 q_completed->un.iq.iq_intx_handler(hw, NULL,1188 0, NULL, (void *)q_completed);1189 }1190 break;1191 default:1192 csio_warn(hw, "Unknown resp type 0x%x received\n",1193 wr_type);1194 CSIO_INC_STATS(q, n_rsp_unknown);1195 break;1196 }1197 1198 /*1199 * Ingress *always* has fixed size WR entries. Therefore,1200 * there should always be complete WRs towards the end of1201 * queue.1202 */1203 if (((uintptr_t)wr + q->wr_sz) == (uintptr_t)q->vwrap) {1204 1205 /* Roll over to start of queue */1206 q->cidx = 0;1207 wr = q->vstart;1208 1209 /* Toggle genbit */1210 q->un.iq.genbit ^= 0x1;1211 1212 CSIO_INC_STATS(q, n_qwrap);1213 } else {1214 q->cidx++;1215 wr = (void *)((uintptr_t)(q->vstart) +1216 (q->cidx * q->wr_sz));1217 }1218 1219 ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +1220 (q->wr_sz - sizeof(*ftr)));1221 q->inc_idx++;1222 1223 } /* while (q->un.iq.genbit == hdr->genbit) */1224 1225 /*1226 * We need to re-arm SGE interrupts in case we got a stray interrupt,1227 * especially in msix mode. With INTx, this may be a common occurence.1228 */1229 if (unlikely(!q->inc_idx)) {1230 CSIO_INC_STATS(q, n_stray_comp);1231 rv = -EINVAL;1232 goto restart;1233 }1234 1235 /* Replenish free list buffers if pending falls below low water mark */1236 if (flq) {1237 uint32_t avail = csio_wr_avail_qcredits(flq);1238 if (avail <= 16) {1239 /* Make sure in FLQ, atleast 1 credit (8 FL buffers)1240 * remains unpopulated otherwise HW thinks1241 * FLQ is empty.1242 */1243 csio_wr_update_fl(hw, flq, (flq->credits - 8) - avail);1244 csio_wr_ring_fldb(hw, flq);1245 }1246 }1247 1248restart:1249 /* Now inform SGE about our incremental index value */1250 csio_wr_reg32(hw, CIDXINC_V(q->inc_idx) |1251 INGRESSQID_V(q->un.iq.physiqid) |1252 TIMERREG_V(csio_sge_timer_reg),1253 MYPF_REG(SGE_PF_GTS_A));1254 q->stats.n_tot_rsps += q->inc_idx;1255 1256 q->inc_idx = 0;1257 1258 return rv;1259}1260 1261int1262csio_wr_process_iq_idx(struct csio_hw *hw, int qidx,1263 void (*iq_handler)(struct csio_hw *, void *,1264 uint32_t, struct csio_fl_dma_buf *,1265 void *),1266 void *priv)1267{1268 struct csio_wrm *wrm = csio_hw_to_wrm(hw);1269 struct csio_q *iq = wrm->q_arr[qidx];1270 1271 return csio_wr_process_iq(hw, iq, iq_handler, priv);1272}1273 1274static int1275csio_closest_timer(struct csio_sge *s, int time)1276{1277 int i, delta, match = 0, min_delta = INT_MAX;1278 1279 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {1280 delta = time - s->timer_val[i];1281 if (delta < 0)1282 delta = -delta;1283 if (delta < min_delta) {1284 min_delta = delta;1285 match = i;1286 }1287 }1288 return match;1289}1290 1291static int1292csio_closest_thresh(struct csio_sge *s, int cnt)1293{1294 int i, delta, match = 0, min_delta = INT_MAX;1295 1296 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {1297 delta = cnt - s->counter_val[i];1298 if (delta < 0)1299 delta = -delta;1300 if (delta < min_delta) {1301 min_delta = delta;1302 match = i;1303 }1304 }1305 return match;1306}1307 1308static void1309csio_wr_fixup_host_params(struct csio_hw *hw)1310{1311 struct csio_wrm *wrm = csio_hw_to_wrm(hw);1312 struct csio_sge *sge = &wrm->sge;1313 uint32_t clsz = L1_CACHE_BYTES;1314 uint32_t s_hps = PAGE_SHIFT - 10;1315 uint32_t stat_len = clsz > 64 ? 128 : 64;1316 u32 fl_align = clsz < 32 ? 32 : clsz;1317 u32 pack_align;1318 u32 ingpad, ingpack;1319 1320 csio_wr_reg32(hw, HOSTPAGESIZEPF0_V(s_hps) | HOSTPAGESIZEPF1_V(s_hps) |1321 HOSTPAGESIZEPF2_V(s_hps) | HOSTPAGESIZEPF3_V(s_hps) |1322 HOSTPAGESIZEPF4_V(s_hps) | HOSTPAGESIZEPF5_V(s_hps) |1323 HOSTPAGESIZEPF6_V(s_hps) | HOSTPAGESIZEPF7_V(s_hps),1324 SGE_HOST_PAGE_SIZE_A);1325 1326 /* T5 introduced the separation of the Free List Padding and1327 * Packing Boundaries. Thus, we can select a smaller Padding1328 * Boundary to avoid uselessly chewing up PCIe Link and Memory1329 * Bandwidth, and use a Packing Boundary which is large enough1330 * to avoid false sharing between CPUs, etc.1331 *1332 * For the PCI Link, the smaller the Padding Boundary the1333 * better. For the Memory Controller, a smaller Padding1334 * Boundary is better until we cross under the Memory Line1335 * Size (the minimum unit of transfer to/from Memory). If we1336 * have a Padding Boundary which is smaller than the Memory1337 * Line Size, that'll involve a Read-Modify-Write cycle on the1338 * Memory Controller which is never good.1339 */1340 1341 /* We want the Packing Boundary to be based on the Cache Line1342 * Size in order to help avoid False Sharing performance1343 * issues between CPUs, etc. We also want the Packing1344 * Boundary to incorporate the PCI-E Maximum Payload Size. We1345 * get best performance when the Packing Boundary is a1346 * multiple of the Maximum Payload Size.1347 */1348 pack_align = fl_align;1349 if (pci_is_pcie(hw->pdev)) {1350 u32 mps, mps_log;1351 u16 devctl;1352 1353 /* The PCIe Device Control Maximum Payload Size field1354 * [bits 7:5] encodes sizes as powers of 2 starting at1355 * 128 bytes.1356 */1357 pcie_capability_read_word(hw->pdev, PCI_EXP_DEVCTL, &devctl);1358 mps_log = ((devctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5) + 7;1359 mps = 1 << mps_log;1360 if (mps > pack_align)1361 pack_align = mps;1362 }1363 1364 /* T5/T6 have a special interpretation of the "0"1365 * value for the Packing Boundary. This corresponds to 161366 * bytes instead of the expected 32 bytes.1367 */1368 if (pack_align <= 16) {1369 ingpack = INGPACKBOUNDARY_16B_X;1370 fl_align = 16;1371 } else if (pack_align == 32) {1372 ingpack = INGPACKBOUNDARY_64B_X;1373 fl_align = 64;1374 } else {1375 u32 pack_align_log = fls(pack_align) - 1;1376 1377 ingpack = pack_align_log - INGPACKBOUNDARY_SHIFT_X;1378 fl_align = pack_align;1379 }1380 1381 /* Use the smallest Ingress Padding which isn't smaller than1382 * the Memory Controller Read/Write Size. We'll take that as1383 * being 8 bytes since we don't know of any system with a1384 * wider Memory Controller Bus Width.1385 */1386 if (csio_is_t5(hw->pdev->device & CSIO_HW_CHIP_MASK))1387 ingpad = INGPADBOUNDARY_32B_X;1388 else1389 ingpad = T6_INGPADBOUNDARY_8B_X;1390 1391 csio_set_reg_field(hw, SGE_CONTROL_A,1392 INGPADBOUNDARY_V(INGPADBOUNDARY_M) |1393 EGRSTATUSPAGESIZE_F,1394 INGPADBOUNDARY_V(ingpad) |1395 EGRSTATUSPAGESIZE_V(stat_len != 64));1396 csio_set_reg_field(hw, SGE_CONTROL2_A,1397 INGPACKBOUNDARY_V(INGPACKBOUNDARY_M),1398 INGPACKBOUNDARY_V(ingpack));1399 1400 /* FL BUFFER SIZE#0 is Page size i,e already aligned to cache line */1401 csio_wr_reg32(hw, PAGE_SIZE, SGE_FL_BUFFER_SIZE0_A);1402 1403 /*1404 * If using hard params, the following will get set correctly1405 * in csio_wr_set_sge().1406 */1407 if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS) {1408 csio_wr_reg32(hw,1409 (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE2_A) +1410 fl_align - 1) & ~(fl_align - 1),1411 SGE_FL_BUFFER_SIZE2_A);1412 csio_wr_reg32(hw,1413 (csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE3_A) +1414 fl_align - 1) & ~(fl_align - 1),1415 SGE_FL_BUFFER_SIZE3_A);1416 }1417 1418 sge->csio_fl_align = fl_align;1419 1420 csio_wr_reg32(hw, HPZ0_V(PAGE_SHIFT - 12), ULP_RX_TDDP_PSZ_A);1421 1422 /* default value of rx_dma_offset of the NIC driver */1423 csio_set_reg_field(hw, SGE_CONTROL_A,1424 PKTSHIFT_V(PKTSHIFT_M),1425 PKTSHIFT_V(CSIO_SGE_RX_DMA_OFFSET));1426 1427 csio_hw_tp_wr_bits_indirect(hw, TP_INGRESS_CONFIG_A,1428 CSUM_HAS_PSEUDO_HDR_F, 0);1429}1430 1431static void1432csio_init_intr_coalesce_parms(struct csio_hw *hw)1433{1434 struct csio_wrm *wrm = csio_hw_to_wrm(hw);1435 struct csio_sge *sge = &wrm->sge;1436 1437 csio_sge_thresh_reg = csio_closest_thresh(sge, csio_intr_coalesce_cnt);1438 if (csio_intr_coalesce_cnt) {1439 csio_sge_thresh_reg = 0;1440 csio_sge_timer_reg = X_TIMERREG_RESTART_COUNTER;1441 return;1442 }1443 1444 csio_sge_timer_reg = csio_closest_timer(sge, csio_intr_coalesce_time);1445}1446 1447/*1448 * csio_wr_get_sge - Get SGE register values.1449 * @hw: HW module.1450 *1451 * Used by non-master functions and by master-functions relying on config file.1452 */1453static void1454csio_wr_get_sge(struct csio_hw *hw)1455{1456 struct csio_wrm *wrm = csio_hw_to_wrm(hw);1457 struct csio_sge *sge = &wrm->sge;1458 uint32_t ingpad;1459 int i;1460 u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5;1461 u32 ingress_rx_threshold;1462 1463 sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A);1464 1465 ingpad = INGPADBOUNDARY_G(sge->sge_control);1466 1467 switch (ingpad) {1468 case X_INGPCIEBOUNDARY_32B:1469 sge->csio_fl_align = 32; break;1470 case X_INGPCIEBOUNDARY_64B:1471 sge->csio_fl_align = 64; break;1472 case X_INGPCIEBOUNDARY_128B:1473 sge->csio_fl_align = 128; break;1474 case X_INGPCIEBOUNDARY_256B:1475 sge->csio_fl_align = 256; break;1476 case X_INGPCIEBOUNDARY_512B:1477 sge->csio_fl_align = 512; break;1478 case X_INGPCIEBOUNDARY_1024B:1479 sge->csio_fl_align = 1024; break;1480 case X_INGPCIEBOUNDARY_2048B:1481 sge->csio_fl_align = 2048; break;1482 case X_INGPCIEBOUNDARY_4096B:1483 sge->csio_fl_align = 4096; break;1484 }1485 1486 for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)1487 csio_get_flbuf_size(hw, sge, i);1488 1489 timer_value_0_and_1 = csio_rd_reg32(hw, SGE_TIMER_VALUE_0_AND_1_A);1490 timer_value_2_and_3 = csio_rd_reg32(hw, SGE_TIMER_VALUE_2_AND_3_A);1491 timer_value_4_and_5 = csio_rd_reg32(hw, SGE_TIMER_VALUE_4_AND_5_A);1492 1493 sge->timer_val[0] = (uint16_t)csio_core_ticks_to_us(hw,1494 TIMERVALUE0_G(timer_value_0_and_1));1495 sge->timer_val[1] = (uint16_t)csio_core_ticks_to_us(hw,1496 TIMERVALUE1_G(timer_value_0_and_1));1497 sge->timer_val[2] = (uint16_t)csio_core_ticks_to_us(hw,1498 TIMERVALUE2_G(timer_value_2_and_3));1499 sge->timer_val[3] = (uint16_t)csio_core_ticks_to_us(hw,1500 TIMERVALUE3_G(timer_value_2_and_3));1501 sge->timer_val[4] = (uint16_t)csio_core_ticks_to_us(hw,1502 TIMERVALUE4_G(timer_value_4_and_5));1503 sge->timer_val[5] = (uint16_t)csio_core_ticks_to_us(hw,1504 TIMERVALUE5_G(timer_value_4_and_5));1505 1506 ingress_rx_threshold = csio_rd_reg32(hw, SGE_INGRESS_RX_THRESHOLD_A);1507 sge->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold);1508 sge->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold);1509 sge->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold);1510 sge->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold);1511 1512 csio_init_intr_coalesce_parms(hw);1513}1514 1515/*1516 * csio_wr_set_sge - Initialize SGE registers1517 * @hw: HW module.1518 *1519 * Used by Master function to initialize SGE registers in the absence1520 * of a config file.1521 */1522static void1523csio_wr_set_sge(struct csio_hw *hw)1524{1525 struct csio_wrm *wrm = csio_hw_to_wrm(hw);1526 struct csio_sge *sge = &wrm->sge;1527 int i;1528 1529 /*1530 * Set up our basic SGE mode to deliver CPL messages to our Ingress1531 * Queue and Packet Date to the Free List.1532 */1533 csio_set_reg_field(hw, SGE_CONTROL_A, RXPKTCPLMODE_F, RXPKTCPLMODE_F);1534 1535 sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A);1536 1537 /* sge->csio_fl_align is set up by csio_wr_fixup_host_params(). */1538 1539 /*1540 * Set up to drop DOORBELL writes when the DOORBELL FIFO overflows1541 * and generate an interrupt when this occurs so we can recover.1542 */1543 csio_set_reg_field(hw, SGE_DBFIFO_STATUS_A,1544 LP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M),1545 LP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH));1546 csio_set_reg_field(hw, SGE_DBFIFO_STATUS2_A,1547 HP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M),1548 HP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH));1549 1550 csio_set_reg_field(hw, SGE_DOORBELL_CONTROL_A, ENABLE_DROP_F,1551 ENABLE_DROP_F);1552 1553 /* SGE_FL_BUFFER_SIZE0 is set up by csio_wr_fixup_host_params(). */1554 1555 CSIO_SET_FLBUF_SIZE(hw, 1, CSIO_SGE_FLBUF_SIZE1);1556 csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE2 + sge->csio_fl_align - 1)1557 & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE2_A);1558 csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE3 + sge->csio_fl_align - 1)1559 & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE3_A);1560 CSIO_SET_FLBUF_SIZE(hw, 4, CSIO_SGE_FLBUF_SIZE4);1561 CSIO_SET_FLBUF_SIZE(hw, 5, CSIO_SGE_FLBUF_SIZE5);1562 CSIO_SET_FLBUF_SIZE(hw, 6, CSIO_SGE_FLBUF_SIZE6);1563 CSIO_SET_FLBUF_SIZE(hw, 7, CSIO_SGE_FLBUF_SIZE7);1564 CSIO_SET_FLBUF_SIZE(hw, 8, CSIO_SGE_FLBUF_SIZE8);1565 1566 for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)1567 csio_get_flbuf_size(hw, sge, i);1568 1569 /* Initialize interrupt coalescing attributes */1570 sge->timer_val[0] = CSIO_SGE_TIMER_VAL_0;1571 sge->timer_val[1] = CSIO_SGE_TIMER_VAL_1;1572 sge->timer_val[2] = CSIO_SGE_TIMER_VAL_2;1573 sge->timer_val[3] = CSIO_SGE_TIMER_VAL_3;1574 sge->timer_val[4] = CSIO_SGE_TIMER_VAL_4;1575 sge->timer_val[5] = CSIO_SGE_TIMER_VAL_5;1576 1577 sge->counter_val[0] = CSIO_SGE_INT_CNT_VAL_0;1578 sge->counter_val[1] = CSIO_SGE_INT_CNT_VAL_1;1579 sge->counter_val[2] = CSIO_SGE_INT_CNT_VAL_2;1580 sge->counter_val[3] = CSIO_SGE_INT_CNT_VAL_3;1581 1582 csio_wr_reg32(hw, THRESHOLD_0_V(sge->counter_val[0]) |1583 THRESHOLD_1_V(sge->counter_val[1]) |1584 THRESHOLD_2_V(sge->counter_val[2]) |1585 THRESHOLD_3_V(sge->counter_val[3]),1586 SGE_INGRESS_RX_THRESHOLD_A);1587 1588 csio_wr_reg32(hw,1589 TIMERVALUE0_V(csio_us_to_core_ticks(hw, sge->timer_val[0])) |1590 TIMERVALUE1_V(csio_us_to_core_ticks(hw, sge->timer_val[1])),1591 SGE_TIMER_VALUE_0_AND_1_A);1592 1593 csio_wr_reg32(hw,1594 TIMERVALUE2_V(csio_us_to_core_ticks(hw, sge->timer_val[2])) |1595 TIMERVALUE3_V(csio_us_to_core_ticks(hw, sge->timer_val[3])),1596 SGE_TIMER_VALUE_2_AND_3_A);1597 1598 csio_wr_reg32(hw,1599 TIMERVALUE4_V(csio_us_to_core_ticks(hw, sge->timer_val[4])) |1600 TIMERVALUE5_V(csio_us_to_core_ticks(hw, sge->timer_val[5])),1601 SGE_TIMER_VALUE_4_AND_5_A);1602 1603 csio_init_intr_coalesce_parms(hw);1604}1605 1606void1607csio_wr_sge_init(struct csio_hw *hw)1608{1609 /*1610 * If we are master and chip is not initialized:1611 * - If we plan to use the config file, we need to fixup some1612 * host specific registers, and read the rest of the SGE1613 * configuration.1614 * - If we dont plan to use the config file, we need to initialize1615 * SGE entirely, including fixing the host specific registers.1616 * If we are master and chip is initialized, just read and work off of1617 * the already initialized SGE values.1618 * If we arent the master, we are only allowed to read and work off of1619 * the already initialized SGE values.1620 *1621 * Therefore, before calling this function, we assume that the master-1622 * ship of the card, state and whether to use config file or not, have1623 * already been decided.1624 */1625 if (csio_is_hw_master(hw)) {1626 if (hw->fw_state != CSIO_DEV_STATE_INIT)1627 csio_wr_fixup_host_params(hw);1628 1629 if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS)1630 csio_wr_get_sge(hw);1631 else1632 csio_wr_set_sge(hw);1633 } else1634 csio_wr_get_sge(hw);1635}1636 1637/*1638 * csio_wrm_init - Initialize Work request module.1639 * @wrm: WR module1640 * @hw: HW pointer1641 *1642 * Allocates memory for an array of queue pointers starting at q_arr.1643 */1644int1645csio_wrm_init(struct csio_wrm *wrm, struct csio_hw *hw)1646{1647 int i;1648 1649 if (!wrm->num_q) {1650 csio_err(hw, "Num queues is not set\n");1651 return -EINVAL;1652 }1653 1654 wrm->q_arr = kcalloc(wrm->num_q, sizeof(struct csio_q *), GFP_KERNEL);1655 if (!wrm->q_arr)1656 goto err;1657 1658 for (i = 0; i < wrm->num_q; i++) {1659 wrm->q_arr[i] = kzalloc(sizeof(struct csio_q), GFP_KERNEL);1660 if (!wrm->q_arr[i]) {1661 while (--i >= 0)1662 kfree(wrm->q_arr[i]);1663 goto err_free_arr;1664 }1665 }1666 wrm->free_qidx = 0;1667 1668 return 0;1669 1670err_free_arr:1671 kfree(wrm->q_arr);1672err:1673 return -ENOMEM;1674}1675 1676/*1677 * csio_wrm_exit - Initialize Work request module.1678 * @wrm: WR module1679 * @hw: HW module1680 *1681 * Uninitialize WR module. Free q_arr and pointers in it.1682 * We have the additional job of freeing the DMA memory associated1683 * with the queues.1684 */1685void1686csio_wrm_exit(struct csio_wrm *wrm, struct csio_hw *hw)1687{1688 int i;1689 uint32_t j;1690 struct csio_q *q;1691 struct csio_dma_buf *buf;1692 1693 for (i = 0; i < wrm->num_q; i++) {1694 q = wrm->q_arr[i];1695 1696 if (wrm->free_qidx && (i < wrm->free_qidx)) {1697 if (q->type == CSIO_FREELIST) {1698 if (!q->un.fl.bufs)1699 continue;1700 for (j = 0; j < q->credits; j++) {1701 buf = &q->un.fl.bufs[j];1702 if (!buf->vaddr)1703 continue;1704 dma_free_coherent(&hw->pdev->dev,1705 buf->len, buf->vaddr,1706 buf->paddr);1707 }1708 kfree(q->un.fl.bufs);1709 }1710 dma_free_coherent(&hw->pdev->dev, q->size,1711 q->vstart, q->pstart);1712 }1713 kfree(q);1714 }1715 1716 hw->flags &= ~CSIO_HWF_Q_MEM_ALLOCED;1717 1718 kfree(wrm->q_arr);1719}1720