972 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Huawei HiNIC PCI Express Linux driver4 * Copyright(c) 2017 Huawei Technologies Co., Ltd5 */6 7#include <linux/kernel.h>8#include <linux/types.h>9#include <linux/pci.h>10#include <linux/device.h>11#include <linux/dma-mapping.h>12#include <linux/vmalloc.h>13#include <linux/errno.h>14#include <linux/sizes.h>15#include <linux/atomic.h>16#include <linux/skbuff.h>17#include <linux/io.h>18#include <asm/barrier.h>19#include <asm/byteorder.h>20 21#include "hinic_common.h"22#include "hinic_hw_if.h"23#include "hinic_hw_wqe.h"24#include "hinic_hw_wq.h"25#include "hinic_hw_qp_ctxt.h"26#include "hinic_hw_qp.h"27#include "hinic_hw_io.h"28 29#define SQ_DB_OFF SZ_2K30 31/* The number of cache line to prefetch Until threshold state */32#define WQ_PREFETCH_MAX 233/* The number of cache line to prefetch After threshold state */34#define WQ_PREFETCH_MIN 135/* Threshold state */36#define WQ_PREFETCH_THRESHOLD 25637 38/* sizes of the SQ/RQ ctxt */39#define Q_CTXT_SIZE 4840#define CTXT_RSVD 24041 42#define SQ_CTXT_OFFSET(max_sqs, max_rqs, q_id) \43 (((max_rqs) + (max_sqs)) * CTXT_RSVD + (q_id) * Q_CTXT_SIZE)44 45#define RQ_CTXT_OFFSET(max_sqs, max_rqs, q_id) \46 (((max_rqs) + (max_sqs)) * CTXT_RSVD + \47 (max_sqs + (q_id)) * Q_CTXT_SIZE)48 49#define SIZE_16BYTES(size) (ALIGN(size, 16) >> 4)50#define SIZE_8BYTES(size) (ALIGN(size, 8) >> 3)51#define SECT_SIZE_FROM_8BYTES(size) ((size) << 3)52 53#define SQ_DB_PI_HI_SHIFT 854#define SQ_DB_PI_HI(prod_idx) ((prod_idx) >> SQ_DB_PI_HI_SHIFT)55 56#define SQ_DB_PI_LOW_MASK 0xFF57#define SQ_DB_PI_LOW(prod_idx) ((prod_idx) & SQ_DB_PI_LOW_MASK)58 59#define SQ_DB_ADDR(sq, pi) ((u64 *)((sq)->db_base) + SQ_DB_PI_LOW(pi))60 61#define SQ_MASKED_IDX(sq, idx) ((idx) & (sq)->wq->mask)62#define RQ_MASKED_IDX(rq, idx) ((idx) & (rq)->wq->mask)63 64enum sq_wqe_type {65 SQ_NORMAL_WQE = 0,66};67 68enum rq_completion_fmt {69 RQ_COMPLETE_SGE = 170};71 72void hinic_qp_prepare_header(struct hinic_qp_ctxt_header *qp_ctxt_hdr,73 enum hinic_qp_ctxt_type ctxt_type,74 u16 num_queues, u16 max_queues)75{76 u16 max_sqs = max_queues;77 u16 max_rqs = max_queues;78 79 qp_ctxt_hdr->num_queues = num_queues;80 qp_ctxt_hdr->queue_type = ctxt_type;81 82 if (ctxt_type == HINIC_QP_CTXT_TYPE_SQ)83 qp_ctxt_hdr->addr_offset = SQ_CTXT_OFFSET(max_sqs, max_rqs, 0);84 else85 qp_ctxt_hdr->addr_offset = RQ_CTXT_OFFSET(max_sqs, max_rqs, 0);86 87 qp_ctxt_hdr->addr_offset = SIZE_16BYTES(qp_ctxt_hdr->addr_offset);88 89 hinic_cpu_to_be32(qp_ctxt_hdr, sizeof(*qp_ctxt_hdr));90}91 92void hinic_sq_prepare_ctxt(struct hinic_sq_ctxt *sq_ctxt,93 struct hinic_sq *sq, u16 global_qid)94{95 u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo;96 u64 wq_page_addr, wq_page_pfn, wq_block_pfn;97 u16 pi_start, ci_start;98 struct hinic_wq *wq;99 100 wq = sq->wq;101 ci_start = atomic_read(&wq->cons_idx);102 pi_start = atomic_read(&wq->prod_idx);103 104 /* Read the first page paddr from the WQ page paddr ptrs */105 wq_page_addr = be64_to_cpu(*wq->block_vaddr);106 107 wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr);108 wq_page_pfn_hi = upper_32_bits(wq_page_pfn);109 wq_page_pfn_lo = lower_32_bits(wq_page_pfn);110 111 /* If only one page, use 0-level CLA */112 if (wq->num_q_pages == 1)113 wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq_page_addr);114 else115 wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr);116 117 wq_block_pfn_hi = upper_32_bits(wq_block_pfn);118 wq_block_pfn_lo = lower_32_bits(wq_block_pfn);119 120 sq_ctxt->ceq_attr = HINIC_SQ_CTXT_CEQ_ATTR_SET(global_qid,121 GLOBAL_SQ_ID) |122 HINIC_SQ_CTXT_CEQ_ATTR_SET(0, EN);123 124 sq_ctxt->ci_wrapped = HINIC_SQ_CTXT_CI_SET(ci_start, IDX) |125 HINIC_SQ_CTXT_CI_SET(1, WRAPPED);126 127 sq_ctxt->wq_hi_pfn_pi =128 HINIC_SQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, HI_PFN) |129 HINIC_SQ_CTXT_WQ_PAGE_SET(pi_start, PI);130 131 sq_ctxt->wq_lo_pfn = wq_page_pfn_lo;132 133 sq_ctxt->pref_cache =134 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) |135 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) |136 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD);137 138 sq_ctxt->pref_wrapped = 1;139 140 sq_ctxt->pref_wq_hi_pfn_ci =141 HINIC_SQ_CTXT_PREF_SET(ci_start, CI) |142 HINIC_SQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN);143 144 sq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo;145 146 sq_ctxt->wq_block_hi_pfn =147 HINIC_SQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN);148 149 sq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo;150 151 hinic_cpu_to_be32(sq_ctxt, sizeof(*sq_ctxt));152}153 154void hinic_rq_prepare_ctxt(struct hinic_rq_ctxt *rq_ctxt,155 struct hinic_rq *rq, u16 global_qid)156{157 u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo;158 u64 wq_page_addr, wq_page_pfn, wq_block_pfn;159 u16 pi_start, ci_start;160 struct hinic_wq *wq;161 162 wq = rq->wq;163 ci_start = atomic_read(&wq->cons_idx);164 pi_start = atomic_read(&wq->prod_idx);165 166 /* Read the first page paddr from the WQ page paddr ptrs */167 wq_page_addr = be64_to_cpu(*wq->block_vaddr);168 169 wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr);170 wq_page_pfn_hi = upper_32_bits(wq_page_pfn);171 wq_page_pfn_lo = lower_32_bits(wq_page_pfn);172 173 wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr);174 wq_block_pfn_hi = upper_32_bits(wq_block_pfn);175 wq_block_pfn_lo = lower_32_bits(wq_block_pfn);176 177 rq_ctxt->ceq_attr = HINIC_RQ_CTXT_CEQ_ATTR_SET(0, EN) |178 HINIC_RQ_CTXT_CEQ_ATTR_SET(1, WRAPPED);179 180 rq_ctxt->pi_intr_attr = HINIC_RQ_CTXT_PI_SET(pi_start, IDX) |181 HINIC_RQ_CTXT_PI_SET(rq->msix_entry, INTR);182 183 rq_ctxt->wq_hi_pfn_ci = HINIC_RQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi,184 HI_PFN) |185 HINIC_RQ_CTXT_WQ_PAGE_SET(ci_start, CI);186 187 rq_ctxt->wq_lo_pfn = wq_page_pfn_lo;188 189 rq_ctxt->pref_cache =190 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) |191 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) |192 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD);193 194 rq_ctxt->pref_wrapped = 1;195 196 rq_ctxt->pref_wq_hi_pfn_ci =197 HINIC_RQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN) |198 HINIC_RQ_CTXT_PREF_SET(ci_start, CI);199 200 rq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo;201 202 rq_ctxt->pi_paddr_hi = upper_32_bits(rq->pi_dma_addr);203 rq_ctxt->pi_paddr_lo = lower_32_bits(rq->pi_dma_addr);204 205 rq_ctxt->wq_block_hi_pfn =206 HINIC_RQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN);207 208 rq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo;209 210 hinic_cpu_to_be32(rq_ctxt, sizeof(*rq_ctxt));211}212 213/**214 * alloc_sq_skb_arr - allocate sq array for saved skb215 * @sq: HW Send Queue216 *217 * Return 0 - Success, negative - Failure218 **/219static int alloc_sq_skb_arr(struct hinic_sq *sq)220{221 struct hinic_wq *wq = sq->wq;222 size_t skb_arr_size;223 224 skb_arr_size = wq->q_depth * sizeof(*sq->saved_skb);225 sq->saved_skb = vzalloc(skb_arr_size);226 if (!sq->saved_skb)227 return -ENOMEM;228 229 return 0;230}231 232/**233 * free_sq_skb_arr - free sq array for saved skb234 * @sq: HW Send Queue235 **/236static void free_sq_skb_arr(struct hinic_sq *sq)237{238 vfree(sq->saved_skb);239}240 241/**242 * alloc_rq_skb_arr - allocate rq array for saved skb243 * @rq: HW Receive Queue244 *245 * Return 0 - Success, negative - Failure246 **/247static int alloc_rq_skb_arr(struct hinic_rq *rq)248{249 struct hinic_wq *wq = rq->wq;250 size_t skb_arr_size;251 252 skb_arr_size = wq->q_depth * sizeof(*rq->saved_skb);253 rq->saved_skb = vzalloc(skb_arr_size);254 if (!rq->saved_skb)255 return -ENOMEM;256 257 return 0;258}259 260/**261 * free_rq_skb_arr - free rq array for saved skb262 * @rq: HW Receive Queue263 **/264static void free_rq_skb_arr(struct hinic_rq *rq)265{266 vfree(rq->saved_skb);267}268 269/**270 * hinic_init_sq - Initialize HW Send Queue271 * @sq: HW Send Queue272 * @hwif: HW Interface for accessing HW273 * @wq: Work Queue for the data of the SQ274 * @entry: msix entry for sq275 * @ci_addr: address for reading the current HW consumer index276 * @ci_dma_addr: dma address for reading the current HW consumer index277 * @db_base: doorbell base address278 *279 * Return 0 - Success, negative - Failure280 **/281int hinic_init_sq(struct hinic_sq *sq, struct hinic_hwif *hwif,282 struct hinic_wq *wq, struct msix_entry *entry,283 void *ci_addr, dma_addr_t ci_dma_addr,284 void __iomem *db_base)285{286 sq->hwif = hwif;287 288 sq->wq = wq;289 290 sq->irq = entry->vector;291 sq->msix_entry = entry->entry;292 293 sq->hw_ci_addr = ci_addr;294 sq->hw_ci_dma_addr = ci_dma_addr;295 296 sq->db_base = db_base + SQ_DB_OFF;297 298 return alloc_sq_skb_arr(sq);299}300 301/**302 * hinic_clean_sq - Clean HW Send Queue's Resources303 * @sq: Send Queue304 **/305void hinic_clean_sq(struct hinic_sq *sq)306{307 free_sq_skb_arr(sq);308}309 310/**311 * alloc_rq_cqe - allocate rq completion queue elements312 * @rq: HW Receive Queue313 *314 * Return 0 - Success, negative - Failure315 **/316static int alloc_rq_cqe(struct hinic_rq *rq)317{318 struct hinic_hwif *hwif = rq->hwif;319 struct pci_dev *pdev = hwif->pdev;320 size_t cqe_dma_size, cqe_size;321 struct hinic_wq *wq = rq->wq;322 int j, i;323 324 cqe_size = wq->q_depth * sizeof(*rq->cqe);325 rq->cqe = vzalloc(cqe_size);326 if (!rq->cqe)327 return -ENOMEM;328 329 cqe_dma_size = wq->q_depth * sizeof(*rq->cqe_dma);330 rq->cqe_dma = vzalloc(cqe_dma_size);331 if (!rq->cqe_dma)332 goto err_cqe_dma_arr_alloc;333 334 for (i = 0; i < wq->q_depth; i++) {335 rq->cqe[i] = dma_alloc_coherent(&pdev->dev,336 sizeof(*rq->cqe[i]),337 &rq->cqe_dma[i], GFP_KERNEL);338 if (!rq->cqe[i])339 goto err_cqe_alloc;340 }341 342 return 0;343 344err_cqe_alloc:345 for (j = 0; j < i; j++)346 dma_free_coherent(&pdev->dev, sizeof(*rq->cqe[j]), rq->cqe[j],347 rq->cqe_dma[j]);348 349 vfree(rq->cqe_dma);350 351err_cqe_dma_arr_alloc:352 vfree(rq->cqe);353 return -ENOMEM;354}355 356/**357 * free_rq_cqe - free rq completion queue elements358 * @rq: HW Receive Queue359 **/360static void free_rq_cqe(struct hinic_rq *rq)361{362 struct hinic_hwif *hwif = rq->hwif;363 struct pci_dev *pdev = hwif->pdev;364 struct hinic_wq *wq = rq->wq;365 int i;366 367 for (i = 0; i < wq->q_depth; i++)368 dma_free_coherent(&pdev->dev, sizeof(*rq->cqe[i]), rq->cqe[i],369 rq->cqe_dma[i]);370 371 vfree(rq->cqe_dma);372 vfree(rq->cqe);373}374 375/**376 * hinic_init_rq - Initialize HW Receive Queue377 * @rq: HW Receive Queue378 * @hwif: HW Interface for accessing HW379 * @wq: Work Queue for the data of the RQ380 * @entry: msix entry for rq381 *382 * Return 0 - Success, negative - Failure383 **/384int hinic_init_rq(struct hinic_rq *rq, struct hinic_hwif *hwif,385 struct hinic_wq *wq, struct msix_entry *entry)386{387 struct pci_dev *pdev = hwif->pdev;388 size_t pi_size;389 int err;390 391 rq->hwif = hwif;392 393 rq->wq = wq;394 395 rq->irq = entry->vector;396 rq->msix_entry = entry->entry;397 398 rq->buf_sz = HINIC_RX_BUF_SZ;399 400 err = alloc_rq_skb_arr(rq);401 if (err) {402 dev_err(&pdev->dev, "Failed to allocate rq priv data\n");403 return err;404 }405 406 err = alloc_rq_cqe(rq);407 if (err) {408 dev_err(&pdev->dev, "Failed to allocate rq cqe\n");409 goto err_alloc_rq_cqe;410 }411 412 /* HW requirements: Must be at least 32 bit */413 pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32));414 rq->pi_virt_addr = dma_alloc_coherent(&pdev->dev, pi_size,415 &rq->pi_dma_addr, GFP_KERNEL);416 if (!rq->pi_virt_addr) {417 err = -ENOMEM;418 goto err_pi_virt;419 }420 421 return 0;422 423err_pi_virt:424 free_rq_cqe(rq);425 426err_alloc_rq_cqe:427 free_rq_skb_arr(rq);428 return err;429}430 431/**432 * hinic_clean_rq - Clean HW Receive Queue's Resources433 * @rq: HW Receive Queue434 **/435void hinic_clean_rq(struct hinic_rq *rq)436{437 struct hinic_hwif *hwif = rq->hwif;438 struct pci_dev *pdev = hwif->pdev;439 size_t pi_size;440 441 pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32));442 dma_free_coherent(&pdev->dev, pi_size, rq->pi_virt_addr,443 rq->pi_dma_addr);444 445 free_rq_cqe(rq);446 free_rq_skb_arr(rq);447}448 449/**450 * hinic_get_sq_free_wqebbs - return number of free wqebbs for use451 * @sq: send queue452 *453 * Return number of free wqebbs454 **/455int hinic_get_sq_free_wqebbs(struct hinic_sq *sq)456{457 struct hinic_wq *wq = sq->wq;458 459 return atomic_read(&wq->delta) - 1;460}461 462/**463 * hinic_get_rq_free_wqebbs - return number of free wqebbs for use464 * @rq: recv queue465 *466 * Return number of free wqebbs467 **/468int hinic_get_rq_free_wqebbs(struct hinic_rq *rq)469{470 struct hinic_wq *wq = rq->wq;471 472 return atomic_read(&wq->delta) - 1;473}474 475static void sq_prepare_ctrl(struct hinic_sq_ctrl *ctrl, int nr_descs)476{477 u32 ctrl_size, task_size, bufdesc_size;478 479 ctrl_size = SIZE_8BYTES(sizeof(struct hinic_sq_ctrl));480 task_size = SIZE_8BYTES(sizeof(struct hinic_sq_task));481 bufdesc_size = nr_descs * sizeof(struct hinic_sq_bufdesc);482 bufdesc_size = SIZE_8BYTES(bufdesc_size);483 484 ctrl->ctrl_info = HINIC_SQ_CTRL_SET(bufdesc_size, BUFDESC_SECT_LEN) |485 HINIC_SQ_CTRL_SET(task_size, TASKSECT_LEN) |486 HINIC_SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT) |487 HINIC_SQ_CTRL_SET(ctrl_size, LEN);488 489 ctrl->queue_info = HINIC_SQ_CTRL_SET(HINIC_MSS_DEFAULT,490 QUEUE_INFO_MSS) |491 HINIC_SQ_CTRL_SET(1, QUEUE_INFO_UC);492}493 494static void sq_prepare_task(struct hinic_sq_task *task)495{496 task->pkt_info0 = 0;497 task->pkt_info1 = 0;498 task->pkt_info2 = 0;499 500 task->ufo_v6_identify = 0;501 502 task->pkt_info4 = HINIC_SQ_TASK_INFO4_SET(HINIC_L2TYPE_ETH, L2TYPE);503 504 task->zero_pad = 0;505}506 507void hinic_task_set_l2hdr(struct hinic_sq_task *task, u32 len)508{509 task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(len, L2HDR_LEN);510}511 512void hinic_task_set_outter_l3(struct hinic_sq_task *task,513 enum hinic_l3_offload_type l3_type,514 u32 network_len)515{516 task->pkt_info2 |= HINIC_SQ_TASK_INFO2_SET(l3_type, OUTER_L3TYPE) |517 HINIC_SQ_TASK_INFO2_SET(network_len, OUTER_L3LEN);518}519 520void hinic_task_set_inner_l3(struct hinic_sq_task *task,521 enum hinic_l3_offload_type l3_type,522 u32 network_len)523{524 task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l3_type, INNER_L3TYPE);525 task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(network_len, INNER_L3LEN);526}527 528void hinic_task_set_tunnel_l4(struct hinic_sq_task *task,529 enum hinic_l4_tunnel_type l4_type,530 u32 tunnel_len)531{532 task->pkt_info2 |= HINIC_SQ_TASK_INFO2_SET(l4_type, TUNNEL_L4TYPE) |533 HINIC_SQ_TASK_INFO2_SET(tunnel_len, TUNNEL_L4LEN);534}535 536void hinic_set_cs_inner_l4(struct hinic_sq_task *task, u32 *queue_info,537 enum hinic_l4_offload_type l4_offload,538 u32 l4_len, u32 offset)539{540 u32 tcp_udp_cs = 0, sctp = 0;541 u32 mss = HINIC_MSS_DEFAULT;542 543 if (l4_offload == TCP_OFFLOAD_ENABLE ||544 l4_offload == UDP_OFFLOAD_ENABLE)545 tcp_udp_cs = 1;546 else if (l4_offload == SCTP_OFFLOAD_ENABLE)547 sctp = 1;548 549 task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l4_offload, L4_OFFLOAD);550 task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(l4_len, INNER_L4LEN);551 552 *queue_info |= HINIC_SQ_CTRL_SET(offset, QUEUE_INFO_PLDOFF) |553 HINIC_SQ_CTRL_SET(tcp_udp_cs, QUEUE_INFO_TCPUDP_CS) |554 HINIC_SQ_CTRL_SET(sctp, QUEUE_INFO_SCTP);555 556 *queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS);557 *queue_info |= HINIC_SQ_CTRL_SET(mss, QUEUE_INFO_MSS);558}559 560void hinic_set_tso_inner_l4(struct hinic_sq_task *task, u32 *queue_info,561 enum hinic_l4_offload_type l4_offload,562 u32 l4_len, u32 offset, u32 ip_ident, u32 mss)563{564 u32 tso = 0, ufo = 0;565 566 if (l4_offload == TCP_OFFLOAD_ENABLE)567 tso = 1;568 else if (l4_offload == UDP_OFFLOAD_ENABLE)569 ufo = 1;570 571 task->ufo_v6_identify = ip_ident;572 573 task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l4_offload, L4_OFFLOAD);574 task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(tso || ufo, TSO_FLAG);575 task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(l4_len, INNER_L4LEN);576 577 *queue_info |= HINIC_SQ_CTRL_SET(offset, QUEUE_INFO_PLDOFF) |578 HINIC_SQ_CTRL_SET(tso, QUEUE_INFO_TSO) |579 HINIC_SQ_CTRL_SET(ufo, QUEUE_INFO_UFO) |580 HINIC_SQ_CTRL_SET(!!l4_offload, QUEUE_INFO_TCPUDP_CS);581 582 /* set MSS value */583 *queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS);584 *queue_info |= HINIC_SQ_CTRL_SET(mss, QUEUE_INFO_MSS);585}586 587/**588 * hinic_sq_prepare_wqe - prepare wqe before insert to the queue589 * @sq: send queue590 * @sq_wqe: wqe to prepare591 * @sges: sges for use by the wqe for send for buf addresses592 * @nr_sges: number of sges593 **/594void hinic_sq_prepare_wqe(struct hinic_sq *sq, struct hinic_sq_wqe *sq_wqe,595 struct hinic_sge *sges, int nr_sges)596{597 int i;598 599 sq_prepare_ctrl(&sq_wqe->ctrl, nr_sges);600 601 sq_prepare_task(&sq_wqe->task);602 603 for (i = 0; i < nr_sges; i++)604 sq_wqe->buf_descs[i].sge = sges[i];605}606 607/**608 * sq_prepare_db - prepare doorbell to write609 * @sq: send queue610 * @prod_idx: pi value for the doorbell611 * @cos: cos of the doorbell612 *613 * Return db value614 **/615static u32 sq_prepare_db(struct hinic_sq *sq, u16 prod_idx, unsigned int cos)616{617 struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq);618 u8 hi_prod_idx = SQ_DB_PI_HI(SQ_MASKED_IDX(sq, prod_idx));619 620 /* Data should be written to HW in Big Endian Format */621 return cpu_to_be32(HINIC_SQ_DB_INFO_SET(hi_prod_idx, PI_HI) |622 HINIC_SQ_DB_INFO_SET(HINIC_DB_SQ_TYPE, TYPE) |623 HINIC_SQ_DB_INFO_SET(HINIC_DATA_PATH, PATH) |624 HINIC_SQ_DB_INFO_SET(cos, COS) |625 HINIC_SQ_DB_INFO_SET(qp->q_id, QID));626}627 628/**629 * hinic_sq_write_db- write doorbell630 * @sq: send queue631 * @prod_idx: pi value for the doorbell632 * @wqe_size: wqe size633 * @cos: cos of the wqe634 **/635void hinic_sq_write_db(struct hinic_sq *sq, u16 prod_idx, unsigned int wqe_size,636 unsigned int cos)637{638 struct hinic_wq *wq = sq->wq;639 640 /* increment prod_idx to the next */641 prod_idx += ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;642 prod_idx = SQ_MASKED_IDX(sq, prod_idx);643 644 wmb(); /* Write all before the doorbell */645 646 writel(sq_prepare_db(sq, prod_idx, cos), SQ_DB_ADDR(sq, prod_idx));647}648 649/**650 * hinic_sq_get_wqe - get wqe ptr in the current pi and update the pi651 * @sq: sq to get wqe from652 * @wqe_size: wqe size653 * @prod_idx: returned pi654 *655 * Return wqe pointer656 **/657struct hinic_sq_wqe *hinic_sq_get_wqe(struct hinic_sq *sq,658 unsigned int wqe_size, u16 *prod_idx)659{660 struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(sq->wq, wqe_size,661 prod_idx);662 663 if (IS_ERR(hw_wqe))664 return NULL;665 666 return &hw_wqe->sq_wqe;667}668 669/**670 * hinic_sq_return_wqe - return the wqe to the sq671 * @sq: send queue672 * @wqe_size: the size of the wqe673 **/674void hinic_sq_return_wqe(struct hinic_sq *sq, unsigned int wqe_size)675{676 hinic_return_wqe(sq->wq, wqe_size);677}678 679/**680 * hinic_sq_write_wqe - write the wqe to the sq681 * @sq: send queue682 * @prod_idx: pi of the wqe683 * @sq_wqe: the wqe to write684 * @skb: skb to save685 * @wqe_size: the size of the wqe686 **/687void hinic_sq_write_wqe(struct hinic_sq *sq, u16 prod_idx,688 struct hinic_sq_wqe *sq_wqe,689 struct sk_buff *skb, unsigned int wqe_size)690{691 struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)sq_wqe;692 693 sq->saved_skb[prod_idx] = skb;694 695 /* The data in the HW should be in Big Endian Format */696 hinic_cpu_to_be32(sq_wqe, wqe_size);697 698 hinic_write_wqe(sq->wq, hw_wqe, wqe_size);699}700 701/**702 * hinic_sq_read_wqebb - read wqe ptr in the current ci and update the ci, the703 * wqe only have one wqebb704 * @sq: send queue705 * @skb: return skb that was saved706 * @wqe_size: the wqe size ptr707 * @cons_idx: consumer index of the wqe708 *709 * Return wqe in ci position710 **/711struct hinic_sq_wqe *hinic_sq_read_wqebb(struct hinic_sq *sq,712 struct sk_buff **skb,713 unsigned int *wqe_size, u16 *cons_idx)714{715 struct hinic_hw_wqe *hw_wqe;716 struct hinic_sq_wqe *sq_wqe;717 struct hinic_sq_ctrl *ctrl;718 unsigned int buf_sect_len;719 u32 ctrl_info;720 721 /* read the ctrl section for getting wqe size */722 hw_wqe = hinic_read_wqe(sq->wq, sizeof(*ctrl), cons_idx);723 if (IS_ERR(hw_wqe))724 return NULL;725 726 *skb = sq->saved_skb[*cons_idx];727 728 sq_wqe = &hw_wqe->sq_wqe;729 ctrl = &sq_wqe->ctrl;730 ctrl_info = be32_to_cpu(ctrl->ctrl_info);731 buf_sect_len = HINIC_SQ_CTRL_GET(ctrl_info, BUFDESC_SECT_LEN);732 733 *wqe_size = sizeof(*ctrl) + sizeof(sq_wqe->task);734 *wqe_size += SECT_SIZE_FROM_8BYTES(buf_sect_len);735 *wqe_size = ALIGN(*wqe_size, sq->wq->wqebb_size);736 737 return &hw_wqe->sq_wqe;738}739 740/**741 * hinic_sq_read_wqe - read wqe ptr in the current ci and update the ci742 * @sq: send queue743 * @skb: return skb that was saved744 * @wqe_size: the size of the wqe745 * @cons_idx: consumer index of the wqe746 *747 * Return wqe in ci position748 **/749struct hinic_sq_wqe *hinic_sq_read_wqe(struct hinic_sq *sq,750 struct sk_buff **skb,751 unsigned int wqe_size, u16 *cons_idx)752{753 struct hinic_hw_wqe *hw_wqe;754 755 hw_wqe = hinic_read_wqe(sq->wq, wqe_size, cons_idx);756 *skb = sq->saved_skb[*cons_idx];757 758 return &hw_wqe->sq_wqe;759}760 761/**762 * hinic_sq_put_wqe - release the ci for new wqes763 * @sq: send queue764 * @wqe_size: the size of the wqe765 **/766void hinic_sq_put_wqe(struct hinic_sq *sq, unsigned int wqe_size)767{768 hinic_put_wqe(sq->wq, wqe_size);769}770 771/**772 * hinic_sq_get_sges - get sges from the wqe773 * @sq_wqe: wqe to get the sges from its buffer addresses774 * @sges: returned sges775 * @nr_sges: number sges to return776 **/777void hinic_sq_get_sges(struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges,778 int nr_sges)779{780 int i;781 782 for (i = 0; i < nr_sges && i < HINIC_MAX_SQ_BUFDESCS; i++) {783 sges[i] = sq_wqe->buf_descs[i].sge;784 hinic_be32_to_cpu(&sges[i], sizeof(sges[i]));785 }786}787 788/**789 * hinic_rq_get_wqe - get wqe ptr in the current pi and update the pi790 * @rq: rq to get wqe from791 * @wqe_size: wqe size792 * @prod_idx: returned pi793 *794 * Return wqe pointer795 **/796struct hinic_rq_wqe *hinic_rq_get_wqe(struct hinic_rq *rq,797 unsigned int wqe_size, u16 *prod_idx)798{799 struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(rq->wq, wqe_size,800 prod_idx);801 802 if (IS_ERR(hw_wqe))803 return NULL;804 805 return &hw_wqe->rq_wqe;806}807 808/**809 * hinic_rq_write_wqe - write the wqe to the rq810 * @rq: recv queue811 * @prod_idx: pi of the wqe812 * @rq_wqe: the wqe to write813 * @skb: skb to save814 **/815void hinic_rq_write_wqe(struct hinic_rq *rq, u16 prod_idx,816 struct hinic_rq_wqe *rq_wqe, struct sk_buff *skb)817{818 struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)rq_wqe;819 820 rq->saved_skb[prod_idx] = skb;821 822 /* The data in the HW should be in Big Endian Format */823 hinic_cpu_to_be32(rq_wqe, sizeof(*rq_wqe));824 825 hinic_write_wqe(rq->wq, hw_wqe, sizeof(*rq_wqe));826}827 828/**829 * hinic_rq_read_wqe - read wqe ptr in the current ci and update the ci830 * @rq: recv queue831 * @wqe_size: the size of the wqe832 * @skb: return saved skb833 * @cons_idx: consumer index of the wqe834 *835 * Return wqe in ci position836 **/837struct hinic_rq_wqe *hinic_rq_read_wqe(struct hinic_rq *rq,838 unsigned int wqe_size,839 struct sk_buff **skb, u16 *cons_idx)840{841 struct hinic_hw_wqe *hw_wqe;842 struct hinic_rq_cqe *cqe;843 int rx_done;844 u32 status;845 846 hw_wqe = hinic_read_wqe(rq->wq, wqe_size, cons_idx);847 if (IS_ERR(hw_wqe))848 return NULL;849 850 cqe = rq->cqe[*cons_idx];851 852 status = be32_to_cpu(cqe->status);853 854 rx_done = HINIC_RQ_CQE_STATUS_GET(status, RXDONE);855 if (!rx_done)856 return NULL;857 858 *skb = rq->saved_skb[*cons_idx];859 860 return &hw_wqe->rq_wqe;861}862 863/**864 * hinic_rq_read_next_wqe - increment ci and read the wqe in ci position865 * @rq: recv queue866 * @wqe_size: the size of the wqe867 * @skb: return saved skb868 * @cons_idx: consumer index in the wq869 *870 * Return wqe in incremented ci position871 **/872struct hinic_rq_wqe *hinic_rq_read_next_wqe(struct hinic_rq *rq,873 unsigned int wqe_size,874 struct sk_buff **skb,875 u16 *cons_idx)876{877 struct hinic_wq *wq = rq->wq;878 struct hinic_hw_wqe *hw_wqe;879 unsigned int num_wqebbs;880 881 wqe_size = ALIGN(wqe_size, wq->wqebb_size);882 num_wqebbs = wqe_size / wq->wqebb_size;883 884 *cons_idx = RQ_MASKED_IDX(rq, *cons_idx + num_wqebbs);885 886 *skb = rq->saved_skb[*cons_idx];887 888 hw_wqe = hinic_read_wqe_direct(wq, *cons_idx);889 890 return &hw_wqe->rq_wqe;891}892 893/**894 * hinic_rq_put_wqe - release the ci for new wqes895 * @rq: recv queue896 * @cons_idx: consumer index of the wqe897 * @wqe_size: the size of the wqe898 **/899void hinic_rq_put_wqe(struct hinic_rq *rq, u16 cons_idx,900 unsigned int wqe_size)901{902 struct hinic_rq_cqe *cqe = rq->cqe[cons_idx];903 u32 status = be32_to_cpu(cqe->status);904 905 status = HINIC_RQ_CQE_STATUS_CLEAR(status, RXDONE);906 907 /* Rx WQE size is 1 WQEBB, no wq shadow*/908 cqe->status = cpu_to_be32(status);909 910 wmb(); /* clear done flag */911 912 hinic_put_wqe(rq->wq, wqe_size);913}914 915/**916 * hinic_rq_get_sge - get sge from the wqe917 * @rq: recv queue918 * @rq_wqe: wqe to get the sge from its buf address919 * @cons_idx: consumer index920 * @sge: returned sge921 **/922void hinic_rq_get_sge(struct hinic_rq *rq, struct hinic_rq_wqe *rq_wqe,923 u16 cons_idx, struct hinic_sge *sge)924{925 struct hinic_rq_cqe *cqe = rq->cqe[cons_idx];926 u32 len = be32_to_cpu(cqe->len);927 928 sge->hi_addr = be32_to_cpu(rq_wqe->buf_desc.hi_addr);929 sge->lo_addr = be32_to_cpu(rq_wqe->buf_desc.lo_addr);930 sge->len = HINIC_RQ_CQE_SGE_GET(len, LEN);931}932 933/**934 * hinic_rq_prepare_wqe - prepare wqe before insert to the queue935 * @rq: recv queue936 * @prod_idx: pi value937 * @rq_wqe: the wqe938 * @sge: sge for use by the wqe for recv buf address939 **/940void hinic_rq_prepare_wqe(struct hinic_rq *rq, u16 prod_idx,941 struct hinic_rq_wqe *rq_wqe, struct hinic_sge *sge)942{943 struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect;944 struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc;945 struct hinic_rq_cqe *cqe = rq->cqe[prod_idx];946 struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl;947 dma_addr_t cqe_dma = rq->cqe_dma[prod_idx];948 949 ctrl->ctrl_info =950 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)), LEN) |951 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)),952 COMPLETE_LEN) |953 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)),954 BUFDESC_SECT_LEN) |955 HINIC_RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT);956 957 hinic_set_sge(&cqe_sect->sge, cqe_dma, sizeof(*cqe));958 959 buf_desc->hi_addr = sge->hi_addr;960 buf_desc->lo_addr = sge->lo_addr;961}962 963/**964 * hinic_rq_update - update pi of the rq965 * @rq: recv queue966 * @prod_idx: pi value967 **/968void hinic_rq_update(struct hinic_rq *rq, u16 prod_idx)969{970 *rq->pi_virt_addr = cpu_to_be16(RQ_MASKED_IDX(rq, prod_idx + 1));971}972