988 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Huawei HiNIC PCI Express Linux driver4 * Copyright(c) 2017 Huawei Technologies Co., Ltd5 */6 7#include <linux/kernel.h>8#include <linux/types.h>9#include <linux/errno.h>10#include <linux/pci.h>11#include <linux/device.h>12#include <linux/slab.h>13#include <linux/dma-mapping.h>14#include <linux/bitops.h>15#include <linux/err.h>16#include <linux/jiffies.h>17#include <linux/delay.h>18#include <linux/log2.h>19#include <linux/semaphore.h>20#include <asm/byteorder.h>21#include <asm/barrier.h>22 23#include "hinic_hw_csr.h"24#include "hinic_hw_if.h"25#include "hinic_hw_api_cmd.h"26 27#define API_CHAIN_NUM_CELLS 3228 29#define API_CMD_CELL_SIZE_SHIFT 630#define API_CMD_CELL_SIZE_MIN (BIT(API_CMD_CELL_SIZE_SHIFT))31 32#define API_CMD_CELL_SIZE(cell_size) \33 (((cell_size) >= API_CMD_CELL_SIZE_MIN) ? \34 (1 << (fls(cell_size - 1))) : API_CMD_CELL_SIZE_MIN)35 36#define API_CMD_CELL_SIZE_VAL(size) \37 ilog2((size) >> API_CMD_CELL_SIZE_SHIFT)38 39#define API_CMD_BUF_SIZE 204840 41/* Sizes of the members in hinic_api_cmd_cell */42#define API_CMD_CELL_DESC_SIZE 843#define API_CMD_CELL_DATA_ADDR_SIZE 844 45#define API_CMD_CELL_ALIGNMENT 846 47#define API_CMD_TIMEOUT 100048 49#define MASKED_IDX(chain, idx) ((idx) & ((chain)->num_cells - 1))50 51#define SIZE_8BYTES(size) (ALIGN((size), 8) >> 3)52#define SIZE_4BYTES(size) (ALIGN((size), 4) >> 2)53 54#define RD_DMA_ATTR_DEFAULT 055#define WR_DMA_ATTR_DEFAULT 056 57enum api_cmd_data_format {58 SGE_DATA = 1, /* cell data is passed by hw address */59};60 61enum api_cmd_type {62 API_CMD_WRITE = 0,63};64 65enum api_cmd_bypass {66 NO_BYPASS = 0,67 BYPASS = 1,68};69 70enum api_cmd_xor_chk_level {71 XOR_CHK_DIS = 0,72 73 XOR_CHK_ALL = 3,74};75 76static u8 xor_chksum_set(void *data)77{78 int idx;79 u8 *val, checksum = 0;80 81 val = data;82 83 for (idx = 0; idx < 7; idx++)84 checksum ^= val[idx];85 86 return checksum;87}88 89static void set_prod_idx(struct hinic_api_cmd_chain *chain)90{91 enum hinic_api_cmd_chain_type chain_type = chain->chain_type;92 struct hinic_hwif *hwif = chain->hwif;93 u32 addr, prod_idx;94 95 addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain_type);96 prod_idx = hinic_hwif_read_reg(hwif, addr);97 98 prod_idx = HINIC_API_CMD_PI_CLEAR(prod_idx, IDX);99 100 prod_idx |= HINIC_API_CMD_PI_SET(chain->prod_idx, IDX);101 102 hinic_hwif_write_reg(hwif, addr, prod_idx);103}104 105static u32 get_hw_cons_idx(struct hinic_api_cmd_chain *chain)106{107 u32 addr, val;108 109 addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type);110 val = hinic_hwif_read_reg(chain->hwif, addr);111 112 return HINIC_API_CMD_STATUS_GET(val, CONS_IDX);113}114 115static void dump_api_chain_reg(struct hinic_api_cmd_chain *chain)116{117 u32 addr, val;118 119 addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type);120 val = hinic_hwif_read_reg(chain->hwif, addr);121 122 dev_err(&chain->hwif->pdev->dev, "Chain type: 0x%x, cpld error: 0x%x, check error: 0x%x, current fsm: 0x%x\n",123 chain->chain_type, HINIC_API_CMD_STATUS_GET(val, CPLD_ERR),124 HINIC_API_CMD_STATUS_GET(val, CHKSUM_ERR),125 HINIC_API_CMD_STATUS_GET(val, FSM));126 127 dev_err(&chain->hwif->pdev->dev, "Chain hw current ci: 0x%x\n",128 HINIC_API_CMD_STATUS_GET(val, CONS_IDX));129 130 addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain->chain_type);131 val = hinic_hwif_read_reg(chain->hwif, addr);132 dev_err(&chain->hwif->pdev->dev, "Chain hw current pi: 0x%x\n", val);133}134 135/**136 * chain_busy - check if the chain is still processing last requests137 * @chain: chain to check138 *139 * Return 0 - Success, negative - Failure140 **/141static int chain_busy(struct hinic_api_cmd_chain *chain)142{143 struct hinic_hwif *hwif = chain->hwif;144 struct pci_dev *pdev = hwif->pdev;145 u32 prod_idx;146 147 switch (chain->chain_type) {148 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:149 chain->cons_idx = get_hw_cons_idx(chain);150 prod_idx = chain->prod_idx;151 152 /* check for a space for a new command */153 if (chain->cons_idx == MASKED_IDX(chain, prod_idx + 1)) {154 dev_err(&pdev->dev, "API CMD chain %d is busy, cons_idx: %d, prod_idx: %d\n",155 chain->chain_type, chain->cons_idx,156 chain->prod_idx);157 dump_api_chain_reg(chain);158 return -EBUSY;159 }160 break;161 162 default:163 dev_err(&pdev->dev, "Unknown API CMD Chain type\n");164 break;165 }166 167 return 0;168}169 170/**171 * get_cell_data_size - get the data size of a specific cell type172 * @type: chain type173 *174 * Return the data(Desc + Address) size in the cell175 **/176static u8 get_cell_data_size(enum hinic_api_cmd_chain_type type)177{178 u8 cell_data_size = 0;179 180 switch (type) {181 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:182 cell_data_size = ALIGN(API_CMD_CELL_DESC_SIZE +183 API_CMD_CELL_DATA_ADDR_SIZE,184 API_CMD_CELL_ALIGNMENT);185 break;186 default:187 break;188 }189 190 return cell_data_size;191}192 193/**194 * prepare_cell_ctrl - prepare the ctrl of the cell for the command195 * @cell_ctrl: the control of the cell to set the control value into it196 * @data_size: the size of the data in the cell197 **/198static void prepare_cell_ctrl(u64 *cell_ctrl, u16 data_size)199{200 u8 chksum;201 u64 ctrl;202 203 ctrl = HINIC_API_CMD_CELL_CTRL_SET(SIZE_8BYTES(data_size), DATA_SZ) |204 HINIC_API_CMD_CELL_CTRL_SET(RD_DMA_ATTR_DEFAULT, RD_DMA_ATTR) |205 HINIC_API_CMD_CELL_CTRL_SET(WR_DMA_ATTR_DEFAULT, WR_DMA_ATTR);206 207 chksum = xor_chksum_set(&ctrl);208 209 ctrl |= HINIC_API_CMD_CELL_CTRL_SET(chksum, XOR_CHKSUM);210 211 /* The data in the HW should be in Big Endian Format */212 *cell_ctrl = cpu_to_be64(ctrl);213}214 215/**216 * prepare_api_cmd - prepare API CMD command217 * @chain: chain for the command218 * @dest: destination node on the card that will receive the command219 * @cmd: command data220 * @cmd_size: the command size221 **/222static void prepare_api_cmd(struct hinic_api_cmd_chain *chain,223 enum hinic_node_id dest,224 void *cmd, u16 cmd_size)225{226 struct hinic_api_cmd_cell *cell = chain->curr_node;227 struct hinic_api_cmd_cell_ctxt *cell_ctxt;228 struct hinic_hwif *hwif = chain->hwif;229 struct pci_dev *pdev = hwif->pdev;230 231 cell_ctxt = &chain->cell_ctxt[chain->prod_idx];232 233 switch (chain->chain_type) {234 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:235 cell->desc = HINIC_API_CMD_DESC_SET(SGE_DATA, API_TYPE) |236 HINIC_API_CMD_DESC_SET(API_CMD_WRITE, RD_WR) |237 HINIC_API_CMD_DESC_SET(NO_BYPASS, MGMT_BYPASS);238 break;239 240 default:241 dev_err(&pdev->dev, "unknown Chain type\n");242 return;243 }244 245 cell->desc |= HINIC_API_CMD_DESC_SET(dest, DEST) |246 HINIC_API_CMD_DESC_SET(SIZE_4BYTES(cmd_size), SIZE);247 248 cell->desc |= HINIC_API_CMD_DESC_SET(xor_chksum_set(&cell->desc),249 XOR_CHKSUM);250 251 /* The data in the HW should be in Big Endian Format */252 cell->desc = cpu_to_be64(cell->desc);253 254 memcpy(cell_ctxt->api_cmd_vaddr, cmd, cmd_size);255}256 257/**258 * prepare_cell - prepare cell ctrl and cmd in the current cell259 * @chain: chain for the command260 * @dest: destination node on the card that will receive the command261 * @cmd: command data262 * @cmd_size: the command size263 *264 * Return 0 - Success, negative - Failure265 **/266static void prepare_cell(struct hinic_api_cmd_chain *chain,267 enum hinic_node_id dest,268 void *cmd, u16 cmd_size)269{270 struct hinic_api_cmd_cell *curr_node = chain->curr_node;271 u16 data_size = get_cell_data_size(chain->chain_type);272 273 prepare_cell_ctrl(&curr_node->ctrl, data_size);274 prepare_api_cmd(chain, dest, cmd, cmd_size);275}276 277static inline void cmd_chain_prod_idx_inc(struct hinic_api_cmd_chain *chain)278{279 chain->prod_idx = MASKED_IDX(chain, chain->prod_idx + 1);280}281 282/**283 * api_cmd_status_update - update the status in the chain struct284 * @chain: chain to update285 **/286static void api_cmd_status_update(struct hinic_api_cmd_chain *chain)287{288 enum hinic_api_cmd_chain_type chain_type;289 struct hinic_api_cmd_status *wb_status;290 struct hinic_hwif *hwif = chain->hwif;291 struct pci_dev *pdev = hwif->pdev;292 u64 status_header;293 u32 status;294 295 wb_status = chain->wb_status;296 status_header = be64_to_cpu(wb_status->header);297 298 status = be32_to_cpu(wb_status->status);299 if (HINIC_API_CMD_STATUS_GET(status, CHKSUM_ERR)) {300 dev_err(&pdev->dev, "API CMD status: Xor check error\n");301 return;302 }303 304 chain_type = HINIC_API_CMD_STATUS_HEADER_GET(status_header, CHAIN_ID);305 if (chain_type >= HINIC_API_CMD_MAX) {306 dev_err(&pdev->dev, "unknown API CMD Chain %d\n", chain_type);307 return;308 }309 310 chain->cons_idx = HINIC_API_CMD_STATUS_GET(status, CONS_IDX);311}312 313/**314 * wait_for_status_poll - wait for write to api cmd command to complete315 * @chain: the chain of the command316 *317 * Return 0 - Success, negative - Failure318 **/319static int wait_for_status_poll(struct hinic_api_cmd_chain *chain)320{321 int err = -ETIMEDOUT;322 unsigned long end;323 324 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);325 do {326 api_cmd_status_update(chain);327 328 /* wait for CI to be updated - sign for completion */329 if (chain->cons_idx == chain->prod_idx) {330 err = 0;331 break;332 }333 334 msleep(20);335 } while (time_before(jiffies, end));336 337 return err;338}339 340/**341 * wait_for_api_cmd_completion - wait for command to complete342 * @chain: chain for the command343 *344 * Return 0 - Success, negative - Failure345 **/346static int wait_for_api_cmd_completion(struct hinic_api_cmd_chain *chain)347{348 struct hinic_hwif *hwif = chain->hwif;349 struct pci_dev *pdev = hwif->pdev;350 int err;351 352 switch (chain->chain_type) {353 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:354 err = wait_for_status_poll(chain);355 if (err) {356 dev_err(&pdev->dev, "API CMD Poll status timeout\n");357 dump_api_chain_reg(chain);358 break;359 }360 break;361 362 default:363 dev_err(&pdev->dev, "unknown API CMD Chain type\n");364 err = -EINVAL;365 break;366 }367 368 return err;369}370 371/**372 * api_cmd - API CMD command373 * @chain: chain for the command374 * @dest: destination node on the card that will receive the command375 * @cmd: command data376 * @cmd_size: the command size377 *378 * Return 0 - Success, negative - Failure379 **/380static int api_cmd(struct hinic_api_cmd_chain *chain,381 enum hinic_node_id dest, u8 *cmd, u16 cmd_size)382{383 struct hinic_api_cmd_cell_ctxt *ctxt;384 int err;385 386 down(&chain->sem);387 if (chain_busy(chain)) {388 up(&chain->sem);389 return -EBUSY;390 }391 392 prepare_cell(chain, dest, cmd, cmd_size);393 cmd_chain_prod_idx_inc(chain);394 395 wmb(); /* inc pi before issue the command */396 397 set_prod_idx(chain); /* issue the command */398 399 ctxt = &chain->cell_ctxt[chain->prod_idx];400 401 chain->curr_node = ctxt->cell_vaddr;402 403 err = wait_for_api_cmd_completion(chain);404 405 up(&chain->sem);406 return err;407}408 409/**410 * hinic_api_cmd_write - Write API CMD command411 * @chain: chain for write command412 * @dest: destination node on the card that will receive the command413 * @cmd: command data414 * @size: the command size415 *416 * Return 0 - Success, negative - Failure417 **/418int hinic_api_cmd_write(struct hinic_api_cmd_chain *chain,419 enum hinic_node_id dest, u8 *cmd, u16 size)420{421 /* Verify the chain type */422 if (chain->chain_type == HINIC_API_CMD_WRITE_TO_MGMT_CPU)423 return api_cmd(chain, dest, cmd, size);424 425 return -EINVAL;426}427 428/**429 * api_cmd_hw_restart - restart the chain in the HW430 * @chain: the API CMD specific chain to restart431 *432 * Return 0 - Success, negative - Failure433 **/434static int api_cmd_hw_restart(struct hinic_api_cmd_chain *chain)435{436 struct hinic_hwif *hwif = chain->hwif;437 int err = -ETIMEDOUT;438 unsigned long end;439 u32 reg_addr, val;440 441 /* Read Modify Write */442 reg_addr = HINIC_CSR_API_CMD_CHAIN_REQ_ADDR(chain->chain_type);443 val = hinic_hwif_read_reg(hwif, reg_addr);444 445 val = HINIC_API_CMD_CHAIN_REQ_CLEAR(val, RESTART);446 val |= HINIC_API_CMD_CHAIN_REQ_SET(1, RESTART);447 448 hinic_hwif_write_reg(hwif, reg_addr, val);449 450 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);451 do {452 val = hinic_hwif_read_reg(hwif, reg_addr);453 454 if (!HINIC_API_CMD_CHAIN_REQ_GET(val, RESTART)) {455 err = 0;456 break;457 }458 459 msleep(20);460 } while (time_before(jiffies, end));461 462 return err;463}464 465/**466 * api_cmd_ctrl_init - set the control register of a chain467 * @chain: the API CMD specific chain to set control register for468 **/469static void api_cmd_ctrl_init(struct hinic_api_cmd_chain *chain)470{471 struct hinic_hwif *hwif = chain->hwif;472 u32 addr, ctrl;473 u16 cell_size;474 475 /* Read Modify Write */476 addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);477 478 cell_size = API_CMD_CELL_SIZE_VAL(chain->cell_size);479 480 ctrl = hinic_hwif_read_reg(hwif, addr);481 482 ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) &483 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) &484 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) &485 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) &486 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);487 488 ctrl |= HINIC_API_CMD_CHAIN_CTRL_SET(1, XOR_ERR) |489 HINIC_API_CMD_CHAIN_CTRL_SET(XOR_CHK_ALL, XOR_CHK_EN) |490 HINIC_API_CMD_CHAIN_CTRL_SET(cell_size, CELL_SIZE);491 492 hinic_hwif_write_reg(hwif, addr, ctrl);493}494 495/**496 * api_cmd_set_status_addr - set the status address of a chain in the HW497 * @chain: the API CMD specific chain to set in HW status address for498 **/499static void api_cmd_set_status_addr(struct hinic_api_cmd_chain *chain)500{501 struct hinic_hwif *hwif = chain->hwif;502 u32 addr, val;503 504 addr = HINIC_CSR_API_CMD_STATUS_HI_ADDR(chain->chain_type);505 val = upper_32_bits(chain->wb_status_paddr);506 hinic_hwif_write_reg(hwif, addr, val);507 508 addr = HINIC_CSR_API_CMD_STATUS_LO_ADDR(chain->chain_type);509 val = lower_32_bits(chain->wb_status_paddr);510 hinic_hwif_write_reg(hwif, addr, val);511}512 513/**514 * api_cmd_set_num_cells - set the number cells of a chain in the HW515 * @chain: the API CMD specific chain to set in HW the number of cells for516 **/517static void api_cmd_set_num_cells(struct hinic_api_cmd_chain *chain)518{519 struct hinic_hwif *hwif = chain->hwif;520 u32 addr, val;521 522 addr = HINIC_CSR_API_CMD_CHAIN_NUM_CELLS_ADDR(chain->chain_type);523 val = chain->num_cells;524 hinic_hwif_write_reg(hwif, addr, val);525}526 527/**528 * api_cmd_head_init - set the head of a chain in the HW529 * @chain: the API CMD specific chain to set in HW the head for530 **/531static void api_cmd_head_init(struct hinic_api_cmd_chain *chain)532{533 struct hinic_hwif *hwif = chain->hwif;534 u32 addr, val;535 536 addr = HINIC_CSR_API_CMD_CHAIN_HEAD_HI_ADDR(chain->chain_type);537 val = upper_32_bits(chain->head_cell_paddr);538 hinic_hwif_write_reg(hwif, addr, val);539 540 addr = HINIC_CSR_API_CMD_CHAIN_HEAD_LO_ADDR(chain->chain_type);541 val = lower_32_bits(chain->head_cell_paddr);542 hinic_hwif_write_reg(hwif, addr, val);543}544 545/**546 * api_cmd_chain_hw_clean - clean the HW547 * @chain: the API CMD specific chain548 **/549static void api_cmd_chain_hw_clean(struct hinic_api_cmd_chain *chain)550{551 struct hinic_hwif *hwif = chain->hwif;552 u32 addr, ctrl;553 554 addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);555 556 ctrl = hinic_hwif_read_reg(hwif, addr);557 ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) &558 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) &559 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) &560 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) &561 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);562 563 hinic_hwif_write_reg(hwif, addr, ctrl);564}565 566/**567 * api_cmd_chain_hw_init - initialize the chain in the HW568 * @chain: the API CMD specific chain to initialize in HW569 *570 * Return 0 - Success, negative - Failure571 **/572static int api_cmd_chain_hw_init(struct hinic_api_cmd_chain *chain)573{574 struct hinic_hwif *hwif = chain->hwif;575 struct pci_dev *pdev = hwif->pdev;576 int err;577 578 api_cmd_chain_hw_clean(chain);579 580 api_cmd_set_status_addr(chain);581 582 err = api_cmd_hw_restart(chain);583 if (err) {584 dev_err(&pdev->dev, "Failed to restart API CMD HW\n");585 return err;586 }587 588 api_cmd_ctrl_init(chain);589 api_cmd_set_num_cells(chain);590 api_cmd_head_init(chain);591 return 0;592}593 594/**595 * free_cmd_buf - free the dma buffer of API CMD command596 * @chain: the API CMD specific chain of the cmd597 * @cell_idx: the cell index of the cmd598 **/599static void free_cmd_buf(struct hinic_api_cmd_chain *chain, int cell_idx)600{601 struct hinic_api_cmd_cell_ctxt *cell_ctxt;602 struct hinic_hwif *hwif = chain->hwif;603 struct pci_dev *pdev = hwif->pdev;604 605 cell_ctxt = &chain->cell_ctxt[cell_idx];606 607 dma_free_coherent(&pdev->dev, API_CMD_BUF_SIZE,608 cell_ctxt->api_cmd_vaddr,609 cell_ctxt->api_cmd_paddr);610}611 612/**613 * alloc_cmd_buf - allocate a dma buffer for API CMD command614 * @chain: the API CMD specific chain for the cmd615 * @cell: the cell in the HW for the cmd616 * @cell_idx: the index of the cell617 *618 * Return 0 - Success, negative - Failure619 **/620static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain,621 struct hinic_api_cmd_cell *cell, int cell_idx)622{623 struct hinic_api_cmd_cell_ctxt *cell_ctxt;624 struct hinic_hwif *hwif = chain->hwif;625 struct pci_dev *pdev = hwif->pdev;626 dma_addr_t cmd_paddr;627 u8 *cmd_vaddr;628 int err = 0;629 630 cmd_vaddr = dma_alloc_coherent(&pdev->dev, API_CMD_BUF_SIZE,631 &cmd_paddr, GFP_KERNEL);632 if (!cmd_vaddr)633 return -ENOMEM;634 635 cell_ctxt = &chain->cell_ctxt[cell_idx];636 637 cell_ctxt->api_cmd_vaddr = cmd_vaddr;638 cell_ctxt->api_cmd_paddr = cmd_paddr;639 640 /* set the cmd DMA address in the cell */641 switch (chain->chain_type) {642 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:643 /* The data in the HW should be in Big Endian Format */644 cell->write.hw_cmd_paddr = cpu_to_be64(cmd_paddr);645 break;646 647 default:648 dev_err(&pdev->dev, "Unsupported API CMD chain type\n");649 free_cmd_buf(chain, cell_idx);650 err = -EINVAL;651 break;652 }653 654 return err;655}656 657/**658 * api_cmd_create_cell - create API CMD cell for specific chain659 * @chain: the API CMD specific chain to create its cell660 * @cell_idx: the index of the cell to create661 * @pre_node: previous cell662 * @node_vaddr: the returned virt addr of the cell663 *664 * Return 0 - Success, negative - Failure665 **/666static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain,667 int cell_idx,668 struct hinic_api_cmd_cell *pre_node,669 struct hinic_api_cmd_cell **node_vaddr)670{671 struct hinic_api_cmd_cell_ctxt *cell_ctxt;672 struct hinic_hwif *hwif = chain->hwif;673 struct pci_dev *pdev = hwif->pdev;674 struct hinic_api_cmd_cell *node;675 dma_addr_t node_paddr;676 int err;677 678 node = dma_alloc_coherent(&pdev->dev, chain->cell_size, &node_paddr,679 GFP_KERNEL);680 if (!node)681 return -ENOMEM;682 683 node->read.hw_wb_resp_paddr = 0;684 685 cell_ctxt = &chain->cell_ctxt[cell_idx];686 cell_ctxt->cell_vaddr = node;687 cell_ctxt->cell_paddr = node_paddr;688 689 if (!pre_node) {690 chain->head_cell_paddr = node_paddr;691 chain->head_node = node;692 } else {693 /* The data in the HW should be in Big Endian Format */694 pre_node->next_cell_paddr = cpu_to_be64(node_paddr);695 }696 697 switch (chain->chain_type) {698 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:699 err = alloc_cmd_buf(chain, node, cell_idx);700 if (err) {701 dev_err(&pdev->dev, "Failed to allocate cmd buffer\n");702 goto err_alloc_cmd_buf;703 }704 break;705 706 default:707 dev_err(&pdev->dev, "Unsupported API CMD chain type\n");708 err = -EINVAL;709 goto err_alloc_cmd_buf;710 }711 712 *node_vaddr = node;713 return 0;714 715err_alloc_cmd_buf:716 dma_free_coherent(&pdev->dev, chain->cell_size, node, node_paddr);717 return err;718}719 720/**721 * api_cmd_destroy_cell - destroy API CMD cell of specific chain722 * @chain: the API CMD specific chain to destroy its cell723 * @cell_idx: the cell to destroy724 **/725static void api_cmd_destroy_cell(struct hinic_api_cmd_chain *chain,726 int cell_idx)727{728 struct hinic_api_cmd_cell_ctxt *cell_ctxt;729 struct hinic_hwif *hwif = chain->hwif;730 struct pci_dev *pdev = hwif->pdev;731 struct hinic_api_cmd_cell *node;732 dma_addr_t node_paddr;733 size_t node_size;734 735 cell_ctxt = &chain->cell_ctxt[cell_idx];736 737 node = cell_ctxt->cell_vaddr;738 node_paddr = cell_ctxt->cell_paddr;739 node_size = chain->cell_size;740 741 if (cell_ctxt->api_cmd_vaddr) {742 switch (chain->chain_type) {743 case HINIC_API_CMD_WRITE_TO_MGMT_CPU:744 free_cmd_buf(chain, cell_idx);745 break;746 default:747 dev_err(&pdev->dev, "Unsupported API CMD chain type\n");748 break;749 }750 751 dma_free_coherent(&pdev->dev, node_size, node,752 node_paddr);753 }754}755 756/**757 * api_cmd_destroy_cells - destroy API CMD cells of specific chain758 * @chain: the API CMD specific chain to destroy its cells759 * @num_cells: number of cells to destroy760 **/761static void api_cmd_destroy_cells(struct hinic_api_cmd_chain *chain,762 int num_cells)763{764 int cell_idx;765 766 for (cell_idx = 0; cell_idx < num_cells; cell_idx++)767 api_cmd_destroy_cell(chain, cell_idx);768}769 770/**771 * api_cmd_create_cells - create API CMD cells for specific chain772 * @chain: the API CMD specific chain773 *774 * Return 0 - Success, negative - Failure775 **/776static int api_cmd_create_cells(struct hinic_api_cmd_chain *chain)777{778 struct hinic_api_cmd_cell *node = NULL, *pre_node = NULL;779 struct hinic_hwif *hwif = chain->hwif;780 struct pci_dev *pdev = hwif->pdev;781 int err, cell_idx;782 783 for (cell_idx = 0; cell_idx < chain->num_cells; cell_idx++) {784 err = api_cmd_create_cell(chain, cell_idx, pre_node, &node);785 if (err) {786 dev_err(&pdev->dev, "Failed to create API CMD cell\n");787 goto err_create_cell;788 }789 790 pre_node = node;791 }792 793 /* set the Final node to point on the start */794 node->next_cell_paddr = cpu_to_be64(chain->head_cell_paddr);795 796 /* set the current node to be the head */797 chain->curr_node = chain->head_node;798 return 0;799 800err_create_cell:801 api_cmd_destroy_cells(chain, cell_idx);802 return err;803}804 805/**806 * api_chain_init - initialize API CMD specific chain807 * @chain: the API CMD specific chain to initialize808 * @attr: attributes to set in the chain809 *810 * Return 0 - Success, negative - Failure811 **/812static int api_chain_init(struct hinic_api_cmd_chain *chain,813 struct hinic_api_cmd_chain_attr *attr)814{815 struct hinic_hwif *hwif = attr->hwif;816 struct pci_dev *pdev = hwif->pdev;817 818 chain->hwif = hwif;819 chain->chain_type = attr->chain_type;820 chain->num_cells = attr->num_cells;821 chain->cell_size = attr->cell_size;822 823 chain->prod_idx = 0;824 chain->cons_idx = 0;825 826 sema_init(&chain->sem, 1);827 828 chain->cell_ctxt = devm_kcalloc(&pdev->dev, chain->num_cells,829 sizeof(*chain->cell_ctxt), GFP_KERNEL);830 if (!chain->cell_ctxt)831 return -ENOMEM;832 833 chain->wb_status = dma_alloc_coherent(&pdev->dev,834 sizeof(*chain->wb_status),835 &chain->wb_status_paddr,836 GFP_KERNEL);837 if (!chain->wb_status) {838 dev_err(&pdev->dev, "Failed to allocate DMA wb status\n");839 return -ENOMEM;840 }841 842 return 0;843}844 845/**846 * api_chain_free - free API CMD specific chain847 * @chain: the API CMD specific chain to free848 **/849static void api_chain_free(struct hinic_api_cmd_chain *chain)850{851 struct hinic_hwif *hwif = chain->hwif;852 struct pci_dev *pdev = hwif->pdev;853 854 dma_free_coherent(&pdev->dev, sizeof(*chain->wb_status),855 chain->wb_status, chain->wb_status_paddr);856}857 858/**859 * api_cmd_create_chain - create API CMD specific chain860 * @attr: attributes to set the chain861 *862 * Return the created chain863 **/864static struct hinic_api_cmd_chain *865 api_cmd_create_chain(struct hinic_api_cmd_chain_attr *attr)866{867 struct hinic_hwif *hwif = attr->hwif;868 struct pci_dev *pdev = hwif->pdev;869 struct hinic_api_cmd_chain *chain;870 int err;871 872 if (attr->num_cells & (attr->num_cells - 1)) {873 dev_err(&pdev->dev, "Invalid number of cells, must be power of 2\n");874 return ERR_PTR(-EINVAL);875 }876 877 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);878 if (!chain)879 return ERR_PTR(-ENOMEM);880 881 err = api_chain_init(chain, attr);882 if (err) {883 dev_err(&pdev->dev, "Failed to initialize chain\n");884 return ERR_PTR(err);885 }886 887 err = api_cmd_create_cells(chain);888 if (err) {889 dev_err(&pdev->dev, "Failed to create cells for API CMD chain\n");890 goto err_create_cells;891 }892 893 err = api_cmd_chain_hw_init(chain);894 if (err) {895 dev_err(&pdev->dev, "Failed to initialize chain HW\n");896 goto err_chain_hw_init;897 }898 899 return chain;900 901err_chain_hw_init:902 api_cmd_destroy_cells(chain, chain->num_cells);903 904err_create_cells:905 api_chain_free(chain);906 return ERR_PTR(err);907}908 909/**910 * api_cmd_destroy_chain - destroy API CMD specific chain911 * @chain: the API CMD specific chain to destroy912 **/913static void api_cmd_destroy_chain(struct hinic_api_cmd_chain *chain)914{915 api_cmd_chain_hw_clean(chain);916 api_cmd_destroy_cells(chain, chain->num_cells);917 api_chain_free(chain);918}919 920/**921 * hinic_api_cmd_init - Initialize all the API CMD chains922 * @chain: the API CMD chains that are initialized923 * @hwif: the hardware interface of a pci function device924 *925 * Return 0 - Success, negative - Failure926 **/927int hinic_api_cmd_init(struct hinic_api_cmd_chain **chain,928 struct hinic_hwif *hwif)929{930 enum hinic_api_cmd_chain_type type, chain_type;931 struct hinic_api_cmd_chain_attr attr;932 struct pci_dev *pdev = hwif->pdev;933 size_t hw_cell_sz;934 int err;935 936 hw_cell_sz = sizeof(struct hinic_api_cmd_cell);937 938 attr.hwif = hwif;939 attr.num_cells = API_CHAIN_NUM_CELLS;940 attr.cell_size = API_CMD_CELL_SIZE(hw_cell_sz);941 942 chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;943 for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) {944 attr.chain_type = chain_type;945 946 if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)947 continue;948 949 chain[chain_type] = api_cmd_create_chain(&attr);950 if (IS_ERR(chain[chain_type])) {951 dev_err(&pdev->dev, "Failed to create chain %d\n",952 chain_type);953 err = PTR_ERR(chain[chain_type]);954 goto err_create_chain;955 }956 }957 958 return 0;959 960err_create_chain:961 type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;962 for ( ; type < chain_type; type++) {963 if (type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)964 continue;965 966 api_cmd_destroy_chain(chain[type]);967 }968 969 return err;970}971 972/**973 * hinic_api_cmd_free - free the API CMD chains974 * @chain: the API CMD chains that are freed975 **/976void hinic_api_cmd_free(struct hinic_api_cmd_chain **chain)977{978 enum hinic_api_cmd_chain_type chain_type;979 980 chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU;981 for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) {982 if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU)983 continue;984 985 api_cmd_destroy_chain(chain[chain_type]);986 }987}988