7140 lines · c
1/*2 * Copyright (c) 2016-2017 Hisilicon Limited.3 *4 * This software is available to you under a choice of one of two5 * licenses. You may choose to be licensed under the terms of the GNU6 * General Public License (GPL) Version 2, available from the file7 * COPYING in the main directory of this source tree, or the8 * OpenIB.org BSD license below:9 *10 * Redistribution and use in source and binary forms, with or11 * without modification, are permitted provided that the following12 * conditions are met:13 *14 * - Redistributions of source code must retain the above15 * copyright notice, this list of conditions and the following16 * disclaimer.17 *18 * - Redistributions in binary form must reproduce the above19 * copyright notice, this list of conditions and the following20 * disclaimer in the documentation and/or other materials21 * provided with the distribution.22 *23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE30 * SOFTWARE.31 */32 33#include <linux/acpi.h>34#include <linux/etherdevice.h>35#include <linux/interrupt.h>36#include <linux/iopoll.h>37#include <linux/kernel.h>38#include <linux/types.h>39#include <linux/workqueue.h>40#include <net/addrconf.h>41#include <rdma/ib_addr.h>42#include <rdma/ib_cache.h>43#include <rdma/ib_umem.h>44#include <rdma/uverbs_ioctl.h>45 46#include "hnae3.h"47#include "hns_roce_common.h"48#include "hns_roce_device.h"49#include "hns_roce_cmd.h"50#include "hns_roce_hem.h"51#include "hns_roce_hw_v2.h"52 53enum {54 CMD_RST_PRC_OTHERS,55 CMD_RST_PRC_SUCCESS,56 CMD_RST_PRC_EBUSY,57};58 59enum ecc_resource_type {60 ECC_RESOURCE_QPC,61 ECC_RESOURCE_CQC,62 ECC_RESOURCE_MPT,63 ECC_RESOURCE_SRQC,64 ECC_RESOURCE_GMV,65 ECC_RESOURCE_QPC_TIMER,66 ECC_RESOURCE_CQC_TIMER,67 ECC_RESOURCE_SCCC,68 ECC_RESOURCE_COUNT,69};70 71static const struct {72 const char *name;73 u8 read_bt0_op;74 u8 write_bt0_op;75} fmea_ram_res[] = {76 { "ECC_RESOURCE_QPC",77 HNS_ROCE_CMD_READ_QPC_BT0, HNS_ROCE_CMD_WRITE_QPC_BT0 },78 { "ECC_RESOURCE_CQC",79 HNS_ROCE_CMD_READ_CQC_BT0, HNS_ROCE_CMD_WRITE_CQC_BT0 },80 { "ECC_RESOURCE_MPT",81 HNS_ROCE_CMD_READ_MPT_BT0, HNS_ROCE_CMD_WRITE_MPT_BT0 },82 { "ECC_RESOURCE_SRQC",83 HNS_ROCE_CMD_READ_SRQC_BT0, HNS_ROCE_CMD_WRITE_SRQC_BT0 },84 /* ECC_RESOURCE_GMV is handled by cmdq, not mailbox */85 { "ECC_RESOURCE_GMV",86 0, 0 },87 { "ECC_RESOURCE_QPC_TIMER",88 HNS_ROCE_CMD_READ_QPC_TIMER_BT0, HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0 },89 { "ECC_RESOURCE_CQC_TIMER",90 HNS_ROCE_CMD_READ_CQC_TIMER_BT0, HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0 },91 { "ECC_RESOURCE_SCCC",92 HNS_ROCE_CMD_READ_SCCC_BT0, HNS_ROCE_CMD_WRITE_SCCC_BT0 },93};94 95static inline void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,96 struct ib_sge *sg)97{98 dseg->lkey = cpu_to_le32(sg->lkey);99 dseg->addr = cpu_to_le64(sg->addr);100 dseg->len = cpu_to_le32(sg->length);101}102 103/*104 * mapped-value = 1 + real-value105 * The hns wr opcode real value is start from 0, In order to distinguish between106 * initialized and uninitialized map values, we plus 1 to the actual value when107 * defining the mapping, so that the validity can be identified by checking the108 * mapped value is greater than 0.109 */110#define HR_OPC_MAP(ib_key, hr_key) \111 [IB_WR_ ## ib_key] = 1 + HNS_ROCE_V2_WQE_OP_ ## hr_key112 113static const u32 hns_roce_op_code[] = {114 HR_OPC_MAP(RDMA_WRITE, RDMA_WRITE),115 HR_OPC_MAP(RDMA_WRITE_WITH_IMM, RDMA_WRITE_WITH_IMM),116 HR_OPC_MAP(SEND, SEND),117 HR_OPC_MAP(SEND_WITH_IMM, SEND_WITH_IMM),118 HR_OPC_MAP(RDMA_READ, RDMA_READ),119 HR_OPC_MAP(ATOMIC_CMP_AND_SWP, ATOM_CMP_AND_SWAP),120 HR_OPC_MAP(ATOMIC_FETCH_AND_ADD, ATOM_FETCH_AND_ADD),121 HR_OPC_MAP(SEND_WITH_INV, SEND_WITH_INV),122 HR_OPC_MAP(MASKED_ATOMIC_CMP_AND_SWP, ATOM_MSK_CMP_AND_SWAP),123 HR_OPC_MAP(MASKED_ATOMIC_FETCH_AND_ADD, ATOM_MSK_FETCH_AND_ADD),124 HR_OPC_MAP(REG_MR, FAST_REG_PMR),125};126 127static u32 to_hr_opcode(u32 ib_opcode)128{129 if (ib_opcode >= ARRAY_SIZE(hns_roce_op_code))130 return HNS_ROCE_V2_WQE_OP_MASK;131 132 return hns_roce_op_code[ib_opcode] ? hns_roce_op_code[ib_opcode] - 1 :133 HNS_ROCE_V2_WQE_OP_MASK;134}135 136static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,137 const struct ib_reg_wr *wr)138{139 struct hns_roce_wqe_frmr_seg *fseg =140 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);141 struct hns_roce_mr *mr = to_hr_mr(wr->mr);142 u64 pbl_ba;143 144 /* use ib_access_flags */145 hr_reg_write_bool(fseg, FRMR_BIND_EN, wr->access & IB_ACCESS_MW_BIND);146 hr_reg_write_bool(fseg, FRMR_ATOMIC,147 wr->access & IB_ACCESS_REMOTE_ATOMIC);148 hr_reg_write_bool(fseg, FRMR_RR, wr->access & IB_ACCESS_REMOTE_READ);149 hr_reg_write_bool(fseg, FRMR_RW, wr->access & IB_ACCESS_REMOTE_WRITE);150 hr_reg_write_bool(fseg, FRMR_LW, wr->access & IB_ACCESS_LOCAL_WRITE);151 152 /* Data structure reuse may lead to confusion */153 pbl_ba = mr->pbl_mtr.hem_cfg.root_ba;154 rc_sq_wqe->msg_len = cpu_to_le32(lower_32_bits(pbl_ba));155 rc_sq_wqe->inv_key = cpu_to_le32(upper_32_bits(pbl_ba));156 157 rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);158 rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);159 rc_sq_wqe->rkey = cpu_to_le32(wr->key);160 rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);161 162 hr_reg_write(fseg, FRMR_PBL_SIZE, mr->npages);163 hr_reg_write(fseg, FRMR_PBL_BUF_PG_SZ,164 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));165 hr_reg_clear(fseg, FRMR_BLK_MODE);166}167 168static void set_atomic_seg(const struct ib_send_wr *wr,169 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,170 unsigned int valid_num_sge)171{172 struct hns_roce_v2_wqe_data_seg *dseg =173 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);174 struct hns_roce_wqe_atomic_seg *aseg =175 (void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg);176 177 set_data_seg_v2(dseg, wr->sg_list);178 179 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {180 aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap);181 aseg->cmp_data = cpu_to_le64(atomic_wr(wr)->compare_add);182 } else {183 aseg->fetchadd_swap_data =184 cpu_to_le64(atomic_wr(wr)->compare_add);185 aseg->cmp_data = 0;186 }187 188 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, valid_num_sge);189}190 191static int fill_ext_sge_inl_data(struct hns_roce_qp *qp,192 const struct ib_send_wr *wr,193 unsigned int *sge_idx, u32 msg_len)194{195 struct ib_device *ibdev = &(to_hr_dev(qp->ibqp.device))->ib_dev;196 unsigned int left_len_in_pg;197 unsigned int idx = *sge_idx;198 unsigned int i = 0;199 unsigned int len;200 void *addr;201 void *dseg;202 203 if (msg_len > qp->sq.ext_sge_cnt * HNS_ROCE_SGE_SIZE) {204 ibdev_err(ibdev,205 "no enough extended sge space for inline data.\n");206 return -EINVAL;207 }208 209 dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));210 left_len_in_pg = hr_hw_page_align((uintptr_t)dseg) - (uintptr_t)dseg;211 len = wr->sg_list[0].length;212 addr = (void *)(unsigned long)(wr->sg_list[0].addr);213 214 /* When copying data to extended sge space, the left length in page may215 * not long enough for current user's sge. So the data should be216 * splited into several parts, one in the first page, and the others in217 * the subsequent pages.218 */219 while (1) {220 if (len <= left_len_in_pg) {221 memcpy(dseg, addr, len);222 223 idx += len / HNS_ROCE_SGE_SIZE;224 225 i++;226 if (i >= wr->num_sge)227 break;228 229 left_len_in_pg -= len;230 len = wr->sg_list[i].length;231 addr = (void *)(unsigned long)(wr->sg_list[i].addr);232 dseg += len;233 } else {234 memcpy(dseg, addr, left_len_in_pg);235 236 len -= left_len_in_pg;237 addr += left_len_in_pg;238 idx += left_len_in_pg / HNS_ROCE_SGE_SIZE;239 dseg = hns_roce_get_extend_sge(qp,240 idx & (qp->sge.sge_cnt - 1));241 left_len_in_pg = 1 << HNS_HW_PAGE_SHIFT;242 }243 }244 245 *sge_idx = idx;246 247 return 0;248}249 250static void set_extend_sge(struct hns_roce_qp *qp, struct ib_sge *sge,251 unsigned int *sge_ind, unsigned int cnt)252{253 struct hns_roce_v2_wqe_data_seg *dseg;254 unsigned int idx = *sge_ind;255 256 while (cnt > 0) {257 dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));258 if (likely(sge->length)) {259 set_data_seg_v2(dseg, sge);260 idx++;261 cnt--;262 }263 sge++;264 }265 266 *sge_ind = idx;267}268 269static bool check_inl_data_len(struct hns_roce_qp *qp, unsigned int len)270{271 struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);272 int mtu = ib_mtu_enum_to_int(qp->path_mtu);273 274 if (mtu < 0 || len > qp->max_inline_data || len > mtu) {275 ibdev_err(&hr_dev->ib_dev,276 "invalid length of data, data len = %u, max inline len = %u, path mtu = %d.\n",277 len, qp->max_inline_data, mtu);278 return false;279 }280 281 return true;282}283 284static int set_rc_inl(struct hns_roce_qp *qp, const struct ib_send_wr *wr,285 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,286 unsigned int *sge_idx)287{288 struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);289 u32 msg_len = le32_to_cpu(rc_sq_wqe->msg_len);290 struct ib_device *ibdev = &hr_dev->ib_dev;291 unsigned int curr_idx = *sge_idx;292 void *dseg = rc_sq_wqe;293 unsigned int i;294 int ret;295 296 if (unlikely(wr->opcode == IB_WR_RDMA_READ)) {297 ibdev_err(ibdev, "invalid inline parameters!\n");298 return -EINVAL;299 }300 301 if (!check_inl_data_len(qp, msg_len))302 return -EINVAL;303 304 dseg += sizeof(struct hns_roce_v2_rc_send_wqe);305 306 if (msg_len <= HNS_ROCE_V2_MAX_RC_INL_INN_SZ) {307 hr_reg_clear(rc_sq_wqe, RC_SEND_WQE_INL_TYPE);308 309 for (i = 0; i < wr->num_sge; i++) {310 memcpy(dseg, ((void *)wr->sg_list[i].addr),311 wr->sg_list[i].length);312 dseg += wr->sg_list[i].length;313 }314 } else {315 hr_reg_enable(rc_sq_wqe, RC_SEND_WQE_INL_TYPE);316 317 ret = fill_ext_sge_inl_data(qp, wr, &curr_idx, msg_len);318 if (ret)319 return ret;320 321 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, curr_idx - *sge_idx);322 }323 324 *sge_idx = curr_idx;325 326 return 0;327}328 329static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,330 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,331 unsigned int *sge_ind,332 unsigned int valid_num_sge)333{334 struct hns_roce_v2_wqe_data_seg *dseg =335 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);336 struct hns_roce_qp *qp = to_hr_qp(ibqp);337 int j = 0;338 int i;339 340 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_MSG_START_SGE_IDX,341 (*sge_ind) & (qp->sge.sge_cnt - 1));342 343 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_INLINE,344 !!(wr->send_flags & IB_SEND_INLINE));345 if (wr->send_flags & IB_SEND_INLINE)346 return set_rc_inl(qp, wr, rc_sq_wqe, sge_ind);347 348 if (valid_num_sge <= HNS_ROCE_SGE_IN_WQE) {349 for (i = 0; i < wr->num_sge; i++) {350 if (likely(wr->sg_list[i].length)) {351 set_data_seg_v2(dseg, wr->sg_list + i);352 dseg++;353 }354 }355 } else {356 for (i = 0; i < wr->num_sge && j < HNS_ROCE_SGE_IN_WQE; i++) {357 if (likely(wr->sg_list[i].length)) {358 set_data_seg_v2(dseg, wr->sg_list + i);359 dseg++;360 j++;361 }362 }363 364 set_extend_sge(qp, wr->sg_list + i, sge_ind,365 valid_num_sge - HNS_ROCE_SGE_IN_WQE);366 }367 368 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, valid_num_sge);369 370 return 0;371}372 373static int check_send_valid(struct hns_roce_dev *hr_dev,374 struct hns_roce_qp *hr_qp)375{376 struct ib_device *ibdev = &hr_dev->ib_dev;377 378 if (unlikely(hr_qp->state == IB_QPS_RESET ||379 hr_qp->state == IB_QPS_INIT ||380 hr_qp->state == IB_QPS_RTR)) {381 ibdev_err(ibdev, "failed to post WQE, QP state %u!\n",382 hr_qp->state);383 return -EINVAL;384 } else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) {385 ibdev_err(ibdev, "failed to post WQE, dev state %d!\n",386 hr_dev->state);387 return -EIO;388 }389 390 return 0;391}392 393static unsigned int calc_wr_sge_num(const struct ib_send_wr *wr,394 unsigned int *sge_len)395{396 unsigned int valid_num = 0;397 unsigned int len = 0;398 int i;399 400 for (i = 0; i < wr->num_sge; i++) {401 if (likely(wr->sg_list[i].length)) {402 len += wr->sg_list[i].length;403 valid_num++;404 }405 }406 407 *sge_len = len;408 return valid_num;409}410 411static __le32 get_immtdata(const struct ib_send_wr *wr)412{413 switch (wr->opcode) {414 case IB_WR_SEND_WITH_IMM:415 case IB_WR_RDMA_WRITE_WITH_IMM:416 return cpu_to_le32(be32_to_cpu(wr->ex.imm_data));417 default:418 return 0;419 }420}421 422static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,423 const struct ib_send_wr *wr)424{425 u32 ib_op = wr->opcode;426 427 if (ib_op != IB_WR_SEND && ib_op != IB_WR_SEND_WITH_IMM)428 return -EINVAL;429 430 ud_sq_wqe->immtdata = get_immtdata(wr);431 432 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_OPCODE, to_hr_opcode(ib_op));433 434 return 0;435}436 437static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,438 struct hns_roce_ah *ah)439{440 struct ib_device *ib_dev = ah->ibah.device;441 struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);442 443 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_UDPSPN, ah->av.udp_sport);444 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_HOPLIMIT, ah->av.hop_limit);445 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_TCLASS, ah->av.tclass);446 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_FLOW_LABEL, ah->av.flowlabel);447 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, ah->av.sl);448 449 ud_sq_wqe->sgid_index = ah->av.gid_index;450 451 memcpy(ud_sq_wqe->dmac, ah->av.mac, ETH_ALEN);452 memcpy(ud_sq_wqe->dgid, ah->av.dgid, GID_LEN_V2);453 454 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)455 return 0;456 457 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_VLAN_EN, ah->av.vlan_en);458 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_VLAN, ah->av.vlan_id);459 460 return 0;461}462 463static inline int set_ud_wqe(struct hns_roce_qp *qp,464 const struct ib_send_wr *wr,465 void *wqe, unsigned int *sge_idx,466 unsigned int owner_bit)467{468 struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);469 struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe;470 unsigned int curr_idx = *sge_idx;471 unsigned int valid_num_sge;472 u32 msg_len = 0;473 int ret;474 475 valid_num_sge = calc_wr_sge_num(wr, &msg_len);476 477 ret = set_ud_opcode(ud_sq_wqe, wr);478 if (WARN_ON(ret))479 return ret;480 481 ud_sq_wqe->msg_len = cpu_to_le32(msg_len);482 483 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_CQE,484 !!(wr->send_flags & IB_SEND_SIGNALED));485 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SE,486 !!(wr->send_flags & IB_SEND_SOLICITED));487 488 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_PD, to_hr_pd(qp->ibqp.pd)->pdn);489 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SGE_NUM, valid_num_sge);490 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_MSG_START_SGE_IDX,491 curr_idx & (qp->sge.sge_cnt - 1));492 493 ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?494 qp->qkey : ud_wr(wr)->remote_qkey);495 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_DQPN, ud_wr(wr)->remote_qpn);496 497 ret = fill_ud_av(ud_sq_wqe, ah);498 if (ret)499 return ret;500 501 qp->sl = to_hr_ah(ud_wr(wr)->ah)->av.sl;502 503 set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge);504 505 /*506 * The pipeline can sequentially post all valid WQEs into WQ buffer,507 * including new WQEs waiting for the doorbell to update the PI again.508 * Therefore, the owner bit of WQE MUST be updated after all fields509 * and extSGEs have been written into DDR instead of cache.510 */511 if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)512 dma_wmb();513 514 *sge_idx = curr_idx;515 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_OWNER, owner_bit);516 517 return 0;518}519 520static int set_rc_opcode(struct hns_roce_dev *hr_dev,521 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,522 const struct ib_send_wr *wr)523{524 u32 ib_op = wr->opcode;525 int ret = 0;526 527 rc_sq_wqe->immtdata = get_immtdata(wr);528 529 switch (ib_op) {530 case IB_WR_RDMA_READ:531 case IB_WR_RDMA_WRITE:532 case IB_WR_RDMA_WRITE_WITH_IMM:533 rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey);534 rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr);535 break;536 case IB_WR_SEND:537 case IB_WR_SEND_WITH_IMM:538 break;539 case IB_WR_ATOMIC_CMP_AND_SWP:540 case IB_WR_ATOMIC_FETCH_AND_ADD:541 rc_sq_wqe->rkey = cpu_to_le32(atomic_wr(wr)->rkey);542 rc_sq_wqe->va = cpu_to_le64(atomic_wr(wr)->remote_addr);543 break;544 case IB_WR_REG_MR:545 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)546 set_frmr_seg(rc_sq_wqe, reg_wr(wr));547 else548 ret = -EOPNOTSUPP;549 break;550 case IB_WR_SEND_WITH_INV:551 rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey);552 break;553 default:554 ret = -EINVAL;555 }556 557 if (unlikely(ret))558 return ret;559 560 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_OPCODE, to_hr_opcode(ib_op));561 562 return ret;563}564 565static inline int set_rc_wqe(struct hns_roce_qp *qp,566 const struct ib_send_wr *wr,567 void *wqe, unsigned int *sge_idx,568 unsigned int owner_bit)569{570 struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);571 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;572 unsigned int curr_idx = *sge_idx;573 unsigned int valid_num_sge;574 u32 msg_len = 0;575 int ret;576 577 valid_num_sge = calc_wr_sge_num(wr, &msg_len);578 579 rc_sq_wqe->msg_len = cpu_to_le32(msg_len);580 581 ret = set_rc_opcode(hr_dev, rc_sq_wqe, wr);582 if (WARN_ON(ret))583 return ret;584 585 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_FENCE,586 (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);587 588 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SE,589 (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);590 591 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_CQE,592 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);593 594 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||595 wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {596 if (msg_len != ATOMIC_WR_LEN)597 return -EINVAL;598 set_atomic_seg(wr, rc_sq_wqe, valid_num_sge);599 } else if (wr->opcode != IB_WR_REG_MR) {600 ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe,601 &curr_idx, valid_num_sge);602 if (ret)603 return ret;604 }605 606 /*607 * The pipeline can sequentially post all valid WQEs into WQ buffer,608 * including new WQEs waiting for the doorbell to update the PI again.609 * Therefore, the owner bit of WQE MUST be updated after all fields610 * and extSGEs have been written into DDR instead of cache.611 */612 if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)613 dma_wmb();614 615 *sge_idx = curr_idx;616 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_OWNER, owner_bit);617 618 return ret;619}620 621static inline void update_sq_db(struct hns_roce_dev *hr_dev,622 struct hns_roce_qp *qp)623{624 if (unlikely(qp->state == IB_QPS_ERR)) {625 flush_cqe(hr_dev, qp);626 } else {627 struct hns_roce_v2_db sq_db = {};628 629 hr_reg_write(&sq_db, DB_TAG, qp->qpn);630 hr_reg_write(&sq_db, DB_CMD, HNS_ROCE_V2_SQ_DB);631 hr_reg_write(&sq_db, DB_PI, qp->sq.head);632 hr_reg_write(&sq_db, DB_SL, qp->sl);633 634 hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg);635 }636}637 638static inline void update_rq_db(struct hns_roce_dev *hr_dev,639 struct hns_roce_qp *qp)640{641 if (unlikely(qp->state == IB_QPS_ERR)) {642 flush_cqe(hr_dev, qp);643 } else {644 if (likely(qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)) {645 *qp->rdb.db_record =646 qp->rq.head & V2_DB_PRODUCER_IDX_M;647 } else {648 struct hns_roce_v2_db rq_db = {};649 650 hr_reg_write(&rq_db, DB_TAG, qp->qpn);651 hr_reg_write(&rq_db, DB_CMD, HNS_ROCE_V2_RQ_DB);652 hr_reg_write(&rq_db, DB_PI, qp->rq.head);653 654 hns_roce_write64(hr_dev, (__le32 *)&rq_db,655 qp->rq.db_reg);656 }657 }658}659 660static void hns_roce_write512(struct hns_roce_dev *hr_dev, u64 *val,661 u64 __iomem *dest)662{663#define HNS_ROCE_WRITE_TIMES 8664 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;665 struct hnae3_handle *handle = priv->handle;666 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;667 int i;668 669 if (!hr_dev->dis_db && !ops->get_hw_reset_stat(handle))670 for (i = 0; i < HNS_ROCE_WRITE_TIMES; i++)671 writeq_relaxed(*(val + i), dest + i);672}673 674static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,675 void *wqe)676{677#define HNS_ROCE_SL_SHIFT 2678 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;679 680 /* All kinds of DirectWQE have the same header field layout */681 hr_reg_enable(rc_sq_wqe, RC_SEND_WQE_FLAG);682 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_DB_SL_L, qp->sl);683 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_DB_SL_H,684 qp->sl >> HNS_ROCE_SL_SHIFT);685 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_WQE_INDEX, qp->sq.head);686 687 hns_roce_write512(hr_dev, wqe, qp->sq.db_reg);688}689 690static int hns_roce_v2_post_send(struct ib_qp *ibqp,691 const struct ib_send_wr *wr,692 const struct ib_send_wr **bad_wr)693{694 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);695 struct ib_device *ibdev = &hr_dev->ib_dev;696 struct hns_roce_qp *qp = to_hr_qp(ibqp);697 unsigned long flags = 0;698 unsigned int owner_bit;699 unsigned int sge_idx;700 unsigned int wqe_idx;701 void *wqe = NULL;702 u32 nreq;703 int ret;704 705 spin_lock_irqsave(&qp->sq.lock, flags);706 707 ret = check_send_valid(hr_dev, qp);708 if (unlikely(ret)) {709 *bad_wr = wr;710 nreq = 0;711 goto out;712 }713 714 sge_idx = qp->next_sge;715 716 for (nreq = 0; wr; ++nreq, wr = wr->next) {717 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {718 ret = -ENOMEM;719 *bad_wr = wr;720 goto out;721 }722 723 wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1);724 725 if (unlikely(wr->num_sge > qp->sq.max_gs)) {726 ibdev_err(ibdev, "num_sge = %d > qp->sq.max_gs = %u.\n",727 wr->num_sge, qp->sq.max_gs);728 ret = -EINVAL;729 *bad_wr = wr;730 goto out;731 }732 733 wqe = hns_roce_get_send_wqe(qp, wqe_idx);734 qp->sq.wrid[wqe_idx] = wr->wr_id;735 owner_bit =736 ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);737 738 /* Corresponding to the QP type, wqe process separately */739 if (ibqp->qp_type == IB_QPT_RC)740 ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit);741 else742 ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit);743 744 if (unlikely(ret)) {745 *bad_wr = wr;746 goto out;747 }748 }749 750out:751 if (likely(nreq)) {752 qp->sq.head += nreq;753 qp->next_sge = sge_idx;754 755 if (nreq == 1 && !ret &&756 (qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE))757 write_dwqe(hr_dev, qp, wqe);758 else759 update_sq_db(hr_dev, qp);760 }761 762 spin_unlock_irqrestore(&qp->sq.lock, flags);763 764 return ret;765}766 767static int check_recv_valid(struct hns_roce_dev *hr_dev,768 struct hns_roce_qp *hr_qp)769{770 if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN))771 return -EIO;772 773 if (hr_qp->state == IB_QPS_RESET)774 return -EINVAL;775 776 return 0;777}778 779static void fill_recv_sge_to_wqe(const struct ib_recv_wr *wr, void *wqe,780 u32 max_sge, bool rsv)781{782 struct hns_roce_v2_wqe_data_seg *dseg = wqe;783 u32 i, cnt;784 785 for (i = 0, cnt = 0; i < wr->num_sge; i++) {786 /* Skip zero-length sge */787 if (!wr->sg_list[i].length)788 continue;789 set_data_seg_v2(dseg + cnt, wr->sg_list + i);790 cnt++;791 }792 793 /* Fill a reserved sge to make hw stop reading remaining segments */794 if (rsv) {795 dseg[cnt].lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);796 dseg[cnt].addr = 0;797 dseg[cnt].len = cpu_to_le32(HNS_ROCE_INVALID_SGE_LENGTH);798 } else {799 /* Clear remaining segments to make ROCEE ignore sges */800 if (cnt < max_sge)801 memset(dseg + cnt, 0,802 (max_sge - cnt) * HNS_ROCE_SGE_SIZE);803 }804}805 806static void fill_rq_wqe(struct hns_roce_qp *hr_qp, const struct ib_recv_wr *wr,807 u32 wqe_idx, u32 max_sge)808{809 void *wqe = NULL;810 811 wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx);812 fill_recv_sge_to_wqe(wr, wqe, max_sge, hr_qp->rq.rsv_sge);813}814 815static int hns_roce_v2_post_recv(struct ib_qp *ibqp,816 const struct ib_recv_wr *wr,817 const struct ib_recv_wr **bad_wr)818{819 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);820 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);821 struct ib_device *ibdev = &hr_dev->ib_dev;822 u32 wqe_idx, nreq, max_sge;823 unsigned long flags;824 int ret;825 826 spin_lock_irqsave(&hr_qp->rq.lock, flags);827 828 ret = check_recv_valid(hr_dev, hr_qp);829 if (unlikely(ret)) {830 *bad_wr = wr;831 nreq = 0;832 goto out;833 }834 835 max_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;836 for (nreq = 0; wr; ++nreq, wr = wr->next) {837 if (unlikely(hns_roce_wq_overflow(&hr_qp->rq, nreq,838 hr_qp->ibqp.recv_cq))) {839 ret = -ENOMEM;840 *bad_wr = wr;841 goto out;842 }843 844 if (unlikely(wr->num_sge > max_sge)) {845 ibdev_err(ibdev, "num_sge = %d >= max_sge = %u.\n",846 wr->num_sge, max_sge);847 ret = -EINVAL;848 *bad_wr = wr;849 goto out;850 }851 852 wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1);853 fill_rq_wqe(hr_qp, wr, wqe_idx, max_sge);854 hr_qp->rq.wrid[wqe_idx] = wr->wr_id;855 }856 857out:858 if (likely(nreq)) {859 hr_qp->rq.head += nreq;860 861 update_rq_db(hr_dev, hr_qp);862 }863 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);864 865 return ret;866}867 868static void *get_srq_wqe_buf(struct hns_roce_srq *srq, u32 n)869{870 return hns_roce_buf_offset(srq->buf_mtr.kmem, n << srq->wqe_shift);871}872 873static void *get_idx_buf(struct hns_roce_idx_que *idx_que, u32 n)874{875 return hns_roce_buf_offset(idx_que->mtr.kmem,876 n << idx_que->entry_shift);877}878 879static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, u32 wqe_index)880{881 /* always called with interrupts disabled. */882 spin_lock(&srq->lock);883 884 bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);885 srq->idx_que.tail++;886 887 spin_unlock(&srq->lock);888}889 890static int hns_roce_srqwq_overflow(struct hns_roce_srq *srq)891{892 struct hns_roce_idx_que *idx_que = &srq->idx_que;893 894 return idx_que->head - idx_que->tail >= srq->wqe_cnt;895}896 897static int check_post_srq_valid(struct hns_roce_srq *srq, u32 max_sge,898 const struct ib_recv_wr *wr)899{900 struct ib_device *ib_dev = srq->ibsrq.device;901 902 if (unlikely(wr->num_sge > max_sge)) {903 ibdev_err(ib_dev,904 "failed to check sge, wr->num_sge = %d, max_sge = %u.\n",905 wr->num_sge, max_sge);906 return -EINVAL;907 }908 909 if (unlikely(hns_roce_srqwq_overflow(srq))) {910 ibdev_err(ib_dev,911 "failed to check srqwq status, srqwq is full.\n");912 return -ENOMEM;913 }914 915 return 0;916}917 918static int get_srq_wqe_idx(struct hns_roce_srq *srq, u32 *wqe_idx)919{920 struct hns_roce_idx_que *idx_que = &srq->idx_que;921 u32 pos;922 923 pos = find_first_zero_bit(idx_que->bitmap, srq->wqe_cnt);924 if (unlikely(pos == srq->wqe_cnt))925 return -ENOSPC;926 927 bitmap_set(idx_que->bitmap, pos, 1);928 *wqe_idx = pos;929 return 0;930}931 932static void fill_wqe_idx(struct hns_roce_srq *srq, unsigned int wqe_idx)933{934 struct hns_roce_idx_que *idx_que = &srq->idx_que;935 unsigned int head;936 __le32 *buf;937 938 head = idx_que->head & (srq->wqe_cnt - 1);939 940 buf = get_idx_buf(idx_que, head);941 *buf = cpu_to_le32(wqe_idx);942 943 idx_que->head++;944}945 946static void update_srq_db(struct hns_roce_srq *srq)947{948 struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device);949 struct hns_roce_v2_db db;950 951 hr_reg_write(&db, DB_TAG, srq->srqn);952 hr_reg_write(&db, DB_CMD, HNS_ROCE_V2_SRQ_DB);953 hr_reg_write(&db, DB_PI, srq->idx_que.head);954 955 hns_roce_write64(hr_dev, (__le32 *)&db, srq->db_reg);956}957 958static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,959 const struct ib_recv_wr *wr,960 const struct ib_recv_wr **bad_wr)961{962 struct hns_roce_srq *srq = to_hr_srq(ibsrq);963 unsigned long flags;964 int ret = 0;965 u32 max_sge;966 u32 wqe_idx;967 void *wqe;968 u32 nreq;969 970 spin_lock_irqsave(&srq->lock, flags);971 972 max_sge = srq->max_gs - srq->rsv_sge;973 for (nreq = 0; wr; ++nreq, wr = wr->next) {974 ret = check_post_srq_valid(srq, max_sge, wr);975 if (ret) {976 *bad_wr = wr;977 break;978 }979 980 ret = get_srq_wqe_idx(srq, &wqe_idx);981 if (unlikely(ret)) {982 *bad_wr = wr;983 break;984 }985 986 wqe = get_srq_wqe_buf(srq, wqe_idx);987 fill_recv_sge_to_wqe(wr, wqe, max_sge, srq->rsv_sge);988 fill_wqe_idx(srq, wqe_idx);989 srq->wrid[wqe_idx] = wr->wr_id;990 }991 992 if (likely(nreq)) {993 if (srq->cap_flags & HNS_ROCE_SRQ_CAP_RECORD_DB)994 *srq->rdb.db_record = srq->idx_que.head &995 V2_DB_PRODUCER_IDX_M;996 else997 update_srq_db(srq);998 }999 1000 spin_unlock_irqrestore(&srq->lock, flags);1001 1002 return ret;1003}1004 1005static u32 hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,1006 unsigned long instance_stage,1007 unsigned long reset_stage)1008{1009 /* When hardware reset has been completed once or more, we should stop1010 * sending mailbox&cmq&doorbell to hardware. If now in .init_instance()1011 * function, we should exit with error. If now at HNAE3_INIT_CLIENT1012 * stage of soft reset process, we should exit with error, and then1013 * HNAE3_INIT_CLIENT related process can rollback the operation like1014 * notifing hardware to free resources, HNAE3_INIT_CLIENT related1015 * process will exit with error to notify NIC driver to reschedule soft1016 * reset process once again.1017 */1018 hr_dev->is_reset = true;1019 hr_dev->dis_db = true;1020 1021 if (reset_stage == HNS_ROCE_STATE_RST_INIT ||1022 instance_stage == HNS_ROCE_STATE_INIT)1023 return CMD_RST_PRC_EBUSY;1024 1025 return CMD_RST_PRC_SUCCESS;1026}1027 1028static u32 hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,1029 unsigned long instance_stage,1030 unsigned long reset_stage)1031{1032#define HW_RESET_TIMEOUT_US 10000001033#define HW_RESET_SLEEP_US 10001034 1035 struct hns_roce_v2_priv *priv = hr_dev->priv;1036 struct hnae3_handle *handle = priv->handle;1037 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;1038 unsigned long val;1039 int ret;1040 1041 /* When hardware reset is detected, we should stop sending mailbox&cmq&1042 * doorbell to hardware. If now in .init_instance() function, we should1043 * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset1044 * process, we should exit with error, and then HNAE3_INIT_CLIENT1045 * related process can rollback the operation like notifing hardware to1046 * free resources, HNAE3_INIT_CLIENT related process will exit with1047 * error to notify NIC driver to reschedule soft reset process once1048 * again.1049 */1050 hr_dev->dis_db = true;1051 1052 ret = read_poll_timeout(ops->ae_dev_reset_cnt, val,1053 val > hr_dev->reset_cnt, HW_RESET_SLEEP_US,1054 HW_RESET_TIMEOUT_US, false, handle);1055 if (!ret)1056 hr_dev->is_reset = true;1057 1058 if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT ||1059 instance_stage == HNS_ROCE_STATE_INIT)1060 return CMD_RST_PRC_EBUSY;1061 1062 return CMD_RST_PRC_SUCCESS;1063}1064 1065static u32 hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)1066{1067 struct hns_roce_v2_priv *priv = hr_dev->priv;1068 struct hnae3_handle *handle = priv->handle;1069 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;1070 1071 /* When software reset is detected at .init_instance() function, we1072 * should stop sending mailbox&cmq&doorbell to hardware, and exit1073 * with error.1074 */1075 hr_dev->dis_db = true;1076 if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt)1077 hr_dev->is_reset = true;1078 1079 return CMD_RST_PRC_EBUSY;1080}1081 1082static u32 check_aedev_reset_status(struct hns_roce_dev *hr_dev,1083 struct hnae3_handle *handle)1084{1085 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;1086 unsigned long instance_stage; /* the current instance stage */1087 unsigned long reset_stage; /* the current reset stage */1088 unsigned long reset_cnt;1089 bool sw_resetting;1090 bool hw_resetting;1091 1092 /* Get information about reset from NIC driver or RoCE driver itself,1093 * the meaning of the following variables from NIC driver are described1094 * as below:1095 * reset_cnt -- The count value of completed hardware reset.1096 * hw_resetting -- Whether hardware device is resetting now.1097 * sw_resetting -- Whether NIC's software reset process is running now.1098 */1099 instance_stage = handle->rinfo.instance_state;1100 reset_stage = handle->rinfo.reset_state;1101 reset_cnt = ops->ae_dev_reset_cnt(handle);1102 if (reset_cnt != hr_dev->reset_cnt)1103 return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage,1104 reset_stage);1105 1106 hw_resetting = ops->get_cmdq_stat(handle);1107 if (hw_resetting)1108 return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage,1109 reset_stage);1110 1111 sw_resetting = ops->ae_dev_resetting(handle);1112 if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT)1113 return hns_roce_v2_cmd_sw_resetting(hr_dev);1114 1115 return CMD_RST_PRC_OTHERS;1116}1117 1118static bool check_device_is_in_reset(struct hns_roce_dev *hr_dev)1119{1120 struct hns_roce_v2_priv *priv = hr_dev->priv;1121 struct hnae3_handle *handle = priv->handle;1122 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;1123 1124 if (hr_dev->reset_cnt != ops->ae_dev_reset_cnt(handle))1125 return true;1126 1127 if (ops->get_hw_reset_stat(handle))1128 return true;1129 1130 if (ops->ae_dev_resetting(handle))1131 return true;1132 1133 return false;1134}1135 1136static bool v2_chk_mbox_is_avail(struct hns_roce_dev *hr_dev, bool *busy)1137{1138 struct hns_roce_v2_priv *priv = hr_dev->priv;1139 u32 status;1140 1141 if (hr_dev->is_reset)1142 status = CMD_RST_PRC_SUCCESS;1143 else1144 status = check_aedev_reset_status(hr_dev, priv->handle);1145 1146 *busy = (status == CMD_RST_PRC_EBUSY);1147 1148 return status == CMD_RST_PRC_OTHERS;1149}1150 1151static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,1152 struct hns_roce_v2_cmq_ring *ring)1153{1154 int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc);1155 1156 ring->desc = dma_alloc_coherent(hr_dev->dev, size,1157 &ring->desc_dma_addr, GFP_KERNEL);1158 if (!ring->desc)1159 return -ENOMEM;1160 1161 return 0;1162}1163 1164static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,1165 struct hns_roce_v2_cmq_ring *ring)1166{1167 dma_free_coherent(hr_dev->dev,1168 ring->desc_num * sizeof(struct hns_roce_cmq_desc),1169 ring->desc, ring->desc_dma_addr);1170 1171 ring->desc_dma_addr = 0;1172}1173 1174static int init_csq(struct hns_roce_dev *hr_dev,1175 struct hns_roce_v2_cmq_ring *csq)1176{1177 dma_addr_t dma;1178 int ret;1179 1180 csq->desc_num = CMD_CSQ_DESC_NUM;1181 spin_lock_init(&csq->lock);1182 csq->flag = TYPE_CSQ;1183 csq->head = 0;1184 1185 ret = hns_roce_alloc_cmq_desc(hr_dev, csq);1186 if (ret)1187 return ret;1188 1189 dma = csq->desc_dma_addr;1190 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, lower_32_bits(dma));1191 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG, upper_32_bits(dma));1192 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,1193 (u32)csq->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);1194 1195 /* Make sure to write CI first and then PI */1196 roce_write(hr_dev, ROCEE_TX_CMQ_CI_REG, 0);1197 roce_write(hr_dev, ROCEE_TX_CMQ_PI_REG, 0);1198 1199 return 0;1200}1201 1202static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)1203{1204 struct hns_roce_v2_priv *priv = hr_dev->priv;1205 int ret;1206 1207 priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;1208 1209 ret = init_csq(hr_dev, &priv->cmq.csq);1210 if (ret)1211 dev_err(hr_dev->dev, "failed to init CSQ, ret = %d.\n", ret);1212 1213 return ret;1214}1215 1216static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)1217{1218 struct hns_roce_v2_priv *priv = hr_dev->priv;1219 1220 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);1221}1222 1223static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,1224 enum hns_roce_opcode_type opcode,1225 bool is_read)1226{1227 memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));1228 desc->opcode = cpu_to_le16(opcode);1229 desc->flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_IN);1230 if (is_read)1231 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);1232 else1233 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);1234}1235 1236static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)1237{1238 u32 tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG);1239 struct hns_roce_v2_priv *priv = hr_dev->priv;1240 1241 return tail == priv->cmq.csq.head;1242}1243 1244static void update_cmdq_status(struct hns_roce_dev *hr_dev)1245{1246 struct hns_roce_v2_priv *priv = hr_dev->priv;1247 struct hnae3_handle *handle = priv->handle;1248 1249 if (handle->rinfo.reset_state == HNS_ROCE_STATE_RST_INIT ||1250 handle->rinfo.instance_state == HNS_ROCE_STATE_INIT)1251 hr_dev->cmd.state = HNS_ROCE_CMDQ_STATE_FATAL_ERR;1252}1253 1254static int hns_roce_cmd_err_convert_errno(u16 desc_ret)1255{1256 struct hns_roce_cmd_errcode errcode_table[] = {1257 {CMD_EXEC_SUCCESS, 0},1258 {CMD_NO_AUTH, -EPERM},1259 {CMD_NOT_EXIST, -EOPNOTSUPP},1260 {CMD_CRQ_FULL, -EXFULL},1261 {CMD_NEXT_ERR, -ENOSR},1262 {CMD_NOT_EXEC, -ENOTBLK},1263 {CMD_PARA_ERR, -EINVAL},1264 {CMD_RESULT_ERR, -ERANGE},1265 {CMD_TIMEOUT, -ETIME},1266 {CMD_HILINK_ERR, -ENOLINK},1267 {CMD_INFO_ILLEGAL, -ENXIO},1268 {CMD_INVALID, -EBADR},1269 };1270 u16 i;1271 1272 for (i = 0; i < ARRAY_SIZE(errcode_table); i++)1273 if (desc_ret == errcode_table[i].return_status)1274 return errcode_table[i].errno;1275 return -EIO;1276}1277 1278static u32 hns_roce_cmdq_tx_timeout(u16 opcode, u32 tx_timeout)1279{1280 static const struct hns_roce_cmdq_tx_timeout_map cmdq_tx_timeout[] = {1281 {HNS_ROCE_OPC_POST_MB, HNS_ROCE_OPC_POST_MB_TIMEOUT},1282 };1283 int i;1284 1285 for (i = 0; i < ARRAY_SIZE(cmdq_tx_timeout); i++)1286 if (cmdq_tx_timeout[i].opcode == opcode)1287 return cmdq_tx_timeout[i].tx_timeout;1288 1289 return tx_timeout;1290}1291 1292static void hns_roce_wait_csq_done(struct hns_roce_dev *hr_dev, u16 opcode)1293{1294 struct hns_roce_v2_priv *priv = hr_dev->priv;1295 u32 tx_timeout = hns_roce_cmdq_tx_timeout(opcode, priv->cmq.tx_timeout);1296 u32 timeout = 0;1297 1298 do {1299 if (hns_roce_cmq_csq_done(hr_dev))1300 break;1301 udelay(1);1302 } while (++timeout < tx_timeout);1303}1304 1305static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,1306 struct hns_roce_cmq_desc *desc, int num)1307{1308 struct hns_roce_v2_priv *priv = hr_dev->priv;1309 struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;1310 u16 desc_ret;1311 u32 tail;1312 int ret;1313 int i;1314 1315 spin_lock_bh(&csq->lock);1316 1317 tail = csq->head;1318 1319 for (i = 0; i < num; i++) {1320 csq->desc[csq->head++] = desc[i];1321 if (csq->head == csq->desc_num)1322 csq->head = 0;1323 }1324 1325 /* Write to hardware */1326 roce_write(hr_dev, ROCEE_TX_CMQ_PI_REG, csq->head);1327 1328 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CMDS_CNT]);1329 1330 hns_roce_wait_csq_done(hr_dev, le16_to_cpu(desc->opcode));1331 if (hns_roce_cmq_csq_done(hr_dev)) {1332 ret = 0;1333 for (i = 0; i < num; i++) {1334 /* check the result of hardware write back */1335 desc[i] = csq->desc[tail++];1336 if (tail == csq->desc_num)1337 tail = 0;1338 1339 desc_ret = le16_to_cpu(desc[i].retval);1340 if (likely(desc_ret == CMD_EXEC_SUCCESS))1341 continue;1342 1343 dev_err_ratelimited(hr_dev->dev,1344 "Cmdq IO error, opcode = 0x%x, return = 0x%x.\n",1345 desc->opcode, desc_ret);1346 ret = hns_roce_cmd_err_convert_errno(desc_ret);1347 }1348 } else {1349 /* FW/HW reset or incorrect number of desc */1350 tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG);1351 dev_warn(hr_dev->dev, "CMDQ move tail from %u to %u.\n",1352 csq->head, tail);1353 csq->head = tail;1354 1355 update_cmdq_status(hr_dev);1356 1357 ret = -EAGAIN;1358 }1359 1360 spin_unlock_bh(&csq->lock);1361 1362 if (ret)1363 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CMDS_ERR_CNT]);1364 1365 return ret;1366}1367 1368static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,1369 struct hns_roce_cmq_desc *desc, int num)1370{1371 bool busy;1372 int ret;1373 1374 if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR)1375 return -EIO;1376 1377 if (!v2_chk_mbox_is_avail(hr_dev, &busy))1378 return busy ? -EBUSY : 0;1379 1380 ret = __hns_roce_cmq_send(hr_dev, desc, num);1381 if (ret) {1382 if (!v2_chk_mbox_is_avail(hr_dev, &busy))1383 return busy ? -EBUSY : 0;1384 }1385 1386 return ret;1387}1388 1389static int config_hem_ba_to_hw(struct hns_roce_dev *hr_dev,1390 dma_addr_t base_addr, u8 cmd, unsigned long tag)1391{1392 struct hns_roce_cmd_mailbox *mbox;1393 int ret;1394 1395 mbox = hns_roce_alloc_cmd_mailbox(hr_dev);1396 if (IS_ERR(mbox))1397 return PTR_ERR(mbox);1398 1399 ret = hns_roce_cmd_mbox(hr_dev, base_addr, mbox->dma, cmd, tag);1400 hns_roce_free_cmd_mailbox(hr_dev, mbox);1401 return ret;1402}1403 1404static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)1405{1406 struct hns_roce_query_version *resp;1407 struct hns_roce_cmq_desc desc;1408 int ret;1409 1410 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true);1411 ret = hns_roce_cmq_send(hr_dev, &desc, 1);1412 if (ret)1413 return ret;1414 1415 resp = (struct hns_roce_query_version *)desc.data;1416 hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version);1417 hr_dev->vendor_id = hr_dev->pci_dev->vendor;1418 1419 return 0;1420}1421 1422static void func_clr_hw_resetting_state(struct hns_roce_dev *hr_dev,1423 struct hnae3_handle *handle)1424{1425 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;1426 unsigned long end;1427 1428 hr_dev->dis_db = true;1429 1430 dev_warn(hr_dev->dev,1431 "func clear is pending, device in resetting state.\n");1432 end = HNS_ROCE_V2_HW_RST_TIMEOUT;1433 while (end) {1434 if (!ops->get_hw_reset_stat(handle)) {1435 hr_dev->is_reset = true;1436 dev_info(hr_dev->dev,1437 "func clear success after reset.\n");1438 return;1439 }1440 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);1441 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;1442 }1443 1444 dev_warn(hr_dev->dev, "func clear failed.\n");1445}1446 1447static void func_clr_sw_resetting_state(struct hns_roce_dev *hr_dev,1448 struct hnae3_handle *handle)1449{1450 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;1451 unsigned long end;1452 1453 hr_dev->dis_db = true;1454 1455 dev_warn(hr_dev->dev,1456 "func clear is pending, device in resetting state.\n");1457 end = HNS_ROCE_V2_HW_RST_TIMEOUT;1458 while (end) {1459 if (ops->ae_dev_reset_cnt(handle) !=1460 hr_dev->reset_cnt) {1461 hr_dev->is_reset = true;1462 dev_info(hr_dev->dev,1463 "func clear success after sw reset\n");1464 return;1465 }1466 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);1467 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;1468 }1469 1470 dev_warn(hr_dev->dev, "func clear failed because of unfinished sw reset\n");1471}1472 1473static void hns_roce_func_clr_rst_proc(struct hns_roce_dev *hr_dev, int retval,1474 int flag)1475{1476 struct hns_roce_v2_priv *priv = hr_dev->priv;1477 struct hnae3_handle *handle = priv->handle;1478 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;1479 1480 if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) {1481 hr_dev->dis_db = true;1482 hr_dev->is_reset = true;1483 dev_info(hr_dev->dev, "func clear success after reset.\n");1484 return;1485 }1486 1487 if (ops->get_hw_reset_stat(handle)) {1488 func_clr_hw_resetting_state(hr_dev, handle);1489 return;1490 }1491 1492 if (ops->ae_dev_resetting(handle) &&1493 handle->rinfo.instance_state == HNS_ROCE_STATE_INIT) {1494 func_clr_sw_resetting_state(hr_dev, handle);1495 return;1496 }1497 1498 if (retval && !flag)1499 dev_warn(hr_dev->dev,1500 "func clear read failed, ret = %d.\n", retval);1501 1502 dev_warn(hr_dev->dev, "func clear failed.\n");1503}1504 1505static void __hns_roce_function_clear(struct hns_roce_dev *hr_dev, int vf_id)1506{1507 bool fclr_write_fail_flag = false;1508 struct hns_roce_func_clear *resp;1509 struct hns_roce_cmq_desc desc;1510 unsigned long end;1511 int ret = 0;1512 1513 if (check_device_is_in_reset(hr_dev))1514 goto out;1515 1516 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false);1517 resp = (struct hns_roce_func_clear *)desc.data;1518 resp->rst_funcid_en = cpu_to_le32(vf_id);1519 1520 ret = hns_roce_cmq_send(hr_dev, &desc, 1);1521 if (ret) {1522 fclr_write_fail_flag = true;1523 dev_err(hr_dev->dev, "func clear write failed, ret = %d.\n",1524 ret);1525 goto out;1526 }1527 1528 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);1529 end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;1530 while (end) {1531 if (check_device_is_in_reset(hr_dev))1532 goto out;1533 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);1534 end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;1535 1536 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR,1537 true);1538 1539 resp->rst_funcid_en = cpu_to_le32(vf_id);1540 ret = hns_roce_cmq_send(hr_dev, &desc, 1);1541 if (ret)1542 continue;1543 1544 if (hr_reg_read(resp, FUNC_CLEAR_RST_FUN_DONE)) {1545 if (vf_id == 0)1546 hr_dev->is_reset = true;1547 return;1548 }1549 }1550 1551out:1552 hns_roce_func_clr_rst_proc(hr_dev, ret, fclr_write_fail_flag);1553}1554 1555static int hns_roce_free_vf_resource(struct hns_roce_dev *hr_dev, int vf_id)1556{1557 enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES;1558 struct hns_roce_cmq_desc desc[2];1559 struct hns_roce_cmq_req *req_a;1560 1561 req_a = (struct hns_roce_cmq_req *)desc[0].data;1562 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false);1563 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);1564 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false);1565 hr_reg_write(req_a, FUNC_RES_A_VF_ID, vf_id);1566 1567 return hns_roce_cmq_send(hr_dev, desc, 2);1568}1569 1570static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)1571{1572 int ret;1573 int i;1574 1575 if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR)1576 return;1577 1578 for (i = hr_dev->func_num - 1; i >= 0; i--) {1579 __hns_roce_function_clear(hr_dev, i);1580 1581 if (i == 0)1582 continue;1583 1584 ret = hns_roce_free_vf_resource(hr_dev, i);1585 if (ret)1586 ibdev_err(&hr_dev->ib_dev,1587 "failed to free vf resource, vf_id = %d, ret = %d.\n",1588 i, ret);1589 }1590}1591 1592static int hns_roce_clear_extdb_list_info(struct hns_roce_dev *hr_dev)1593{1594 struct hns_roce_cmq_desc desc;1595 int ret;1596 1597 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLEAR_EXTDB_LIST_INFO,1598 false);1599 ret = hns_roce_cmq_send(hr_dev, &desc, 1);1600 if (ret)1601 ibdev_err(&hr_dev->ib_dev,1602 "failed to clear extended doorbell info, ret = %d.\n",1603 ret);1604 1605 return ret;1606}1607 1608static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)1609{1610 struct hns_roce_query_fw_info *resp;1611 struct hns_roce_cmq_desc desc;1612 int ret;1613 1614 hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true);1615 ret = hns_roce_cmq_send(hr_dev, &desc, 1);1616 if (ret)1617 return ret;1618 1619 resp = (struct hns_roce_query_fw_info *)desc.data;1620 hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver));1621 1622 return 0;1623}1624 1625static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev)1626{1627 struct hns_roce_cmq_desc desc;1628 int ret;1629 1630 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {1631 hr_dev->func_num = 1;1632 return 0;1633 }1634 1635 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_FUNC_INFO,1636 true);1637 ret = hns_roce_cmq_send(hr_dev, &desc, 1);1638 if (ret) {1639 hr_dev->func_num = 1;1640 return ret;1641 }1642 1643 hr_dev->func_num = le32_to_cpu(desc.func_info.own_func_num);1644 hr_dev->cong_algo_tmpl_id = le32_to_cpu(desc.func_info.own_mac_id);1645 1646 return 0;1647}1648 1649static int hns_roce_hw_v2_query_counter(struct hns_roce_dev *hr_dev,1650 u64 *stats, u32 port, int *num_counters)1651{1652#define CNT_PER_DESC 31653 struct hns_roce_cmq_desc *desc;1654 int bd_idx, cnt_idx;1655 __le64 *cnt_data;1656 int desc_num;1657 int ret;1658 int i;1659 1660 if (port > hr_dev->caps.num_ports)1661 return -EINVAL;1662 1663 desc_num = DIV_ROUND_UP(HNS_ROCE_HW_CNT_TOTAL, CNT_PER_DESC);1664 desc = kcalloc(desc_num, sizeof(*desc), GFP_KERNEL);1665 if (!desc)1666 return -ENOMEM;1667 1668 for (i = 0; i < desc_num; i++) {1669 hns_roce_cmq_setup_basic_desc(&desc[i],1670 HNS_ROCE_OPC_QUERY_COUNTER, true);1671 if (i != desc_num - 1)1672 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);1673 }1674 1675 ret = hns_roce_cmq_send(hr_dev, desc, desc_num);1676 if (ret) {1677 ibdev_err(&hr_dev->ib_dev,1678 "failed to get counter, ret = %d.\n", ret);1679 goto err_out;1680 }1681 1682 for (i = 0; i < HNS_ROCE_HW_CNT_TOTAL && i < *num_counters; i++) {1683 bd_idx = i / CNT_PER_DESC;1684 if (bd_idx != HNS_ROCE_HW_CNT_TOTAL / CNT_PER_DESC &&1685 !(desc[bd_idx].flag & cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT)))1686 break;1687 1688 cnt_data = (__le64 *)&desc[bd_idx].data[0];1689 cnt_idx = i % CNT_PER_DESC;1690 stats[i] = le64_to_cpu(cnt_data[cnt_idx]);1691 }1692 *num_counters = i;1693 1694err_out:1695 kfree(desc);1696 return ret;1697}1698 1699static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)1700{1701 struct hns_roce_cmq_desc desc;1702 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;1703 u32 clock_cycles_of_1us;1704 1705 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,1706 false);1707 1708 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)1709 clock_cycles_of_1us = HNS_ROCE_1NS_CFG;1710 else1711 clock_cycles_of_1us = HNS_ROCE_1US_CFG;1712 1713 hr_reg_write(req, CFG_GLOBAL_PARAM_1US_CYCLES, clock_cycles_of_1us);1714 hr_reg_write(req, CFG_GLOBAL_PARAM_UDP_PORT, ROCE_V2_UDP_DPORT);1715 1716 return hns_roce_cmq_send(hr_dev, &desc, 1);1717}1718 1719static int load_func_res_caps(struct hns_roce_dev *hr_dev, bool is_vf)1720{1721 struct hns_roce_cmq_desc desc[2];1722 struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;1723 struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;1724 struct hns_roce_caps *caps = &hr_dev->caps;1725 enum hns_roce_opcode_type opcode;1726 u32 func_num;1727 int ret;1728 1729 if (is_vf) {1730 opcode = HNS_ROCE_OPC_QUERY_VF_RES;1731 func_num = 1;1732 } else {1733 opcode = HNS_ROCE_OPC_QUERY_PF_RES;1734 func_num = hr_dev->func_num;1735 }1736 1737 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, true);1738 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);1739 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, true);1740 1741 ret = hns_roce_cmq_send(hr_dev, desc, 2);1742 if (ret)1743 return ret;1744 1745 caps->qpc_bt_num = hr_reg_read(r_a, FUNC_RES_A_QPC_BT_NUM) / func_num;1746 caps->srqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_SRQC_BT_NUM) / func_num;1747 caps->cqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_CQC_BT_NUM) / func_num;1748 caps->mpt_bt_num = hr_reg_read(r_a, FUNC_RES_A_MPT_BT_NUM) / func_num;1749 caps->eqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_EQC_BT_NUM) / func_num;1750 caps->smac_bt_num = hr_reg_read(r_b, FUNC_RES_B_SMAC_NUM) / func_num;1751 caps->sgid_bt_num = hr_reg_read(r_b, FUNC_RES_B_SGID_NUM) / func_num;1752 caps->sccc_bt_num = hr_reg_read(r_b, FUNC_RES_B_SCCC_BT_NUM) / func_num;1753 1754 if (is_vf) {1755 caps->sl_num = hr_reg_read(r_b, FUNC_RES_V_QID_NUM) / func_num;1756 caps->gmv_bt_num = hr_reg_read(r_b, FUNC_RES_V_GMV_BT_NUM) /1757 func_num;1758 } else {1759 caps->sl_num = hr_reg_read(r_b, FUNC_RES_B_QID_NUM) / func_num;1760 caps->gmv_bt_num = hr_reg_read(r_b, FUNC_RES_B_GMV_BT_NUM) /1761 func_num;1762 }1763 1764 return 0;1765}1766 1767static int load_pf_timer_res_caps(struct hns_roce_dev *hr_dev)1768{1769 struct hns_roce_cmq_desc desc;1770 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;1771 struct hns_roce_caps *caps = &hr_dev->caps;1772 int ret;1773 1774 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_PF_TIMER_RES,1775 true);1776 1777 ret = hns_roce_cmq_send(hr_dev, &desc, 1);1778 if (ret)1779 return ret;1780 1781 caps->qpc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_QPC_ITEM_NUM);1782 caps->cqc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_CQC_ITEM_NUM);1783 1784 return 0;1785}1786 1787static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)1788{1789 struct device *dev = hr_dev->dev;1790 int ret;1791 1792 ret = load_func_res_caps(hr_dev, false);1793 if (ret) {1794 dev_err(dev, "failed to load pf res caps, ret = %d.\n", ret);1795 return ret;1796 }1797 1798 ret = load_pf_timer_res_caps(hr_dev);1799 if (ret)1800 dev_err(dev, "failed to load pf timer resource, ret = %d.\n",1801 ret);1802 1803 return ret;1804}1805 1806static int hns_roce_query_vf_resource(struct hns_roce_dev *hr_dev)1807{1808 struct device *dev = hr_dev->dev;1809 int ret;1810 1811 ret = load_func_res_caps(hr_dev, true);1812 if (ret)1813 dev_err(dev, "failed to load vf res caps, ret = %d.\n", ret);1814 1815 return ret;1816}1817 1818static int __hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev,1819 u32 vf_id)1820{1821 struct hns_roce_vf_switch *swt;1822 struct hns_roce_cmq_desc desc;1823 int ret;1824 1825 swt = (struct hns_roce_vf_switch *)desc.data;1826 hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true);1827 swt->rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL);1828 hr_reg_write(swt, VF_SWITCH_VF_ID, vf_id);1829 ret = hns_roce_cmq_send(hr_dev, &desc, 1);1830 if (ret)1831 return ret;1832 1833 desc.flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_IN);1834 desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);1835 hr_reg_enable(swt, VF_SWITCH_ALW_LPBK);1836 hr_reg_clear(swt, VF_SWITCH_ALW_LCL_LPBK);1837 hr_reg_enable(swt, VF_SWITCH_ALW_DST_OVRD);1838 1839 return hns_roce_cmq_send(hr_dev, &desc, 1);1840}1841 1842static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev)1843{1844 u32 vf_id;1845 int ret;1846 1847 for (vf_id = 0; vf_id < hr_dev->func_num; vf_id++) {1848 ret = __hns_roce_set_vf_switch_param(hr_dev, vf_id);1849 if (ret)1850 return ret;1851 }1852 return 0;1853}1854 1855static int config_vf_hem_resource(struct hns_roce_dev *hr_dev, int vf_id)1856{1857 struct hns_roce_cmq_desc desc[2];1858 struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;1859 struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;1860 enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES;1861 struct hns_roce_caps *caps = &hr_dev->caps;1862 1863 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false);1864 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);1865 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false);1866 1867 hr_reg_write(r_a, FUNC_RES_A_VF_ID, vf_id);1868 1869 hr_reg_write(r_a, FUNC_RES_A_QPC_BT_NUM, caps->qpc_bt_num);1870 hr_reg_write(r_a, FUNC_RES_A_QPC_BT_IDX, vf_id * caps->qpc_bt_num);1871 hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_NUM, caps->srqc_bt_num);1872 hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_IDX, vf_id * caps->srqc_bt_num);1873 hr_reg_write(r_a, FUNC_RES_A_CQC_BT_NUM, caps->cqc_bt_num);1874 hr_reg_write(r_a, FUNC_RES_A_CQC_BT_IDX, vf_id * caps->cqc_bt_num);1875 hr_reg_write(r_a, FUNC_RES_A_MPT_BT_NUM, caps->mpt_bt_num);1876 hr_reg_write(r_a, FUNC_RES_A_MPT_BT_IDX, vf_id * caps->mpt_bt_num);1877 hr_reg_write(r_a, FUNC_RES_A_EQC_BT_NUM, caps->eqc_bt_num);1878 hr_reg_write(r_a, FUNC_RES_A_EQC_BT_IDX, vf_id * caps->eqc_bt_num);1879 hr_reg_write(r_b, FUNC_RES_V_QID_NUM, caps->sl_num);1880 hr_reg_write(r_b, FUNC_RES_B_QID_IDX, vf_id * caps->sl_num);1881 hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_NUM, caps->sccc_bt_num);1882 hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_IDX, vf_id * caps->sccc_bt_num);1883 1884 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {1885 hr_reg_write(r_b, FUNC_RES_V_GMV_BT_NUM, caps->gmv_bt_num);1886 hr_reg_write(r_b, FUNC_RES_B_GMV_BT_IDX,1887 vf_id * caps->gmv_bt_num);1888 } else {1889 hr_reg_write(r_b, FUNC_RES_B_SGID_NUM, caps->sgid_bt_num);1890 hr_reg_write(r_b, FUNC_RES_B_SGID_IDX,1891 vf_id * caps->sgid_bt_num);1892 hr_reg_write(r_b, FUNC_RES_B_SMAC_NUM, caps->smac_bt_num);1893 hr_reg_write(r_b, FUNC_RES_B_SMAC_IDX,1894 vf_id * caps->smac_bt_num);1895 }1896 1897 return hns_roce_cmq_send(hr_dev, desc, 2);1898}1899 1900static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)1901{1902 u32 func_num = max_t(u32, 1, hr_dev->func_num);1903 u32 vf_id;1904 int ret;1905 1906 for (vf_id = 0; vf_id < func_num; vf_id++) {1907 ret = config_vf_hem_resource(hr_dev, vf_id);1908 if (ret) {1909 dev_err(hr_dev->dev,1910 "failed to config vf-%u hem res, ret = %d.\n",1911 vf_id, ret);1912 return ret;1913 }1914 }1915 1916 return 0;1917}1918 1919static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)1920{1921 struct hns_roce_cmq_desc desc;1922 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;1923 struct hns_roce_caps *caps = &hr_dev->caps;1924 1925 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);1926 1927 hr_reg_write(req, CFG_BT_ATTR_QPC_BA_PGSZ,1928 caps->qpc_ba_pg_sz + PG_SHIFT_OFFSET);1929 hr_reg_write(req, CFG_BT_ATTR_QPC_BUF_PGSZ,1930 caps->qpc_buf_pg_sz + PG_SHIFT_OFFSET);1931 hr_reg_write(req, CFG_BT_ATTR_QPC_HOPNUM,1932 to_hr_hem_hopnum(caps->qpc_hop_num, caps->num_qps));1933 1934 hr_reg_write(req, CFG_BT_ATTR_SRQC_BA_PGSZ,1935 caps->srqc_ba_pg_sz + PG_SHIFT_OFFSET);1936 hr_reg_write(req, CFG_BT_ATTR_SRQC_BUF_PGSZ,1937 caps->srqc_buf_pg_sz + PG_SHIFT_OFFSET);1938 hr_reg_write(req, CFG_BT_ATTR_SRQC_HOPNUM,1939 to_hr_hem_hopnum(caps->srqc_hop_num, caps->num_srqs));1940 1941 hr_reg_write(req, CFG_BT_ATTR_CQC_BA_PGSZ,1942 caps->cqc_ba_pg_sz + PG_SHIFT_OFFSET);1943 hr_reg_write(req, CFG_BT_ATTR_CQC_BUF_PGSZ,1944 caps->cqc_buf_pg_sz + PG_SHIFT_OFFSET);1945 hr_reg_write(req, CFG_BT_ATTR_CQC_HOPNUM,1946 to_hr_hem_hopnum(caps->cqc_hop_num, caps->num_cqs));1947 1948 hr_reg_write(req, CFG_BT_ATTR_MPT_BA_PGSZ,1949 caps->mpt_ba_pg_sz + PG_SHIFT_OFFSET);1950 hr_reg_write(req, CFG_BT_ATTR_MPT_BUF_PGSZ,1951 caps->mpt_buf_pg_sz + PG_SHIFT_OFFSET);1952 hr_reg_write(req, CFG_BT_ATTR_MPT_HOPNUM,1953 to_hr_hem_hopnum(caps->mpt_hop_num, caps->num_mtpts));1954 1955 hr_reg_write(req, CFG_BT_ATTR_SCCC_BA_PGSZ,1956 caps->sccc_ba_pg_sz + PG_SHIFT_OFFSET);1957 hr_reg_write(req, CFG_BT_ATTR_SCCC_BUF_PGSZ,1958 caps->sccc_buf_pg_sz + PG_SHIFT_OFFSET);1959 hr_reg_write(req, CFG_BT_ATTR_SCCC_HOPNUM,1960 to_hr_hem_hopnum(caps->sccc_hop_num, caps->num_qps));1961 1962 return hns_roce_cmq_send(hr_dev, &desc, 1);1963}1964 1965static void calc_pg_sz(u32 obj_num, u32 obj_size, u32 hop_num, u32 ctx_bt_num,1966 u32 *buf_page_size, u32 *bt_page_size, u32 hem_type)1967{1968 u64 obj_per_chunk;1969 u64 bt_chunk_size = PAGE_SIZE;1970 u64 buf_chunk_size = PAGE_SIZE;1971 u64 obj_per_chunk_default = buf_chunk_size / obj_size;1972 1973 *buf_page_size = 0;1974 *bt_page_size = 0;1975 1976 switch (hop_num) {1977 case 3:1978 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *1979 (bt_chunk_size / BA_BYTE_LEN) *1980 (bt_chunk_size / BA_BYTE_LEN) *1981 obj_per_chunk_default;1982 break;1983 case 2:1984 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *1985 (bt_chunk_size / BA_BYTE_LEN) *1986 obj_per_chunk_default;1987 break;1988 case 1:1989 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *1990 obj_per_chunk_default;1991 break;1992 case HNS_ROCE_HOP_NUM_0:1993 obj_per_chunk = ctx_bt_num * obj_per_chunk_default;1994 break;1995 default:1996 pr_err("table %u not support hop_num = %u!\n", hem_type,1997 hop_num);1998 return;1999 }2000 2001 if (hem_type >= HEM_TYPE_MTT)2002 *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));2003 else2004 *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));2005}2006 2007static void set_hem_page_size(struct hns_roce_dev *hr_dev)2008{2009 struct hns_roce_caps *caps = &hr_dev->caps;2010 2011 /* EQ */2012 caps->eqe_ba_pg_sz = 0;2013 caps->eqe_buf_pg_sz = 0;2014 2015 /* Link Table */2016 caps->llm_buf_pg_sz = 0;2017 2018 /* MR */2019 caps->mpt_ba_pg_sz = 0;2020 caps->mpt_buf_pg_sz = 0;2021 caps->pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K;2022 caps->pbl_buf_pg_sz = 0;2023 calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num,2024 caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz,2025 HEM_TYPE_MTPT);2026 2027 /* QP */2028 caps->qpc_ba_pg_sz = 0;2029 caps->qpc_buf_pg_sz = 0;2030 caps->qpc_timer_ba_pg_sz = 0;2031 caps->qpc_timer_buf_pg_sz = 0;2032 caps->sccc_ba_pg_sz = 0;2033 caps->sccc_buf_pg_sz = 0;2034 caps->mtt_ba_pg_sz = 0;2035 caps->mtt_buf_pg_sz = 0;2036 calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num,2037 caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz,2038 HEM_TYPE_QPC);2039 2040 if (caps->flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)2041 calc_pg_sz(caps->num_qps, caps->sccc_sz, caps->sccc_hop_num,2042 caps->sccc_bt_num, &caps->sccc_buf_pg_sz,2043 &caps->sccc_ba_pg_sz, HEM_TYPE_SCCC);2044 2045 /* CQ */2046 caps->cqc_ba_pg_sz = 0;2047 caps->cqc_buf_pg_sz = 0;2048 caps->cqc_timer_ba_pg_sz = 0;2049 caps->cqc_timer_buf_pg_sz = 0;2050 caps->cqe_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K;2051 caps->cqe_buf_pg_sz = 0;2052 calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num,2053 caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz,2054 HEM_TYPE_CQC);2055 calc_pg_sz(caps->max_cqes, caps->cqe_sz, caps->cqe_hop_num,2056 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE);2057 2058 /* SRQ */2059 if (caps->flags & HNS_ROCE_CAP_FLAG_SRQ) {2060 caps->srqc_ba_pg_sz = 0;2061 caps->srqc_buf_pg_sz = 0;2062 caps->srqwqe_ba_pg_sz = 0;2063 caps->srqwqe_buf_pg_sz = 0;2064 caps->idx_ba_pg_sz = 0;2065 caps->idx_buf_pg_sz = 0;2066 calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz,2067 caps->srqc_hop_num, caps->srqc_bt_num,2068 &caps->srqc_buf_pg_sz, &caps->srqc_ba_pg_sz,2069 HEM_TYPE_SRQC);2070 calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz,2071 caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz,2072 &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE);2073 calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz,2074 caps->idx_hop_num, 1, &caps->idx_buf_pg_sz,2075 &caps->idx_ba_pg_sz, HEM_TYPE_IDX);2076 }2077 2078 /* GMV */2079 caps->gmv_ba_pg_sz = 0;2080 caps->gmv_buf_pg_sz = 0;2081}2082 2083/* Apply all loaded caps before setting to hardware */2084static void apply_func_caps(struct hns_roce_dev *hr_dev)2085{2086#define MAX_GID_TBL_LEN 2562087 struct hns_roce_caps *caps = &hr_dev->caps;2088 struct hns_roce_v2_priv *priv = hr_dev->priv;2089 2090 /* The following configurations don't need to be got from firmware. */2091 caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;2092 caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;2093 caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;2094 2095 caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM;2096 caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;2097 caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;2098 2099 caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;2100 caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;2101 2102 if (!caps->num_comp_vectors)2103 caps->num_comp_vectors =2104 min_t(u32, caps->eqc_bt_num - HNS_ROCE_V2_AEQE_VEC_NUM,2105 (u32)priv->handle->rinfo.num_vectors -2106 (HNS_ROCE_V2_AEQE_VEC_NUM + HNS_ROCE_V2_ABNORMAL_VEC_NUM));2107 2108 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {2109 caps->eqe_hop_num = HNS_ROCE_V3_EQE_HOP_NUM;2110 caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE;2111 caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE;2112 2113 /* The following configurations will be overwritten */2114 caps->qpc_sz = HNS_ROCE_V3_QPC_SZ;2115 caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE;2116 caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ;2117 2118 /* The following configurations are not got from firmware */2119 caps->gmv_entry_sz = HNS_ROCE_V3_GMV_ENTRY_SZ;2120 2121 caps->gmv_hop_num = HNS_ROCE_HOP_NUM_0;2122 2123 /* It's meaningless to support excessively large gid_table_len,2124 * as the type of sgid_index in kernel struct ib_global_route2125 * and userspace struct ibv_global_route are u8/uint8_t (0-255).2126 */2127 caps->gid_table_len[0] = min_t(u32, MAX_GID_TBL_LEN,2128 caps->gmv_bt_num *2129 (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz));2130 2131 caps->gmv_entry_num = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE /2132 caps->gmv_entry_sz);2133 } else {2134 u32 func_num = max_t(u32, 1, hr_dev->func_num);2135 2136 caps->eqe_hop_num = HNS_ROCE_V2_EQE_HOP_NUM;2137 caps->ceqe_size = HNS_ROCE_CEQE_SIZE;2138 caps->aeqe_size = HNS_ROCE_AEQE_SIZE;2139 caps->gid_table_len[0] /= func_num;2140 }2141 2142 if (hr_dev->is_vf) {2143 caps->default_aeq_arm_st = 0x3;2144 caps->default_ceq_arm_st = 0x3;2145 caps->default_ceq_max_cnt = 0x1;2146 caps->default_ceq_period = 0x10;2147 caps->default_aeq_max_cnt = 0x1;2148 caps->default_aeq_period = 0x10;2149 }2150 2151 set_hem_page_size(hr_dev);2152}2153 2154static int hns_roce_query_caps(struct hns_roce_dev *hr_dev)2155{2156 struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM];2157 struct hns_roce_caps *caps = &hr_dev->caps;2158 struct hns_roce_query_pf_caps_a *resp_a;2159 struct hns_roce_query_pf_caps_b *resp_b;2160 struct hns_roce_query_pf_caps_c *resp_c;2161 struct hns_roce_query_pf_caps_d *resp_d;2162 struct hns_roce_query_pf_caps_e *resp_e;2163 enum hns_roce_opcode_type cmd;2164 int ctx_hop_num;2165 int pbl_hop_num;2166 int ret;2167 int i;2168 2169 cmd = hr_dev->is_vf ? HNS_ROCE_OPC_QUERY_VF_CAPS_NUM :2170 HNS_ROCE_OPC_QUERY_PF_CAPS_NUM;2171 2172 for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) {2173 hns_roce_cmq_setup_basic_desc(&desc[i], cmd, true);2174 if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1))2175 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);2176 else2177 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);2178 }2179 2180 ret = hns_roce_cmq_send(hr_dev, desc, HNS_ROCE_QUERY_PF_CAPS_CMD_NUM);2181 if (ret)2182 return ret;2183 2184 resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data;2185 resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data;2186 resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data;2187 resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data;2188 resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data;2189 2190 caps->local_ca_ack_delay = resp_a->local_ca_ack_delay;2191 caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg);2192 caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline);2193 caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg);2194 caps->max_rq_sg = roundup_pow_of_two(caps->max_rq_sg);2195 caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges);2196 caps->max_srq_sges = roundup_pow_of_two(caps->max_srq_sges);2197 caps->num_aeq_vectors = resp_a->num_aeq_vectors;2198 caps->num_other_vectors = resp_a->num_other_vectors;2199 caps->max_sq_desc_sz = resp_a->max_sq_desc_sz;2200 caps->max_rq_desc_sz = resp_a->max_rq_desc_sz;2201 2202 caps->mtpt_entry_sz = resp_b->mtpt_entry_sz;2203 caps->irrl_entry_sz = resp_b->irrl_entry_sz;2204 caps->trrl_entry_sz = resp_b->trrl_entry_sz;2205 caps->cqc_entry_sz = resp_b->cqc_entry_sz;2206 caps->srqc_entry_sz = resp_b->srqc_entry_sz;2207 caps->idx_entry_sz = resp_b->idx_entry_sz;2208 caps->sccc_sz = resp_b->sccc_sz;2209 caps->max_mtu = resp_b->max_mtu;2210 caps->min_cqes = resp_b->min_cqes;2211 caps->min_wqes = resp_b->min_wqes;2212 caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap);2213 caps->pkey_table_len[0] = resp_b->pkey_table_len;2214 caps->phy_num_uars = resp_b->phy_num_uars;2215 ctx_hop_num = resp_b->ctx_hop_num;2216 pbl_hop_num = resp_b->pbl_hop_num;2217 2218 caps->num_pds = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_PDS);2219 2220 caps->flags = hr_reg_read(resp_c, PF_CAPS_C_CAP_FLAGS);2221 caps->flags |= le16_to_cpu(resp_d->cap_flags_ex) <<2222 HNS_ROCE_CAP_FLAGS_EX_SHIFT;2223 2224 caps->num_cqs = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_CQS);2225 caps->gid_table_len[0] = hr_reg_read(resp_c, PF_CAPS_C_MAX_GID);2226 caps->max_cqes = 1 << hr_reg_read(resp_c, PF_CAPS_C_CQ_DEPTH);2227 caps->num_xrcds = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_XRCDS);2228 caps->num_mtpts = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_MRWS);2229 caps->num_qps = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_QPS);2230 caps->max_qp_init_rdma = hr_reg_read(resp_c, PF_CAPS_C_MAX_ORD);2231 caps->max_qp_dest_rdma = caps->max_qp_init_rdma;2232 caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth);2233 2234 caps->num_srqs = 1 << hr_reg_read(resp_d, PF_CAPS_D_NUM_SRQS);2235 caps->cong_cap = hr_reg_read(resp_d, PF_CAPS_D_CONG_CAP);2236 caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth);2237 caps->ceqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_CEQ_DEPTH);2238 caps->num_comp_vectors = hr_reg_read(resp_d, PF_CAPS_D_NUM_CEQS);2239 caps->aeqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_AEQ_DEPTH);2240 caps->default_cong_type = hr_reg_read(resp_d, PF_CAPS_D_DEFAULT_ALG);2241 caps->reserved_pds = hr_reg_read(resp_d, PF_CAPS_D_RSV_PDS);2242 caps->num_uars = 1 << hr_reg_read(resp_d, PF_CAPS_D_NUM_UARS);2243 caps->reserved_qps = hr_reg_read(resp_d, PF_CAPS_D_RSV_QPS);2244 caps->reserved_uars = hr_reg_read(resp_d, PF_CAPS_D_RSV_UARS);2245 2246 caps->reserved_mrws = hr_reg_read(resp_e, PF_CAPS_E_RSV_MRWS);2247 caps->chunk_sz = 1 << hr_reg_read(resp_e, PF_CAPS_E_CHUNK_SIZE_SHIFT);2248 caps->reserved_cqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_CQS);2249 caps->reserved_xrcds = hr_reg_read(resp_e, PF_CAPS_E_RSV_XRCDS);2250 caps->reserved_srqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_SRQS);2251 caps->reserved_lkey = hr_reg_read(resp_e, PF_CAPS_E_RSV_LKEYS);2252 2253 caps->qpc_hop_num = ctx_hop_num;2254 caps->sccc_hop_num = ctx_hop_num;2255 caps->srqc_hop_num = ctx_hop_num;2256 caps->cqc_hop_num = ctx_hop_num;2257 caps->mpt_hop_num = ctx_hop_num;2258 caps->mtt_hop_num = pbl_hop_num;2259 caps->cqe_hop_num = pbl_hop_num;2260 caps->srqwqe_hop_num = pbl_hop_num;2261 caps->idx_hop_num = pbl_hop_num;2262 caps->wqe_sq_hop_num = hr_reg_read(resp_d, PF_CAPS_D_SQWQE_HOP_NUM);2263 caps->wqe_sge_hop_num = hr_reg_read(resp_d, PF_CAPS_D_EX_SGE_HOP_NUM);2264 caps->wqe_rq_hop_num = hr_reg_read(resp_d, PF_CAPS_D_RQWQE_HOP_NUM);2265 2266 if (!(caps->page_size_cap & PAGE_SIZE))2267 caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;2268 2269 if (!hr_dev->is_vf) {2270 caps->cqe_sz = resp_a->cqe_sz;2271 caps->qpc_sz = le16_to_cpu(resp_b->qpc_sz);2272 caps->default_aeq_arm_st =2273 hr_reg_read(resp_d, PF_CAPS_D_AEQ_ARM_ST);2274 caps->default_ceq_arm_st =2275 hr_reg_read(resp_d, PF_CAPS_D_CEQ_ARM_ST);2276 caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt);2277 caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period);2278 caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt);2279 caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period);2280 }2281 2282 return 0;2283}2284 2285static int config_hem_entry_size(struct hns_roce_dev *hr_dev, u32 type, u32 val)2286{2287 struct hns_roce_cmq_desc desc;2288 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;2289 2290 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE,2291 false);2292 2293 hr_reg_write(req, CFG_HEM_ENTRY_SIZE_TYPE, type);2294 hr_reg_write(req, CFG_HEM_ENTRY_SIZE_VALUE, val);2295 2296 return hns_roce_cmq_send(hr_dev, &desc, 1);2297}2298 2299static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev)2300{2301 struct hns_roce_caps *caps = &hr_dev->caps;2302 int ret;2303 2304 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)2305 return 0;2306 2307 ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_QPC_SIZE,2308 caps->qpc_sz);2309 if (ret) {2310 dev_err(hr_dev->dev, "failed to cfg qpc sz, ret = %d.\n", ret);2311 return ret;2312 }2313 2314 ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_SCCC_SIZE,2315 caps->sccc_sz);2316 if (ret)2317 dev_err(hr_dev->dev, "failed to cfg sccc sz, ret = %d.\n", ret);2318 2319 return ret;2320}2321 2322static int hns_roce_v2_vf_profile(struct hns_roce_dev *hr_dev)2323{2324 struct device *dev = hr_dev->dev;2325 int ret;2326 2327 hr_dev->func_num = 1;2328 2329 ret = hns_roce_query_caps(hr_dev);2330 if (ret) {2331 dev_err(dev, "failed to query VF caps, ret = %d.\n", ret);2332 return ret;2333 }2334 2335 ret = hns_roce_query_vf_resource(hr_dev);2336 if (ret) {2337 dev_err(dev, "failed to query VF resource, ret = %d.\n", ret);2338 return ret;2339 }2340 2341 apply_func_caps(hr_dev);2342 2343 ret = hns_roce_v2_set_bt(hr_dev);2344 if (ret)2345 dev_err(dev, "failed to config VF BA table, ret = %d.\n", ret);2346 2347 return ret;2348}2349 2350static int hns_roce_v2_pf_profile(struct hns_roce_dev *hr_dev)2351{2352 struct device *dev = hr_dev->dev;2353 int ret;2354 2355 ret = hns_roce_query_func_info(hr_dev);2356 if (ret) {2357 dev_err(dev, "failed to query func info, ret = %d.\n", ret);2358 return ret;2359 }2360 2361 ret = hns_roce_config_global_param(hr_dev);2362 if (ret) {2363 dev_err(dev, "failed to config global param, ret = %d.\n", ret);2364 return ret;2365 }2366 2367 ret = hns_roce_set_vf_switch_param(hr_dev);2368 if (ret) {2369 dev_err(dev, "failed to set switch param, ret = %d.\n", ret);2370 return ret;2371 }2372 2373 ret = hns_roce_query_caps(hr_dev);2374 if (ret) {2375 dev_err(dev, "failed to query PF caps, ret = %d.\n", ret);2376 return ret;2377 }2378 2379 ret = hns_roce_query_pf_resource(hr_dev);2380 if (ret) {2381 dev_err(dev, "failed to query pf resource, ret = %d.\n", ret);2382 return ret;2383 }2384 2385 apply_func_caps(hr_dev);2386 2387 ret = hns_roce_alloc_vf_resource(hr_dev);2388 if (ret) {2389 dev_err(dev, "failed to alloc vf resource, ret = %d.\n", ret);2390 return ret;2391 }2392 2393 ret = hns_roce_v2_set_bt(hr_dev);2394 if (ret) {2395 dev_err(dev, "failed to config BA table, ret = %d.\n", ret);2396 return ret;2397 }2398 2399 /* Configure the size of QPC, SCCC, etc. */2400 return hns_roce_config_entry_size(hr_dev);2401}2402 2403static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)2404{2405 struct device *dev = hr_dev->dev;2406 int ret;2407 2408 ret = hns_roce_cmq_query_hw_info(hr_dev);2409 if (ret) {2410 dev_err(dev, "failed to query hardware info, ret = %d.\n", ret);2411 return ret;2412 }2413 2414 ret = hns_roce_query_fw_ver(hr_dev);2415 if (ret) {2416 dev_err(dev, "failed to query firmware info, ret = %d.\n", ret);2417 return ret;2418 }2419 2420 hr_dev->vendor_part_id = hr_dev->pci_dev->device;2421 hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);2422 2423 if (hr_dev->is_vf)2424 return hns_roce_v2_vf_profile(hr_dev);2425 else2426 return hns_roce_v2_pf_profile(hr_dev);2427}2428 2429static void config_llm_table(struct hns_roce_buf *data_buf, void *cfg_buf)2430{2431 u32 i, next_ptr, page_num;2432 __le64 *entry = cfg_buf;2433 dma_addr_t addr;2434 u64 val;2435 2436 page_num = data_buf->npages;2437 for (i = 0; i < page_num; i++) {2438 addr = hns_roce_buf_page(data_buf, i);2439 if (i == (page_num - 1))2440 next_ptr = 0;2441 else2442 next_ptr = i + 1;2443 2444 val = HNS_ROCE_EXT_LLM_ENTRY(addr, (u64)next_ptr);2445 entry[i] = cpu_to_le64(val);2446 }2447}2448 2449static int set_llm_cfg_to_hw(struct hns_roce_dev *hr_dev,2450 struct hns_roce_link_table *table)2451{2452 struct hns_roce_cmq_desc desc[2];2453 struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;2454 struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;2455 struct hns_roce_buf *buf = table->buf;2456 enum hns_roce_opcode_type opcode;2457 dma_addr_t addr;2458 2459 opcode = HNS_ROCE_OPC_CFG_EXT_LLM;2460 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false);2461 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);2462 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false);2463 2464 hr_reg_write(r_a, CFG_LLM_A_BA_L, lower_32_bits(table->table.map));2465 hr_reg_write(r_a, CFG_LLM_A_BA_H, upper_32_bits(table->table.map));2466 hr_reg_write(r_a, CFG_LLM_A_DEPTH, buf->npages);2467 hr_reg_write(r_a, CFG_LLM_A_PGSZ, to_hr_hw_page_shift(buf->page_shift));2468 hr_reg_enable(r_a, CFG_LLM_A_INIT_EN);2469 2470 addr = to_hr_hw_page_addr(hns_roce_buf_page(buf, 0));2471 hr_reg_write(r_a, CFG_LLM_A_HEAD_BA_L, lower_32_bits(addr));2472 hr_reg_write(r_a, CFG_LLM_A_HEAD_BA_H, upper_32_bits(addr));2473 hr_reg_write(r_a, CFG_LLM_A_HEAD_NXTPTR, 1);2474 hr_reg_write(r_a, CFG_LLM_A_HEAD_PTR, 0);2475 2476 addr = to_hr_hw_page_addr(hns_roce_buf_page(buf, buf->npages - 1));2477 hr_reg_write(r_b, CFG_LLM_B_TAIL_BA_L, lower_32_bits(addr));2478 hr_reg_write(r_b, CFG_LLM_B_TAIL_BA_H, upper_32_bits(addr));2479 hr_reg_write(r_b, CFG_LLM_B_TAIL_PTR, buf->npages - 1);2480 2481 return hns_roce_cmq_send(hr_dev, desc, 2);2482}2483 2484static struct hns_roce_link_table *2485alloc_link_table_buf(struct hns_roce_dev *hr_dev)2486{2487 u16 total_sl = hr_dev->caps.sl_num * hr_dev->func_num;2488 struct hns_roce_v2_priv *priv = hr_dev->priv;2489 struct hns_roce_link_table *link_tbl;2490 u32 pg_shift, size, min_size;2491 2492 link_tbl = &priv->ext_llm;2493 pg_shift = hr_dev->caps.llm_buf_pg_sz + PAGE_SHIFT;2494 size = hr_dev->caps.num_qps * hr_dev->func_num *2495 HNS_ROCE_V2_EXT_LLM_ENTRY_SZ;2496 min_size = HNS_ROCE_EXT_LLM_MIN_PAGES(total_sl) << pg_shift;2497 2498 /* Alloc data table */2499 size = max(size, min_size);2500 link_tbl->buf = hns_roce_buf_alloc(hr_dev, size, pg_shift, 0);2501 if (IS_ERR(link_tbl->buf))2502 return ERR_PTR(-ENOMEM);2503 2504 /* Alloc config table */2505 size = link_tbl->buf->npages * sizeof(u64);2506 link_tbl->table.buf = dma_alloc_coherent(hr_dev->dev, size,2507 &link_tbl->table.map,2508 GFP_KERNEL);2509 if (!link_tbl->table.buf) {2510 hns_roce_buf_free(hr_dev, link_tbl->buf);2511 return ERR_PTR(-ENOMEM);2512 }2513 2514 return link_tbl;2515}2516 2517static void free_link_table_buf(struct hns_roce_dev *hr_dev,2518 struct hns_roce_link_table *tbl)2519{2520 if (tbl->buf) {2521 u32 size = tbl->buf->npages * sizeof(u64);2522 2523 dma_free_coherent(hr_dev->dev, size, tbl->table.buf,2524 tbl->table.map);2525 }2526 2527 hns_roce_buf_free(hr_dev, tbl->buf);2528}2529 2530static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev)2531{2532 struct hns_roce_link_table *link_tbl;2533 int ret;2534 2535 link_tbl = alloc_link_table_buf(hr_dev);2536 if (IS_ERR(link_tbl))2537 return -ENOMEM;2538 2539 if (WARN_ON(link_tbl->buf->npages > HNS_ROCE_V2_EXT_LLM_MAX_DEPTH)) {2540 ret = -EINVAL;2541 goto err_alloc;2542 }2543 2544 config_llm_table(link_tbl->buf, link_tbl->table.buf);2545 ret = set_llm_cfg_to_hw(hr_dev, link_tbl);2546 if (ret)2547 goto err_alloc;2548 2549 return 0;2550 2551err_alloc:2552 free_link_table_buf(hr_dev, link_tbl);2553 return ret;2554}2555 2556static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev)2557{2558 struct hns_roce_v2_priv *priv = hr_dev->priv;2559 2560 free_link_table_buf(hr_dev, &priv->ext_llm);2561}2562 2563static void free_dip_list(struct hns_roce_dev *hr_dev)2564{2565 struct hns_roce_dip *hr_dip;2566 struct hns_roce_dip *tmp;2567 unsigned long flags;2568 2569 spin_lock_irqsave(&hr_dev->dip_list_lock, flags);2570 2571 list_for_each_entry_safe(hr_dip, tmp, &hr_dev->dip_list, node) {2572 list_del(&hr_dip->node);2573 kfree(hr_dip);2574 }2575 2576 spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags);2577}2578 2579static struct ib_pd *free_mr_init_pd(struct hns_roce_dev *hr_dev)2580{2581 struct hns_roce_v2_priv *priv = hr_dev->priv;2582 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;2583 struct ib_device *ibdev = &hr_dev->ib_dev;2584 struct hns_roce_pd *hr_pd;2585 struct ib_pd *pd;2586 2587 hr_pd = kzalloc(sizeof(*hr_pd), GFP_KERNEL);2588 if (ZERO_OR_NULL_PTR(hr_pd))2589 return NULL;2590 pd = &hr_pd->ibpd;2591 pd->device = ibdev;2592 2593 if (hns_roce_alloc_pd(pd, NULL)) {2594 ibdev_err(ibdev, "failed to create pd for free mr.\n");2595 kfree(hr_pd);2596 return NULL;2597 }2598 free_mr->rsv_pd = to_hr_pd(pd);2599 free_mr->rsv_pd->ibpd.device = &hr_dev->ib_dev;2600 free_mr->rsv_pd->ibpd.uobject = NULL;2601 free_mr->rsv_pd->ibpd.__internal_mr = NULL;2602 atomic_set(&free_mr->rsv_pd->ibpd.usecnt, 0);2603 2604 return pd;2605}2606 2607static struct ib_cq *free_mr_init_cq(struct hns_roce_dev *hr_dev)2608{2609 struct hns_roce_v2_priv *priv = hr_dev->priv;2610 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;2611 struct ib_device *ibdev = &hr_dev->ib_dev;2612 struct ib_cq_init_attr cq_init_attr = {};2613 struct hns_roce_cq *hr_cq;2614 struct ib_cq *cq;2615 2616 cq_init_attr.cqe = HNS_ROCE_FREE_MR_USED_CQE_NUM;2617 2618 hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL);2619 if (ZERO_OR_NULL_PTR(hr_cq))2620 return NULL;2621 2622 cq = &hr_cq->ib_cq;2623 cq->device = ibdev;2624 2625 if (hns_roce_create_cq(cq, &cq_init_attr, NULL)) {2626 ibdev_err(ibdev, "failed to create cq for free mr.\n");2627 kfree(hr_cq);2628 return NULL;2629 }2630 free_mr->rsv_cq = to_hr_cq(cq);2631 free_mr->rsv_cq->ib_cq.device = &hr_dev->ib_dev;2632 free_mr->rsv_cq->ib_cq.uobject = NULL;2633 free_mr->rsv_cq->ib_cq.comp_handler = NULL;2634 free_mr->rsv_cq->ib_cq.event_handler = NULL;2635 free_mr->rsv_cq->ib_cq.cq_context = NULL;2636 atomic_set(&free_mr->rsv_cq->ib_cq.usecnt, 0);2637 2638 return cq;2639}2640 2641static int free_mr_init_qp(struct hns_roce_dev *hr_dev, struct ib_cq *cq,2642 struct ib_qp_init_attr *init_attr, int i)2643{2644 struct hns_roce_v2_priv *priv = hr_dev->priv;2645 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;2646 struct ib_device *ibdev = &hr_dev->ib_dev;2647 struct hns_roce_qp *hr_qp;2648 struct ib_qp *qp;2649 int ret;2650 2651 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);2652 if (ZERO_OR_NULL_PTR(hr_qp))2653 return -ENOMEM;2654 2655 qp = &hr_qp->ibqp;2656 qp->device = ibdev;2657 2658 ret = hns_roce_create_qp(qp, init_attr, NULL);2659 if (ret) {2660 ibdev_err(ibdev, "failed to create qp for free mr.\n");2661 kfree(hr_qp);2662 return ret;2663 }2664 2665 free_mr->rsv_qp[i] = hr_qp;2666 free_mr->rsv_qp[i]->ibqp.recv_cq = cq;2667 free_mr->rsv_qp[i]->ibqp.send_cq = cq;2668 2669 return 0;2670}2671 2672static void free_mr_exit(struct hns_roce_dev *hr_dev)2673{2674 struct hns_roce_v2_priv *priv = hr_dev->priv;2675 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;2676 struct ib_qp *qp;2677 int i;2678 2679 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {2680 if (free_mr->rsv_qp[i]) {2681 qp = &free_mr->rsv_qp[i]->ibqp;2682 hns_roce_v2_destroy_qp(qp, NULL);2683 kfree(free_mr->rsv_qp[i]);2684 free_mr->rsv_qp[i] = NULL;2685 }2686 }2687 2688 if (free_mr->rsv_cq) {2689 hns_roce_destroy_cq(&free_mr->rsv_cq->ib_cq, NULL);2690 kfree(free_mr->rsv_cq);2691 free_mr->rsv_cq = NULL;2692 }2693 2694 if (free_mr->rsv_pd) {2695 hns_roce_dealloc_pd(&free_mr->rsv_pd->ibpd, NULL);2696 kfree(free_mr->rsv_pd);2697 free_mr->rsv_pd = NULL;2698 }2699 2700 mutex_destroy(&free_mr->mutex);2701}2702 2703static int free_mr_alloc_res(struct hns_roce_dev *hr_dev)2704{2705 struct hns_roce_v2_priv *priv = hr_dev->priv;2706 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;2707 struct ib_qp_init_attr qp_init_attr = {};2708 struct ib_pd *pd;2709 struct ib_cq *cq;2710 int ret;2711 int i;2712 2713 pd = free_mr_init_pd(hr_dev);2714 if (!pd)2715 return -ENOMEM;2716 2717 cq = free_mr_init_cq(hr_dev);2718 if (!cq) {2719 ret = -ENOMEM;2720 goto create_failed_cq;2721 }2722 2723 qp_init_attr.qp_type = IB_QPT_RC;2724 qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;2725 qp_init_attr.send_cq = cq;2726 qp_init_attr.recv_cq = cq;2727 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {2728 qp_init_attr.cap.max_send_wr = HNS_ROCE_FREE_MR_USED_SQWQE_NUM;2729 qp_init_attr.cap.max_send_sge = HNS_ROCE_FREE_MR_USED_SQSGE_NUM;2730 qp_init_attr.cap.max_recv_wr = HNS_ROCE_FREE_MR_USED_RQWQE_NUM;2731 qp_init_attr.cap.max_recv_sge = HNS_ROCE_FREE_MR_USED_RQSGE_NUM;2732 2733 ret = free_mr_init_qp(hr_dev, cq, &qp_init_attr, i);2734 if (ret)2735 goto create_failed_qp;2736 }2737 2738 return 0;2739 2740create_failed_qp:2741 for (i--; i >= 0; i--) {2742 hns_roce_v2_destroy_qp(&free_mr->rsv_qp[i]->ibqp, NULL);2743 kfree(free_mr->rsv_qp[i]);2744 }2745 hns_roce_destroy_cq(cq, NULL);2746 kfree(cq);2747 2748create_failed_cq:2749 hns_roce_dealloc_pd(pd, NULL);2750 kfree(pd);2751 2752 return ret;2753}2754 2755static int free_mr_modify_rsv_qp(struct hns_roce_dev *hr_dev,2756 struct ib_qp_attr *attr, int sl_num)2757{2758 struct hns_roce_v2_priv *priv = hr_dev->priv;2759 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;2760 struct ib_device *ibdev = &hr_dev->ib_dev;2761 struct hns_roce_qp *hr_qp;2762 int loopback;2763 int mask;2764 int ret;2765 2766 hr_qp = to_hr_qp(&free_mr->rsv_qp[sl_num]->ibqp);2767 hr_qp->free_mr_en = 1;2768 hr_qp->ibqp.device = ibdev;2769 hr_qp->ibqp.qp_type = IB_QPT_RC;2770 2771 mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS;2772 attr->qp_state = IB_QPS_INIT;2773 attr->port_num = 1;2774 attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;2775 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT,2776 IB_QPS_INIT, NULL);2777 if (ret) {2778 ibdev_err(ibdev, "failed to modify qp to init, ret = %d.\n",2779 ret);2780 return ret;2781 }2782 2783 loopback = hr_dev->loop_idc;2784 /* Set qpc lbi = 1 incidate loopback IO */2785 hr_dev->loop_idc = 1;2786 2787 mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN |2788 IB_QP_RQ_PSN | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER;2789 attr->qp_state = IB_QPS_RTR;2790 attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;2791 attr->path_mtu = IB_MTU_256;2792 attr->dest_qp_num = hr_qp->qpn;2793 attr->rq_psn = HNS_ROCE_FREE_MR_USED_PSN;2794 2795 rdma_ah_set_sl(&attr->ah_attr, (u8)sl_num);2796 2797 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT,2798 IB_QPS_RTR, NULL);2799 hr_dev->loop_idc = loopback;2800 if (ret) {2801 ibdev_err(ibdev, "failed to modify qp to rtr, ret = %d.\n",2802 ret);2803 return ret;2804 }2805 2806 mask = IB_QP_STATE | IB_QP_SQ_PSN | IB_QP_RETRY_CNT | IB_QP_TIMEOUT |2807 IB_QP_RNR_RETRY | IB_QP_MAX_QP_RD_ATOMIC;2808 attr->qp_state = IB_QPS_RTS;2809 attr->sq_psn = HNS_ROCE_FREE_MR_USED_PSN;2810 attr->retry_cnt = HNS_ROCE_FREE_MR_USED_QP_RETRY_CNT;2811 attr->timeout = HNS_ROCE_FREE_MR_USED_QP_TIMEOUT;2812 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_RTR,2813 IB_QPS_RTS, NULL);2814 if (ret)2815 ibdev_err(ibdev, "failed to modify qp to rts, ret = %d.\n",2816 ret);2817 2818 return ret;2819}2820 2821static int free_mr_modify_qp(struct hns_roce_dev *hr_dev)2822{2823 struct hns_roce_v2_priv *priv = hr_dev->priv;2824 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;2825 struct ib_qp_attr attr = {};2826 int ret;2827 int i;2828 2829 rdma_ah_set_grh(&attr.ah_attr, NULL, 0, 0, 1, 0);2830 rdma_ah_set_static_rate(&attr.ah_attr, 3);2831 rdma_ah_set_port_num(&attr.ah_attr, 1);2832 2833 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {2834 ret = free_mr_modify_rsv_qp(hr_dev, &attr, i);2835 if (ret)2836 return ret;2837 }2838 2839 return 0;2840}2841 2842static int free_mr_init(struct hns_roce_dev *hr_dev)2843{2844 struct hns_roce_v2_priv *priv = hr_dev->priv;2845 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;2846 int ret;2847 2848 mutex_init(&free_mr->mutex);2849 2850 ret = free_mr_alloc_res(hr_dev);2851 if (ret) {2852 mutex_destroy(&free_mr->mutex);2853 return ret;2854 }2855 2856 ret = free_mr_modify_qp(hr_dev);2857 if (ret)2858 goto err_modify_qp;2859 2860 return 0;2861 2862err_modify_qp:2863 free_mr_exit(hr_dev);2864 2865 return ret;2866}2867 2868static int get_hem_table(struct hns_roce_dev *hr_dev)2869{2870 unsigned int qpc_count;2871 unsigned int cqc_count;2872 unsigned int gmv_count;2873 int ret;2874 int i;2875 2876 /* Alloc memory for source address table buffer space chunk */2877 for (gmv_count = 0; gmv_count < hr_dev->caps.gmv_entry_num;2878 gmv_count++) {2879 ret = hns_roce_table_get(hr_dev, &hr_dev->gmv_table, gmv_count);2880 if (ret)2881 goto err_gmv_failed;2882 }2883 2884 if (hr_dev->is_vf)2885 return 0;2886 2887 /* Alloc memory for QPC Timer buffer space chunk */2888 for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;2889 qpc_count++) {2890 ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table,2891 qpc_count);2892 if (ret) {2893 dev_err(hr_dev->dev, "QPC Timer get failed\n");2894 goto err_qpc_timer_failed;2895 }2896 }2897 2898 /* Alloc memory for CQC Timer buffer space chunk */2899 for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num;2900 cqc_count++) {2901 ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table,2902 cqc_count);2903 if (ret) {2904 dev_err(hr_dev->dev, "CQC Timer get failed\n");2905 goto err_cqc_timer_failed;2906 }2907 }2908 2909 return 0;2910 2911err_cqc_timer_failed:2912 for (i = 0; i < cqc_count; i++)2913 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);2914 2915err_qpc_timer_failed:2916 for (i = 0; i < qpc_count; i++)2917 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);2918 2919err_gmv_failed:2920 for (i = 0; i < gmv_count; i++)2921 hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i);2922 2923 return ret;2924}2925 2926static void put_hem_table(struct hns_roce_dev *hr_dev)2927{2928 int i;2929 2930 for (i = 0; i < hr_dev->caps.gmv_entry_num; i++)2931 hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i);2932 2933 if (hr_dev->is_vf)2934 return;2935 2936 for (i = 0; i < hr_dev->caps.qpc_timer_bt_num; i++)2937 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);2938 2939 for (i = 0; i < hr_dev->caps.cqc_timer_bt_num; i++)2940 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);2941}2942 2943static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)2944{2945 int ret;2946 2947 /* The hns ROCEE requires the extdb info to be cleared before using */2948 ret = hns_roce_clear_extdb_list_info(hr_dev);2949 if (ret)2950 return ret;2951 2952 ret = get_hem_table(hr_dev);2953 if (ret)2954 return ret;2955 2956 if (hr_dev->is_vf)2957 return 0;2958 2959 ret = hns_roce_init_link_table(hr_dev);2960 if (ret) {2961 dev_err(hr_dev->dev, "failed to init llm, ret = %d.\n", ret);2962 goto err_llm_init_failed;2963 }2964 2965 return 0;2966 2967err_llm_init_failed:2968 put_hem_table(hr_dev);2969 2970 return ret;2971}2972 2973static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)2974{2975 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)2976 free_mr_exit(hr_dev);2977 2978 hns_roce_function_clear(hr_dev);2979 2980 if (!hr_dev->is_vf)2981 hns_roce_free_link_table(hr_dev);2982 2983 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP09)2984 free_dip_list(hr_dev);2985}2986 2987static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev,2988 struct hns_roce_mbox_msg *mbox_msg)2989{2990 struct hns_roce_cmq_desc desc;2991 struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data;2992 2993 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false);2994 2995 mb->in_param_l = cpu_to_le32(mbox_msg->in_param);2996 mb->in_param_h = cpu_to_le32(mbox_msg->in_param >> 32);2997 mb->out_param_l = cpu_to_le32(mbox_msg->out_param);2998 mb->out_param_h = cpu_to_le32(mbox_msg->out_param >> 32);2999 mb->cmd_tag = cpu_to_le32(mbox_msg->tag << 8 | mbox_msg->cmd);3000 mb->token_event_en = cpu_to_le32(mbox_msg->event_en << 16 |3001 mbox_msg->token);3002 3003 return hns_roce_cmq_send(hr_dev, &desc, 1);3004}3005 3006static int v2_wait_mbox_complete(struct hns_roce_dev *hr_dev, u32 timeout,3007 u8 *complete_status)3008{3009 struct hns_roce_mbox_status *mb_st;3010 struct hns_roce_cmq_desc desc;3011 unsigned long end;3012 int ret = -EBUSY;3013 u32 status;3014 bool busy;3015 3016 mb_st = (struct hns_roce_mbox_status *)desc.data;3017 end = msecs_to_jiffies(timeout) + jiffies;3018 while (v2_chk_mbox_is_avail(hr_dev, &busy)) {3019 if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR)3020 return -EIO;3021 3022 status = 0;3023 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST,3024 true);3025 ret = __hns_roce_cmq_send(hr_dev, &desc, 1);3026 if (!ret) {3027 status = le32_to_cpu(mb_st->mb_status_hw_run);3028 /* No pending message exists in ROCEE mbox. */3029 if (!(status & MB_ST_HW_RUN_M))3030 break;3031 } else if (!v2_chk_mbox_is_avail(hr_dev, &busy)) {3032 break;3033 }3034 3035 if (time_after(jiffies, end)) {3036 dev_err_ratelimited(hr_dev->dev,3037 "failed to wait mbox status 0x%x\n",3038 status);3039 return -ETIMEDOUT;3040 }3041 3042 cond_resched();3043 ret = -EBUSY;3044 }3045 3046 if (!ret) {3047 *complete_status = (u8)(status & MB_ST_COMPLETE_M);3048 } else if (!v2_chk_mbox_is_avail(hr_dev, &busy)) {3049 /* Ignore all errors if the mbox is unavailable. */3050 ret = 0;3051 *complete_status = MB_ST_COMPLETE_M;3052 }3053 3054 return ret;3055}3056 3057static int v2_post_mbox(struct hns_roce_dev *hr_dev,3058 struct hns_roce_mbox_msg *mbox_msg)3059{3060 u8 status = 0;3061 int ret;3062 3063 /* Waiting for the mbox to be idle */3064 ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS,3065 &status);3066 if (unlikely(ret)) {3067 dev_err_ratelimited(hr_dev->dev,3068 "failed to check post mbox status = 0x%x, ret = %d.\n",3069 status, ret);3070 return ret;3071 }3072 3073 /* Post new message to mbox */3074 ret = hns_roce_mbox_post(hr_dev, mbox_msg);3075 if (ret)3076 dev_err_ratelimited(hr_dev->dev,3077 "failed to post mailbox, ret = %d.\n", ret);3078 3079 return ret;3080}3081 3082static int v2_poll_mbox_done(struct hns_roce_dev *hr_dev)3083{3084 u8 status = 0;3085 int ret;3086 3087 ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_CMD_TIMEOUT_MSECS,3088 &status);3089 if (!ret) {3090 if (status != MB_ST_COMPLETE_SUCC)3091 return -EBUSY;3092 } else {3093 dev_err_ratelimited(hr_dev->dev,3094 "failed to check mbox status = 0x%x, ret = %d.\n",3095 status, ret);3096 }3097 3098 return ret;3099}3100 3101static void copy_gid(void *dest, const union ib_gid *gid)3102{3103#define GID_SIZE 43104 const union ib_gid *src = gid;3105 __le32 (*p)[GID_SIZE] = dest;3106 int i;3107 3108 if (!gid)3109 src = &zgid;3110 3111 for (i = 0; i < GID_SIZE; i++)3112 (*p)[i] = cpu_to_le32(*(u32 *)&src->raw[i * sizeof(u32)]);3113}3114 3115static int config_sgid_table(struct hns_roce_dev *hr_dev,3116 int gid_index, const union ib_gid *gid,3117 enum hns_roce_sgid_type sgid_type)3118{3119 struct hns_roce_cmq_desc desc;3120 struct hns_roce_cfg_sgid_tb *sgid_tb =3121 (struct hns_roce_cfg_sgid_tb *)desc.data;3122 3123 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);3124 3125 hr_reg_write(sgid_tb, CFG_SGID_TB_TABLE_IDX, gid_index);3126 hr_reg_write(sgid_tb, CFG_SGID_TB_VF_SGID_TYPE, sgid_type);3127 3128 copy_gid(&sgid_tb->vf_sgid_l, gid);3129 3130 return hns_roce_cmq_send(hr_dev, &desc, 1);3131}3132 3133static int config_gmv_table(struct hns_roce_dev *hr_dev,3134 int gid_index, const union ib_gid *gid,3135 enum hns_roce_sgid_type sgid_type,3136 const struct ib_gid_attr *attr)3137{3138 struct hns_roce_cmq_desc desc[2];3139 struct hns_roce_cfg_gmv_tb_a *tb_a =3140 (struct hns_roce_cfg_gmv_tb_a *)desc[0].data;3141 struct hns_roce_cfg_gmv_tb_b *tb_b =3142 (struct hns_roce_cfg_gmv_tb_b *)desc[1].data;3143 3144 u16 vlan_id = VLAN_CFI_MASK;3145 u8 mac[ETH_ALEN] = {};3146 int ret;3147 3148 if (gid) {3149 ret = rdma_read_gid_l2_fields(attr, &vlan_id, mac);3150 if (ret)3151 return ret;3152 }3153 3154 hns_roce_cmq_setup_basic_desc(&desc[0], HNS_ROCE_OPC_CFG_GMV_TBL, false);3155 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);3156 3157 hns_roce_cmq_setup_basic_desc(&desc[1], HNS_ROCE_OPC_CFG_GMV_TBL, false);3158 3159 copy_gid(&tb_a->vf_sgid_l, gid);3160 3161 hr_reg_write(tb_a, GMV_TB_A_VF_SGID_TYPE, sgid_type);3162 hr_reg_write(tb_a, GMV_TB_A_VF_VLAN_EN, vlan_id < VLAN_CFI_MASK);3163 hr_reg_write(tb_a, GMV_TB_A_VF_VLAN_ID, vlan_id);3164 3165 tb_b->vf_smac_l = cpu_to_le32(*(u32 *)mac);3166 3167 hr_reg_write(tb_b, GMV_TB_B_SMAC_H, *(u16 *)&mac[4]);3168 hr_reg_write(tb_b, GMV_TB_B_SGID_IDX, gid_index);3169 3170 return hns_roce_cmq_send(hr_dev, desc, 2);3171}3172 3173static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, int gid_index,3174 const union ib_gid *gid,3175 const struct ib_gid_attr *attr)3176{3177 enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;3178 int ret;3179 3180 if (gid) {3181 if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {3182 if (ipv6_addr_v4mapped((void *)gid))3183 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;3184 else3185 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;3186 } else if (attr->gid_type == IB_GID_TYPE_ROCE) {3187 sgid_type = GID_TYPE_FLAG_ROCE_V1;3188 }3189 }3190 3191 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)3192 ret = config_gmv_table(hr_dev, gid_index, gid, sgid_type, attr);3193 else3194 ret = config_sgid_table(hr_dev, gid_index, gid, sgid_type);3195 3196 if (ret)3197 ibdev_err(&hr_dev->ib_dev, "failed to set gid, ret = %d!\n",3198 ret);3199 3200 return ret;3201}3202 3203static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,3204 const u8 *addr)3205{3206 struct hns_roce_cmq_desc desc;3207 struct hns_roce_cfg_smac_tb *smac_tb =3208 (struct hns_roce_cfg_smac_tb *)desc.data;3209 u16 reg_smac_h;3210 u32 reg_smac_l;3211 3212 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false);3213 3214 reg_smac_l = *(u32 *)(&addr[0]);3215 reg_smac_h = *(u16 *)(&addr[4]);3216 3217 hr_reg_write(smac_tb, CFG_SMAC_TB_IDX, phy_port);3218 hr_reg_write(smac_tb, CFG_SMAC_TB_VF_SMAC_H, reg_smac_h);3219 smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l);3220 3221 return hns_roce_cmq_send(hr_dev, &desc, 1);3222}3223 3224static int set_mtpt_pbl(struct hns_roce_dev *hr_dev,3225 struct hns_roce_v2_mpt_entry *mpt_entry,3226 struct hns_roce_mr *mr)3227{3228 u64 pages[HNS_ROCE_V2_MAX_INNER_MTPT_NUM] = { 0 };3229 struct ib_device *ibdev = &hr_dev->ib_dev;3230 dma_addr_t pbl_ba;3231 int ret;3232 int i;3233 3234 ret = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,3235 min_t(int, ARRAY_SIZE(pages), mr->npages));3236 if (ret) {3237 ibdev_err(ibdev, "failed to find PBL mtr, ret = %d.\n", ret);3238 return ret;3239 }3240 3241 /* Aligned to the hardware address access unit */3242 for (i = 0; i < ARRAY_SIZE(pages); i++)3243 pages[i] >>= MPT_PBL_BUF_ADDR_S;3244 3245 pbl_ba = hns_roce_get_mtr_ba(&mr->pbl_mtr);3246 3247 mpt_entry->pbl_size = cpu_to_le32(mr->npages);3248 mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> MPT_PBL_BA_ADDR_S);3249 hr_reg_write(mpt_entry, MPT_PBL_BA_H,3250 upper_32_bits(pbl_ba >> MPT_PBL_BA_ADDR_S));3251 3252 mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));3253 hr_reg_write(mpt_entry, MPT_PA0_H, upper_32_bits(pages[0]));3254 3255 mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1]));3256 hr_reg_write(mpt_entry, MPT_PA1_H, upper_32_bits(pages[1]));3257 hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ,3258 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));3259 3260 return 0;3261}3262 3263static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev,3264 void *mb_buf, struct hns_roce_mr *mr)3265{3266 struct hns_roce_v2_mpt_entry *mpt_entry;3267 3268 mpt_entry = mb_buf;3269 memset(mpt_entry, 0, sizeof(*mpt_entry));3270 3271 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID);3272 hr_reg_write(mpt_entry, MPT_PD, mr->pd);3273 3274 hr_reg_write_bool(mpt_entry, MPT_BIND_EN,3275 mr->access & IB_ACCESS_MW_BIND);3276 hr_reg_write_bool(mpt_entry, MPT_ATOMIC_EN,3277 mr->access & IB_ACCESS_REMOTE_ATOMIC);3278 hr_reg_write_bool(mpt_entry, MPT_RR_EN,3279 mr->access & IB_ACCESS_REMOTE_READ);3280 hr_reg_write_bool(mpt_entry, MPT_RW_EN,3281 mr->access & IB_ACCESS_REMOTE_WRITE);3282 hr_reg_write_bool(mpt_entry, MPT_LW_EN,3283 mr->access & IB_ACCESS_LOCAL_WRITE);3284 3285 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));3286 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));3287 mpt_entry->lkey = cpu_to_le32(mr->key);3288 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));3289 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));3290 3291 if (mr->type != MR_TYPE_MR)3292 hr_reg_enable(mpt_entry, MPT_PA);3293 3294 if (mr->type == MR_TYPE_DMA)3295 return 0;3296 3297 if (mr->pbl_hop_num != HNS_ROCE_HOP_NUM_0)3298 hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, mr->pbl_hop_num);3299 3300 hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ,3301 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));3302 hr_reg_enable(mpt_entry, MPT_INNER_PA_VLD);3303 3304 return set_mtpt_pbl(hr_dev, mpt_entry, mr);3305}3306 3307static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,3308 struct hns_roce_mr *mr, int flags,3309 void *mb_buf)3310{3311 struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;3312 u32 mr_access_flags = mr->access;3313 int ret = 0;3314 3315 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID);3316 hr_reg_write(mpt_entry, MPT_PD, mr->pd);3317 3318 if (flags & IB_MR_REREG_ACCESS) {3319 hr_reg_write(mpt_entry, MPT_BIND_EN,3320 (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0));3321 hr_reg_write(mpt_entry, MPT_ATOMIC_EN,3322 mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);3323 hr_reg_write(mpt_entry, MPT_RR_EN,3324 mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0);3325 hr_reg_write(mpt_entry, MPT_RW_EN,3326 mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0);3327 hr_reg_write(mpt_entry, MPT_LW_EN,3328 mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0);3329 }3330 3331 if (flags & IB_MR_REREG_TRANS) {3332 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));3333 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));3334 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));3335 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));3336 3337 ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);3338 }3339 3340 return ret;3341}3342 3343static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)3344{3345 dma_addr_t pbl_ba = hns_roce_get_mtr_ba(&mr->pbl_mtr);3346 struct hns_roce_v2_mpt_entry *mpt_entry;3347 3348 mpt_entry = mb_buf;3349 memset(mpt_entry, 0, sizeof(*mpt_entry));3350 3351 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_FREE);3352 hr_reg_write(mpt_entry, MPT_PD, mr->pd);3353 3354 hr_reg_enable(mpt_entry, MPT_RA_EN);3355 hr_reg_enable(mpt_entry, MPT_R_INV_EN);3356 3357 hr_reg_enable(mpt_entry, MPT_FRE);3358 hr_reg_clear(mpt_entry, MPT_MR_MW);3359 hr_reg_enable(mpt_entry, MPT_BPD);3360 hr_reg_clear(mpt_entry, MPT_PA);3361 3362 hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, 1);3363 hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ,3364 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));3365 hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ,3366 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));3367 3368 mpt_entry->pbl_size = cpu_to_le32(mr->npages);3369 3370 mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >>3371 MPT_PBL_BA_ADDR_S));3372 hr_reg_write(mpt_entry, MPT_PBL_BA_H,3373 upper_32_bits(pbl_ba >> MPT_PBL_BA_ADDR_S));3374 3375 return 0;3376}3377 3378static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)3379{3380 struct hns_roce_v2_mpt_entry *mpt_entry;3381 3382 mpt_entry = mb_buf;3383 memset(mpt_entry, 0, sizeof(*mpt_entry));3384 3385 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_FREE);3386 hr_reg_write(mpt_entry, MPT_PD, mw->pdn);3387 3388 hr_reg_enable(mpt_entry, MPT_R_INV_EN);3389 hr_reg_enable(mpt_entry, MPT_LW_EN);3390 3391 hr_reg_enable(mpt_entry, MPT_MR_MW);3392 hr_reg_enable(mpt_entry, MPT_BPD);3393 hr_reg_clear(mpt_entry, MPT_PA);3394 hr_reg_write(mpt_entry, MPT_BQP,3395 mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1);3396 3397 mpt_entry->lkey = cpu_to_le32(mw->rkey);3398 3399 hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM,3400 mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :3401 mw->pbl_hop_num);3402 hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ,3403 mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET);3404 hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ,3405 mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET);3406 3407 return 0;3408}3409 3410static int free_mr_post_send_lp_wqe(struct hns_roce_qp *hr_qp)3411{3412 struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device);3413 struct ib_device *ibdev = &hr_dev->ib_dev;3414 const struct ib_send_wr *bad_wr;3415 struct ib_rdma_wr rdma_wr = {};3416 struct ib_send_wr *send_wr;3417 int ret;3418 3419 send_wr = &rdma_wr.wr;3420 send_wr->opcode = IB_WR_RDMA_WRITE;3421 3422 ret = hns_roce_v2_post_send(&hr_qp->ibqp, send_wr, &bad_wr);3423 if (ret) {3424 ibdev_err(ibdev, "failed to post wqe for free mr, ret = %d.\n",3425 ret);3426 return ret;3427 }3428 3429 return 0;3430}3431 3432static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,3433 struct ib_wc *wc);3434 3435static void free_mr_send_cmd_to_hw(struct hns_roce_dev *hr_dev)3436{3437 struct hns_roce_v2_priv *priv = hr_dev->priv;3438 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;3439 struct ib_wc wc[ARRAY_SIZE(free_mr->rsv_qp)];3440 struct ib_device *ibdev = &hr_dev->ib_dev;3441 struct hns_roce_qp *hr_qp;3442 unsigned long end;3443 int cqe_cnt = 0;3444 int npolled;3445 int ret;3446 int i;3447 3448 /*3449 * If the device initialization is not complete or in the uninstall3450 * process, then there is no need to execute free mr.3451 */3452 if (priv->handle->rinfo.reset_state == HNS_ROCE_STATE_RST_INIT ||3453 priv->handle->rinfo.instance_state == HNS_ROCE_STATE_INIT ||3454 hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT)3455 return;3456 3457 mutex_lock(&free_mr->mutex);3458 3459 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {3460 hr_qp = free_mr->rsv_qp[i];3461 3462 ret = free_mr_post_send_lp_wqe(hr_qp);3463 if (ret) {3464 ibdev_err(ibdev,3465 "failed to send wqe (qp:0x%lx) for free mr, ret = %d.\n",3466 hr_qp->qpn, ret);3467 break;3468 }3469 3470 cqe_cnt++;3471 }3472 3473 end = msecs_to_jiffies(HNS_ROCE_V2_FREE_MR_TIMEOUT) + jiffies;3474 while (cqe_cnt) {3475 npolled = hns_roce_v2_poll_cq(&free_mr->rsv_cq->ib_cq, cqe_cnt, wc);3476 if (npolled < 0) {3477 ibdev_err(ibdev,3478 "failed to poll cqe for free mr, remain %d cqe.\n",3479 cqe_cnt);3480 goto out;3481 }3482 3483 if (time_after(jiffies, end)) {3484 ibdev_err(ibdev,3485 "failed to poll cqe for free mr and timeout, remain %d cqe.\n",3486 cqe_cnt);3487 goto out;3488 }3489 cqe_cnt -= npolled;3490 }3491 3492out:3493 mutex_unlock(&free_mr->mutex);3494}3495 3496static void hns_roce_v2_dereg_mr(struct hns_roce_dev *hr_dev)3497{3498 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)3499 free_mr_send_cmd_to_hw(hr_dev);3500}3501 3502static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)3503{3504 return hns_roce_buf_offset(hr_cq->mtr.kmem, n * hr_cq->cqe_size);3505}3506 3507static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, unsigned int n)3508{3509 struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);3510 3511 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */3512 return (hr_reg_read(cqe, CQE_OWNER) ^ !!(n & hr_cq->cq_depth)) ? cqe :3513 NULL;3514}3515 3516static inline void update_cq_db(struct hns_roce_dev *hr_dev,3517 struct hns_roce_cq *hr_cq)3518{3519 if (likely(hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB)) {3520 *hr_cq->set_ci_db = hr_cq->cons_index & V2_CQ_DB_CONS_IDX_M;3521 } else {3522 struct hns_roce_v2_db cq_db = {};3523 3524 hr_reg_write(&cq_db, DB_TAG, hr_cq->cqn);3525 hr_reg_write(&cq_db, DB_CMD, HNS_ROCE_V2_CQ_DB);3526 hr_reg_write(&cq_db, DB_CQ_CI, hr_cq->cons_index);3527 hr_reg_write(&cq_db, DB_CQ_CMD_SN, 1);3528 3529 hns_roce_write64(hr_dev, (__le32 *)&cq_db, hr_cq->db_reg);3530 }3531}3532 3533static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,3534 struct hns_roce_srq *srq)3535{3536 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);3537 struct hns_roce_v2_cqe *cqe, *dest;3538 u32 prod_index;3539 int nfreed = 0;3540 int wqe_index;3541 u8 owner_bit;3542 3543 for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);3544 ++prod_index) {3545 if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe)3546 break;3547 }3548 3549 /*3550 * Now backwards through the CQ, removing CQ entries3551 * that match our QP by overwriting them with next entries.3552 */3553 while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {3554 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);3555 if (hr_reg_read(cqe, CQE_LCL_QPN) == qpn) {3556 if (srq && hr_reg_read(cqe, CQE_S_R)) {3557 wqe_index = hr_reg_read(cqe, CQE_WQE_IDX);3558 hns_roce_free_srq_wqe(srq, wqe_index);3559 }3560 ++nfreed;3561 } else if (nfreed) {3562 dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &3563 hr_cq->ib_cq.cqe);3564 owner_bit = hr_reg_read(dest, CQE_OWNER);3565 memcpy(dest, cqe, hr_cq->cqe_size);3566 hr_reg_write(dest, CQE_OWNER, owner_bit);3567 }3568 }3569 3570 if (nfreed) {3571 hr_cq->cons_index += nfreed;3572 update_cq_db(hr_dev, hr_cq);3573 }3574}3575 3576static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,3577 struct hns_roce_srq *srq)3578{3579 spin_lock_irq(&hr_cq->lock);3580 __hns_roce_v2_cq_clean(hr_cq, qpn, srq);3581 spin_unlock_irq(&hr_cq->lock);3582}3583 3584static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,3585 struct hns_roce_cq *hr_cq, void *mb_buf,3586 u64 *mtts, dma_addr_t dma_handle)3587{3588 struct hns_roce_v2_cq_context *cq_context;3589 3590 cq_context = mb_buf;3591 memset(cq_context, 0, sizeof(*cq_context));3592 3593 hr_reg_write(cq_context, CQC_CQ_ST, V2_CQ_STATE_VALID);3594 hr_reg_write(cq_context, CQC_ARM_ST, NO_ARMED);3595 hr_reg_write(cq_context, CQC_SHIFT, ilog2(hr_cq->cq_depth));3596 hr_reg_write(cq_context, CQC_CEQN, hr_cq->vector);3597 hr_reg_write(cq_context, CQC_CQN, hr_cq->cqn);3598 3599 if (hr_cq->cqe_size == HNS_ROCE_V3_CQE_SIZE)3600 hr_reg_write(cq_context, CQC_CQE_SIZE, CQE_SIZE_64B);3601 3602 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH)3603 hr_reg_enable(cq_context, CQC_STASH);3604 3605 hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_L,3606 to_hr_hw_page_addr(mtts[0]));3607 hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_H,3608 upper_32_bits(to_hr_hw_page_addr(mtts[0])));3609 hr_reg_write(cq_context, CQC_CQE_HOP_NUM, hr_dev->caps.cqe_hop_num ==3610 HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);3611 hr_reg_write(cq_context, CQC_CQE_NEX_BLK_ADDR_L,3612 to_hr_hw_page_addr(mtts[1]));3613 hr_reg_write(cq_context, CQC_CQE_NEX_BLK_ADDR_H,3614 upper_32_bits(to_hr_hw_page_addr(mtts[1])));3615 hr_reg_write(cq_context, CQC_CQE_BAR_PG_SZ,3616 to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.ba_pg_shift));3617 hr_reg_write(cq_context, CQC_CQE_BUF_PG_SZ,3618 to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.buf_pg_shift));3619 hr_reg_write(cq_context, CQC_CQE_BA_L, dma_handle >> CQC_CQE_BA_L_S);3620 hr_reg_write(cq_context, CQC_CQE_BA_H, dma_handle >> CQC_CQE_BA_H_S);3621 hr_reg_write_bool(cq_context, CQC_DB_RECORD_EN,3622 hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB);3623 hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_L,3624 ((u32)hr_cq->db.dma) >> 1);3625 hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_H,3626 hr_cq->db.dma >> CQC_CQE_DB_RECORD_ADDR_H_S);3627 hr_reg_write(cq_context, CQC_CQ_MAX_CNT,3628 HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);3629 hr_reg_write(cq_context, CQC_CQ_PERIOD,3630 HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);3631}3632 3633static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,3634 enum ib_cq_notify_flags flags)3635{3636 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);3637 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);3638 struct hns_roce_v2_db cq_db = {};3639 u32 notify_flag;3640 3641 /*3642 * flags = 0, then notify_flag : next3643 * flags = 1, then notify flag : solocited3644 */3645 notify_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?3646 V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;3647 3648 hr_reg_write(&cq_db, DB_TAG, hr_cq->cqn);3649 hr_reg_write(&cq_db, DB_CMD, HNS_ROCE_V2_CQ_DB_NOTIFY);3650 hr_reg_write(&cq_db, DB_CQ_CI, hr_cq->cons_index);3651 hr_reg_write(&cq_db, DB_CQ_CMD_SN, hr_cq->arm_sn);3652 hr_reg_write(&cq_db, DB_CQ_NOTIFY, notify_flag);3653 3654 hns_roce_write64(hr_dev, (__le32 *)&cq_db, hr_cq->db_reg);3655 3656 return 0;3657}3658 3659static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq,3660 int num_entries, struct ib_wc *wc)3661{3662 unsigned int left;3663 int npolled = 0;3664 3665 left = wq->head - wq->tail;3666 if (left == 0)3667 return 0;3668 3669 left = min_t(unsigned int, (unsigned int)num_entries, left);3670 while (npolled < left) {3671 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];3672 wc->status = IB_WC_WR_FLUSH_ERR;3673 wc->vendor_err = 0;3674 wc->qp = &hr_qp->ibqp;3675 3676 wq->tail++;3677 wc++;3678 npolled++;3679 }3680 3681 return npolled;3682}3683 3684static int hns_roce_v2_sw_poll_cq(struct hns_roce_cq *hr_cq, int num_entries,3685 struct ib_wc *wc)3686{3687 struct hns_roce_qp *hr_qp;3688 int npolled = 0;3689 3690 list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) {3691 npolled += sw_comp(hr_qp, &hr_qp->sq,3692 num_entries - npolled, wc + npolled);3693 if (npolled >= num_entries)3694 goto out;3695 }3696 3697 list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) {3698 npolled += sw_comp(hr_qp, &hr_qp->rq,3699 num_entries - npolled, wc + npolled);3700 if (npolled >= num_entries)3701 goto out;3702 }3703 3704out:3705 return npolled;3706}3707 3708static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,3709 struct hns_roce_cq *cq, struct hns_roce_v2_cqe *cqe,3710 struct ib_wc *wc)3711{3712 static const struct {3713 u32 cqe_status;3714 enum ib_wc_status wc_status;3715 } map[] = {3716 { HNS_ROCE_CQE_V2_SUCCESS, IB_WC_SUCCESS },3717 { HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR, IB_WC_LOC_LEN_ERR },3718 { HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR },3719 { HNS_ROCE_CQE_V2_LOCAL_PROT_ERR, IB_WC_LOC_PROT_ERR },3720 { HNS_ROCE_CQE_V2_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR },3721 { HNS_ROCE_CQE_V2_MW_BIND_ERR, IB_WC_MW_BIND_ERR },3722 { HNS_ROCE_CQE_V2_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR },3723 { HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR },3724 { HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR, IB_WC_REM_INV_REQ_ERR },3725 { HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR, IB_WC_REM_ACCESS_ERR },3726 { HNS_ROCE_CQE_V2_REMOTE_OP_ERR, IB_WC_REM_OP_ERR },3727 { HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR,3728 IB_WC_RETRY_EXC_ERR },3729 { HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR },3730 { HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR },3731 { HNS_ROCE_CQE_V2_GENERAL_ERR, IB_WC_GENERAL_ERR}3732 };3733 3734 u32 cqe_status = hr_reg_read(cqe, CQE_STATUS);3735 int i;3736 3737 wc->status = IB_WC_GENERAL_ERR;3738 for (i = 0; i < ARRAY_SIZE(map); i++)3739 if (cqe_status == map[i].cqe_status) {3740 wc->status = map[i].wc_status;3741 break;3742 }3743 3744 if (likely(wc->status == IB_WC_SUCCESS ||3745 wc->status == IB_WC_WR_FLUSH_ERR))3746 return;3747 3748 ibdev_err_ratelimited(&hr_dev->ib_dev, "error cqe status 0x%x:\n",3749 cqe_status);3750 print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_NONE, 16, 4, cqe,3751 cq->cqe_size, false);3752 wc->vendor_err = hr_reg_read(cqe, CQE_SUB_STATUS);3753 3754 /*3755 * For hns ROCEE, GENERAL_ERR is an error type that is not defined in3756 * the standard protocol, the driver must ignore it and needn't to set3757 * the QP to an error state.3758 */3759 if (cqe_status == HNS_ROCE_CQE_V2_GENERAL_ERR)3760 return;3761 3762 flush_cqe(hr_dev, qp);3763}3764 3765static int get_cur_qp(struct hns_roce_cq *hr_cq, struct hns_roce_v2_cqe *cqe,3766 struct hns_roce_qp **cur_qp)3767{3768 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);3769 struct hns_roce_qp *hr_qp = *cur_qp;3770 u32 qpn;3771 3772 qpn = hr_reg_read(cqe, CQE_LCL_QPN);3773 3774 if (!hr_qp || qpn != hr_qp->qpn) {3775 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);3776 if (unlikely(!hr_qp)) {3777 ibdev_err(&hr_dev->ib_dev,3778 "CQ %06lx with entry for unknown QPN %06x\n",3779 hr_cq->cqn, qpn);3780 return -EINVAL;3781 }3782 *cur_qp = hr_qp;3783 }3784 3785 return 0;3786}3787 3788/*3789 * mapped-value = 1 + real-value3790 * The ib wc opcode's real value is start from 0, In order to distinguish3791 * between initialized and uninitialized map values, we plus 1 to the actual3792 * value when defining the mapping, so that the validity can be identified by3793 * checking whether the mapped value is greater than 0.3794 */3795#define HR_WC_OP_MAP(hr_key, ib_key) \3796 [HNS_ROCE_V2_WQE_OP_ ## hr_key] = 1 + IB_WC_ ## ib_key3797 3798static const u32 wc_send_op_map[] = {3799 HR_WC_OP_MAP(SEND, SEND),3800 HR_WC_OP_MAP(SEND_WITH_INV, SEND),3801 HR_WC_OP_MAP(SEND_WITH_IMM, SEND),3802 HR_WC_OP_MAP(RDMA_READ, RDMA_READ),3803 HR_WC_OP_MAP(RDMA_WRITE, RDMA_WRITE),3804 HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM, RDMA_WRITE),3805 HR_WC_OP_MAP(ATOM_CMP_AND_SWAP, COMP_SWAP),3806 HR_WC_OP_MAP(ATOM_FETCH_AND_ADD, FETCH_ADD),3807 HR_WC_OP_MAP(ATOM_MSK_CMP_AND_SWAP, MASKED_COMP_SWAP),3808 HR_WC_OP_MAP(ATOM_MSK_FETCH_AND_ADD, MASKED_FETCH_ADD),3809 HR_WC_OP_MAP(FAST_REG_PMR, REG_MR),3810 HR_WC_OP_MAP(BIND_MW, REG_MR),3811};3812 3813static int to_ib_wc_send_op(u32 hr_opcode)3814{3815 if (hr_opcode >= ARRAY_SIZE(wc_send_op_map))3816 return -EINVAL;3817 3818 return wc_send_op_map[hr_opcode] ? wc_send_op_map[hr_opcode] - 1 :3819 -EINVAL;3820}3821 3822static const u32 wc_recv_op_map[] = {3823 HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM, WITH_IMM),3824 HR_WC_OP_MAP(SEND, RECV),3825 HR_WC_OP_MAP(SEND_WITH_IMM, WITH_IMM),3826 HR_WC_OP_MAP(SEND_WITH_INV, RECV),3827};3828 3829static int to_ib_wc_recv_op(u32 hr_opcode)3830{3831 if (hr_opcode >= ARRAY_SIZE(wc_recv_op_map))3832 return -EINVAL;3833 3834 return wc_recv_op_map[hr_opcode] ? wc_recv_op_map[hr_opcode] - 1 :3835 -EINVAL;3836}3837 3838static void fill_send_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe)3839{3840 u32 hr_opcode;3841 int ib_opcode;3842 3843 wc->wc_flags = 0;3844 3845 hr_opcode = hr_reg_read(cqe, CQE_OPCODE);3846 switch (hr_opcode) {3847 case HNS_ROCE_V2_WQE_OP_RDMA_READ:3848 wc->byte_len = le32_to_cpu(cqe->byte_cnt);3849 break;3850 case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM:3851 case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM:3852 wc->wc_flags |= IB_WC_WITH_IMM;3853 break;3854 case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP:3855 case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD:3856 case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP:3857 case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD:3858 wc->byte_len = 8;3859 break;3860 default:3861 break;3862 }3863 3864 ib_opcode = to_ib_wc_send_op(hr_opcode);3865 if (ib_opcode < 0)3866 wc->status = IB_WC_GENERAL_ERR;3867 else3868 wc->opcode = ib_opcode;3869}3870 3871static int fill_recv_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe)3872{3873 u32 hr_opcode;3874 int ib_opcode;3875 3876 wc->byte_len = le32_to_cpu(cqe->byte_cnt);3877 3878 hr_opcode = hr_reg_read(cqe, CQE_OPCODE);3879 switch (hr_opcode) {3880 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:3881 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:3882 wc->wc_flags = IB_WC_WITH_IMM;3883 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(cqe->immtdata));3884 break;3885 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:3886 wc->wc_flags = IB_WC_WITH_INVALIDATE;3887 wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey);3888 break;3889 default:3890 wc->wc_flags = 0;3891 }3892 3893 ib_opcode = to_ib_wc_recv_op(hr_opcode);3894 if (ib_opcode < 0)3895 wc->status = IB_WC_GENERAL_ERR;3896 else3897 wc->opcode = ib_opcode;3898 3899 wc->sl = hr_reg_read(cqe, CQE_SL);3900 wc->src_qp = hr_reg_read(cqe, CQE_RMT_QPN);3901 wc->slid = 0;3902 wc->wc_flags |= hr_reg_read(cqe, CQE_GRH) ? IB_WC_GRH : 0;3903 wc->port_num = hr_reg_read(cqe, CQE_PORTN);3904 wc->pkey_index = 0;3905 3906 if (hr_reg_read(cqe, CQE_VID_VLD)) {3907 wc->vlan_id = hr_reg_read(cqe, CQE_VID);3908 wc->wc_flags |= IB_WC_WITH_VLAN;3909 } else {3910 wc->vlan_id = 0xffff;3911 }3912 3913 wc->network_hdr_type = hr_reg_read(cqe, CQE_PORT_TYPE);3914 3915 return 0;3916}3917 3918static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,3919 struct hns_roce_qp **cur_qp, struct ib_wc *wc)3920{3921 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);3922 struct hns_roce_qp *qp = *cur_qp;3923 struct hns_roce_srq *srq = NULL;3924 struct hns_roce_v2_cqe *cqe;3925 struct hns_roce_wq *wq;3926 int is_send;3927 u16 wqe_idx;3928 int ret;3929 3930 cqe = get_sw_cqe_v2(hr_cq, hr_cq->cons_index);3931 if (!cqe)3932 return -EAGAIN;3933 3934 ++hr_cq->cons_index;3935 /* Memory barrier */3936 rmb();3937 3938 ret = get_cur_qp(hr_cq, cqe, &qp);3939 if (ret)3940 return ret;3941 3942 wc->qp = &qp->ibqp;3943 wc->vendor_err = 0;3944 3945 wqe_idx = hr_reg_read(cqe, CQE_WQE_IDX);3946 3947 is_send = !hr_reg_read(cqe, CQE_S_R);3948 if (is_send) {3949 wq = &qp->sq;3950 3951 /* If sg_signal_bit is set, tail pointer will be updated to3952 * the WQE corresponding to the current CQE.3953 */3954 if (qp->sq_signal_bits)3955 wq->tail += (wqe_idx - (u16)wq->tail) &3956 (wq->wqe_cnt - 1);3957 3958 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];3959 ++wq->tail;3960 3961 fill_send_wc(wc, cqe);3962 } else {3963 if (qp->ibqp.srq) {3964 srq = to_hr_srq(qp->ibqp.srq);3965 wc->wr_id = srq->wrid[wqe_idx];3966 hns_roce_free_srq_wqe(srq, wqe_idx);3967 } else {3968 wq = &qp->rq;3969 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];3970 ++wq->tail;3971 }3972 3973 ret = fill_recv_wc(wc, cqe);3974 }3975 3976 get_cqe_status(hr_dev, qp, hr_cq, cqe, wc);3977 if (unlikely(wc->status != IB_WC_SUCCESS))3978 return 0;3979 3980 return ret;3981}3982 3983static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,3984 struct ib_wc *wc)3985{3986 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);3987 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);3988 struct hns_roce_qp *cur_qp = NULL;3989 unsigned long flags;3990 int npolled;3991 3992 spin_lock_irqsave(&hr_cq->lock, flags);3993 3994 /*3995 * When the device starts to reset, the state is RST_DOWN. At this time,3996 * there may still be some valid CQEs in the hardware that are not3997 * polled. Therefore, it is not allowed to switch to the software mode3998 * immediately. When the state changes to UNINIT, CQE no longer exists3999 * in the hardware, and then switch to software mode.4000 */4001 if (hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) {4002 npolled = hns_roce_v2_sw_poll_cq(hr_cq, num_entries, wc);4003 goto out;4004 }4005 4006 for (npolled = 0; npolled < num_entries; ++npolled) {4007 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))4008 break;4009 }4010 4011 if (npolled)4012 update_cq_db(hr_dev, hr_cq);4013 4014out:4015 spin_unlock_irqrestore(&hr_cq->lock, flags);4016 4017 return npolled;4018}4019 4020static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,4021 u32 step_idx, u8 *mbox_cmd)4022{4023 u8 cmd;4024 4025 switch (type) {4026 case HEM_TYPE_QPC:4027 cmd = HNS_ROCE_CMD_WRITE_QPC_BT0;4028 break;4029 case HEM_TYPE_MTPT:4030 cmd = HNS_ROCE_CMD_WRITE_MPT_BT0;4031 break;4032 case HEM_TYPE_CQC:4033 cmd = HNS_ROCE_CMD_WRITE_CQC_BT0;4034 break;4035 case HEM_TYPE_SRQC:4036 cmd = HNS_ROCE_CMD_WRITE_SRQC_BT0;4037 break;4038 case HEM_TYPE_SCCC:4039 cmd = HNS_ROCE_CMD_WRITE_SCCC_BT0;4040 break;4041 case HEM_TYPE_QPC_TIMER:4042 cmd = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;4043 break;4044 case HEM_TYPE_CQC_TIMER:4045 cmd = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;4046 break;4047 default:4048 dev_warn(hr_dev->dev, "failed to check hem type %u.\n", type);4049 return -EINVAL;4050 }4051 4052 *mbox_cmd = cmd + step_idx;4053 4054 return 0;4055}4056 4057static int config_gmv_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj,4058 dma_addr_t base_addr)4059{4060 struct hns_roce_cmq_desc desc;4061 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;4062 u32 idx = obj / (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz);4063 u64 addr = to_hr_hw_page_addr(base_addr);4064 4065 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false);4066 4067 hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr));4068 hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr));4069 hr_reg_write(req, CFG_GMV_BT_IDX, idx);4070 4071 return hns_roce_cmq_send(hr_dev, &desc, 1);4072}4073 4074static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj,4075 dma_addr_t base_addr, u32 hem_type, u32 step_idx)4076{4077 int ret;4078 u8 cmd;4079 4080 if (unlikely(hem_type == HEM_TYPE_GMV))4081 return config_gmv_ba_to_hw(hr_dev, obj, base_addr);4082 4083 if (unlikely(hem_type == HEM_TYPE_SCCC && step_idx))4084 return 0;4085 4086 ret = get_op_for_set_hem(hr_dev, hem_type, step_idx, &cmd);4087 if (ret < 0)4088 return ret;4089 4090 return config_hem_ba_to_hw(hr_dev, base_addr, cmd, obj);4091}4092 4093static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,4094 struct hns_roce_hem_table *table, int obj,4095 u32 step_idx)4096{4097 struct hns_roce_hem_mhop mhop;4098 struct hns_roce_hem *hem;4099 unsigned long mhop_obj = obj;4100 int i, j, k;4101 int ret = 0;4102 u64 hem_idx = 0;4103 u64 l1_idx = 0;4104 u64 bt_ba = 0;4105 u32 chunk_ba_num;4106 u32 hop_num;4107 4108 if (!hns_roce_check_whether_mhop(hr_dev, table->type))4109 return 0;4110 4111 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);4112 i = mhop.l0_idx;4113 j = mhop.l1_idx;4114 k = mhop.l2_idx;4115 hop_num = mhop.hop_num;4116 chunk_ba_num = mhop.bt_chunk_size / 8;4117 4118 if (hop_num == 2) {4119 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num +4120 k;4121 l1_idx = i * chunk_ba_num + j;4122 } else if (hop_num == 1) {4123 hem_idx = i * chunk_ba_num + j;4124 } else if (hop_num == HNS_ROCE_HOP_NUM_0) {4125 hem_idx = i;4126 }4127 4128 if (table->type == HEM_TYPE_SCCC)4129 obj = mhop.l0_idx;4130 4131 if (check_whether_last_step(hop_num, step_idx)) {4132 hem = table->hem[hem_idx];4133 4134 ret = set_hem_to_hw(hr_dev, obj, hem->dma, table->type, step_idx);4135 } else {4136 if (step_idx == 0)4137 bt_ba = table->bt_l0_dma_addr[i];4138 else if (step_idx == 1 && hop_num == 2)4139 bt_ba = table->bt_l1_dma_addr[l1_idx];4140 4141 ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, step_idx);4142 }4143 4144 return ret;4145}4146 4147static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,4148 struct hns_roce_hem_table *table,4149 int tag, u32 step_idx)4150{4151 struct hns_roce_cmd_mailbox *mailbox;4152 struct device *dev = hr_dev->dev;4153 u8 cmd = 0xff;4154 int ret;4155 4156 if (!hns_roce_check_whether_mhop(hr_dev, table->type))4157 return 0;4158 4159 switch (table->type) {4160 case HEM_TYPE_QPC:4161 cmd = HNS_ROCE_CMD_DESTROY_QPC_BT0;4162 break;4163 case HEM_TYPE_MTPT:4164 cmd = HNS_ROCE_CMD_DESTROY_MPT_BT0;4165 break;4166 case HEM_TYPE_CQC:4167 cmd = HNS_ROCE_CMD_DESTROY_CQC_BT0;4168 break;4169 case HEM_TYPE_SRQC:4170 cmd = HNS_ROCE_CMD_DESTROY_SRQC_BT0;4171 break;4172 case HEM_TYPE_SCCC:4173 case HEM_TYPE_QPC_TIMER:4174 case HEM_TYPE_CQC_TIMER:4175 case HEM_TYPE_GMV:4176 return 0;4177 default:4178 dev_warn(dev, "table %u not to be destroyed by mailbox!\n",4179 table->type);4180 return 0;4181 }4182 4183 cmd += step_idx;4184 4185 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);4186 if (IS_ERR(mailbox))4187 return PTR_ERR(mailbox);4188 4189 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, cmd, tag);4190 4191 hns_roce_free_cmd_mailbox(hr_dev, mailbox);4192 return ret;4193}4194 4195static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,4196 struct hns_roce_v2_qp_context *context,4197 struct hns_roce_v2_qp_context *qpc_mask,4198 struct hns_roce_qp *hr_qp)4199{4200 struct hns_roce_cmd_mailbox *mailbox;4201 int qpc_size;4202 int ret;4203 4204 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);4205 if (IS_ERR(mailbox))4206 return PTR_ERR(mailbox);4207 4208 /* The qpc size of HIP08 is only 256B, which is half of HIP09 */4209 qpc_size = hr_dev->caps.qpc_sz;4210 memcpy(mailbox->buf, context, qpc_size);4211 memcpy(mailbox->buf + qpc_size, qpc_mask, qpc_size);4212 4213 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0,4214 HNS_ROCE_CMD_MODIFY_QPC, hr_qp->qpn);4215 4216 hns_roce_free_cmd_mailbox(hr_dev, mailbox);4217 4218 return ret;4219}4220 4221static void set_access_flags(struct hns_roce_qp *hr_qp,4222 struct hns_roce_v2_qp_context *context,4223 struct hns_roce_v2_qp_context *qpc_mask,4224 const struct ib_qp_attr *attr, int attr_mask)4225{4226 u8 dest_rd_atomic;4227 u32 access_flags;4228 4229 dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ?4230 attr->max_dest_rd_atomic : hr_qp->resp_depth;4231 4232 access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ?4233 attr->qp_access_flags : hr_qp->atomic_rd_en;4234 4235 if (!dest_rd_atomic)4236 access_flags &= IB_ACCESS_REMOTE_WRITE;4237 4238 hr_reg_write_bool(context, QPC_RRE,4239 access_flags & IB_ACCESS_REMOTE_READ);4240 hr_reg_clear(qpc_mask, QPC_RRE);4241 4242 hr_reg_write_bool(context, QPC_RWE,4243 access_flags & IB_ACCESS_REMOTE_WRITE);4244 hr_reg_clear(qpc_mask, QPC_RWE);4245 4246 hr_reg_write_bool(context, QPC_ATE,4247 access_flags & IB_ACCESS_REMOTE_ATOMIC);4248 hr_reg_clear(qpc_mask, QPC_ATE);4249 hr_reg_write_bool(context, QPC_EXT_ATE,4250 access_flags & IB_ACCESS_REMOTE_ATOMIC);4251 hr_reg_clear(qpc_mask, QPC_EXT_ATE);4252}4253 4254static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp,4255 struct hns_roce_v2_qp_context *context)4256{4257 hr_reg_write(context, QPC_SGE_SHIFT,4258 to_hr_hem_entries_shift(hr_qp->sge.sge_cnt,4259 hr_qp->sge.sge_shift));4260 4261 hr_reg_write(context, QPC_SQ_SHIFT, ilog2(hr_qp->sq.wqe_cnt));4262 4263 hr_reg_write(context, QPC_RQ_SHIFT, ilog2(hr_qp->rq.wqe_cnt));4264}4265 4266static inline int get_cqn(struct ib_cq *ib_cq)4267{4268 return ib_cq ? to_hr_cq(ib_cq)->cqn : 0;4269}4270 4271static inline int get_pdn(struct ib_pd *ib_pd)4272{4273 return ib_pd ? to_hr_pd(ib_pd)->pdn : 0;4274}4275 4276static void modify_qp_reset_to_init(struct ib_qp *ibqp,4277 struct hns_roce_v2_qp_context *context,4278 struct hns_roce_v2_qp_context *qpc_mask)4279{4280 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);4281 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);4282 4283 /*4284 * In v2 engine, software pass context and context mask to hardware4285 * when modifying qp. If software need modify some fields in context,4286 * we should set all bits of the relevant fields in context mask to4287 * 0 at the same time, else set them to 0x1.4288 */4289 hr_reg_write(context, QPC_TST, to_hr_qp_type(ibqp->qp_type));4290 4291 hr_reg_write(context, QPC_PD, get_pdn(ibqp->pd));4292 4293 hr_reg_write(context, QPC_RQWS, ilog2(hr_qp->rq.max_gs));4294 4295 set_qpc_wqe_cnt(hr_qp, context);4296 4297 /* No VLAN need to set 0xFFF */4298 hr_reg_write(context, QPC_VLAN_ID, 0xfff);4299 4300 if (ibqp->qp_type == IB_QPT_XRC_TGT) {4301 context->qkey_xrcd = cpu_to_le32(hr_qp->xrcdn);4302 4303 hr_reg_enable(context, QPC_XRC_QP_TYPE);4304 }4305 4306 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)4307 hr_reg_enable(context, QPC_RQ_RECORD_EN);4308 4309 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)4310 hr_reg_enable(context, QPC_OWNER_MODE);4311 4312 hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_L,4313 lower_32_bits(hr_qp->rdb.dma) >> 1);4314 hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_H,4315 upper_32_bits(hr_qp->rdb.dma));4316 4317 hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq));4318 4319 if (ibqp->srq) {4320 hr_reg_enable(context, QPC_SRQ_EN);4321 hr_reg_write(context, QPC_SRQN, to_hr_srq(ibqp->srq)->srqn);4322 }4323 4324 hr_reg_enable(context, QPC_FRE);4325 4326 hr_reg_write(context, QPC_TX_CQN, get_cqn(ibqp->send_cq));4327 4328 if (hr_dev->caps.qpc_sz < HNS_ROCE_V3_QPC_SZ)4329 return;4330 4331 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH)4332 hr_reg_enable(&context->ext, QPCEX_STASH);4333}4334 4335static void modify_qp_init_to_init(struct ib_qp *ibqp,4336 struct hns_roce_v2_qp_context *context,4337 struct hns_roce_v2_qp_context *qpc_mask)4338{4339 /*4340 * In v2 engine, software pass context and context mask to hardware4341 * when modifying qp. If software need modify some fields in context,4342 * we should set all bits of the relevant fields in context mask to4343 * 0 at the same time, else set them to 0x1.4344 */4345 hr_reg_write(context, QPC_TST, to_hr_qp_type(ibqp->qp_type));4346 hr_reg_clear(qpc_mask, QPC_TST);4347 4348 hr_reg_write(context, QPC_PD, get_pdn(ibqp->pd));4349 hr_reg_clear(qpc_mask, QPC_PD);4350 4351 hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq));4352 hr_reg_clear(qpc_mask, QPC_RX_CQN);4353 4354 hr_reg_write(context, QPC_TX_CQN, get_cqn(ibqp->send_cq));4355 hr_reg_clear(qpc_mask, QPC_TX_CQN);4356 4357 if (ibqp->srq) {4358 hr_reg_enable(context, QPC_SRQ_EN);4359 hr_reg_clear(qpc_mask, QPC_SRQ_EN);4360 hr_reg_write(context, QPC_SRQN, to_hr_srq(ibqp->srq)->srqn);4361 hr_reg_clear(qpc_mask, QPC_SRQN);4362 }4363}4364 4365static int config_qp_rq_buf(struct hns_roce_dev *hr_dev,4366 struct hns_roce_qp *hr_qp,4367 struct hns_roce_v2_qp_context *context,4368 struct hns_roce_v2_qp_context *qpc_mask)4369{4370 u64 mtts[MTT_MIN_COUNT] = { 0 };4371 u64 wqe_sge_ba;4372 int ret;4373 4374 /* Search qp buf's mtts */4375 ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts,4376 MTT_MIN_COUNT);4377 if (hr_qp->rq.wqe_cnt && ret) {4378 ibdev_err(&hr_dev->ib_dev,4379 "failed to find QP(0x%lx) RQ WQE buf, ret = %d.\n",4380 hr_qp->qpn, ret);4381 return ret;4382 }4383 4384 wqe_sge_ba = hns_roce_get_mtr_ba(&hr_qp->mtr);4385 4386 context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);4387 qpc_mask->wqe_sge_ba = 0;4388 4389 /*4390 * In v2 engine, software pass context and context mask to hardware4391 * when modifying qp. If software need modify some fields in context,4392 * we should set all bits of the relevant fields in context mask to4393 * 0 at the same time, else set them to 0x1.4394 */4395 hr_reg_write(context, QPC_WQE_SGE_BA_H, wqe_sge_ba >> (32 + 3));4396 hr_reg_clear(qpc_mask, QPC_WQE_SGE_BA_H);4397 4398 hr_reg_write(context, QPC_SQ_HOP_NUM,4399 to_hr_hem_hopnum(hr_dev->caps.wqe_sq_hop_num,4400 hr_qp->sq.wqe_cnt));4401 hr_reg_clear(qpc_mask, QPC_SQ_HOP_NUM);4402 4403 hr_reg_write(context, QPC_SGE_HOP_NUM,4404 to_hr_hem_hopnum(hr_dev->caps.wqe_sge_hop_num,4405 hr_qp->sge.sge_cnt));4406 hr_reg_clear(qpc_mask, QPC_SGE_HOP_NUM);4407 4408 hr_reg_write(context, QPC_RQ_HOP_NUM,4409 to_hr_hem_hopnum(hr_dev->caps.wqe_rq_hop_num,4410 hr_qp->rq.wqe_cnt));4411 4412 hr_reg_clear(qpc_mask, QPC_RQ_HOP_NUM);4413 4414 hr_reg_write(context, QPC_WQE_SGE_BA_PG_SZ,4415 to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.ba_pg_shift));4416 hr_reg_clear(qpc_mask, QPC_WQE_SGE_BA_PG_SZ);4417 4418 hr_reg_write(context, QPC_WQE_SGE_BUF_PG_SZ,4419 to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.buf_pg_shift));4420 hr_reg_clear(qpc_mask, QPC_WQE_SGE_BUF_PG_SZ);4421 4422 context->rq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));4423 qpc_mask->rq_cur_blk_addr = 0;4424 4425 hr_reg_write(context, QPC_RQ_CUR_BLK_ADDR_H,4426 upper_32_bits(to_hr_hw_page_addr(mtts[0])));4427 hr_reg_clear(qpc_mask, QPC_RQ_CUR_BLK_ADDR_H);4428 4429 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {4430 context->rq_nxt_blk_addr =4431 cpu_to_le32(to_hr_hw_page_addr(mtts[1]));4432 qpc_mask->rq_nxt_blk_addr = 0;4433 hr_reg_write(context, QPC_RQ_NXT_BLK_ADDR_H,4434 upper_32_bits(to_hr_hw_page_addr(mtts[1])));4435 hr_reg_clear(qpc_mask, QPC_RQ_NXT_BLK_ADDR_H);4436 }4437 4438 return 0;4439}4440 4441static int config_qp_sq_buf(struct hns_roce_dev *hr_dev,4442 struct hns_roce_qp *hr_qp,4443 struct hns_roce_v2_qp_context *context,4444 struct hns_roce_v2_qp_context *qpc_mask)4445{4446 struct ib_device *ibdev = &hr_dev->ib_dev;4447 u64 sge_cur_blk = 0;4448 u64 sq_cur_blk = 0;4449 int ret;4450 4451 /* search qp buf's mtts */4452 ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->sq.offset,4453 &sq_cur_blk, 1);4454 if (ret) {4455 ibdev_err(ibdev, "failed to find QP(0x%lx) SQ WQE buf, ret = %d.\n",4456 hr_qp->qpn, ret);4457 return ret;4458 }4459 if (hr_qp->sge.sge_cnt > 0) {4460 ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,4461 hr_qp->sge.offset, &sge_cur_blk, 1);4462 if (ret) {4463 ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf, ret = %d.\n",4464 hr_qp->qpn, ret);4465 return ret;4466 }4467 }4468 4469 /*4470 * In v2 engine, software pass context and context mask to hardware4471 * when modifying qp. If software need modify some fields in context,4472 * we should set all bits of the relevant fields in context mask to4473 * 0 at the same time, else set them to 0x1.4474 */4475 hr_reg_write(context, QPC_SQ_CUR_BLK_ADDR_L,4476 lower_32_bits(to_hr_hw_page_addr(sq_cur_blk)));4477 hr_reg_write(context, QPC_SQ_CUR_BLK_ADDR_H,4478 upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));4479 hr_reg_clear(qpc_mask, QPC_SQ_CUR_BLK_ADDR_L);4480 hr_reg_clear(qpc_mask, QPC_SQ_CUR_BLK_ADDR_H);4481 4482 hr_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_L,4483 lower_32_bits(to_hr_hw_page_addr(sge_cur_blk)));4484 hr_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_H,4485 upper_32_bits(to_hr_hw_page_addr(sge_cur_blk)));4486 hr_reg_clear(qpc_mask, QPC_SQ_CUR_SGE_BLK_ADDR_L);4487 hr_reg_clear(qpc_mask, QPC_SQ_CUR_SGE_BLK_ADDR_H);4488 4489 hr_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_L,4490 lower_32_bits(to_hr_hw_page_addr(sq_cur_blk)));4491 hr_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_H,4492 upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));4493 hr_reg_clear(qpc_mask, QPC_RX_SQ_CUR_BLK_ADDR_L);4494 hr_reg_clear(qpc_mask, QPC_RX_SQ_CUR_BLK_ADDR_H);4495 4496 return 0;4497}4498 4499static inline enum ib_mtu get_mtu(struct ib_qp *ibqp,4500 const struct ib_qp_attr *attr)4501{4502 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)4503 return IB_MTU_4096;4504 4505 return attr->path_mtu;4506}4507 4508static int modify_qp_init_to_rtr(struct ib_qp *ibqp,4509 const struct ib_qp_attr *attr, int attr_mask,4510 struct hns_roce_v2_qp_context *context,4511 struct hns_roce_v2_qp_context *qpc_mask,4512 struct ib_udata *udata)4513{4514 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata,4515 struct hns_roce_ucontext, ibucontext);4516 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);4517 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);4518 struct ib_device *ibdev = &hr_dev->ib_dev;4519 dma_addr_t trrl_ba;4520 dma_addr_t irrl_ba;4521 enum ib_mtu ib_mtu;4522 const u8 *smac;4523 u8 lp_pktn_ini;4524 u64 *mtts;4525 u8 *dmac;4526 u32 port;4527 int mtu;4528 int ret;4529 4530 ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask);4531 if (ret) {4532 ibdev_err(ibdev, "failed to config rq buf, ret = %d.\n", ret);4533 return ret;4534 }4535 4536 /* Search IRRL's mtts */4537 mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,4538 hr_qp->qpn, &irrl_ba);4539 if (!mtts) {4540 ibdev_err(ibdev, "failed to find qp irrl_table.\n");4541 return -EINVAL;4542 }4543 4544 /* Search TRRL's mtts */4545 mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,4546 hr_qp->qpn, &trrl_ba);4547 if (!mtts) {4548 ibdev_err(ibdev, "failed to find qp trrl_table.\n");4549 return -EINVAL;4550 }4551 4552 if (attr_mask & IB_QP_ALT_PATH) {4553 ibdev_err(ibdev, "INIT2RTR attr_mask (0x%x) error.\n",4554 attr_mask);4555 return -EINVAL;4556 }4557 4558 hr_reg_write(context, QPC_TRRL_BA_L, trrl_ba >> QPC_TRRL_BA_L_S);4559 hr_reg_clear(qpc_mask, QPC_TRRL_BA_L);4560 context->trrl_ba = cpu_to_le32(trrl_ba >> QPC_TRRL_BA_M_S);4561 qpc_mask->trrl_ba = 0;4562 hr_reg_write(context, QPC_TRRL_BA_H, trrl_ba >> QPC_TRRL_BA_H_S);4563 hr_reg_clear(qpc_mask, QPC_TRRL_BA_H);4564 4565 context->irrl_ba = cpu_to_le32(irrl_ba >> QPC_IRRL_BA_L_S);4566 qpc_mask->irrl_ba = 0;4567 hr_reg_write(context, QPC_IRRL_BA_H, irrl_ba >> QPC_IRRL_BA_H_S);4568 hr_reg_clear(qpc_mask, QPC_IRRL_BA_H);4569 4570 hr_reg_enable(context, QPC_RMT_E2E);4571 hr_reg_clear(qpc_mask, QPC_RMT_E2E);4572 4573 hr_reg_write(context, QPC_SIG_TYPE, hr_qp->sq_signal_bits);4574 hr_reg_clear(qpc_mask, QPC_SIG_TYPE);4575 4576 port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;4577 4578 smac = (const u8 *)hr_dev->dev_addr[port];4579 dmac = (u8 *)attr->ah_attr.roce.dmac;4580 /* when dmac equals smac or loop_idc is 1, it should loopback */4581 if (ether_addr_equal_unaligned(dmac, smac) ||4582 hr_dev->loop_idc == 0x1) {4583 hr_reg_write(context, QPC_LBI, hr_dev->loop_idc);4584 hr_reg_clear(qpc_mask, QPC_LBI);4585 }4586 4587 if (attr_mask & IB_QP_DEST_QPN) {4588 hr_reg_write(context, QPC_DQPN, attr->dest_qp_num);4589 hr_reg_clear(qpc_mask, QPC_DQPN);4590 }4591 4592 memcpy(&context->dmac, dmac, sizeof(u32));4593 hr_reg_write(context, QPC_DMAC_H, *((u16 *)(&dmac[4])));4594 qpc_mask->dmac = 0;4595 hr_reg_clear(qpc_mask, QPC_DMAC_H);4596 4597 ib_mtu = get_mtu(ibqp, attr);4598 hr_qp->path_mtu = ib_mtu;4599 4600 mtu = ib_mtu_enum_to_int(ib_mtu);4601 if (WARN_ON(mtu <= 0))4602 return -EINVAL;4603#define MIN_LP_MSG_LEN 10244604 /* mtu * (2 ^ lp_pktn_ini) should be in the range of 1024 to mtu */4605 lp_pktn_ini = ilog2(max(mtu, MIN_LP_MSG_LEN) / mtu);4606 4607 if (attr_mask & IB_QP_PATH_MTU) {4608 hr_reg_write(context, QPC_MTU, ib_mtu);4609 hr_reg_clear(qpc_mask, QPC_MTU);4610 }4611 4612 hr_reg_write(context, QPC_LP_PKTN_INI, lp_pktn_ini);4613 hr_reg_clear(qpc_mask, QPC_LP_PKTN_INI);4614 4615 /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */4616 hr_reg_write(context, QPC_ACK_REQ_FREQ, lp_pktn_ini);4617 hr_reg_clear(qpc_mask, QPC_ACK_REQ_FREQ);4618 4619 hr_reg_clear(qpc_mask, QPC_RX_REQ_PSN_ERR);4620 hr_reg_clear(qpc_mask, QPC_RX_REQ_MSN);4621 hr_reg_clear(qpc_mask, QPC_RX_REQ_LAST_OPTYPE);4622 4623 context->rq_rnr_timer = 0;4624 qpc_mask->rq_rnr_timer = 0;4625 4626 hr_reg_clear(qpc_mask, QPC_TRRL_HEAD_MAX);4627 hr_reg_clear(qpc_mask, QPC_TRRL_TAIL_MAX);4628 4629#define MAX_LP_SGEN 34630 /* rocee send 2^lp_sgen_ini segs every time */4631 hr_reg_write(context, QPC_LP_SGEN_INI, MAX_LP_SGEN);4632 hr_reg_clear(qpc_mask, QPC_LP_SGEN_INI);4633 4634 if (udata && ibqp->qp_type == IB_QPT_RC &&4635 (uctx->config & HNS_ROCE_RQ_INLINE_FLAGS)) {4636 hr_reg_write_bool(context, QPC_RQIE,4637 hr_dev->caps.flags &4638 HNS_ROCE_CAP_FLAG_RQ_INLINE);4639 hr_reg_clear(qpc_mask, QPC_RQIE);4640 }4641 4642 if (udata &&4643 (ibqp->qp_type == IB_QPT_RC || ibqp->qp_type == IB_QPT_XRC_TGT) &&4644 (uctx->config & HNS_ROCE_CQE_INLINE_FLAGS)) {4645 hr_reg_write_bool(context, QPC_CQEIE,4646 hr_dev->caps.flags &4647 HNS_ROCE_CAP_FLAG_CQE_INLINE);4648 hr_reg_clear(qpc_mask, QPC_CQEIE);4649 4650 hr_reg_write(context, QPC_CQEIS, 0);4651 hr_reg_clear(qpc_mask, QPC_CQEIS);4652 }4653 4654 return 0;4655}4656 4657static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, int attr_mask,4658 struct hns_roce_v2_qp_context *context,4659 struct hns_roce_v2_qp_context *qpc_mask)4660{4661 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);4662 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);4663 struct ib_device *ibdev = &hr_dev->ib_dev;4664 int ret;4665 4666 /* Not support alternate path and path migration */4667 if (attr_mask & (IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE)) {4668 ibdev_err(ibdev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);4669 return -EINVAL;4670 }4671 4672 ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask);4673 if (ret) {4674 ibdev_err(ibdev, "failed to config sq buf, ret = %d.\n", ret);4675 return ret;4676 }4677 4678 /*4679 * Set some fields in context to zero, Because the default values4680 * of all fields in context are zero, we need not set them to 0 again.4681 * but we should set the relevant fields of context mask to 0.4682 */4683 hr_reg_clear(qpc_mask, QPC_IRRL_SGE_IDX);4684 4685 hr_reg_clear(qpc_mask, QPC_RX_ACK_MSN);4686 4687 hr_reg_clear(qpc_mask, QPC_ACK_LAST_OPTYPE);4688 hr_reg_clear(qpc_mask, QPC_IRRL_PSN_VLD);4689 hr_reg_clear(qpc_mask, QPC_IRRL_PSN);4690 4691 hr_reg_clear(qpc_mask, QPC_IRRL_TAIL_REAL);4692 4693 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_MSN);4694 4695 hr_reg_clear(qpc_mask, QPC_RNR_RETRY_FLAG);4696 4697 hr_reg_clear(qpc_mask, QPC_CHECK_FLG);4698 4699 hr_reg_clear(qpc_mask, QPC_V2_IRRL_HEAD);4700 4701 return 0;4702}4703 4704static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr,4705 u32 *dip_idx)4706{4707 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);4708 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);4709 u32 *spare_idx = hr_dev->qp_table.idx_table.spare_idx;4710 u32 *head = &hr_dev->qp_table.idx_table.head;4711 u32 *tail = &hr_dev->qp_table.idx_table.tail;4712 struct hns_roce_dip *hr_dip;4713 unsigned long flags;4714 int ret = 0;4715 4716 spin_lock_irqsave(&hr_dev->dip_list_lock, flags);4717 4718 spare_idx[*tail] = ibqp->qp_num;4719 *tail = (*tail == hr_dev->caps.num_qps - 1) ? 0 : (*tail + 1);4720 4721 list_for_each_entry(hr_dip, &hr_dev->dip_list, node) {4722 if (!memcmp(grh->dgid.raw, hr_dip->dgid, GID_LEN_V2)) {4723 *dip_idx = hr_dip->dip_idx;4724 goto out;4725 }4726 }4727 4728 /* If no dgid is found, a new dip and a mapping between dgid and4729 * dip_idx will be created.4730 */4731 hr_dip = kzalloc(sizeof(*hr_dip), GFP_ATOMIC);4732 if (!hr_dip) {4733 ret = -ENOMEM;4734 goto out;4735 }4736 4737 memcpy(hr_dip->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));4738 hr_dip->dip_idx = *dip_idx = spare_idx[*head];4739 *head = (*head == hr_dev->caps.num_qps - 1) ? 0 : (*head + 1);4740 list_add_tail(&hr_dip->node, &hr_dev->dip_list);4741 4742out:4743 spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags);4744 return ret;4745}4746 4747enum {4748 CONG_DCQCN,4749 CONG_WINDOW,4750};4751 4752enum {4753 UNSUPPORT_CONG_LEVEL,4754 SUPPORT_CONG_LEVEL,4755};4756 4757enum {4758 CONG_LDCP,4759 CONG_HC3,4760};4761 4762enum {4763 DIP_INVALID,4764 DIP_VALID,4765};4766 4767enum {4768 WND_LIMIT,4769 WND_UNLIMIT,4770};4771 4772static int check_cong_type(struct ib_qp *ibqp,4773 struct hns_roce_congestion_algorithm *cong_alg)4774{4775 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);4776 4777 /* different congestion types match different configurations */4778 switch (hr_qp->cong_type) {4779 case CONG_TYPE_DCQCN:4780 cong_alg->alg_sel = CONG_DCQCN;4781 cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL;4782 cong_alg->dip_vld = DIP_INVALID;4783 cong_alg->wnd_mode_sel = WND_LIMIT;4784 break;4785 case CONG_TYPE_LDCP:4786 cong_alg->alg_sel = CONG_WINDOW;4787 cong_alg->alg_sub_sel = CONG_LDCP;4788 cong_alg->dip_vld = DIP_INVALID;4789 cong_alg->wnd_mode_sel = WND_UNLIMIT;4790 break;4791 case CONG_TYPE_HC3:4792 cong_alg->alg_sel = CONG_WINDOW;4793 cong_alg->alg_sub_sel = CONG_HC3;4794 cong_alg->dip_vld = DIP_INVALID;4795 cong_alg->wnd_mode_sel = WND_LIMIT;4796 break;4797 case CONG_TYPE_DIP:4798 cong_alg->alg_sel = CONG_DCQCN;4799 cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL;4800 cong_alg->dip_vld = DIP_VALID;4801 cong_alg->wnd_mode_sel = WND_LIMIT;4802 break;4803 default:4804 hr_qp->cong_type = CONG_TYPE_DCQCN;4805 cong_alg->alg_sel = CONG_DCQCN;4806 cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL;4807 cong_alg->dip_vld = DIP_INVALID;4808 cong_alg->wnd_mode_sel = WND_LIMIT;4809 break;4810 }4811 4812 return 0;4813}4814 4815static int fill_cong_field(struct ib_qp *ibqp, const struct ib_qp_attr *attr,4816 struct hns_roce_v2_qp_context *context,4817 struct hns_roce_v2_qp_context *qpc_mask)4818{4819 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);4820 struct hns_roce_congestion_algorithm cong_field;4821 struct ib_device *ibdev = ibqp->device;4822 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);4823 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);4824 u32 dip_idx = 0;4825 int ret;4826 4827 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 ||4828 grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE)4829 return 0;4830 4831 ret = check_cong_type(ibqp, &cong_field);4832 if (ret)4833 return ret;4834 4835 hr_reg_write(context, QPC_CONG_ALGO_TMPL_ID, hr_dev->cong_algo_tmpl_id +4836 hr_qp->cong_type * HNS_ROCE_CONG_SIZE);4837 hr_reg_clear(qpc_mask, QPC_CONG_ALGO_TMPL_ID);4838 hr_reg_write(&context->ext, QPCEX_CONG_ALG_SEL, cong_field.alg_sel);4839 hr_reg_clear(&qpc_mask->ext, QPCEX_CONG_ALG_SEL);4840 hr_reg_write(&context->ext, QPCEX_CONG_ALG_SUB_SEL,4841 cong_field.alg_sub_sel);4842 hr_reg_clear(&qpc_mask->ext, QPCEX_CONG_ALG_SUB_SEL);4843 hr_reg_write(&context->ext, QPCEX_DIP_CTX_IDX_VLD, cong_field.dip_vld);4844 hr_reg_clear(&qpc_mask->ext, QPCEX_DIP_CTX_IDX_VLD);4845 hr_reg_write(&context->ext, QPCEX_SQ_RQ_NOT_FORBID_EN,4846 cong_field.wnd_mode_sel);4847 hr_reg_clear(&qpc_mask->ext, QPCEX_SQ_RQ_NOT_FORBID_EN);4848 4849 /* if dip is disabled, there is no need to set dip idx */4850 if (cong_field.dip_vld == 0)4851 return 0;4852 4853 ret = get_dip_ctx_idx(ibqp, attr, &dip_idx);4854 if (ret) {4855 ibdev_err(ibdev, "failed to fill cong field, ret = %d.\n", ret);4856 return ret;4857 }4858 4859 hr_reg_write(&context->ext, QPCEX_DIP_CTX_IDX, dip_idx);4860 hr_reg_write(&qpc_mask->ext, QPCEX_DIP_CTX_IDX, 0);4861 4862 return 0;4863}4864 4865static int hns_roce_hw_v2_get_dscp(struct hns_roce_dev *hr_dev, u8 dscp,4866 u8 *tc_mode, u8 *priority)4867{4868 struct hns_roce_v2_priv *priv = hr_dev->priv;4869 struct hnae3_handle *handle = priv->handle;4870 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;4871 4872 if (!ops->get_dscp_prio)4873 return -EOPNOTSUPP;4874 4875 return ops->get_dscp_prio(handle, dscp, tc_mode, priority);4876}4877 4878bool check_sl_valid(struct hns_roce_dev *hr_dev, u8 sl)4879{4880 u32 max_sl;4881 4882 max_sl = min_t(u32, MAX_SERVICE_LEVEL, hr_dev->caps.sl_num - 1);4883 if (unlikely(sl > max_sl)) {4884 ibdev_err_ratelimited(&hr_dev->ib_dev,4885 "failed to set SL(%u). Shouldn't be larger than %u.\n",4886 sl, max_sl);4887 return false;4888 }4889 4890 return true;4891}4892 4893static int hns_roce_set_sl(struct ib_qp *ibqp,4894 const struct ib_qp_attr *attr,4895 struct hns_roce_v2_qp_context *context,4896 struct hns_roce_v2_qp_context *qpc_mask)4897{4898 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);4899 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);4900 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);4901 struct ib_device *ibdev = &hr_dev->ib_dev;4902 int ret;4903 4904 ret = hns_roce_hw_v2_get_dscp(hr_dev, get_tclass(&attr->ah_attr.grh),4905 &hr_qp->tc_mode, &hr_qp->priority);4906 if (ret && ret != -EOPNOTSUPP &&4907 grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {4908 ibdev_err_ratelimited(ibdev,4909 "failed to get dscp, ret = %d.\n", ret);4910 return ret;4911 }4912 4913 if (hr_qp->tc_mode == HNAE3_TC_MAP_MODE_DSCP &&4914 grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)4915 hr_qp->sl = hr_qp->priority;4916 else4917 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);4918 4919 if (!check_sl_valid(hr_dev, hr_qp->sl))4920 return -EINVAL;4921 4922 hr_reg_write(context, QPC_SL, hr_qp->sl);4923 hr_reg_clear(qpc_mask, QPC_SL);4924 4925 return 0;4926}4927 4928static int hns_roce_v2_set_path(struct ib_qp *ibqp,4929 const struct ib_qp_attr *attr,4930 int attr_mask,4931 struct hns_roce_v2_qp_context *context,4932 struct hns_roce_v2_qp_context *qpc_mask)4933{4934 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);4935 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);4936 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);4937 struct ib_device *ibdev = &hr_dev->ib_dev;4938 const struct ib_gid_attr *gid_attr = NULL;4939 u8 sl = rdma_ah_get_sl(&attr->ah_attr);4940 int is_roce_protocol;4941 u16 vlan_id = 0xffff;4942 bool is_udp = false;4943 u8 ib_port;4944 u8 hr_port;4945 int ret;4946 4947 /*4948 * If free_mr_en of qp is set, it means that this qp comes from4949 * free mr. This qp will perform the loopback operation.4950 * In the loopback scenario, only sl needs to be set.4951 */4952 if (hr_qp->free_mr_en) {4953 if (!check_sl_valid(hr_dev, sl))4954 return -EINVAL;4955 hr_reg_write(context, QPC_SL, sl);4956 hr_reg_clear(qpc_mask, QPC_SL);4957 hr_qp->sl = sl;4958 return 0;4959 }4960 4961 ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1;4962 hr_port = ib_port - 1;4963 is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&4964 rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;4965 4966 if (is_roce_protocol) {4967 gid_attr = attr->ah_attr.grh.sgid_attr;4968 ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);4969 if (ret)4970 return ret;4971 4972 is_udp = (gid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP);4973 }4974 4975 /* Only HIP08 needs to set the vlan_en bits in QPC */4976 if (vlan_id < VLAN_N_VID &&4977 hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {4978 hr_reg_enable(context, QPC_RQ_VLAN_EN);4979 hr_reg_clear(qpc_mask, QPC_RQ_VLAN_EN);4980 hr_reg_enable(context, QPC_SQ_VLAN_EN);4981 hr_reg_clear(qpc_mask, QPC_SQ_VLAN_EN);4982 }4983 4984 hr_reg_write(context, QPC_VLAN_ID, vlan_id);4985 hr_reg_clear(qpc_mask, QPC_VLAN_ID);4986 4987 if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {4988 ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n",4989 grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]);4990 return -EINVAL;4991 }4992 4993 if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {4994 ibdev_err(ibdev, "ah attr is not RDMA roce type\n");4995 return -EINVAL;4996 }4997 4998 hr_reg_write(context, QPC_UDPSPN,4999 is_udp ? rdma_get_udp_sport(grh->flow_label, ibqp->qp_num,5000 attr->dest_qp_num) :5001 0);5002 5003 hr_reg_clear(qpc_mask, QPC_UDPSPN);5004 5005 hr_reg_write(context, QPC_GMV_IDX, grh->sgid_index);5006 5007 hr_reg_clear(qpc_mask, QPC_GMV_IDX);5008 5009 hr_reg_write(context, QPC_HOPLIMIT, grh->hop_limit);5010 hr_reg_clear(qpc_mask, QPC_HOPLIMIT);5011 5012 ret = fill_cong_field(ibqp, attr, context, qpc_mask);5013 if (ret)5014 return ret;5015 5016 hr_reg_write(context, QPC_TC, get_tclass(&attr->ah_attr.grh));5017 hr_reg_clear(qpc_mask, QPC_TC);5018 5019 hr_reg_write(context, QPC_FL, grh->flow_label);5020 hr_reg_clear(qpc_mask, QPC_FL);5021 memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));5022 memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));5023 5024 return hns_roce_set_sl(ibqp, attr, context, qpc_mask);5025}5026 5027static bool check_qp_state(enum ib_qp_state cur_state,5028 enum ib_qp_state new_state)5029{5030 static const bool sm[][IB_QPS_ERR + 1] = {5031 [IB_QPS_RESET] = { [IB_QPS_RESET] = true,5032 [IB_QPS_INIT] = true },5033 [IB_QPS_INIT] = { [IB_QPS_RESET] = true,5034 [IB_QPS_INIT] = true,5035 [IB_QPS_RTR] = true,5036 [IB_QPS_ERR] = true },5037 [IB_QPS_RTR] = { [IB_QPS_RESET] = true,5038 [IB_QPS_RTS] = true,5039 [IB_QPS_ERR] = true },5040 [IB_QPS_RTS] = { [IB_QPS_RESET] = true,5041 [IB_QPS_RTS] = true,5042 [IB_QPS_ERR] = true },5043 [IB_QPS_SQD] = {},5044 [IB_QPS_SQE] = {},5045 [IB_QPS_ERR] = { [IB_QPS_RESET] = true,5046 [IB_QPS_ERR] = true }5047 };5048 5049 return sm[cur_state][new_state];5050}5051 5052static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,5053 const struct ib_qp_attr *attr,5054 int attr_mask,5055 enum ib_qp_state cur_state,5056 enum ib_qp_state new_state,5057 struct hns_roce_v2_qp_context *context,5058 struct hns_roce_v2_qp_context *qpc_mask,5059 struct ib_udata *udata)5060{5061 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);5062 int ret = 0;5063 5064 if (!check_qp_state(cur_state, new_state)) {5065 ibdev_err(&hr_dev->ib_dev, "Illegal state for QP!\n");5066 return -EINVAL;5067 }5068 5069 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {5070 memset(qpc_mask, 0, hr_dev->caps.qpc_sz);5071 modify_qp_reset_to_init(ibqp, context, qpc_mask);5072 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {5073 modify_qp_init_to_init(ibqp, context, qpc_mask);5074 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {5075 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,5076 qpc_mask, udata);5077 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {5078 ret = modify_qp_rtr_to_rts(ibqp, attr_mask, context, qpc_mask);5079 }5080 5081 return ret;5082}5083 5084static bool check_qp_timeout_cfg_range(struct hns_roce_dev *hr_dev, u8 *timeout)5085{5086#define QP_ACK_TIMEOUT_MAX_HIP08 205087#define QP_ACK_TIMEOUT_MAX 315088 5089 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {5090 if (*timeout > QP_ACK_TIMEOUT_MAX_HIP08) {5091 ibdev_warn(&hr_dev->ib_dev,5092 "local ACK timeout shall be 0 to 20.\n");5093 return false;5094 }5095 *timeout += HNS_ROCE_V2_QP_ACK_TIMEOUT_OFS_HIP08;5096 } else if (hr_dev->pci_dev->revision > PCI_REVISION_ID_HIP08) {5097 if (*timeout > QP_ACK_TIMEOUT_MAX) {5098 ibdev_warn(&hr_dev->ib_dev,5099 "local ACK timeout shall be 0 to 31.\n");5100 return false;5101 }5102 }5103 5104 return true;5105}5106 5107static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,5108 const struct ib_qp_attr *attr,5109 int attr_mask,5110 struct hns_roce_v2_qp_context *context,5111 struct hns_roce_v2_qp_context *qpc_mask)5112{5113 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);5114 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);5115 int ret = 0;5116 u8 timeout;5117 5118 if (attr_mask & IB_QP_AV) {5119 ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context,5120 qpc_mask);5121 if (ret)5122 return ret;5123 }5124 5125 if (attr_mask & IB_QP_TIMEOUT) {5126 timeout = attr->timeout;5127 if (check_qp_timeout_cfg_range(hr_dev, &timeout)) {5128 hr_reg_write(context, QPC_AT, timeout);5129 hr_reg_clear(qpc_mask, QPC_AT);5130 }5131 }5132 5133 if (attr_mask & IB_QP_RETRY_CNT) {5134 hr_reg_write(context, QPC_RETRY_NUM_INIT, attr->retry_cnt);5135 hr_reg_clear(qpc_mask, QPC_RETRY_NUM_INIT);5136 5137 hr_reg_write(context, QPC_RETRY_CNT, attr->retry_cnt);5138 hr_reg_clear(qpc_mask, QPC_RETRY_CNT);5139 }5140 5141 if (attr_mask & IB_QP_RNR_RETRY) {5142 hr_reg_write(context, QPC_RNR_NUM_INIT, attr->rnr_retry);5143 hr_reg_clear(qpc_mask, QPC_RNR_NUM_INIT);5144 5145 hr_reg_write(context, QPC_RNR_CNT, attr->rnr_retry);5146 hr_reg_clear(qpc_mask, QPC_RNR_CNT);5147 }5148 5149 if (attr_mask & IB_QP_SQ_PSN) {5150 hr_reg_write(context, QPC_SQ_CUR_PSN, attr->sq_psn);5151 hr_reg_clear(qpc_mask, QPC_SQ_CUR_PSN);5152 5153 hr_reg_write(context, QPC_SQ_MAX_PSN, attr->sq_psn);5154 hr_reg_clear(qpc_mask, QPC_SQ_MAX_PSN);5155 5156 hr_reg_write(context, QPC_RETRY_MSG_PSN_L, attr->sq_psn);5157 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_PSN_L);5158 5159 hr_reg_write(context, QPC_RETRY_MSG_PSN_H,5160 attr->sq_psn >> RETRY_MSG_PSN_SHIFT);5161 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_PSN_H);5162 5163 hr_reg_write(context, QPC_RETRY_MSG_FPKT_PSN, attr->sq_psn);5164 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_FPKT_PSN);5165 5166 hr_reg_write(context, QPC_RX_ACK_EPSN, attr->sq_psn);5167 hr_reg_clear(qpc_mask, QPC_RX_ACK_EPSN);5168 }5169 5170 if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&5171 attr->max_dest_rd_atomic) {5172 hr_reg_write(context, QPC_RR_MAX,5173 fls(attr->max_dest_rd_atomic - 1));5174 hr_reg_clear(qpc_mask, QPC_RR_MAX);5175 }5176 5177 if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {5178 hr_reg_write(context, QPC_SR_MAX, fls(attr->max_rd_atomic - 1));5179 hr_reg_clear(qpc_mask, QPC_SR_MAX);5180 }5181 5182 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))5183 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);5184 5185 if (attr_mask & IB_QP_MIN_RNR_TIMER) {5186 hr_reg_write(context, QPC_MIN_RNR_TIME,5187 hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 ?5188 HNS_ROCE_RNR_TIMER_10NS : attr->min_rnr_timer);5189 hr_reg_clear(qpc_mask, QPC_MIN_RNR_TIME);5190 }5191 5192 if (attr_mask & IB_QP_RQ_PSN) {5193 hr_reg_write(context, QPC_RX_REQ_EPSN, attr->rq_psn);5194 hr_reg_clear(qpc_mask, QPC_RX_REQ_EPSN);5195 5196 hr_reg_write(context, QPC_RAQ_PSN, attr->rq_psn - 1);5197 hr_reg_clear(qpc_mask, QPC_RAQ_PSN);5198 }5199 5200 if (attr_mask & IB_QP_QKEY) {5201 context->qkey_xrcd = cpu_to_le32(attr->qkey);5202 qpc_mask->qkey_xrcd = 0;5203 hr_qp->qkey = attr->qkey;5204 }5205 5206 return ret;5207}5208 5209static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp,5210 const struct ib_qp_attr *attr,5211 int attr_mask)5212{5213 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);5214 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);5215 5216 if (attr_mask & IB_QP_ACCESS_FLAGS)5217 hr_qp->atomic_rd_en = attr->qp_access_flags;5218 5219 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)5220 hr_qp->resp_depth = attr->max_dest_rd_atomic;5221 if (attr_mask & IB_QP_PORT) {5222 hr_qp->port = attr->port_num - 1;5223 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];5224 }5225}5226 5227static void clear_qp(struct hns_roce_qp *hr_qp)5228{5229 struct ib_qp *ibqp = &hr_qp->ibqp;5230 5231 if (ibqp->send_cq)5232 hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq),5233 hr_qp->qpn, NULL);5234 5235 if (ibqp->recv_cq && ibqp->recv_cq != ibqp->send_cq)5236 hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq),5237 hr_qp->qpn, ibqp->srq ?5238 to_hr_srq(ibqp->srq) : NULL);5239 5240 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)5241 *hr_qp->rdb.db_record = 0;5242 5243 hr_qp->rq.head = 0;5244 hr_qp->rq.tail = 0;5245 hr_qp->sq.head = 0;5246 hr_qp->sq.tail = 0;5247 hr_qp->next_sge = 0;5248}5249 5250static void v2_set_flushed_fields(struct ib_qp *ibqp,5251 struct hns_roce_v2_qp_context *context,5252 struct hns_roce_v2_qp_context *qpc_mask)5253{5254 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);5255 unsigned long sq_flag = 0;5256 unsigned long rq_flag = 0;5257 5258 if (ibqp->qp_type == IB_QPT_XRC_TGT)5259 return;5260 5261 spin_lock_irqsave(&hr_qp->sq.lock, sq_flag);5262 hr_reg_write(context, QPC_SQ_PRODUCER_IDX, hr_qp->sq.head);5263 hr_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX);5264 hr_qp->state = IB_QPS_ERR;5265 spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag);5266 5267 if (ibqp->srq || ibqp->qp_type == IB_QPT_XRC_INI) /* no RQ */5268 return;5269 5270 spin_lock_irqsave(&hr_qp->rq.lock, rq_flag);5271 hr_reg_write(context, QPC_RQ_PRODUCER_IDX, hr_qp->rq.head);5272 hr_reg_clear(qpc_mask, QPC_RQ_PRODUCER_IDX);5273 spin_unlock_irqrestore(&hr_qp->rq.lock, rq_flag);5274}5275 5276static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,5277 const struct ib_qp_attr *attr,5278 int attr_mask, enum ib_qp_state cur_state,5279 enum ib_qp_state new_state, struct ib_udata *udata)5280{5281 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);5282 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);5283 struct hns_roce_v2_qp_context ctx[2];5284 struct hns_roce_v2_qp_context *context = ctx;5285 struct hns_roce_v2_qp_context *qpc_mask = ctx + 1;5286 struct ib_device *ibdev = &hr_dev->ib_dev;5287 int ret;5288 5289 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)5290 return -EOPNOTSUPP;5291 5292 /*5293 * In v2 engine, software pass context and context mask to hardware5294 * when modifying qp. If software need modify some fields in context,5295 * we should set all bits of the relevant fields in context mask to5296 * 0 at the same time, else set them to 0x1.5297 */5298 memset(context, 0, hr_dev->caps.qpc_sz);5299 memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz);5300 5301 ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state,5302 new_state, context, qpc_mask, udata);5303 if (ret)5304 goto out;5305 5306 /* When QP state is err, SQ and RQ WQE should be flushed */5307 if (new_state == IB_QPS_ERR)5308 v2_set_flushed_fields(ibqp, context, qpc_mask);5309 5310 /* Configure the optional fields */5311 ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context,5312 qpc_mask);5313 if (ret)5314 goto out;5315 5316 hr_reg_write_bool(context, QPC_INV_CREDIT,5317 to_hr_qp_type(hr_qp->ibqp.qp_type) == SERV_TYPE_XRC ||5318 ibqp->srq);5319 hr_reg_clear(qpc_mask, QPC_INV_CREDIT);5320 5321 /* Every status migrate must change state */5322 hr_reg_write(context, QPC_QP_ST, new_state);5323 hr_reg_clear(qpc_mask, QPC_QP_ST);5324 5325 /* SW pass context to HW */5326 ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp);5327 if (ret) {5328 ibdev_err(ibdev, "failed to modify QP, ret = %d.\n", ret);5329 goto out;5330 }5331 5332 hr_qp->state = new_state;5333 5334 hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask);5335 5336 if (new_state == IB_QPS_RESET && !ibqp->uobject)5337 clear_qp(hr_qp);5338 5339out:5340 return ret;5341}5342 5343static int to_ib_qp_st(enum hns_roce_v2_qp_state state)5344{5345 static const enum ib_qp_state map[] = {5346 [HNS_ROCE_QP_ST_RST] = IB_QPS_RESET,5347 [HNS_ROCE_QP_ST_INIT] = IB_QPS_INIT,5348 [HNS_ROCE_QP_ST_RTR] = IB_QPS_RTR,5349 [HNS_ROCE_QP_ST_RTS] = IB_QPS_RTS,5350 [HNS_ROCE_QP_ST_SQD] = IB_QPS_SQD,5351 [HNS_ROCE_QP_ST_SQER] = IB_QPS_SQE,5352 [HNS_ROCE_QP_ST_ERR] = IB_QPS_ERR,5353 [HNS_ROCE_QP_ST_SQ_DRAINING] = IB_QPS_SQD5354 };5355 5356 return (state < ARRAY_SIZE(map)) ? map[state] : -1;5357}5358 5359static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, u32 qpn,5360 void *buffer)5361{5362 struct hns_roce_cmd_mailbox *mailbox;5363 int ret;5364 5365 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);5366 if (IS_ERR(mailbox))5367 return PTR_ERR(mailbox);5368 5369 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_QPC,5370 qpn);5371 if (ret)5372 goto out;5373 5374 memcpy(buffer, mailbox->buf, hr_dev->caps.qpc_sz);5375 5376out:5377 hns_roce_free_cmd_mailbox(hr_dev, mailbox);5378 return ret;5379}5380 5381static int hns_roce_v2_query_srqc(struct hns_roce_dev *hr_dev, u32 srqn,5382 void *buffer)5383{5384 struct hns_roce_srq_context *context;5385 struct hns_roce_cmd_mailbox *mailbox;5386 int ret;5387 5388 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);5389 if (IS_ERR(mailbox))5390 return PTR_ERR(mailbox);5391 5392 context = mailbox->buf;5393 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_SRQC,5394 srqn);5395 if (ret)5396 goto out;5397 5398 memcpy(buffer, context, sizeof(*context));5399 5400out:5401 hns_roce_free_cmd_mailbox(hr_dev, mailbox);5402 return ret;5403}5404 5405static int hns_roce_v2_query_sccc(struct hns_roce_dev *hr_dev, u32 qpn,5406 void *buffer)5407{5408 struct hns_roce_v2_scc_context *context;5409 struct hns_roce_cmd_mailbox *mailbox;5410 int ret;5411 5412 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);5413 if (IS_ERR(mailbox))5414 return PTR_ERR(mailbox);5415 5416 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_SCCC,5417 qpn);5418 if (ret)5419 goto out;5420 5421 context = mailbox->buf;5422 memcpy(buffer, context, sizeof(*context));5423 5424out:5425 hns_roce_free_cmd_mailbox(hr_dev, mailbox);5426 return ret;5427}5428 5429static u8 get_qp_timeout_attr(struct hns_roce_dev *hr_dev,5430 struct hns_roce_v2_qp_context *context)5431{5432 u8 timeout;5433 5434 timeout = (u8)hr_reg_read(context, QPC_AT);5435 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)5436 timeout -= HNS_ROCE_V2_QP_ACK_TIMEOUT_OFS_HIP08;5437 5438 return timeout;5439}5440 5441static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,5442 int qp_attr_mask,5443 struct ib_qp_init_attr *qp_init_attr)5444{5445 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);5446 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);5447 struct hns_roce_v2_qp_context context = {};5448 struct ib_device *ibdev = &hr_dev->ib_dev;5449 int tmp_qp_state;5450 int state;5451 int ret;5452 5453 memset(qp_attr, 0, sizeof(*qp_attr));5454 memset(qp_init_attr, 0, sizeof(*qp_init_attr));5455 5456 mutex_lock(&hr_qp->mutex);5457 5458 if (hr_qp->state == IB_QPS_RESET) {5459 qp_attr->qp_state = IB_QPS_RESET;5460 ret = 0;5461 goto done;5462 }5463 5464 ret = hns_roce_v2_query_qpc(hr_dev, hr_qp->qpn, &context);5465 if (ret) {5466 ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret);5467 ret = -EINVAL;5468 goto out;5469 }5470 5471 state = hr_reg_read(&context, QPC_QP_ST);5472 tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);5473 if (tmp_qp_state == -1) {5474 ibdev_err(ibdev, "Illegal ib_qp_state\n");5475 ret = -EINVAL;5476 goto out;5477 }5478 hr_qp->state = (u8)tmp_qp_state;5479 qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;5480 qp_attr->path_mtu = (enum ib_mtu)hr_reg_read(&context, QPC_MTU);5481 qp_attr->path_mig_state = IB_MIG_ARMED;5482 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;5483 if (hr_qp->ibqp.qp_type == IB_QPT_UD)5484 qp_attr->qkey = le32_to_cpu(context.qkey_xrcd);5485 5486 qp_attr->rq_psn = hr_reg_read(&context, QPC_RX_REQ_EPSN);5487 qp_attr->sq_psn = (u32)hr_reg_read(&context, QPC_SQ_CUR_PSN);5488 qp_attr->dest_qp_num = hr_reg_read(&context, QPC_DQPN);5489 qp_attr->qp_access_flags =5490 ((hr_reg_read(&context, QPC_RRE)) << V2_QP_RRE_S) |5491 ((hr_reg_read(&context, QPC_RWE)) << V2_QP_RWE_S) |5492 ((hr_reg_read(&context, QPC_ATE)) << V2_QP_ATE_S);5493 5494 if (hr_qp->ibqp.qp_type == IB_QPT_RC ||5495 hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||5496 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT) {5497 struct ib_global_route *grh =5498 rdma_ah_retrieve_grh(&qp_attr->ah_attr);5499 5500 rdma_ah_set_sl(&qp_attr->ah_attr,5501 hr_reg_read(&context, QPC_SL));5502 rdma_ah_set_port_num(&qp_attr->ah_attr, hr_qp->port + 1);5503 rdma_ah_set_ah_flags(&qp_attr->ah_attr, IB_AH_GRH);5504 grh->flow_label = hr_reg_read(&context, QPC_FL);5505 grh->sgid_index = hr_reg_read(&context, QPC_GMV_IDX);5506 grh->hop_limit = hr_reg_read(&context, QPC_HOPLIMIT);5507 grh->traffic_class = hr_reg_read(&context, QPC_TC);5508 5509 memcpy(grh->dgid.raw, context.dgid, sizeof(grh->dgid.raw));5510 }5511 5512 qp_attr->port_num = hr_qp->port + 1;5513 qp_attr->sq_draining = 0;5514 qp_attr->max_rd_atomic = 1 << hr_reg_read(&context, QPC_SR_MAX);5515 qp_attr->max_dest_rd_atomic = 1 << hr_reg_read(&context, QPC_RR_MAX);5516 5517 qp_attr->min_rnr_timer = (u8)hr_reg_read(&context, QPC_MIN_RNR_TIME);5518 qp_attr->timeout = get_qp_timeout_attr(hr_dev, &context);5519 qp_attr->retry_cnt = hr_reg_read(&context, QPC_RETRY_NUM_INIT);5520 qp_attr->rnr_retry = hr_reg_read(&context, QPC_RNR_NUM_INIT);5521 5522done:5523 qp_attr->cur_qp_state = qp_attr->qp_state;5524 qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;5525 qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;5526 qp_attr->cap.max_inline_data = hr_qp->max_inline_data;5527 5528 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;5529 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;5530 5531 qp_init_attr->qp_context = ibqp->qp_context;5532 qp_init_attr->qp_type = ibqp->qp_type;5533 qp_init_attr->recv_cq = ibqp->recv_cq;5534 qp_init_attr->send_cq = ibqp->send_cq;5535 qp_init_attr->srq = ibqp->srq;5536 qp_init_attr->cap = qp_attr->cap;5537 qp_init_attr->sq_sig_type = hr_qp->sq_signal_bits;5538 5539out:5540 mutex_unlock(&hr_qp->mutex);5541 return ret;5542}5543 5544static inline int modify_qp_is_ok(struct hns_roce_qp *hr_qp)5545{5546 return ((hr_qp->ibqp.qp_type == IB_QPT_RC ||5547 hr_qp->ibqp.qp_type == IB_QPT_UD ||5548 hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||5549 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT) &&5550 hr_qp->state != IB_QPS_RESET);5551}5552 5553static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,5554 struct hns_roce_qp *hr_qp,5555 struct ib_udata *udata)5556{5557 struct ib_device *ibdev = &hr_dev->ib_dev;5558 struct hns_roce_cq *send_cq, *recv_cq;5559 unsigned long flags;5560 int ret = 0;5561 5562 if (modify_qp_is_ok(hr_qp)) {5563 /* Modify qp to reset before destroying qp */5564 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,5565 hr_qp->state, IB_QPS_RESET, udata);5566 if (ret)5567 ibdev_err(ibdev,5568 "failed to modify QP to RST, ret = %d.\n",5569 ret);5570 }5571 5572 send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL;5573 recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL;5574 5575 spin_lock_irqsave(&hr_dev->qp_list_lock, flags);5576 hns_roce_lock_cqs(send_cq, recv_cq);5577 5578 if (!udata) {5579 if (recv_cq)5580 __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn,5581 (hr_qp->ibqp.srq ?5582 to_hr_srq(hr_qp->ibqp.srq) :5583 NULL));5584 5585 if (send_cq && send_cq != recv_cq)5586 __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL);5587 }5588 5589 hns_roce_qp_remove(hr_dev, hr_qp);5590 5591 hns_roce_unlock_cqs(send_cq, recv_cq);5592 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);5593 5594 return ret;5595}5596 5597int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)5598{5599 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);5600 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);5601 int ret;5602 5603 ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);5604 if (ret)5605 ibdev_err(&hr_dev->ib_dev,5606 "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n",5607 hr_qp->qpn, ret);5608 5609 hns_roce_qp_destroy(hr_dev, hr_qp, udata);5610 5611 return 0;5612}5613 5614static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,5615 struct hns_roce_qp *hr_qp)5616{5617 struct ib_device *ibdev = &hr_dev->ib_dev;5618 struct hns_roce_sccc_clr_done *resp;5619 struct hns_roce_sccc_clr *clr;5620 struct hns_roce_cmq_desc desc;5621 int ret, i;5622 5623 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)5624 return 0;5625 5626 mutex_lock(&hr_dev->qp_table.scc_mutex);5627 5628 /* set scc ctx clear done flag */5629 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false);5630 ret = hns_roce_cmq_send(hr_dev, &desc, 1);5631 if (ret) {5632 ibdev_err(ibdev, "failed to reset SCC ctx, ret = %d.\n", ret);5633 goto out;5634 }5635 5636 /* clear scc context */5637 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false);5638 clr = (struct hns_roce_sccc_clr *)desc.data;5639 clr->qpn = cpu_to_le32(hr_qp->qpn);5640 ret = hns_roce_cmq_send(hr_dev, &desc, 1);5641 if (ret) {5642 ibdev_err(ibdev, "failed to clear SCC ctx, ret = %d.\n", ret);5643 goto out;5644 }5645 5646 /* query scc context clear is done or not */5647 resp = (struct hns_roce_sccc_clr_done *)desc.data;5648 for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) {5649 hns_roce_cmq_setup_basic_desc(&desc,5650 HNS_ROCE_OPC_QUERY_SCCC, true);5651 ret = hns_roce_cmq_send(hr_dev, &desc, 1);5652 if (ret) {5653 ibdev_err(ibdev, "failed to query clr cmq, ret = %d\n",5654 ret);5655 goto out;5656 }5657 5658 if (resp->clr_done)5659 goto out;5660 5661 msleep(20);5662 }5663 5664 ibdev_err(ibdev, "query SCC clr done flag overtime.\n");5665 ret = -ETIMEDOUT;5666 5667out:5668 mutex_unlock(&hr_dev->qp_table.scc_mutex);5669 return ret;5670}5671 5672#define DMA_IDX_SHIFT 35673#define DMA_WQE_SHIFT 35674 5675static int hns_roce_v2_write_srqc_index_queue(struct hns_roce_srq *srq,5676 struct hns_roce_srq_context *ctx)5677{5678 struct hns_roce_idx_que *idx_que = &srq->idx_que;5679 struct ib_device *ibdev = srq->ibsrq.device;5680 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);5681 u64 mtts_idx[MTT_MIN_COUNT] = {};5682 dma_addr_t dma_handle_idx;5683 int ret;5684 5685 /* Get physical address of idx que buf */5686 ret = hns_roce_mtr_find(hr_dev, &idx_que->mtr, 0, mtts_idx,5687 ARRAY_SIZE(mtts_idx));5688 if (ret) {5689 ibdev_err(ibdev, "failed to find mtr for SRQ idx, ret = %d.\n",5690 ret);5691 return ret;5692 }5693 5694 dma_handle_idx = hns_roce_get_mtr_ba(&idx_que->mtr);5695 5696 hr_reg_write(ctx, SRQC_IDX_HOP_NUM,5697 to_hr_hem_hopnum(hr_dev->caps.idx_hop_num, srq->wqe_cnt));5698 5699 hr_reg_write(ctx, SRQC_IDX_BT_BA_L, dma_handle_idx >> DMA_IDX_SHIFT);5700 hr_reg_write(ctx, SRQC_IDX_BT_BA_H,5701 upper_32_bits(dma_handle_idx >> DMA_IDX_SHIFT));5702 5703 hr_reg_write(ctx, SRQC_IDX_BA_PG_SZ,5704 to_hr_hw_page_shift(idx_que->mtr.hem_cfg.ba_pg_shift));5705 hr_reg_write(ctx, SRQC_IDX_BUF_PG_SZ,5706 to_hr_hw_page_shift(idx_que->mtr.hem_cfg.buf_pg_shift));5707 5708 hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_L,5709 to_hr_hw_page_addr(mtts_idx[0]));5710 hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_H,5711 upper_32_bits(to_hr_hw_page_addr(mtts_idx[0])));5712 5713 hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_L,5714 to_hr_hw_page_addr(mtts_idx[1]));5715 hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_H,5716 upper_32_bits(to_hr_hw_page_addr(mtts_idx[1])));5717 5718 return 0;5719}5720 5721static int hns_roce_v2_write_srqc(struct hns_roce_srq *srq, void *mb_buf)5722{5723 struct ib_device *ibdev = srq->ibsrq.device;5724 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);5725 struct hns_roce_srq_context *ctx = mb_buf;5726 u64 mtts_wqe[MTT_MIN_COUNT] = {};5727 dma_addr_t dma_handle_wqe;5728 int ret;5729 5730 memset(ctx, 0, sizeof(*ctx));5731 5732 /* Get the physical address of srq buf */5733 ret = hns_roce_mtr_find(hr_dev, &srq->buf_mtr, 0, mtts_wqe,5734 ARRAY_SIZE(mtts_wqe));5735 if (ret) {5736 ibdev_err(ibdev, "failed to find mtr for SRQ WQE, ret = %d.\n",5737 ret);5738 return ret;5739 }5740 5741 dma_handle_wqe = hns_roce_get_mtr_ba(&srq->buf_mtr);5742 5743 hr_reg_write(ctx, SRQC_SRQ_ST, 1);5744 hr_reg_write_bool(ctx, SRQC_SRQ_TYPE,5745 srq->ibsrq.srq_type == IB_SRQT_XRC);5746 hr_reg_write(ctx, SRQC_PD, to_hr_pd(srq->ibsrq.pd)->pdn);5747 hr_reg_write(ctx, SRQC_SRQN, srq->srqn);5748 hr_reg_write(ctx, SRQC_XRCD, srq->xrcdn);5749 hr_reg_write(ctx, SRQC_XRC_CQN, srq->cqn);5750 hr_reg_write(ctx, SRQC_SHIFT, ilog2(srq->wqe_cnt));5751 hr_reg_write(ctx, SRQC_RQWS,5752 srq->max_gs <= 0 ? 0 : fls(srq->max_gs - 1));5753 5754 hr_reg_write(ctx, SRQC_WQE_HOP_NUM,5755 to_hr_hem_hopnum(hr_dev->caps.srqwqe_hop_num,5756 srq->wqe_cnt));5757 5758 hr_reg_write(ctx, SRQC_WQE_BT_BA_L, dma_handle_wqe >> DMA_WQE_SHIFT);5759 hr_reg_write(ctx, SRQC_WQE_BT_BA_H,5760 upper_32_bits(dma_handle_wqe >> DMA_WQE_SHIFT));5761 5762 hr_reg_write(ctx, SRQC_WQE_BA_PG_SZ,5763 to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.ba_pg_shift));5764 hr_reg_write(ctx, SRQC_WQE_BUF_PG_SZ,5765 to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.buf_pg_shift));5766 5767 if (srq->cap_flags & HNS_ROCE_SRQ_CAP_RECORD_DB) {5768 hr_reg_enable(ctx, SRQC_DB_RECORD_EN);5769 hr_reg_write(ctx, SRQC_DB_RECORD_ADDR_L,5770 lower_32_bits(srq->rdb.dma) >> 1);5771 hr_reg_write(ctx, SRQC_DB_RECORD_ADDR_H,5772 upper_32_bits(srq->rdb.dma));5773 }5774 5775 return hns_roce_v2_write_srqc_index_queue(srq, ctx);5776}5777 5778static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,5779 struct ib_srq_attr *srq_attr,5780 enum ib_srq_attr_mask srq_attr_mask,5781 struct ib_udata *udata)5782{5783 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);5784 struct hns_roce_srq *srq = to_hr_srq(ibsrq);5785 struct hns_roce_srq_context *srq_context;5786 struct hns_roce_srq_context *srqc_mask;5787 struct hns_roce_cmd_mailbox *mailbox;5788 int ret = 0;5789 5790 /* Resizing SRQs is not supported yet */5791 if (srq_attr_mask & IB_SRQ_MAX_WR) {5792 ret = -EOPNOTSUPP;5793 goto out;5794 }5795 5796 if (srq_attr_mask & IB_SRQ_LIMIT) {5797 if (srq_attr->srq_limit > srq->wqe_cnt) {5798 ret = -EINVAL;5799 goto out;5800 }5801 5802 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);5803 if (IS_ERR(mailbox)) {5804 ret = PTR_ERR(mailbox);5805 goto out;5806 }5807 5808 srq_context = mailbox->buf;5809 srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;5810 5811 memset(srqc_mask, 0xff, sizeof(*srqc_mask));5812 5813 hr_reg_write(srq_context, SRQC_LIMIT_WL, srq_attr->srq_limit);5814 hr_reg_clear(srqc_mask, SRQC_LIMIT_WL);5815 5816 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0,5817 HNS_ROCE_CMD_MODIFY_SRQC, srq->srqn);5818 hns_roce_free_cmd_mailbox(hr_dev, mailbox);5819 if (ret)5820 ibdev_err(&hr_dev->ib_dev,5821 "failed to handle cmd of modifying SRQ, ret = %d.\n",5822 ret);5823 }5824 5825out:5826 if (ret)5827 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_SRQ_MODIFY_ERR_CNT]);5828 5829 return ret;5830}5831 5832static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)5833{5834 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);5835 struct hns_roce_srq *srq = to_hr_srq(ibsrq);5836 struct hns_roce_srq_context *srq_context;5837 struct hns_roce_cmd_mailbox *mailbox;5838 int ret;5839 5840 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);5841 if (IS_ERR(mailbox))5842 return PTR_ERR(mailbox);5843 5844 srq_context = mailbox->buf;5845 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma,5846 HNS_ROCE_CMD_QUERY_SRQC, srq->srqn);5847 if (ret) {5848 ibdev_err(&hr_dev->ib_dev,5849 "failed to process cmd of querying SRQ, ret = %d.\n",5850 ret);5851 goto out;5852 }5853 5854 attr->srq_limit = hr_reg_read(srq_context, SRQC_LIMIT_WL);5855 attr->max_wr = srq->wqe_cnt;5856 attr->max_sge = srq->max_gs - srq->rsv_sge;5857 5858out:5859 hns_roce_free_cmd_mailbox(hr_dev, mailbox);5860 return ret;5861}5862 5863static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)5864{5865 struct hns_roce_dev *hr_dev = to_hr_dev(cq->device);5866 struct hns_roce_v2_cq_context *cq_context;5867 struct hns_roce_cq *hr_cq = to_hr_cq(cq);5868 struct hns_roce_v2_cq_context *cqc_mask;5869 struct hns_roce_cmd_mailbox *mailbox;5870 int ret;5871 5872 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);5873 ret = PTR_ERR_OR_ZERO(mailbox);5874 if (ret)5875 goto err_out;5876 5877 cq_context = mailbox->buf;5878 cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1;5879 5880 memset(cqc_mask, 0xff, sizeof(*cqc_mask));5881 5882 hr_reg_write(cq_context, CQC_CQ_MAX_CNT, cq_count);5883 hr_reg_clear(cqc_mask, CQC_CQ_MAX_CNT);5884 5885 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {5886 if (cq_period * HNS_ROCE_CLOCK_ADJUST > USHRT_MAX) {5887 dev_info(hr_dev->dev,5888 "cq_period(%u) reached the upper limit, adjusted to 65.\n",5889 cq_period);5890 cq_period = HNS_ROCE_MAX_CQ_PERIOD_HIP08;5891 }5892 cq_period *= HNS_ROCE_CLOCK_ADJUST;5893 }5894 hr_reg_write(cq_context, CQC_CQ_PERIOD, cq_period);5895 hr_reg_clear(cqc_mask, CQC_CQ_PERIOD);5896 5897 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0,5898 HNS_ROCE_CMD_MODIFY_CQC, hr_cq->cqn);5899 hns_roce_free_cmd_mailbox(hr_dev, mailbox);5900 if (ret)5901 ibdev_err(&hr_dev->ib_dev,5902 "failed to process cmd when modifying CQ, ret = %d.\n",5903 ret);5904 5905err_out:5906 if (ret)5907 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CQ_MODIFY_ERR_CNT]);5908 5909 return ret;5910}5911 5912static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn,5913 void *buffer)5914{5915 struct hns_roce_v2_cq_context *context;5916 struct hns_roce_cmd_mailbox *mailbox;5917 int ret;5918 5919 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);5920 if (IS_ERR(mailbox))5921 return PTR_ERR(mailbox);5922 5923 context = mailbox->buf;5924 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma,5925 HNS_ROCE_CMD_QUERY_CQC, cqn);5926 if (ret) {5927 ibdev_err(&hr_dev->ib_dev,5928 "failed to process cmd when querying CQ, ret = %d.\n",5929 ret);5930 goto err_mailbox;5931 }5932 5933 memcpy(buffer, context, sizeof(*context));5934 5935err_mailbox:5936 hns_roce_free_cmd_mailbox(hr_dev, mailbox);5937 5938 return ret;5939}5940 5941static int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key,5942 void *buffer)5943{5944 struct hns_roce_v2_mpt_entry *context;5945 struct hns_roce_cmd_mailbox *mailbox;5946 int ret;5947 5948 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);5949 if (IS_ERR(mailbox))5950 return PTR_ERR(mailbox);5951 5952 context = mailbox->buf;5953 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT,5954 key_to_hw_index(key));5955 if (ret) {5956 ibdev_err(&hr_dev->ib_dev,5957 "failed to process cmd when querying MPT, ret = %d.\n",5958 ret);5959 goto err_mailbox;5960 }5961 5962 memcpy(buffer, context, sizeof(*context));5963 5964err_mailbox:5965 hns_roce_free_cmd_mailbox(hr_dev, mailbox);5966 5967 return ret;5968}5969 5970static void hns_roce_irq_work_handle(struct work_struct *work)5971{5972 struct hns_roce_work *irq_work =5973 container_of(work, struct hns_roce_work, work);5974 struct ib_device *ibdev = &irq_work->hr_dev->ib_dev;5975 5976 switch (irq_work->event_type) {5977 case HNS_ROCE_EVENT_TYPE_PATH_MIG:5978 ibdev_info(ibdev, "path migrated succeeded.\n");5979 break;5980 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:5981 ibdev_warn(ibdev, "path migration failed.\n");5982 break;5983 case HNS_ROCE_EVENT_TYPE_COMM_EST:5984 break;5985 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:5986 ibdev_dbg(ibdev, "send queue drained.\n");5987 break;5988 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:5989 ibdev_err(ibdev, "local work queue 0x%x catast error, sub_event type is: %d\n",5990 irq_work->queue_num, irq_work->sub_type);5991 break;5992 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:5993 ibdev_err(ibdev, "invalid request local work queue 0x%x error.\n",5994 irq_work->queue_num);5995 break;5996 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:5997 ibdev_err(ibdev, "local access violation work queue 0x%x error, sub_event type is: %d\n",5998 irq_work->queue_num, irq_work->sub_type);5999 break;6000 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:6001 ibdev_dbg(ibdev, "SRQ limit reach.\n");6002 break;6003 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:6004 ibdev_dbg(ibdev, "SRQ last wqe reach.\n");6005 break;6006 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:6007 ibdev_err(ibdev, "SRQ catas error.\n");6008 break;6009 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:6010 ibdev_err(ibdev, "CQ 0x%x access err.\n", irq_work->queue_num);6011 break;6012 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:6013 ibdev_warn(ibdev, "CQ 0x%x overflow\n", irq_work->queue_num);6014 break;6015 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:6016 ibdev_warn(ibdev, "DB overflow.\n");6017 break;6018 case HNS_ROCE_EVENT_TYPE_FLR:6019 ibdev_warn(ibdev, "function level reset.\n");6020 break;6021 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:6022 ibdev_err(ibdev, "xrc domain violation error.\n");6023 break;6024 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:6025 ibdev_err(ibdev, "invalid xrceth error.\n");6026 break;6027 default:6028 break;6029 }6030 6031 kfree(irq_work);6032}6033 6034static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,6035 struct hns_roce_eq *eq, u32 queue_num)6036{6037 struct hns_roce_work *irq_work;6038 6039 irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC);6040 if (!irq_work)6041 return;6042 6043 INIT_WORK(&irq_work->work, hns_roce_irq_work_handle);6044 irq_work->hr_dev = hr_dev;6045 irq_work->event_type = eq->event_type;6046 irq_work->sub_type = eq->sub_type;6047 irq_work->queue_num = queue_num;6048 queue_work(hr_dev->irq_workq, &irq_work->work);6049}6050 6051static void update_eq_db(struct hns_roce_eq *eq)6052{6053 struct hns_roce_dev *hr_dev = eq->hr_dev;6054 struct hns_roce_v2_db eq_db = {};6055 6056 if (eq->type_flag == HNS_ROCE_AEQ) {6057 hr_reg_write(&eq_db, EQ_DB_CMD,6058 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?6059 HNS_ROCE_EQ_DB_CMD_AEQ :6060 HNS_ROCE_EQ_DB_CMD_AEQ_ARMED);6061 } else {6062 hr_reg_write(&eq_db, EQ_DB_TAG, eq->eqn);6063 6064 hr_reg_write(&eq_db, EQ_DB_CMD,6065 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?6066 HNS_ROCE_EQ_DB_CMD_CEQ :6067 HNS_ROCE_EQ_DB_CMD_CEQ_ARMED);6068 }6069 6070 hr_reg_write(&eq_db, EQ_DB_CI, eq->cons_index);6071 6072 hns_roce_write64(hr_dev, (__le32 *)&eq_db, eq->db_reg);6073}6074 6075static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq)6076{6077 struct hns_roce_aeqe *aeqe;6078 6079 aeqe = hns_roce_buf_offset(eq->mtr.kmem,6080 (eq->cons_index & (eq->entries - 1)) *6081 eq->eqe_size);6082 6083 return (hr_reg_read(aeqe, AEQE_OWNER) ^6084 !!(eq->cons_index & eq->entries)) ? aeqe : NULL;6085}6086 6087static irqreturn_t hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,6088 struct hns_roce_eq *eq)6089{6090 struct device *dev = hr_dev->dev;6091 struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq);6092 irqreturn_t aeqe_found = IRQ_NONE;6093 int event_type;6094 u32 queue_num;6095 int sub_type;6096 6097 while (aeqe) {6098 /* Make sure we read AEQ entry after we have checked the6099 * ownership bit6100 */6101 dma_rmb();6102 6103 event_type = hr_reg_read(aeqe, AEQE_EVENT_TYPE);6104 sub_type = hr_reg_read(aeqe, AEQE_SUB_TYPE);6105 queue_num = hr_reg_read(aeqe, AEQE_EVENT_QUEUE_NUM);6106 6107 switch (event_type) {6108 case HNS_ROCE_EVENT_TYPE_PATH_MIG:6109 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:6110 case HNS_ROCE_EVENT_TYPE_COMM_EST:6111 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:6112 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:6113 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:6114 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:6115 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:6116 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:6117 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:6118 hns_roce_qp_event(hr_dev, queue_num, event_type);6119 break;6120 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:6121 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:6122 hns_roce_srq_event(hr_dev, queue_num, event_type);6123 break;6124 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:6125 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:6126 hns_roce_cq_event(hr_dev, queue_num, event_type);6127 break;6128 case HNS_ROCE_EVENT_TYPE_MB:6129 hns_roce_cmd_event(hr_dev,6130 le16_to_cpu(aeqe->event.cmd.token),6131 aeqe->event.cmd.status,6132 le64_to_cpu(aeqe->event.cmd.out_param));6133 break;6134 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:6135 case HNS_ROCE_EVENT_TYPE_FLR:6136 break;6137 default:6138 dev_err(dev, "unhandled event %d on EQ %d at idx %u.\n",6139 event_type, eq->eqn, eq->cons_index);6140 break;6141 }6142 6143 eq->event_type = event_type;6144 eq->sub_type = sub_type;6145 ++eq->cons_index;6146 aeqe_found = IRQ_HANDLED;6147 6148 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_AEQE_CNT]);6149 6150 hns_roce_v2_init_irq_work(hr_dev, eq, queue_num);6151 6152 aeqe = next_aeqe_sw_v2(eq);6153 }6154 6155 update_eq_db(eq);6156 6157 return IRQ_RETVAL(aeqe_found);6158}6159 6160static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq)6161{6162 struct hns_roce_ceqe *ceqe;6163 6164 ceqe = hns_roce_buf_offset(eq->mtr.kmem,6165 (eq->cons_index & (eq->entries - 1)) *6166 eq->eqe_size);6167 6168 return (hr_reg_read(ceqe, CEQE_OWNER) ^6169 !!(eq->cons_index & eq->entries)) ? ceqe : NULL;6170}6171 6172static irqreturn_t hns_roce_v2_ceq_int(struct hns_roce_eq *eq)6173{6174 queue_work(system_bh_wq, &eq->work);6175 6176 return IRQ_HANDLED;6177}6178 6179static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr)6180{6181 struct hns_roce_eq *eq = eq_ptr;6182 struct hns_roce_dev *hr_dev = eq->hr_dev;6183 irqreturn_t int_work;6184 6185 if (eq->type_flag == HNS_ROCE_CEQ)6186 /* Completion event interrupt */6187 int_work = hns_roce_v2_ceq_int(eq);6188 else6189 /* Asynchronous event interrupt */6190 int_work = hns_roce_v2_aeq_int(hr_dev, eq);6191 6192 return IRQ_RETVAL(int_work);6193}6194 6195static irqreturn_t abnormal_interrupt_basic(struct hns_roce_dev *hr_dev,6196 u32 int_st)6197{6198 struct pci_dev *pdev = hr_dev->pci_dev;6199 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);6200 const struct hnae3_ae_ops *ops = ae_dev->ops;6201 enum hnae3_reset_type reset_type;6202 irqreturn_t int_work = IRQ_NONE;6203 u32 int_en;6204 6205 int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);6206 6207 if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {6208 dev_err(hr_dev->dev, "AEQ overflow!\n");6209 6210 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG,6211 1 << HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S);6212 6213 reset_type = hr_dev->is_vf ?6214 HNAE3_VF_FUNC_RESET : HNAE3_FUNC_RESET;6215 6216 /* Set reset level for reset_event() */6217 if (ops->set_default_reset_request)6218 ops->set_default_reset_request(ae_dev, reset_type);6219 if (ops->reset_event)6220 ops->reset_event(pdev, NULL);6221 6222 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;6223 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);6224 6225 int_work = IRQ_HANDLED;6226 } else {6227 dev_err(hr_dev->dev, "there is no basic abn irq found.\n");6228 }6229 6230 return IRQ_RETVAL(int_work);6231}6232 6233static int fmea_ram_ecc_query(struct hns_roce_dev *hr_dev,6234 struct fmea_ram_ecc *ecc_info)6235{6236 struct hns_roce_cmq_desc desc;6237 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;6238 int ret;6239 6240 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_QUERY_RAM_ECC, true);6241 ret = hns_roce_cmq_send(hr_dev, &desc, 1);6242 if (ret)6243 return ret;6244 6245 ecc_info->is_ecc_err = hr_reg_read(req, QUERY_RAM_ECC_1BIT_ERR);6246 ecc_info->res_type = hr_reg_read(req, QUERY_RAM_ECC_RES_TYPE);6247 ecc_info->index = hr_reg_read(req, QUERY_RAM_ECC_TAG);6248 6249 return 0;6250}6251 6252static int fmea_recover_gmv(struct hns_roce_dev *hr_dev, u32 idx)6253{6254 struct hns_roce_cmq_desc desc;6255 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;6256 u32 addr_upper;6257 u32 addr_low;6258 int ret;6259 6260 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, true);6261 hr_reg_write(req, CFG_GMV_BT_IDX, idx);6262 6263 ret = hns_roce_cmq_send(hr_dev, &desc, 1);6264 if (ret) {6265 dev_err(hr_dev->dev,6266 "failed to execute cmd to read gmv, ret = %d.\n", ret);6267 return ret;6268 }6269 6270 addr_low = hr_reg_read(req, CFG_GMV_BT_BA_L);6271 addr_upper = hr_reg_read(req, CFG_GMV_BT_BA_H);6272 6273 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false);6274 hr_reg_write(req, CFG_GMV_BT_BA_L, addr_low);6275 hr_reg_write(req, CFG_GMV_BT_BA_H, addr_upper);6276 hr_reg_write(req, CFG_GMV_BT_IDX, idx);6277 6278 return hns_roce_cmq_send(hr_dev, &desc, 1);6279}6280 6281static u64 fmea_get_ram_res_addr(u32 res_type, __le64 *data)6282{6283 if (res_type == ECC_RESOURCE_QPC_TIMER ||6284 res_type == ECC_RESOURCE_CQC_TIMER ||6285 res_type == ECC_RESOURCE_SCCC)6286 return le64_to_cpu(*data);6287 6288 return le64_to_cpu(*data) << HNS_HW_PAGE_SHIFT;6289}6290 6291static int fmea_recover_others(struct hns_roce_dev *hr_dev, u32 res_type,6292 u32 index)6293{6294 u8 write_bt0_op = fmea_ram_res[res_type].write_bt0_op;6295 u8 read_bt0_op = fmea_ram_res[res_type].read_bt0_op;6296 struct hns_roce_cmd_mailbox *mailbox;6297 u64 addr;6298 int ret;6299 6300 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);6301 if (IS_ERR(mailbox))6302 return PTR_ERR(mailbox);6303 6304 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, read_bt0_op, index);6305 if (ret) {6306 dev_err(hr_dev->dev,6307 "failed to execute cmd to read fmea ram, ret = %d.\n",6308 ret);6309 goto out;6310 }6311 6312 addr = fmea_get_ram_res_addr(res_type, mailbox->buf);6313 6314 ret = hns_roce_cmd_mbox(hr_dev, addr, 0, write_bt0_op, index);6315 if (ret)6316 dev_err(hr_dev->dev,6317 "failed to execute cmd to write fmea ram, ret = %d.\n",6318 ret);6319 6320out:6321 hns_roce_free_cmd_mailbox(hr_dev, mailbox);6322 return ret;6323}6324 6325static void fmea_ram_ecc_recover(struct hns_roce_dev *hr_dev,6326 struct fmea_ram_ecc *ecc_info)6327{6328 u32 res_type = ecc_info->res_type;6329 u32 index = ecc_info->index;6330 int ret;6331 6332 BUILD_BUG_ON(ARRAY_SIZE(fmea_ram_res) != ECC_RESOURCE_COUNT);6333 6334 if (res_type >= ECC_RESOURCE_COUNT) {6335 dev_err(hr_dev->dev, "unsupported fmea ram ecc type %u.\n",6336 res_type);6337 return;6338 }6339 6340 if (res_type == ECC_RESOURCE_GMV)6341 ret = fmea_recover_gmv(hr_dev, index);6342 else6343 ret = fmea_recover_others(hr_dev, res_type, index);6344 if (ret)6345 dev_err(hr_dev->dev,6346 "failed to recover %s, index = %u, ret = %d.\n",6347 fmea_ram_res[res_type].name, index, ret);6348}6349 6350static void fmea_ram_ecc_work(struct work_struct *ecc_work)6351{6352 struct hns_roce_dev *hr_dev =6353 container_of(ecc_work, struct hns_roce_dev, ecc_work);6354 struct fmea_ram_ecc ecc_info = {};6355 6356 if (fmea_ram_ecc_query(hr_dev, &ecc_info)) {6357 dev_err(hr_dev->dev, "failed to query fmea ram ecc.\n");6358 return;6359 }6360 6361 if (!ecc_info.is_ecc_err) {6362 dev_err(hr_dev->dev, "there is no fmea ram ecc err found.\n");6363 return;6364 }6365 6366 fmea_ram_ecc_recover(hr_dev, &ecc_info);6367}6368 6369static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id)6370{6371 struct hns_roce_dev *hr_dev = dev_id;6372 irqreturn_t int_work = IRQ_NONE;6373 u32 int_st;6374 6375 int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);6376 6377 if (int_st) {6378 int_work = abnormal_interrupt_basic(hr_dev, int_st);6379 } else if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {6380 queue_work(hr_dev->irq_workq, &hr_dev->ecc_work);6381 int_work = IRQ_HANDLED;6382 } else {6383 dev_err(hr_dev->dev, "there is no abnormal irq found.\n");6384 }6385 6386 return IRQ_RETVAL(int_work);6387}6388 6389static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev,6390 int eq_num, u32 enable_flag)6391{6392 int i;6393 6394 for (i = 0; i < eq_num; i++)6395 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +6396 i * EQ_REG_OFFSET, enable_flag);6397 6398 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, enable_flag);6399 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, enable_flag);6400}6401 6402static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)6403{6404 hns_roce_mtr_destroy(hr_dev, &eq->mtr);6405}6406 6407static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev,6408 struct hns_roce_eq *eq)6409{6410 struct device *dev = hr_dev->dev;6411 int eqn = eq->eqn;6412 int ret;6413 u8 cmd;6414 6415 if (eqn < hr_dev->caps.num_comp_vectors)6416 cmd = HNS_ROCE_CMD_DESTROY_CEQC;6417 else6418 cmd = HNS_ROCE_CMD_DESTROY_AEQC;6419 6420 ret = hns_roce_destroy_hw_ctx(hr_dev, cmd, eqn & HNS_ROCE_V2_EQN_M);6421 if (ret)6422 dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);6423 6424 free_eq_buf(hr_dev, eq);6425}6426 6427static void init_eq_config(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)6428{6429 eq->db_reg = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;6430 eq->cons_index = 0;6431 eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;6432 eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;6433 eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;6434 eq->shift = ilog2((unsigned int)eq->entries);6435}6436 6437static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq,6438 void *mb_buf)6439{6440 u64 eqe_ba[MTT_MIN_COUNT] = { 0 };6441 struct hns_roce_eq_context *eqc;6442 u64 bt_ba = 0;6443 int ret;6444 6445 eqc = mb_buf;6446 memset(eqc, 0, sizeof(struct hns_roce_eq_context));6447 6448 init_eq_config(hr_dev, eq);6449 6450 /* if not multi-hop, eqe buffer only use one trunk */6451 ret = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, eqe_ba,6452 ARRAY_SIZE(eqe_ba));6453 if (ret) {6454 dev_err(hr_dev->dev, "failed to find EQE mtr, ret = %d\n", ret);6455 return ret;6456 }6457 6458 bt_ba = hns_roce_get_mtr_ba(&eq->mtr);6459 6460 hr_reg_write(eqc, EQC_EQ_ST, HNS_ROCE_V2_EQ_STATE_VALID);6461 hr_reg_write(eqc, EQC_EQE_HOP_NUM, eq->hop_num);6462 hr_reg_write(eqc, EQC_OVER_IGNORE, eq->over_ignore);6463 hr_reg_write(eqc, EQC_COALESCE, eq->coalesce);6464 hr_reg_write(eqc, EQC_ARM_ST, eq->arm_st);6465 hr_reg_write(eqc, EQC_EQN, eq->eqn);6466 hr_reg_write(eqc, EQC_EQE_CNT, HNS_ROCE_EQ_INIT_EQE_CNT);6467 hr_reg_write(eqc, EQC_EQE_BA_PG_SZ,6468 to_hr_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift));6469 hr_reg_write(eqc, EQC_EQE_BUF_PG_SZ,6470 to_hr_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift));6471 hr_reg_write(eqc, EQC_EQ_PROD_INDX, HNS_ROCE_EQ_INIT_PROD_IDX);6472 hr_reg_write(eqc, EQC_EQ_MAX_CNT, eq->eq_max_cnt);6473 6474 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {6475 if (eq->eq_period * HNS_ROCE_CLOCK_ADJUST > USHRT_MAX) {6476 dev_info(hr_dev->dev, "eq_period(%u) reached the upper limit, adjusted to 65.\n",6477 eq->eq_period);6478 eq->eq_period = HNS_ROCE_MAX_EQ_PERIOD;6479 }6480 eq->eq_period *= HNS_ROCE_CLOCK_ADJUST;6481 }6482 6483 hr_reg_write(eqc, EQC_EQ_PERIOD, eq->eq_period);6484 hr_reg_write(eqc, EQC_EQE_REPORT_TIMER, HNS_ROCE_EQ_INIT_REPORT_TIMER);6485 hr_reg_write(eqc, EQC_EQE_BA_L, bt_ba >> 3);6486 hr_reg_write(eqc, EQC_EQE_BA_H, bt_ba >> 35);6487 hr_reg_write(eqc, EQC_SHIFT, eq->shift);6488 hr_reg_write(eqc, EQC_MSI_INDX, HNS_ROCE_EQ_INIT_MSI_IDX);6489 hr_reg_write(eqc, EQC_CUR_EQE_BA_L, eqe_ba[0] >> 12);6490 hr_reg_write(eqc, EQC_CUR_EQE_BA_M, eqe_ba[0] >> 28);6491 hr_reg_write(eqc, EQC_CUR_EQE_BA_H, eqe_ba[0] >> 60);6492 hr_reg_write(eqc, EQC_EQ_CONS_INDX, HNS_ROCE_EQ_INIT_CONS_IDX);6493 hr_reg_write(eqc, EQC_NEX_EQE_BA_L, eqe_ba[1] >> 12);6494 hr_reg_write(eqc, EQC_NEX_EQE_BA_H, eqe_ba[1] >> 44);6495 hr_reg_write(eqc, EQC_EQE_SIZE, eq->eqe_size == HNS_ROCE_V3_EQE_SIZE);6496 6497 return 0;6498}6499 6500static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)6501{6502 struct hns_roce_buf_attr buf_attr = {};6503 int err;6504 6505 if (hr_dev->caps.eqe_hop_num == HNS_ROCE_HOP_NUM_0)6506 eq->hop_num = 0;6507 else6508 eq->hop_num = hr_dev->caps.eqe_hop_num;6509 6510 buf_attr.page_shift = hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT;6511 buf_attr.region[0].size = eq->entries * eq->eqe_size;6512 buf_attr.region[0].hopnum = eq->hop_num;6513 buf_attr.region_count = 1;6514 6515 err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr,6516 hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, NULL,6517 0);6518 if (err)6519 dev_err(hr_dev->dev, "failed to alloc EQE mtr, err %d\n", err);6520 6521 return err;6522}6523 6524static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,6525 struct hns_roce_eq *eq, u8 eq_cmd)6526{6527 struct hns_roce_cmd_mailbox *mailbox;6528 int ret;6529 6530 /* Allocate mailbox memory */6531 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);6532 if (IS_ERR(mailbox))6533 return PTR_ERR(mailbox);6534 6535 ret = alloc_eq_buf(hr_dev, eq);6536 if (ret)6537 goto free_cmd_mbox;6538 6539 ret = config_eqc(hr_dev, eq, mailbox->buf);6540 if (ret)6541 goto err_cmd_mbox;6542 6543 ret = hns_roce_create_hw_ctx(hr_dev, mailbox, eq_cmd, eq->eqn);6544 if (ret) {6545 dev_err(hr_dev->dev, "[mailbox cmd] create eqc failed.\n");6546 goto err_cmd_mbox;6547 }6548 6549 hns_roce_free_cmd_mailbox(hr_dev, mailbox);6550 6551 return 0;6552 6553err_cmd_mbox:6554 free_eq_buf(hr_dev, eq);6555 6556free_cmd_mbox:6557 hns_roce_free_cmd_mailbox(hr_dev, mailbox);6558 6559 return ret;6560}6561 6562static void hns_roce_ceq_work(struct work_struct *work)6563{6564 struct hns_roce_eq *eq = from_work(eq, work, work);6565 struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq);6566 struct hns_roce_dev *hr_dev = eq->hr_dev;6567 int ceqe_num = 0;6568 u32 cqn;6569 6570 while (ceqe && ceqe_num < hr_dev->caps.ceqe_depth) {6571 /* Make sure we read CEQ entry after we have checked the6572 * ownership bit6573 */6574 dma_rmb();6575 6576 cqn = hr_reg_read(ceqe, CEQE_CQN);6577 6578 hns_roce_cq_completion(hr_dev, cqn);6579 6580 ++eq->cons_index;6581 ++ceqe_num;6582 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CEQE_CNT]);6583 6584 ceqe = next_ceqe_sw_v2(eq);6585 }6586 6587 update_eq_db(eq);6588}6589 6590static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num,6591 int comp_num, int aeq_num, int other_num)6592{6593 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;6594 int i, j;6595 int ret;6596 6597 for (i = 0; i < irq_num; i++) {6598 hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,6599 GFP_KERNEL);6600 if (!hr_dev->irq_names[i]) {6601 ret = -ENOMEM;6602 goto err_kzalloc_failed;6603 }6604 }6605 6606 /* irq contains: abnormal + AEQ + CEQ */6607 for (j = 0; j < other_num; j++)6608 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,6609 "hns-%s-abn-%d", pci_name(hr_dev->pci_dev), j);6610 6611 for (j = other_num; j < (other_num + aeq_num); j++)6612 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,6613 "hns-%s-aeq-%d", pci_name(hr_dev->pci_dev), j - other_num);6614 6615 for (j = (other_num + aeq_num); j < irq_num; j++)6616 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,6617 "hns-%s-ceq-%d", pci_name(hr_dev->pci_dev),6618 j - other_num - aeq_num);6619 6620 for (j = 0; j < irq_num; j++) {6621 if (j < other_num) {6622 ret = request_irq(hr_dev->irq[j],6623 hns_roce_v2_msix_interrupt_abn,6624 0, hr_dev->irq_names[j], hr_dev);6625 } else if (j < (other_num + comp_num)) {6626 INIT_WORK(&eq_table->eq[j - other_num].work,6627 hns_roce_ceq_work);6628 ret = request_irq(eq_table->eq[j - other_num].irq,6629 hns_roce_v2_msix_interrupt_eq,6630 0, hr_dev->irq_names[j + aeq_num],6631 &eq_table->eq[j - other_num]);6632 } else {6633 ret = request_irq(eq_table->eq[j - other_num].irq,6634 hns_roce_v2_msix_interrupt_eq,6635 0, hr_dev->irq_names[j - comp_num],6636 &eq_table->eq[j - other_num]);6637 }6638 6639 if (ret) {6640 dev_err(hr_dev->dev, "request irq error!\n");6641 goto err_request_failed;6642 }6643 }6644 6645 return 0;6646 6647err_request_failed:6648 for (j -= 1; j >= 0; j--) {6649 if (j < other_num) {6650 free_irq(hr_dev->irq[j], hr_dev);6651 continue;6652 }6653 free_irq(eq_table->eq[j - other_num].irq,6654 &eq_table->eq[j - other_num]);6655 if (j < other_num + comp_num)6656 cancel_work_sync(&eq_table->eq[j - other_num].work);6657 }6658 6659err_kzalloc_failed:6660 for (i -= 1; i >= 0; i--)6661 kfree(hr_dev->irq_names[i]);6662 6663 return ret;6664}6665 6666static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev)6667{6668 int irq_num;6669 int eq_num;6670 int i;6671 6672 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;6673 irq_num = eq_num + hr_dev->caps.num_other_vectors;6674 6675 for (i = 0; i < hr_dev->caps.num_other_vectors; i++)6676 free_irq(hr_dev->irq[i], hr_dev);6677 6678 for (i = 0; i < eq_num; i++) {6679 free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]);6680 if (i < hr_dev->caps.num_comp_vectors)6681 cancel_work_sync(&hr_dev->eq_table.eq[i].work);6682 }6683 6684 for (i = 0; i < irq_num; i++)6685 kfree(hr_dev->irq_names[i]);6686}6687 6688static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)6689{6690 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;6691 struct device *dev = hr_dev->dev;6692 struct hns_roce_eq *eq;6693 int other_num;6694 int comp_num;6695 int aeq_num;6696 int irq_num;6697 int eq_num;6698 u8 eq_cmd;6699 int ret;6700 int i;6701 6702 other_num = hr_dev->caps.num_other_vectors;6703 comp_num = hr_dev->caps.num_comp_vectors;6704 aeq_num = hr_dev->caps.num_aeq_vectors;6705 6706 eq_num = comp_num + aeq_num;6707 irq_num = eq_num + other_num;6708 6709 eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);6710 if (!eq_table->eq)6711 return -ENOMEM;6712 6713 /* create eq */6714 for (i = 0; i < eq_num; i++) {6715 eq = &eq_table->eq[i];6716 eq->hr_dev = hr_dev;6717 eq->eqn = i;6718 if (i < comp_num) {6719 /* CEQ */6720 eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;6721 eq->type_flag = HNS_ROCE_CEQ;6722 eq->entries = hr_dev->caps.ceqe_depth;6723 eq->eqe_size = hr_dev->caps.ceqe_size;6724 eq->irq = hr_dev->irq[i + other_num + aeq_num];6725 eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;6726 eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;6727 } else {6728 /* AEQ */6729 eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;6730 eq->type_flag = HNS_ROCE_AEQ;6731 eq->entries = hr_dev->caps.aeqe_depth;6732 eq->eqe_size = hr_dev->caps.aeqe_size;6733 eq->irq = hr_dev->irq[i - comp_num + other_num];6734 eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;6735 eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;6736 }6737 6738 ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd);6739 if (ret) {6740 dev_err(dev, "failed to create eq.\n");6741 goto err_create_eq_fail;6742 }6743 }6744 6745 INIT_WORK(&hr_dev->ecc_work, fmea_ram_ecc_work);6746 6747 hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0);6748 if (!hr_dev->irq_workq) {6749 dev_err(dev, "failed to create irq workqueue.\n");6750 ret = -ENOMEM;6751 goto err_create_eq_fail;6752 }6753 6754 ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num, aeq_num,6755 other_num);6756 if (ret) {6757 dev_err(dev, "failed to request irq.\n");6758 goto err_request_irq_fail;6759 }6760 6761 /* enable irq */6762 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);6763 6764 return 0;6765 6766err_request_irq_fail:6767 destroy_workqueue(hr_dev->irq_workq);6768 6769err_create_eq_fail:6770 for (i -= 1; i >= 0; i--)6771 hns_roce_v2_destroy_eqc(hr_dev, &eq_table->eq[i]);6772 kfree(eq_table->eq);6773 6774 return ret;6775}6776 6777static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)6778{6779 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;6780 int eq_num;6781 int i;6782 6783 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;6784 6785 /* Disable irq */6786 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);6787 6788 __hns_roce_free_irq(hr_dev);6789 destroy_workqueue(hr_dev->irq_workq);6790 6791 for (i = 0; i < eq_num; i++)6792 hns_roce_v2_destroy_eqc(hr_dev, &eq_table->eq[i]);6793 6794 kfree(eq_table->eq);6795}6796 6797static const struct ib_device_ops hns_roce_v2_dev_ops = {6798 .destroy_qp = hns_roce_v2_destroy_qp,6799 .modify_cq = hns_roce_v2_modify_cq,6800 .poll_cq = hns_roce_v2_poll_cq,6801 .post_recv = hns_roce_v2_post_recv,6802 .post_send = hns_roce_v2_post_send,6803 .query_qp = hns_roce_v2_query_qp,6804 .req_notify_cq = hns_roce_v2_req_notify_cq,6805};6806 6807static const struct ib_device_ops hns_roce_v2_dev_srq_ops = {6808 .modify_srq = hns_roce_v2_modify_srq,6809 .post_srq_recv = hns_roce_v2_post_srq_recv,6810 .query_srq = hns_roce_v2_query_srq,6811};6812 6813static const struct hns_roce_hw hns_roce_hw_v2 = {6814 .cmq_init = hns_roce_v2_cmq_init,6815 .cmq_exit = hns_roce_v2_cmq_exit,6816 .hw_profile = hns_roce_v2_profile,6817 .hw_init = hns_roce_v2_init,6818 .hw_exit = hns_roce_v2_exit,6819 .post_mbox = v2_post_mbox,6820 .poll_mbox_done = v2_poll_mbox_done,6821 .chk_mbox_avail = v2_chk_mbox_is_avail,6822 .set_gid = hns_roce_v2_set_gid,6823 .set_mac = hns_roce_v2_set_mac,6824 .write_mtpt = hns_roce_v2_write_mtpt,6825 .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,6826 .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,6827 .mw_write_mtpt = hns_roce_v2_mw_write_mtpt,6828 .write_cqc = hns_roce_v2_write_cqc,6829 .set_hem = hns_roce_v2_set_hem,6830 .clear_hem = hns_roce_v2_clear_hem,6831 .modify_qp = hns_roce_v2_modify_qp,6832 .dereg_mr = hns_roce_v2_dereg_mr,6833 .qp_flow_control_init = hns_roce_v2_qp_flow_control_init,6834 .init_eq = hns_roce_v2_init_eq_table,6835 .cleanup_eq = hns_roce_v2_cleanup_eq_table,6836 .write_srqc = hns_roce_v2_write_srqc,6837 .query_cqc = hns_roce_v2_query_cqc,6838 .query_qpc = hns_roce_v2_query_qpc,6839 .query_mpt = hns_roce_v2_query_mpt,6840 .query_srqc = hns_roce_v2_query_srqc,6841 .query_sccc = hns_roce_v2_query_sccc,6842 .query_hw_counter = hns_roce_hw_v2_query_counter,6843 .get_dscp = hns_roce_hw_v2_get_dscp,6844 .hns_roce_dev_ops = &hns_roce_v2_dev_ops,6845 .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,6846};6847 6848static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {6849 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},6850 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},6851 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},6852 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},6853 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},6854 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0},6855 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),6856 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},6857 /* required last entry */6858 {0, }6859};6860 6861MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);6862 6863static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,6864 struct hnae3_handle *handle)6865{6866 struct hns_roce_v2_priv *priv = hr_dev->priv;6867 const struct pci_device_id *id;6868 int i;6869 6870 hr_dev->pci_dev = handle->pdev;6871 id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev->pci_dev);6872 hr_dev->is_vf = id->driver_data;6873 hr_dev->dev = &handle->pdev->dev;6874 hr_dev->hw = &hns_roce_hw_v2;6875 hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;6876 hr_dev->odb_offset = hr_dev->sdb_offset;6877 6878 /* Get info from NIC driver. */6879 hr_dev->reg_base = handle->rinfo.roce_io_base;6880 hr_dev->mem_base = handle->rinfo.roce_mem_base;6881 hr_dev->caps.num_ports = 1;6882 hr_dev->iboe.netdevs[0] = handle->rinfo.netdev;6883 hr_dev->iboe.phy_port[0] = 0;6884 6885 addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid,6886 hr_dev->iboe.netdevs[0]->dev_addr);6887 6888 for (i = 0; i < handle->rinfo.num_vectors; i++)6889 hr_dev->irq[i] = pci_irq_vector(handle->pdev,6890 i + handle->rinfo.base_vector);6891 6892 /* cmd issue mode: 0 is poll, 1 is event */6893 hr_dev->cmd_mod = 1;6894 hr_dev->loop_idc = 0;6895 6896 hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle);6897 priv->handle = handle;6898}6899 6900static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)6901{6902 struct hns_roce_dev *hr_dev;6903 int ret;6904 6905 hr_dev = ib_alloc_device(hns_roce_dev, ib_dev);6906 if (!hr_dev)6907 return -ENOMEM;6908 6909 hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL);6910 if (!hr_dev->priv) {6911 ret = -ENOMEM;6912 goto error_failed_kzalloc;6913 }6914 6915 hns_roce_hw_v2_get_cfg(hr_dev, handle);6916 6917 ret = hns_roce_init(hr_dev);6918 if (ret) {6919 dev_err(hr_dev->dev, "RoCE Engine init failed!\n");6920 goto error_failed_roce_init;6921 }6922 6923 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {6924 ret = free_mr_init(hr_dev);6925 if (ret) {6926 dev_err(hr_dev->dev, "failed to init free mr!\n");6927 goto error_failed_free_mr_init;6928 }6929 }6930 6931 handle->priv = hr_dev;6932 6933 return 0;6934 6935error_failed_free_mr_init:6936 hns_roce_exit(hr_dev);6937 6938error_failed_roce_init:6939 kfree(hr_dev->priv);6940 6941error_failed_kzalloc:6942 ib_dealloc_device(&hr_dev->ib_dev);6943 6944 return ret;6945}6946 6947static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,6948 bool reset)6949{6950 struct hns_roce_dev *hr_dev = handle->priv;6951 6952 if (!hr_dev)6953 return;6954 6955 handle->priv = NULL;6956 6957 hr_dev->state = HNS_ROCE_DEVICE_STATE_UNINIT;6958 hns_roce_handle_device_err(hr_dev);6959 6960 hns_roce_exit(hr_dev);6961 kfree(hr_dev->priv);6962 ib_dealloc_device(&hr_dev->ib_dev);6963}6964 6965static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)6966{6967 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;6968 const struct pci_device_id *id;6969 struct device *dev = &handle->pdev->dev;6970 int ret;6971 6972 handle->rinfo.instance_state = HNS_ROCE_STATE_INIT;6973 6974 if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) {6975 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;6976 goto reset_chk_err;6977 }6978 6979 id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev);6980 if (!id)6981 return 0;6982 6983 if (id->driver_data && handle->pdev->revision == PCI_REVISION_ID_HIP08)6984 return 0;6985 6986 ret = __hns_roce_hw_v2_init_instance(handle);6987 if (ret) {6988 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;6989 dev_err(dev, "RoCE instance init failed! ret = %d\n", ret);6990 if (ops->ae_dev_resetting(handle) ||6991 ops->get_hw_reset_stat(handle))6992 goto reset_chk_err;6993 else6994 return ret;6995 }6996 6997 handle->rinfo.instance_state = HNS_ROCE_STATE_INITED;6998 6999 return 0;7000 7001reset_chk_err:7002 dev_err(dev, "Device is busy in resetting state.\n"7003 "please retry later.\n");7004 7005 return -EBUSY;7006}7007 7008static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,7009 bool reset)7010{7011 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)7012 return;7013 7014 handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;7015 7016 __hns_roce_hw_v2_uninit_instance(handle, reset);7017 7018 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;7019}7020static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)7021{7022 struct hns_roce_dev *hr_dev;7023 7024 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) {7025 set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);7026 return 0;7027 }7028 7029 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN;7030 clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);7031 7032 hr_dev = handle->priv;7033 if (!hr_dev)7034 return 0;7035 7036 hr_dev->active = false;7037 hr_dev->dis_db = true;7038 hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN;7039 7040 return 0;7041}7042 7043static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)7044{7045 struct device *dev = &handle->pdev->dev;7046 int ret;7047 7048 if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN,7049 &handle->rinfo.state)) {7050 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;7051 return 0;7052 }7053 7054 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT;7055 7056 dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n");7057 ret = __hns_roce_hw_v2_init_instance(handle);7058 if (ret) {7059 /* when reset notify type is HNAE3_INIT_CLIENT In reset notify7060 * callback function, RoCE Engine reinitialize. If RoCE reinit7061 * failed, we should inform NIC driver.7062 */7063 handle->priv = NULL;7064 dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret);7065 } else {7066 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;7067 dev_info(dev, "reset done, RoCE client reinit finished.\n");7068 }7069 7070 return ret;7071}7072 7073static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)7074{7075 if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state))7076 return 0;7077 7078 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT;7079 dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n");7080 msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY);7081 __hns_roce_hw_v2_uninit_instance(handle, false);7082 7083 return 0;7084}7085 7086static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle,7087 enum hnae3_reset_notify_type type)7088{7089 int ret = 0;7090 7091 switch (type) {7092 case HNAE3_DOWN_CLIENT:7093 ret = hns_roce_hw_v2_reset_notify_down(handle);7094 break;7095 case HNAE3_INIT_CLIENT:7096 ret = hns_roce_hw_v2_reset_notify_init(handle);7097 break;7098 case HNAE3_UNINIT_CLIENT:7099 ret = hns_roce_hw_v2_reset_notify_uninit(handle);7100 break;7101 default:7102 break;7103 }7104 7105 return ret;7106}7107 7108static const struct hnae3_client_ops hns_roce_hw_v2_ops = {7109 .init_instance = hns_roce_hw_v2_init_instance,7110 .uninit_instance = hns_roce_hw_v2_uninit_instance,7111 .reset_notify = hns_roce_hw_v2_reset_notify,7112};7113 7114static struct hnae3_client hns_roce_hw_v2_client = {7115 .name = "hns_roce_hw_v2",7116 .type = HNAE3_CLIENT_ROCE,7117 .ops = &hns_roce_hw_v2_ops,7118};7119 7120static int __init hns_roce_hw_v2_init(void)7121{7122 hns_roce_init_debugfs();7123 return hnae3_register_client(&hns_roce_hw_v2_client);7124}7125 7126static void __exit hns_roce_hw_v2_exit(void)7127{7128 hnae3_unregister_client(&hns_roce_hw_v2_client);7129 hns_roce_cleanup_debugfs();7130}7131 7132module_init(hns_roce_hw_v2_init);7133module_exit(hns_roce_hw_v2_exit);7134 7135MODULE_LICENSE("Dual BSD/GPL");7136MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");7137MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");7138MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>");7139MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver");7140