1311 lines · c
1/*2 * Copyright (c) 2016 Hisilicon Limited.3 *4 * This software is available to you under a choice of one of two5 * licenses. You may choose to be licensed under the terms of the GNU6 * General Public License (GPL) Version 2, available from the file7 * COPYING in the main directory of this source tree, or the8 * OpenIB.org BSD license below:9 *10 * Redistribution and use in source and binary forms, with or11 * without modification, are permitted provided that the following12 * conditions are met:13 *14 * - Redistributions of source code must retain the above15 * copyright notice, this list of conditions and the following16 * disclaimer.17 *18 * - Redistributions in binary form must reproduce the above19 * copyright notice, this list of conditions and the following20 * disclaimer in the documentation and/or other materials21 * provided with the distribution.22 *23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE30 * SOFTWARE.31 */32 33#ifndef _HNS_ROCE_DEVICE_H34#define _HNS_ROCE_DEVICE_H35 36#include <rdma/ib_verbs.h>37#include <rdma/hns-abi.h>38#include "hns_roce_debugfs.h"39 40#define PCI_REVISION_ID_HIP08 0x2141#define PCI_REVISION_ID_HIP09 0x3042 43#define HNS_ROCE_MAX_MSG_LEN 0x8000000044 45#define HNS_ROCE_IB_MIN_SQ_STRIDE 646 47#define BA_BYTE_LEN 848 49#define HNS_ROCE_MIN_CQE_NUM 0x4050#define HNS_ROCE_MIN_SRQ_WQE_NUM 151 52#define HNS_ROCE_MAX_IRQ_NUM 12853 54#define HNS_ROCE_SGE_IN_WQE 255#define HNS_ROCE_SGE_SHIFT 456 57#define EQ_ENABLE 158#define EQ_DISABLE 059 60#define HNS_ROCE_CEQ 061#define HNS_ROCE_AEQ 162 63#define HNS_ROCE_CEQE_SIZE 0x464#define HNS_ROCE_AEQE_SIZE 0x1065 66#define HNS_ROCE_V3_EQE_SIZE 0x4067 68#define HNS_ROCE_V2_CQE_SIZE 3269#define HNS_ROCE_V3_CQE_SIZE 6470 71#define HNS_ROCE_V2_QPC_SZ 25672#define HNS_ROCE_V3_QPC_SZ 51273 74#define HNS_ROCE_MAX_PORTS 675#define HNS_ROCE_GID_SIZE 1676#define HNS_ROCE_SGE_SIZE 1677#define HNS_ROCE_DWQE_SIZE 6553678 79#define HNS_ROCE_HOP_NUM_0 0xff80 81#define MR_TYPE_MR 0x0082#define MR_TYPE_FRMR 0x0183#define MR_TYPE_DMA 0x0384 85#define HNS_ROCE_FRMR_MAX_PA 51286#define HNS_ROCE_FRMR_ALIGN_SIZE 12887 88#define PKEY_ID 0xffff89#define NODE_DESC_SIZE 6490#define DB_REG_OFFSET 0x100091 92/* Configure to HW for PAGE_SIZE larger than 4KB */93#define PG_SHIFT_OFFSET (PAGE_SHIFT - 12)94 95#define ATOMIC_WR_LEN 896 97#define HNS_ROCE_IDX_QUE_ENTRY_SZ 498#define SRQ_DB_REG 0x23099 100#define HNS_ROCE_QP_BANK_NUM 8101#define HNS_ROCE_CQ_BANK_NUM 4102 103#define CQ_BANKID_SHIFT 2104#define CQ_BANKID_MASK GENMASK(1, 0)105 106#define HNS_ROCE_MAX_CQ_COUNT 0xFFFF107#define HNS_ROCE_MAX_CQ_PERIOD 0xFFFF108 109enum {110 SERV_TYPE_RC,111 SERV_TYPE_UC,112 SERV_TYPE_RD,113 SERV_TYPE_UD,114 SERV_TYPE_XRC = 5,115};116 117enum hns_roce_event {118 HNS_ROCE_EVENT_TYPE_PATH_MIG = 0x01,119 HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED = 0x02,120 HNS_ROCE_EVENT_TYPE_COMM_EST = 0x03,121 HNS_ROCE_EVENT_TYPE_SQ_DRAINED = 0x04,122 HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR = 0x05,123 HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR = 0x06,124 HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR = 0x07,125 HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH = 0x08,126 HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH = 0x09,127 HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR = 0x0a,128 HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR = 0x0b,129 HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW = 0x0c,130 HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID = 0x0d,131 HNS_ROCE_EVENT_TYPE_PORT_CHANGE = 0x0f,132 /* 0x10 and 0x11 is unused in currently application case */133 HNS_ROCE_EVENT_TYPE_DB_OVERFLOW = 0x12,134 HNS_ROCE_EVENT_TYPE_MB = 0x13,135 HNS_ROCE_EVENT_TYPE_FLR = 0x15,136 HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION = 0x16,137 HNS_ROCE_EVENT_TYPE_INVALID_XRCETH = 0x17,138};139 140enum {141 HNS_ROCE_CAP_FLAG_REREG_MR = BIT(0),142 HNS_ROCE_CAP_FLAG_ROCE_V1_V2 = BIT(1),143 HNS_ROCE_CAP_FLAG_RQ_INLINE = BIT(2),144 HNS_ROCE_CAP_FLAG_CQ_RECORD_DB = BIT(3),145 HNS_ROCE_CAP_FLAG_QP_RECORD_DB = BIT(4),146 HNS_ROCE_CAP_FLAG_SRQ = BIT(5),147 HNS_ROCE_CAP_FLAG_XRC = BIT(6),148 HNS_ROCE_CAP_FLAG_MW = BIT(7),149 HNS_ROCE_CAP_FLAG_FRMR = BIT(8),150 HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL = BIT(9),151 HNS_ROCE_CAP_FLAG_ATOMIC = BIT(10),152 HNS_ROCE_CAP_FLAG_DIRECT_WQE = BIT(12),153 HNS_ROCE_CAP_FLAG_SDI_MODE = BIT(14),154 HNS_ROCE_CAP_FLAG_STASH = BIT(17),155 HNS_ROCE_CAP_FLAG_CQE_INLINE = BIT(19),156 HNS_ROCE_CAP_FLAG_SRQ_RECORD_DB = BIT(22),157};158 159#define HNS_ROCE_DB_TYPE_COUNT 2160#define HNS_ROCE_DB_UNIT_SIZE 4161 162enum {163 HNS_ROCE_DB_PER_PAGE = PAGE_SIZE / 4164};165 166enum hns_roce_reset_stage {167 HNS_ROCE_STATE_NON_RST,168 HNS_ROCE_STATE_RST_BEF_DOWN,169 HNS_ROCE_STATE_RST_DOWN,170 HNS_ROCE_STATE_RST_UNINIT,171 HNS_ROCE_STATE_RST_INIT,172 HNS_ROCE_STATE_RST_INITED,173};174 175enum hns_roce_instance_state {176 HNS_ROCE_STATE_NON_INIT,177 HNS_ROCE_STATE_INIT,178 HNS_ROCE_STATE_INITED,179 HNS_ROCE_STATE_UNINIT,180};181 182enum {183 HNS_ROCE_RST_DIRECT_RETURN = 0,184};185 186#define HNS_ROCE_CMD_SUCCESS 1187 188#define HNS_ROCE_MAX_HOP_NUM 3189/* The minimum page size is 4K for hardware */190#define HNS_HW_PAGE_SHIFT 12191#define HNS_HW_PAGE_SIZE (1 << HNS_HW_PAGE_SHIFT)192 193#define HNS_HW_MAX_PAGE_SHIFT 27194#define HNS_HW_MAX_PAGE_SIZE (1 << HNS_HW_MAX_PAGE_SHIFT)195 196struct hns_roce_uar {197 u64 pfn;198 unsigned long index;199 unsigned long logic_idx;200};201 202enum hns_roce_mmap_type {203 HNS_ROCE_MMAP_TYPE_DB = 1,204 HNS_ROCE_MMAP_TYPE_DWQE,205};206 207struct hns_user_mmap_entry {208 struct rdma_user_mmap_entry rdma_entry;209 enum hns_roce_mmap_type mmap_type;210 u64 address;211};212 213struct hns_roce_ucontext {214 struct ib_ucontext ibucontext;215 struct hns_roce_uar uar;216 struct list_head page_list;217 struct mutex page_mutex;218 struct hns_user_mmap_entry *db_mmap_entry;219 u32 config;220};221 222struct hns_roce_pd {223 struct ib_pd ibpd;224 unsigned long pdn;225};226 227struct hns_roce_xrcd {228 struct ib_xrcd ibxrcd;229 u32 xrcdn;230};231 232struct hns_roce_bitmap {233 /* Bitmap Traversal last a bit which is 1 */234 unsigned long last;235 unsigned long top;236 unsigned long max;237 unsigned long reserved_top;238 unsigned long mask;239 spinlock_t lock;240 unsigned long *table;241};242 243struct hns_roce_ida {244 struct ida ida;245 u32 min; /* Lowest ID to allocate. */246 u32 max; /* Highest ID to allocate. */247};248 249/* For Hardware Entry Memory */250struct hns_roce_hem_table {251 /* HEM type: 0 = qpc, 1 = mtt, 2 = cqc, 3 = srq, 4 = other */252 u32 type;253 /* HEM array elment num */254 unsigned long num_hem;255 /* Single obj size */256 unsigned long obj_size;257 unsigned long table_chunk_size;258 struct mutex mutex;259 struct hns_roce_hem **hem;260 u64 **bt_l1;261 dma_addr_t *bt_l1_dma_addr;262 u64 **bt_l0;263 dma_addr_t *bt_l0_dma_addr;264};265 266struct hns_roce_buf_region {267 u32 offset; /* page offset */268 u32 count; /* page count */269 int hopnum; /* addressing hop num */270};271 272#define HNS_ROCE_MAX_BT_REGION 3273#define HNS_ROCE_MAX_BT_LEVEL 3274struct hns_roce_hem_list {275 struct list_head root_bt;276 /* link all bt dma mem by hop config */277 struct list_head mid_bt[HNS_ROCE_MAX_BT_REGION][HNS_ROCE_MAX_BT_LEVEL];278 struct list_head btm_bt; /* link all bottom bt in @mid_bt */279 dma_addr_t root_ba; /* pointer to the root ba table */280};281 282enum mtr_type {283 MTR_DEFAULT = 0,284 MTR_PBL,285};286 287struct hns_roce_buf_attr {288 struct {289 size_t size; /* region size */290 int hopnum; /* multi-hop addressing hop num */291 } region[HNS_ROCE_MAX_BT_REGION];292 unsigned int region_count; /* valid region count */293 unsigned int page_shift; /* buffer page shift */294 unsigned int user_access; /* umem access flag */295 u64 iova;296 enum mtr_type type;297 bool mtt_only; /* only alloc buffer-required MTT memory */298 bool adaptive; /* adaptive for page_shift and hopnum */299};300 301struct hns_roce_hem_cfg {302 dma_addr_t root_ba; /* root BA table's address */303 bool is_direct; /* addressing without BA table */304 unsigned int ba_pg_shift; /* BA table page shift */305 unsigned int buf_pg_shift; /* buffer page shift */306 unsigned int buf_pg_count; /* buffer page count */307 struct hns_roce_buf_region region[HNS_ROCE_MAX_BT_REGION];308 unsigned int region_count;309};310 311/* memory translate region */312struct hns_roce_mtr {313 struct hns_roce_hem_list hem_list; /* multi-hop addressing resource */314 struct ib_umem *umem; /* user space buffer */315 struct hns_roce_buf *kmem; /* kernel space buffer */316 struct hns_roce_hem_cfg hem_cfg; /* config for hardware addressing */317};318 319struct hns_roce_mw {320 struct ib_mw ibmw;321 u32 pdn;322 u32 rkey;323 int enabled; /* MW's active status */324 u32 pbl_hop_num;325 u32 pbl_ba_pg_sz;326 u32 pbl_buf_pg_sz;327};328 329struct hns_roce_mr {330 struct ib_mr ibmr;331 u64 iova; /* MR's virtual original addr */332 u64 size; /* Address range of MR */333 u32 key; /* Key of MR */334 u32 pd; /* PD num of MR */335 u32 access; /* Access permission of MR */336 int enabled; /* MR's active status */337 int type; /* MR's register type */338 u32 pbl_hop_num; /* multi-hop number */339 struct hns_roce_mtr pbl_mtr;340 u32 npages;341 dma_addr_t *page_list;342};343 344struct hns_roce_mr_table {345 struct hns_roce_ida mtpt_ida;346 struct hns_roce_hem_table mtpt_table;347};348 349struct hns_roce_wq {350 u64 *wrid; /* Work request ID */351 spinlock_t lock;352 u32 wqe_cnt; /* WQE num */353 u32 max_gs;354 u32 rsv_sge;355 u32 offset;356 u32 wqe_shift; /* WQE size */357 u32 head;358 u32 tail;359 void __iomem *db_reg;360 u32 ext_sge_cnt;361};362 363struct hns_roce_sge {364 unsigned int sge_cnt; /* SGE num */365 u32 offset;366 u32 sge_shift; /* SGE size */367};368 369struct hns_roce_buf_list {370 void *buf;371 dma_addr_t map;372};373 374/*375 * %HNS_ROCE_BUF_DIRECT indicates that the all memory must be in a continuous376 * dma address range.377 *378 * %HNS_ROCE_BUF_NOSLEEP indicates that the caller cannot sleep.379 *380 * %HNS_ROCE_BUF_NOFAIL allocation only failed when allocated size is zero, even381 * the allocated size is smaller than the required size.382 */383enum {384 HNS_ROCE_BUF_DIRECT = BIT(0),385 HNS_ROCE_BUF_NOSLEEP = BIT(1),386 HNS_ROCE_BUF_NOFAIL = BIT(2),387};388 389struct hns_roce_buf {390 struct hns_roce_buf_list *trunk_list;391 u32 ntrunks;392 u32 npages;393 unsigned int trunk_shift;394 unsigned int page_shift;395};396 397struct hns_roce_db_pgdir {398 struct list_head list;399 DECLARE_BITMAP(order0, HNS_ROCE_DB_PER_PAGE);400 DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT);401 unsigned long *bits[HNS_ROCE_DB_TYPE_COUNT];402 u32 *page;403 dma_addr_t db_dma;404};405 406struct hns_roce_user_db_page {407 struct list_head list;408 struct ib_umem *umem;409 unsigned long user_virt;410 refcount_t refcount;411};412 413struct hns_roce_db {414 u32 *db_record;415 union {416 struct hns_roce_db_pgdir *pgdir;417 struct hns_roce_user_db_page *user_page;418 } u;419 dma_addr_t dma;420 void *virt_addr;421 unsigned long index;422 unsigned long order;423};424 425struct hns_roce_cq {426 struct ib_cq ib_cq;427 struct hns_roce_mtr mtr;428 struct hns_roce_db db;429 u32 flags;430 spinlock_t lock;431 u32 cq_depth;432 u32 cons_index;433 u32 *set_ci_db;434 void __iomem *db_reg;435 int arm_sn;436 int cqe_size;437 unsigned long cqn;438 u32 vector;439 refcount_t refcount;440 struct completion free;441 struct list_head sq_list; /* all qps on this send cq */442 struct list_head rq_list; /* all qps on this recv cq */443 int is_armed; /* cq is armed */444 struct list_head node; /* all armed cqs are on a list */445};446 447struct hns_roce_idx_que {448 struct hns_roce_mtr mtr;449 u32 entry_shift;450 unsigned long *bitmap;451 u32 head;452 u32 tail;453};454 455struct hns_roce_srq {456 struct ib_srq ibsrq;457 unsigned long srqn;458 u32 wqe_cnt;459 int max_gs;460 u32 rsv_sge;461 u32 wqe_shift;462 u32 cqn;463 u32 xrcdn;464 void __iomem *db_reg;465 466 refcount_t refcount;467 struct completion free;468 469 struct hns_roce_mtr buf_mtr;470 471 u64 *wrid;472 struct hns_roce_idx_que idx_que;473 spinlock_t lock;474 struct mutex mutex;475 void (*event)(struct hns_roce_srq *srq, enum hns_roce_event event);476 struct hns_roce_db rdb;477 u32 cap_flags;478};479 480struct hns_roce_uar_table {481 struct hns_roce_bitmap bitmap;482};483 484struct hns_roce_bank {485 struct ida ida;486 u32 inuse; /* Number of IDs allocated */487 u32 min; /* Lowest ID to allocate. */488 u32 max; /* Highest ID to allocate. */489 u32 next; /* Next ID to allocate. */490};491 492struct hns_roce_idx_table {493 u32 *spare_idx;494 u32 head;495 u32 tail;496};497 498struct hns_roce_qp_table {499 struct hns_roce_hem_table qp_table;500 struct hns_roce_hem_table irrl_table;501 struct hns_roce_hem_table trrl_table;502 struct hns_roce_hem_table sccc_table;503 struct mutex scc_mutex;504 struct hns_roce_bank bank[HNS_ROCE_QP_BANK_NUM];505 struct mutex bank_mutex;506 struct hns_roce_idx_table idx_table;507};508 509struct hns_roce_cq_table {510 struct xarray array;511 struct hns_roce_hem_table table;512 struct hns_roce_bank bank[HNS_ROCE_CQ_BANK_NUM];513 struct mutex bank_mutex;514};515 516struct hns_roce_srq_table {517 struct hns_roce_ida srq_ida;518 struct xarray xa;519 struct hns_roce_hem_table table;520};521 522struct hns_roce_av {523 u8 port;524 u8 gid_index;525 u8 stat_rate;526 u8 hop_limit;527 u32 flowlabel;528 u16 udp_sport;529 u8 sl;530 u8 tclass;531 u8 dgid[HNS_ROCE_GID_SIZE];532 u8 mac[ETH_ALEN];533 u16 vlan_id;534 u8 vlan_en;535};536 537struct hns_roce_ah {538 struct ib_ah ibah;539 struct hns_roce_av av;540};541 542struct hns_roce_cmd_context {543 struct completion done;544 int result;545 int next;546 u64 out_param;547 u16 token;548 u16 busy;549};550 551enum hns_roce_cmdq_state {552 HNS_ROCE_CMDQ_STATE_NORMAL,553 HNS_ROCE_CMDQ_STATE_FATAL_ERR,554};555 556struct hns_roce_cmdq {557 struct dma_pool *pool;558 struct semaphore poll_sem;559 /*560 * Event mode: cmd register mutex protection,561 * ensure to not exceed max_cmds and user use limit region562 */563 struct semaphore event_sem;564 int max_cmds;565 spinlock_t context_lock;566 int free_head;567 struct hns_roce_cmd_context *context;568 /*569 * Process whether use event mode, init default non-zero570 * After the event queue of cmd event ready,571 * can switch into event mode572 * close device, switch into poll mode(non event mode)573 */574 u8 use_events;575 enum hns_roce_cmdq_state state;576};577 578struct hns_roce_cmd_mailbox {579 void *buf;580 dma_addr_t dma;581};582 583struct hns_roce_mbox_msg {584 u64 in_param;585 u64 out_param;586 u8 cmd;587 u32 tag;588 u16 token;589 u8 event_en;590};591 592struct hns_roce_dev;593 594enum {595 HNS_ROCE_FLUSH_FLAG = 0,596};597 598struct hns_roce_work {599 struct hns_roce_dev *hr_dev;600 struct work_struct work;601 int event_type;602 int sub_type;603 u32 queue_num;604};605 606enum hns_roce_cong_type {607 CONG_TYPE_DCQCN,608 CONG_TYPE_LDCP,609 CONG_TYPE_HC3,610 CONG_TYPE_DIP,611};612 613struct hns_roce_qp {614 struct ib_qp ibqp;615 struct hns_roce_wq rq;616 struct hns_roce_db rdb;617 struct hns_roce_db sdb;618 unsigned long en_flags;619 enum ib_sig_type sq_signal_bits;620 struct hns_roce_wq sq;621 622 struct hns_roce_mtr mtr;623 624 u32 buff_size;625 struct mutex mutex;626 u8 port;627 u8 phy_port;628 u8 sl;629 u8 resp_depth;630 u8 state;631 u32 atomic_rd_en;632 u32 qkey;633 void (*event)(struct hns_roce_qp *qp,634 enum hns_roce_event event_type);635 unsigned long qpn;636 637 u32 xrcdn;638 639 refcount_t refcount;640 struct completion free;641 642 struct hns_roce_sge sge;643 u32 next_sge;644 enum ib_mtu path_mtu;645 u32 max_inline_data;646 u8 free_mr_en;647 648 /* 0: flush needed, 1: unneeded */649 unsigned long flush_flag;650 struct hns_roce_work flush_work;651 struct list_head node; /* all qps are on a list */652 struct list_head rq_node; /* all recv qps are on a list */653 struct list_head sq_node; /* all send qps are on a list */654 struct hns_user_mmap_entry *dwqe_mmap_entry;655 u32 config;656 enum hns_roce_cong_type cong_type;657 u8 tc_mode;658 u8 priority;659};660 661struct hns_roce_ib_iboe {662 spinlock_t lock;663 struct net_device *netdevs[HNS_ROCE_MAX_PORTS];664 struct notifier_block nb;665 u8 phy_port[HNS_ROCE_MAX_PORTS];666};667 668struct hns_roce_ceqe {669 __le32 comp;670 __le32 rsv[15];671};672 673#define CEQE_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_ceqe, h, l)674 675#define CEQE_CQN CEQE_FIELD_LOC(23, 0)676#define CEQE_OWNER CEQE_FIELD_LOC(31, 31)677 678struct hns_roce_aeqe {679 __le32 asyn;680 union {681 struct {682 __le32 num;683 u32 rsv0;684 u32 rsv1;685 } queue_event;686 687 struct {688 __le64 out_param;689 __le16 token;690 u8 status;691 u8 rsv0;692 } __packed cmd;693 } event;694 __le32 rsv[12];695};696 697#define AEQE_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_aeqe, h, l)698 699#define AEQE_EVENT_TYPE AEQE_FIELD_LOC(7, 0)700#define AEQE_SUB_TYPE AEQE_FIELD_LOC(15, 8)701#define AEQE_OWNER AEQE_FIELD_LOC(31, 31)702#define AEQE_EVENT_QUEUE_NUM AEQE_FIELD_LOC(55, 32)703 704struct hns_roce_eq {705 struct hns_roce_dev *hr_dev;706 void __iomem *db_reg;707 708 int type_flag; /* Aeq:1 ceq:0 */709 int eqn;710 u32 entries;711 int eqe_size;712 int irq;713 u32 cons_index;714 int over_ignore;715 int coalesce;716 int arm_st;717 int hop_num;718 struct hns_roce_mtr mtr;719 u16 eq_max_cnt;720 u32 eq_period;721 int shift;722 int event_type;723 int sub_type;724 struct work_struct work;725};726 727struct hns_roce_eq_table {728 struct hns_roce_eq *eq;729};730 731struct hns_roce_caps {732 u64 fw_ver;733 u8 num_ports;734 int gid_table_len[HNS_ROCE_MAX_PORTS];735 int pkey_table_len[HNS_ROCE_MAX_PORTS];736 int local_ca_ack_delay;737 int num_uars;738 u32 phy_num_uars;739 u32 max_sq_sg;740 u32 max_sq_inline;741 u32 max_rq_sg;742 u32 rsv0;743 u32 num_qps;744 u32 reserved_qps;745 u32 num_srqs;746 u32 max_wqes;747 u32 max_srq_wrs;748 u32 max_srq_sges;749 u32 max_sq_desc_sz;750 u32 max_rq_desc_sz;751 u32 rsv2;752 int max_qp_init_rdma;753 int max_qp_dest_rdma;754 u32 num_cqs;755 u32 max_cqes;756 u32 min_cqes;757 u32 min_wqes;758 u32 reserved_cqs;759 u32 reserved_srqs;760 int num_aeq_vectors;761 int num_comp_vectors;762 int num_other_vectors;763 u32 num_mtpts;764 u32 rsv1;765 u32 num_srqwqe_segs;766 u32 num_idx_segs;767 int reserved_mrws;768 int reserved_uars;769 int num_pds;770 int reserved_pds;771 u32 num_xrcds;772 u32 reserved_xrcds;773 u32 mtt_entry_sz;774 u32 cqe_sz;775 u32 page_size_cap;776 u32 reserved_lkey;777 int mtpt_entry_sz;778 int qpc_sz;779 int irrl_entry_sz;780 int trrl_entry_sz;781 int cqc_entry_sz;782 int sccc_sz;783 int qpc_timer_entry_sz;784 int cqc_timer_entry_sz;785 int srqc_entry_sz;786 int idx_entry_sz;787 u32 pbl_ba_pg_sz;788 u32 pbl_buf_pg_sz;789 u32 pbl_hop_num;790 int aeqe_depth;791 int ceqe_depth;792 u32 aeqe_size;793 u32 ceqe_size;794 enum ib_mtu max_mtu;795 u32 qpc_bt_num;796 u32 qpc_timer_bt_num;797 u32 srqc_bt_num;798 u32 cqc_bt_num;799 u32 cqc_timer_bt_num;800 u32 mpt_bt_num;801 u32 eqc_bt_num;802 u32 smac_bt_num;803 u32 sgid_bt_num;804 u32 sccc_bt_num;805 u32 gmv_bt_num;806 u32 qpc_ba_pg_sz;807 u32 qpc_buf_pg_sz;808 u32 qpc_hop_num;809 u32 srqc_ba_pg_sz;810 u32 srqc_buf_pg_sz;811 u32 srqc_hop_num;812 u32 cqc_ba_pg_sz;813 u32 cqc_buf_pg_sz;814 u32 cqc_hop_num;815 u32 mpt_ba_pg_sz;816 u32 mpt_buf_pg_sz;817 u32 mpt_hop_num;818 u32 mtt_ba_pg_sz;819 u32 mtt_buf_pg_sz;820 u32 mtt_hop_num;821 u32 wqe_sq_hop_num;822 u32 wqe_sge_hop_num;823 u32 wqe_rq_hop_num;824 u32 sccc_ba_pg_sz;825 u32 sccc_buf_pg_sz;826 u32 sccc_hop_num;827 u32 qpc_timer_ba_pg_sz;828 u32 qpc_timer_buf_pg_sz;829 u32 qpc_timer_hop_num;830 u32 cqc_timer_ba_pg_sz;831 u32 cqc_timer_buf_pg_sz;832 u32 cqc_timer_hop_num;833 u32 cqe_ba_pg_sz; /* page_size = 4K*(2^cqe_ba_pg_sz) */834 u32 cqe_buf_pg_sz;835 u32 cqe_hop_num;836 u32 srqwqe_ba_pg_sz;837 u32 srqwqe_buf_pg_sz;838 u32 srqwqe_hop_num;839 u32 idx_ba_pg_sz;840 u32 idx_buf_pg_sz;841 u32 idx_hop_num;842 u32 eqe_ba_pg_sz;843 u32 eqe_buf_pg_sz;844 u32 eqe_hop_num;845 u32 gmv_entry_num;846 u32 gmv_entry_sz;847 u32 gmv_ba_pg_sz;848 u32 gmv_buf_pg_sz;849 u32 gmv_hop_num;850 u32 sl_num;851 u32 llm_buf_pg_sz;852 u32 chunk_sz; /* chunk size in non multihop mode */853 u64 flags;854 u16 default_ceq_max_cnt;855 u16 default_ceq_period;856 u16 default_aeq_max_cnt;857 u16 default_aeq_period;858 u16 default_aeq_arm_st;859 u16 default_ceq_arm_st;860 u8 cong_cap;861 enum hns_roce_cong_type default_cong_type;862};863 864enum hns_roce_device_state {865 HNS_ROCE_DEVICE_STATE_INITED,866 HNS_ROCE_DEVICE_STATE_RST_DOWN,867 HNS_ROCE_DEVICE_STATE_UNINIT,868};869 870enum hns_roce_hw_pkt_stat_index {871 HNS_ROCE_HW_RX_RC_PKT_CNT,872 HNS_ROCE_HW_RX_UC_PKT_CNT,873 HNS_ROCE_HW_RX_UD_PKT_CNT,874 HNS_ROCE_HW_RX_XRC_PKT_CNT,875 HNS_ROCE_HW_RX_PKT_CNT,876 HNS_ROCE_HW_RX_ERR_PKT_CNT,877 HNS_ROCE_HW_RX_CNP_PKT_CNT,878 HNS_ROCE_HW_TX_RC_PKT_CNT,879 HNS_ROCE_HW_TX_UC_PKT_CNT,880 HNS_ROCE_HW_TX_UD_PKT_CNT,881 HNS_ROCE_HW_TX_XRC_PKT_CNT,882 HNS_ROCE_HW_TX_PKT_CNT,883 HNS_ROCE_HW_TX_ERR_PKT_CNT,884 HNS_ROCE_HW_TX_CNP_PKT_CNT,885 HNS_ROCE_HW_TRP_GET_MPT_ERR_PKT_CNT,886 HNS_ROCE_HW_TRP_GET_IRRL_ERR_PKT_CNT,887 HNS_ROCE_HW_ECN_DB_CNT,888 HNS_ROCE_HW_RX_BUF_CNT,889 HNS_ROCE_HW_TRP_RX_SOF_CNT,890 HNS_ROCE_HW_CQ_CQE_CNT,891 HNS_ROCE_HW_CQ_POE_CNT,892 HNS_ROCE_HW_CQ_NOTIFY_CNT,893 HNS_ROCE_HW_CNT_TOTAL894};895 896enum hns_roce_sw_dfx_stat_index {897 HNS_ROCE_DFX_AEQE_CNT,898 HNS_ROCE_DFX_CEQE_CNT,899 HNS_ROCE_DFX_CMDS_CNT,900 HNS_ROCE_DFX_CMDS_ERR_CNT,901 HNS_ROCE_DFX_MBX_POSTED_CNT,902 HNS_ROCE_DFX_MBX_POLLED_CNT,903 HNS_ROCE_DFX_MBX_EVENT_CNT,904 HNS_ROCE_DFX_QP_CREATE_ERR_CNT,905 HNS_ROCE_DFX_QP_MODIFY_ERR_CNT,906 HNS_ROCE_DFX_CQ_CREATE_ERR_CNT,907 HNS_ROCE_DFX_CQ_MODIFY_ERR_CNT,908 HNS_ROCE_DFX_SRQ_CREATE_ERR_CNT,909 HNS_ROCE_DFX_SRQ_MODIFY_ERR_CNT,910 HNS_ROCE_DFX_XRCD_ALLOC_ERR_CNT,911 HNS_ROCE_DFX_MR_REG_ERR_CNT,912 HNS_ROCE_DFX_MR_REREG_ERR_CNT,913 HNS_ROCE_DFX_AH_CREATE_ERR_CNT,914 HNS_ROCE_DFX_MMAP_ERR_CNT,915 HNS_ROCE_DFX_UCTX_ALLOC_ERR_CNT,916 HNS_ROCE_DFX_CNT_TOTAL917};918 919struct hns_roce_hw {920 int (*cmq_init)(struct hns_roce_dev *hr_dev);921 void (*cmq_exit)(struct hns_roce_dev *hr_dev);922 int (*hw_profile)(struct hns_roce_dev *hr_dev);923 int (*hw_init)(struct hns_roce_dev *hr_dev);924 void (*hw_exit)(struct hns_roce_dev *hr_dev);925 int (*post_mbox)(struct hns_roce_dev *hr_dev,926 struct hns_roce_mbox_msg *mbox_msg);927 int (*poll_mbox_done)(struct hns_roce_dev *hr_dev);928 bool (*chk_mbox_avail)(struct hns_roce_dev *hr_dev, bool *is_busy);929 int (*set_gid)(struct hns_roce_dev *hr_dev, int gid_index,930 const union ib_gid *gid, const struct ib_gid_attr *attr);931 int (*set_mac)(struct hns_roce_dev *hr_dev, u8 phy_port,932 const u8 *addr);933 int (*write_mtpt)(struct hns_roce_dev *hr_dev, void *mb_buf,934 struct hns_roce_mr *mr);935 int (*rereg_write_mtpt)(struct hns_roce_dev *hr_dev,936 struct hns_roce_mr *mr, int flags,937 void *mb_buf);938 int (*frmr_write_mtpt)(void *mb_buf, struct hns_roce_mr *mr);939 int (*mw_write_mtpt)(void *mb_buf, struct hns_roce_mw *mw);940 void (*write_cqc)(struct hns_roce_dev *hr_dev,941 struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts,942 dma_addr_t dma_handle);943 int (*set_hem)(struct hns_roce_dev *hr_dev,944 struct hns_roce_hem_table *table, int obj, u32 step_idx);945 int (*clear_hem)(struct hns_roce_dev *hr_dev,946 struct hns_roce_hem_table *table, int obj,947 u32 step_idx);948 int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr,949 int attr_mask, enum ib_qp_state cur_state,950 enum ib_qp_state new_state, struct ib_udata *udata);951 int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev,952 struct hns_roce_qp *hr_qp);953 void (*dereg_mr)(struct hns_roce_dev *hr_dev);954 int (*init_eq)(struct hns_roce_dev *hr_dev);955 void (*cleanup_eq)(struct hns_roce_dev *hr_dev);956 int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);957 int (*query_cqc)(struct hns_roce_dev *hr_dev, u32 cqn, void *buffer);958 int (*query_qpc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer);959 int (*query_mpt)(struct hns_roce_dev *hr_dev, u32 key, void *buffer);960 int (*query_srqc)(struct hns_roce_dev *hr_dev, u32 srqn, void *buffer);961 int (*query_sccc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer);962 int (*query_hw_counter)(struct hns_roce_dev *hr_dev,963 u64 *stats, u32 port, int *hw_counters);964 int (*get_dscp)(struct hns_roce_dev *hr_dev, u8 dscp,965 u8 *tc_mode, u8 *priority);966 const struct ib_device_ops *hns_roce_dev_ops;967 const struct ib_device_ops *hns_roce_dev_srq_ops;968};969 970struct hns_roce_dev {971 struct ib_device ib_dev;972 struct pci_dev *pci_dev;973 struct device *dev;974 struct hns_roce_uar priv_uar;975 const char *irq_names[HNS_ROCE_MAX_IRQ_NUM];976 spinlock_t sm_lock;977 bool active;978 bool is_reset;979 bool dis_db;980 unsigned long reset_cnt;981 struct hns_roce_ib_iboe iboe;982 enum hns_roce_device_state state;983 struct list_head qp_list; /* list of all qps on this dev */984 spinlock_t qp_list_lock; /* protect qp_list */985 struct list_head dip_list; /* list of all dest ips on this dev */986 spinlock_t dip_list_lock; /* protect dip_list */987 988 struct list_head pgdir_list;989 struct mutex pgdir_mutex;990 int irq[HNS_ROCE_MAX_IRQ_NUM];991 u8 __iomem *reg_base;992 void __iomem *mem_base;993 struct hns_roce_caps caps;994 struct xarray qp_table_xa;995 996 unsigned char dev_addr[HNS_ROCE_MAX_PORTS][ETH_ALEN];997 u64 sys_image_guid;998 u32 vendor_id;999 u32 vendor_part_id;1000 u32 hw_rev;1001 void __iomem *priv_addr;1002 1003 struct hns_roce_cmdq cmd;1004 struct hns_roce_ida pd_ida;1005 struct hns_roce_ida xrcd_ida;1006 struct hns_roce_ida uar_ida;1007 struct hns_roce_mr_table mr_table;1008 struct hns_roce_cq_table cq_table;1009 struct hns_roce_srq_table srq_table;1010 struct hns_roce_qp_table qp_table;1011 struct hns_roce_eq_table eq_table;1012 struct hns_roce_hem_table qpc_timer_table;1013 struct hns_roce_hem_table cqc_timer_table;1014 /* GMV is the memory area that the driver allocates for the hardware1015 * to store SGID, SMAC and VLAN information.1016 */1017 struct hns_roce_hem_table gmv_table;1018 1019 int cmd_mod;1020 int loop_idc;1021 u32 sdb_offset;1022 u32 odb_offset;1023 const struct hns_roce_hw *hw;1024 void *priv;1025 struct workqueue_struct *irq_workq;1026 struct work_struct ecc_work;1027 u32 func_num;1028 u32 is_vf;1029 u32 cong_algo_tmpl_id;1030 u64 dwqe_page;1031 struct hns_roce_dev_debugfs dbgfs;1032 atomic64_t *dfx_cnt;1033};1034 1035static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev)1036{1037 return container_of(ib_dev, struct hns_roce_dev, ib_dev);1038}1039 1040static inline struct hns_roce_ucontext1041 *to_hr_ucontext(struct ib_ucontext *ibucontext)1042{1043 return container_of(ibucontext, struct hns_roce_ucontext, ibucontext);1044}1045 1046static inline struct hns_roce_pd *to_hr_pd(struct ib_pd *ibpd)1047{1048 return container_of(ibpd, struct hns_roce_pd, ibpd);1049}1050 1051static inline struct hns_roce_xrcd *to_hr_xrcd(struct ib_xrcd *ibxrcd)1052{1053 return container_of(ibxrcd, struct hns_roce_xrcd, ibxrcd);1054}1055 1056static inline struct hns_roce_ah *to_hr_ah(struct ib_ah *ibah)1057{1058 return container_of(ibah, struct hns_roce_ah, ibah);1059}1060 1061static inline struct hns_roce_mr *to_hr_mr(struct ib_mr *ibmr)1062{1063 return container_of(ibmr, struct hns_roce_mr, ibmr);1064}1065 1066static inline struct hns_roce_mw *to_hr_mw(struct ib_mw *ibmw)1067{1068 return container_of(ibmw, struct hns_roce_mw, ibmw);1069}1070 1071static inline struct hns_roce_qp *to_hr_qp(struct ib_qp *ibqp)1072{1073 return container_of(ibqp, struct hns_roce_qp, ibqp);1074}1075 1076static inline struct hns_roce_cq *to_hr_cq(struct ib_cq *ib_cq)1077{1078 return container_of(ib_cq, struct hns_roce_cq, ib_cq);1079}1080 1081static inline struct hns_roce_srq *to_hr_srq(struct ib_srq *ibsrq)1082{1083 return container_of(ibsrq, struct hns_roce_srq, ibsrq);1084}1085 1086static inline struct hns_user_mmap_entry *1087to_hns_mmap(struct rdma_user_mmap_entry *rdma_entry)1088{1089 return container_of(rdma_entry, struct hns_user_mmap_entry, rdma_entry);1090}1091 1092static inline void hns_roce_write64_k(__le32 val[2], void __iomem *dest)1093{1094 writeq(*(u64 *)val, dest);1095}1096 1097static inline struct hns_roce_qp1098 *__hns_roce_qp_lookup(struct hns_roce_dev *hr_dev, u32 qpn)1099{1100 return xa_load(&hr_dev->qp_table_xa, qpn);1101}1102 1103static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf,1104 unsigned int offset)1105{1106 return (char *)(buf->trunk_list[offset >> buf->trunk_shift].buf) +1107 (offset & ((1 << buf->trunk_shift) - 1));1108}1109 1110static inline dma_addr_t hns_roce_buf_dma_addr(struct hns_roce_buf *buf,1111 unsigned int offset)1112{1113 return buf->trunk_list[offset >> buf->trunk_shift].map +1114 (offset & ((1 << buf->trunk_shift) - 1));1115}1116 1117static inline dma_addr_t hns_roce_buf_page(struct hns_roce_buf *buf, u32 idx)1118{1119 return hns_roce_buf_dma_addr(buf, idx << buf->page_shift);1120}1121 1122#define hr_hw_page_align(x) ALIGN(x, 1 << HNS_HW_PAGE_SHIFT)1123 1124static inline u64 to_hr_hw_page_addr(u64 addr)1125{1126 return addr >> HNS_HW_PAGE_SHIFT;1127}1128 1129static inline u32 to_hr_hw_page_shift(u32 page_shift)1130{1131 return page_shift - HNS_HW_PAGE_SHIFT;1132}1133 1134static inline u32 to_hr_hem_hopnum(u32 hopnum, u32 count)1135{1136 if (count > 0)1137 return hopnum == HNS_ROCE_HOP_NUM_0 ? 0 : hopnum;1138 1139 return 0;1140}1141 1142static inline u32 to_hr_hem_entries_size(u32 count, u32 buf_shift)1143{1144 return hr_hw_page_align(count << buf_shift);1145}1146 1147static inline u32 to_hr_hem_entries_count(u32 count, u32 buf_shift)1148{1149 return hr_hw_page_align(count << buf_shift) >> buf_shift;1150}1151 1152static inline u32 to_hr_hem_entries_shift(u32 count, u32 buf_shift)1153{1154 if (!count)1155 return 0;1156 1157 return ilog2(to_hr_hem_entries_count(count, buf_shift));1158}1159 1160#define DSCP_SHIFT 21161 1162static inline u8 get_tclass(const struct ib_global_route *grh)1163{1164 return grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP ?1165 grh->traffic_class >> DSCP_SHIFT : grh->traffic_class;1166}1167 1168void hns_roce_init_uar_table(struct hns_roce_dev *dev);1169int hns_roce_uar_alloc(struct hns_roce_dev *dev, struct hns_roce_uar *uar);1170 1171int hns_roce_cmd_init(struct hns_roce_dev *hr_dev);1172void hns_roce_cmd_cleanup(struct hns_roce_dev *hr_dev);1173void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status,1174 u64 out_param);1175int hns_roce_cmd_use_events(struct hns_roce_dev *hr_dev);1176void hns_roce_cmd_use_polling(struct hns_roce_dev *hr_dev);1177 1178/* hns roce hw need current block and next block addr from mtt */1179#define MTT_MIN_COUNT 21180static inline dma_addr_t hns_roce_get_mtr_ba(struct hns_roce_mtr *mtr)1181{1182 return mtr->hem_cfg.root_ba;1183}1184 1185int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,1186 u32 offset, u64 *mtt_buf, int mtt_max);1187int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,1188 struct hns_roce_buf_attr *buf_attr,1189 unsigned int page_shift, struct ib_udata *udata,1190 unsigned long user_addr);1191void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev,1192 struct hns_roce_mtr *mtr);1193int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,1194 dma_addr_t *pages, unsigned int page_cnt);1195 1196void hns_roce_init_pd_table(struct hns_roce_dev *hr_dev);1197void hns_roce_init_mr_table(struct hns_roce_dev *hr_dev);1198void hns_roce_init_cq_table(struct hns_roce_dev *hr_dev);1199int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev);1200void hns_roce_init_srq_table(struct hns_roce_dev *hr_dev);1201void hns_roce_init_xrcd_table(struct hns_roce_dev *hr_dev);1202 1203void hns_roce_cleanup_cq_table(struct hns_roce_dev *hr_dev);1204void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev);1205 1206void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev);1207 1208int hns_roce_create_ah(struct ib_ah *ah, struct rdma_ah_init_attr *init_attr,1209 struct ib_udata *udata);1210int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);1211static inline int hns_roce_destroy_ah(struct ib_ah *ah, u32 flags)1212{1213 return 0;1214}1215 1216int hns_roce_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);1217int hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);1218 1219struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc);1220struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,1221 u64 virt_addr, int access_flags,1222 struct ib_udata *udata);1223struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *mr, int flags, u64 start,1224 u64 length, u64 virt_addr,1225 int mr_access_flags, struct ib_pd *pd,1226 struct ib_udata *udata);1227struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,1228 u32 max_num_sg);1229int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,1230 unsigned int *sg_offset);1231int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);1232unsigned long key_to_hw_index(u32 key);1233 1234int hns_roce_alloc_mw(struct ib_mw *mw, struct ib_udata *udata);1235int hns_roce_dealloc_mw(struct ib_mw *ibmw);1236 1237void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf);1238struct hns_roce_buf *hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size,1239 u32 page_shift, u32 flags);1240 1241int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs,1242 int buf_cnt, struct hns_roce_buf *buf,1243 unsigned int page_shift);1244int hns_roce_get_umem_bufs(dma_addr_t *bufs,1245 int buf_cnt, struct ib_umem *umem,1246 unsigned int page_shift);1247 1248int hns_roce_create_srq(struct ib_srq *srq,1249 struct ib_srq_init_attr *srq_init_attr,1250 struct ib_udata *udata);1251int hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);1252 1253int hns_roce_alloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata);1254int hns_roce_dealloc_xrcd(struct ib_xrcd *ib_xrcd, struct ib_udata *udata);1255 1256int hns_roce_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *init_attr,1257 struct ib_udata *udata);1258int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,1259 int attr_mask, struct ib_udata *udata);1260void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);1261void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n);1262void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n);1263void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n);1264bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq,1265 struct ib_cq *ib_cq);1266void hns_roce_lock_cqs(struct hns_roce_cq *send_cq,1267 struct hns_roce_cq *recv_cq);1268void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,1269 struct hns_roce_cq *recv_cq);1270void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp);1271void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,1272 struct ib_udata *udata);1273__be32 send_ieth(const struct ib_send_wr *wr);1274int to_hr_qp_type(int qp_type);1275 1276int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr,1277 struct uverbs_attr_bundle *attrs);1278 1279int hns_roce_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);1280int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,1281 struct hns_roce_db *db);1282void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,1283 struct hns_roce_db *db);1284int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db,1285 int order);1286void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db);1287 1288void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn);1289void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type);1290void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp);1291void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type);1292void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type);1293void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev);1294int hns_roce_init(struct hns_roce_dev *hr_dev);1295void hns_roce_exit(struct hns_roce_dev *hr_dev);1296int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);1297int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);1298int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp);1299int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp);1300int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr);1301int hns_roce_fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ib_mr);1302int hns_roce_fill_res_srq_entry(struct sk_buff *msg, struct ib_srq *ib_srq);1303int hns_roce_fill_res_srq_entry_raw(struct sk_buff *msg, struct ib_srq *ib_srq);1304struct hns_user_mmap_entry *1305hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,1306 size_t length,1307 enum hns_roce_mmap_type mmap_type);1308bool check_sl_valid(struct hns_roce_dev *hr_dev, u8 sl);1309 1310#endif /* _HNS_ROCE_DEVICE_H */1311