834 lines · c
1/*2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.3 *4 * This software is available to you under a choice of one of two5 * licenses. You may choose to be licensed under the terms of the GNU6 * General Public License (GPL) Version 2, available from the file7 * COPYING in the main directory of this source tree, or the8 * OpenIB.org BSD license below:9 *10 * Redistribution and use in source and binary forms, with or11 * without modification, are permitted provided that the following12 * conditions are met:13 *14 * - Redistributions of source code must retain the above15 * copyright notice, this list of conditions and the following16 * disclaimer.17 * - Redistributions in binary form must reproduce the above18 * copyright notice, this list of conditions and the following19 * disclaimer in the documentation and/or other materials20 * provided with the distribution.21 *22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,23 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF24 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND25 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS26 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN27 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN28 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE29 * SOFTWARE.30 */31#ifndef __T4_H__32#define __T4_H__33 34#include "t4_hw.h"35#include "t4_regs.h"36#include "t4_values.h"37#include "t4_msg.h"38#include "t4_tcb.h"39#include "t4fw_ri_api.h"40 41#define T4_MAX_NUM_PD 6553642#define T4_MAX_MR_SIZE (~0ULL)43#define T4_PAGESIZE_MASK 0xffff000 /* 4KB-128MB */44#define T4_STAG_UNSET 0xffffffff45#define T4_FW_MAJ 046#define PCIE_MA_SYNC_A 0x30b447 48struct t4_status_page {49 __be32 rsvd1; /* flit 0 - hw owns */50 __be16 rsvd2;51 __be16 qid;52 __be16 cidx;53 __be16 pidx;54 u8 qp_err; /* flit 1 - sw owns */55 u8 db_off;56 u8 pad[2];57 u16 host_wq_pidx;58 u16 host_cidx;59 u16 host_pidx;60 u16 pad2;61 u32 srqidx;62};63 64#define T4_RQT_ENTRY_SHIFT 665#define T4_RQT_ENTRY_SIZE BIT(T4_RQT_ENTRY_SHIFT)66#define T4_EQ_ENTRY_SIZE 6467 68#define T4_SQ_NUM_SLOTS 569#define T4_SQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_SQ_NUM_SLOTS)70#define T4_MAX_SEND_SGE ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_send_wr) - \71 sizeof(struct fw_ri_isgl)) / sizeof(struct fw_ri_sge))72#define T4_MAX_SEND_INLINE ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_send_wr) - \73 sizeof(struct fw_ri_immd)))74#define T4_MAX_WRITE_INLINE ((T4_SQ_NUM_BYTES - \75 sizeof(struct fw_ri_rdma_write_wr) - \76 sizeof(struct fw_ri_immd)))77#define T4_MAX_WRITE_SGE ((T4_SQ_NUM_BYTES - \78 sizeof(struct fw_ri_rdma_write_wr) - \79 sizeof(struct fw_ri_isgl)) / sizeof(struct fw_ri_sge))80#define T4_MAX_FR_IMMD ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_fr_nsmr_wr) - \81 sizeof(struct fw_ri_immd)) & ~31UL)82#define T4_MAX_FR_IMMD_DEPTH (T4_MAX_FR_IMMD / sizeof(u64))83#define T4_MAX_FR_DSGL 102484#define T4_MAX_FR_DSGL_DEPTH (T4_MAX_FR_DSGL / sizeof(u64))85 86static inline int t4_max_fr_depth(int use_dsgl)87{88 return use_dsgl ? T4_MAX_FR_DSGL_DEPTH : T4_MAX_FR_IMMD_DEPTH;89}90 91#define T4_RQ_NUM_SLOTS 292#define T4_RQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_RQ_NUM_SLOTS)93#define T4_MAX_RECV_SGE 494 95#define T4_WRITE_CMPL_MAX_SGL 496#define T4_WRITE_CMPL_MAX_CQE 1697 98union t4_wr {99 struct fw_ri_res_wr res;100 struct fw_ri_wr ri;101 struct fw_ri_rdma_write_wr write;102 struct fw_ri_send_wr send;103 struct fw_ri_rdma_read_wr read;104 struct fw_ri_bind_mw_wr bind;105 struct fw_ri_fr_nsmr_wr fr;106 struct fw_ri_fr_nsmr_tpte_wr fr_tpte;107 struct fw_ri_inv_lstag_wr inv;108 struct fw_ri_rdma_write_cmpl_wr write_cmpl;109 struct t4_status_page status;110 __be64 flits[T4_EQ_ENTRY_SIZE / sizeof(__be64) * T4_SQ_NUM_SLOTS];111};112 113union t4_recv_wr {114 struct fw_ri_recv_wr recv;115 struct t4_status_page status;116 __be64 flits[T4_EQ_ENTRY_SIZE / sizeof(__be64) * T4_RQ_NUM_SLOTS];117};118 119static inline void init_wr_hdr(union t4_wr *wqe, u16 wrid,120 enum fw_wr_opcodes opcode, u8 flags, u8 len16)121{122 wqe->send.opcode = (u8)opcode;123 wqe->send.flags = flags;124 wqe->send.wrid = wrid;125 wqe->send.r1[0] = 0;126 wqe->send.r1[1] = 0;127 wqe->send.r1[2] = 0;128 wqe->send.len16 = len16;129}130 131/* CQE/AE status codes */132#define T4_ERR_SUCCESS 0x0133#define T4_ERR_STAG 0x1 /* STAG invalid: either the */134 /* STAG is offlimt, being 0, */135 /* or STAG_key mismatch */136#define T4_ERR_PDID 0x2 /* PDID mismatch */137#define T4_ERR_QPID 0x3 /* QPID mismatch */138#define T4_ERR_ACCESS 0x4 /* Invalid access right */139#define T4_ERR_WRAP 0x5 /* Wrap error */140#define T4_ERR_BOUND 0x6 /* base and bounds voilation */141#define T4_ERR_INVALIDATE_SHARED_MR 0x7 /* attempt to invalidate a */142 /* shared memory region */143#define T4_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8 /* attempt to invalidate a */144 /* shared memory region */145#define T4_ERR_ECC 0x9 /* ECC error detected */146#define T4_ERR_ECC_PSTAG 0xA /* ECC error detected when */147 /* reading PSTAG for a MW */148 /* Invalidate */149#define T4_ERR_PBL_ADDR_BOUND 0xB /* pbl addr out of bounds: */150 /* software error */151#define T4_ERR_SWFLUSH 0xC /* SW FLUSHED */152#define T4_ERR_CRC 0x10 /* CRC error */153#define T4_ERR_MARKER 0x11 /* Marker error */154#define T4_ERR_PDU_LEN_ERR 0x12 /* invalid PDU length */155#define T4_ERR_OUT_OF_RQE 0x13 /* out of RQE */156#define T4_ERR_DDP_VERSION 0x14 /* wrong DDP version */157#define T4_ERR_RDMA_VERSION 0x15 /* wrong RDMA version */158#define T4_ERR_OPCODE 0x16 /* invalid rdma opcode */159#define T4_ERR_DDP_QUEUE_NUM 0x17 /* invalid ddp queue number */160#define T4_ERR_MSN 0x18 /* MSN error */161#define T4_ERR_TBIT 0x19 /* tag bit not set correctly */162#define T4_ERR_MO 0x1A /* MO not 0 for TERMINATE */163 /* or READ_REQ */164#define T4_ERR_MSN_GAP 0x1B165#define T4_ERR_MSN_RANGE 0x1C166#define T4_ERR_IRD_OVERFLOW 0x1D167#define T4_ERR_RQE_ADDR_BOUND 0x1E /* RQE addr out of bounds: */168 /* software error */169#define T4_ERR_INTERNAL_ERR 0x1F /* internal error (opcode */170 /* mismatch) */171/*172 * CQE defs173 */174struct t4_cqe {175 __be32 header;176 __be32 len;177 union {178 struct {179 __be32 stag;180 __be32 msn;181 } rcqe;182 struct {183 __be32 stag;184 u16 nada2;185 u16 cidx;186 } scqe;187 struct {188 __be32 wrid_hi;189 __be32 wrid_low;190 } gen;191 struct {192 __be32 stag;193 __be32 msn;194 __be32 reserved;195 __be32 abs_rqe_idx;196 } srcqe;197 struct {198 __be32 mo;199 __be32 msn;200 /*201 * Use union for immediate data to be consistent with202 * stack's 32 bit data and iWARP spec's 64 bit data.203 */204 union {205 struct {206 __be32 imm_data32;207 u32 reserved;208 } ib_imm_data;209 __be64 imm_data64;210 } iw_imm_data;211 } imm_data_rcqe;212 213 u64 drain_cookie;214 __be64 flits[3];215 } u;216 __be64 reserved[3];217 __be64 bits_type_ts;218};219 220/* macros for flit 0 of the cqe */221 222#define CQE_QPID_S 12223#define CQE_QPID_M 0xFFFFF224#define CQE_QPID_G(x) ((((x) >> CQE_QPID_S)) & CQE_QPID_M)225#define CQE_QPID_V(x) ((x)<<CQE_QPID_S)226 227#define CQE_SWCQE_S 11228#define CQE_SWCQE_M 0x1229#define CQE_SWCQE_G(x) ((((x) >> CQE_SWCQE_S)) & CQE_SWCQE_M)230#define CQE_SWCQE_V(x) ((x)<<CQE_SWCQE_S)231 232#define CQE_DRAIN_S 10233#define CQE_DRAIN_M 0x1234#define CQE_DRAIN_G(x) ((((x) >> CQE_DRAIN_S)) & CQE_DRAIN_M)235#define CQE_DRAIN_V(x) ((x)<<CQE_DRAIN_S)236 237#define CQE_STATUS_S 5238#define CQE_STATUS_M 0x1F239#define CQE_STATUS_G(x) ((((x) >> CQE_STATUS_S)) & CQE_STATUS_M)240#define CQE_STATUS_V(x) ((x)<<CQE_STATUS_S)241 242#define CQE_TYPE_S 4243#define CQE_TYPE_M 0x1244#define CQE_TYPE_G(x) ((((x) >> CQE_TYPE_S)) & CQE_TYPE_M)245#define CQE_TYPE_V(x) ((x)<<CQE_TYPE_S)246 247#define CQE_OPCODE_S 0248#define CQE_OPCODE_M 0xF249#define CQE_OPCODE_G(x) ((((x) >> CQE_OPCODE_S)) & CQE_OPCODE_M)250#define CQE_OPCODE_V(x) ((x)<<CQE_OPCODE_S)251 252#define SW_CQE(x) (CQE_SWCQE_G(be32_to_cpu((x)->header)))253#define DRAIN_CQE(x) (CQE_DRAIN_G(be32_to_cpu((x)->header)))254#define CQE_QPID(x) (CQE_QPID_G(be32_to_cpu((x)->header)))255#define CQE_TYPE(x) (CQE_TYPE_G(be32_to_cpu((x)->header)))256#define SQ_TYPE(x) (CQE_TYPE((x)))257#define RQ_TYPE(x) (!CQE_TYPE((x)))258#define CQE_STATUS(x) (CQE_STATUS_G(be32_to_cpu((x)->header)))259#define CQE_OPCODE(x) (CQE_OPCODE_G(be32_to_cpu((x)->header)))260 261#define CQE_SEND_OPCODE(x)( \262 (CQE_OPCODE_G(be32_to_cpu((x)->header)) == FW_RI_SEND) || \263 (CQE_OPCODE_G(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_SE) || \264 (CQE_OPCODE_G(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_INV) || \265 (CQE_OPCODE_G(be32_to_cpu((x)->header)) == FW_RI_SEND_WITH_SE_INV))266 267#define CQE_LEN(x) (be32_to_cpu((x)->len))268 269/* used for RQ completion processing */270#define CQE_WRID_STAG(x) (be32_to_cpu((x)->u.rcqe.stag))271#define CQE_WRID_MSN(x) (be32_to_cpu((x)->u.rcqe.msn))272#define CQE_ABS_RQE_IDX(x) (be32_to_cpu((x)->u.srcqe.abs_rqe_idx))273#define CQE_IMM_DATA(x)( \274 (x)->u.imm_data_rcqe.iw_imm_data.ib_imm_data.imm_data32)275 276/* used for SQ completion processing */277#define CQE_WRID_SQ_IDX(x) ((x)->u.scqe.cidx)278#define CQE_WRID_FR_STAG(x) (be32_to_cpu((x)->u.scqe.stag))279 280/* generic accessor macros */281#define CQE_WRID_HI(x) (be32_to_cpu((x)->u.gen.wrid_hi))282#define CQE_WRID_LOW(x) (be32_to_cpu((x)->u.gen.wrid_low))283#define CQE_DRAIN_COOKIE(x) ((x)->u.drain_cookie)284 285/* macros for flit 3 of the cqe */286#define CQE_GENBIT_S 63287#define CQE_GENBIT_M 0x1288#define CQE_GENBIT_G(x) (((x) >> CQE_GENBIT_S) & CQE_GENBIT_M)289#define CQE_GENBIT_V(x) ((x)<<CQE_GENBIT_S)290 291#define CQE_OVFBIT_S 62292#define CQE_OVFBIT_M 0x1293#define CQE_OVFBIT_G(x) ((((x) >> CQE_OVFBIT_S)) & CQE_OVFBIT_M)294 295#define CQE_IQTYPE_S 60296#define CQE_IQTYPE_M 0x3297#define CQE_IQTYPE_G(x) ((((x) >> CQE_IQTYPE_S)) & CQE_IQTYPE_M)298 299#define CQE_TS_M 0x0fffffffffffffffULL300#define CQE_TS_G(x) ((x) & CQE_TS_M)301 302#define CQE_OVFBIT(x) ((unsigned)CQE_OVFBIT_G(be64_to_cpu((x)->bits_type_ts)))303#define CQE_GENBIT(x) ((unsigned)CQE_GENBIT_G(be64_to_cpu((x)->bits_type_ts)))304#define CQE_TS(x) (CQE_TS_G(be64_to_cpu((x)->bits_type_ts)))305 306struct t4_swsqe {307 u64 wr_id;308 struct t4_cqe cqe;309 int read_len;310 int opcode;311 int complete;312 int signaled;313 u16 idx;314 int flushed;315 ktime_t host_time;316 u64 sge_ts;317};318 319static inline pgprot_t t4_pgprot_wc(pgprot_t prot)320{321#if defined(__i386__) || defined(__x86_64__) || defined(CONFIG_PPC64)322 return pgprot_writecombine(prot);323#else324 return pgprot_noncached(prot);325#endif326}327 328enum {329 T4_SQ_ONCHIP = (1<<0),330};331 332struct t4_sq {333 union t4_wr *queue;334 dma_addr_t dma_addr;335 DEFINE_DMA_UNMAP_ADDR(mapping);336 unsigned long phys_addr;337 struct t4_swsqe *sw_sq;338 struct t4_swsqe *oldest_read;339 void __iomem *bar2_va;340 u64 bar2_pa;341 size_t memsize;342 u32 bar2_qid;343 u32 qid;344 u16 in_use;345 u16 size;346 u16 cidx;347 u16 pidx;348 u16 wq_pidx;349 u16 wq_pidx_inc;350 u16 flags;351 short flush_cidx;352};353 354struct t4_swrqe {355 u64 wr_id;356 ktime_t host_time;357 u64 sge_ts;358 int valid;359};360 361struct t4_rq {362 union t4_recv_wr *queue;363 dma_addr_t dma_addr;364 DEFINE_DMA_UNMAP_ADDR(mapping);365 struct t4_swrqe *sw_rq;366 void __iomem *bar2_va;367 u64 bar2_pa;368 size_t memsize;369 u32 bar2_qid;370 u32 qid;371 u32 msn;372 u32 rqt_hwaddr;373 u16 rqt_size;374 u16 in_use;375 u16 size;376 u16 cidx;377 u16 pidx;378 u16 wq_pidx;379 u16 wq_pidx_inc;380};381 382struct t4_wq {383 struct t4_sq sq;384 struct t4_rq rq;385 void __iomem *db;386 struct c4iw_rdev *rdev;387 int flushed;388 u8 *qp_errp;389 u32 *srqidxp;390};391 392struct t4_srq_pending_wr {393 u64 wr_id;394 union t4_recv_wr wqe;395 u8 len16;396};397 398struct t4_srq {399 union t4_recv_wr *queue;400 dma_addr_t dma_addr;401 DEFINE_DMA_UNMAP_ADDR(mapping);402 struct t4_swrqe *sw_rq;403 void __iomem *bar2_va;404 u64 bar2_pa;405 size_t memsize;406 u32 bar2_qid;407 u32 qid;408 u32 msn;409 u32 rqt_hwaddr;410 u32 rqt_abs_idx;411 u16 rqt_size;412 u16 size;413 u16 cidx;414 u16 pidx;415 u16 wq_pidx;416 u16 wq_pidx_inc;417 u16 in_use;418 struct t4_srq_pending_wr *pending_wrs;419 u16 pending_cidx;420 u16 pending_pidx;421 u16 pending_in_use;422 u16 ooo_count;423};424 425static inline u32 t4_srq_avail(struct t4_srq *srq)426{427 return srq->size - 1 - srq->in_use;428}429 430static inline void t4_srq_produce(struct t4_srq *srq, u8 len16)431{432 srq->in_use++;433 if (++srq->pidx == srq->size)434 srq->pidx = 0;435 srq->wq_pidx += DIV_ROUND_UP(len16 * 16, T4_EQ_ENTRY_SIZE);436 if (srq->wq_pidx >= srq->size * T4_RQ_NUM_SLOTS)437 srq->wq_pidx %= srq->size * T4_RQ_NUM_SLOTS;438 srq->queue[srq->size].status.host_pidx = srq->pidx;439}440 441static inline void t4_srq_produce_pending_wr(struct t4_srq *srq)442{443 srq->pending_in_use++;444 srq->in_use++;445 if (++srq->pending_pidx == srq->size)446 srq->pending_pidx = 0;447}448 449static inline void t4_srq_consume_pending_wr(struct t4_srq *srq)450{451 srq->pending_in_use--;452 srq->in_use--;453 if (++srq->pending_cidx == srq->size)454 srq->pending_cidx = 0;455}456 457static inline void t4_srq_produce_ooo(struct t4_srq *srq)458{459 srq->in_use--;460 srq->ooo_count++;461}462 463static inline void t4_srq_consume_ooo(struct t4_srq *srq)464{465 srq->cidx++;466 if (srq->cidx == srq->size)467 srq->cidx = 0;468 srq->queue[srq->size].status.host_cidx = srq->cidx;469 srq->ooo_count--;470}471 472static inline void t4_srq_consume(struct t4_srq *srq)473{474 srq->in_use--;475 if (++srq->cidx == srq->size)476 srq->cidx = 0;477 srq->queue[srq->size].status.host_cidx = srq->cidx;478}479 480static inline int t4_rqes_posted(struct t4_wq *wq)481{482 return wq->rq.in_use;483}484 485static inline int t4_rq_empty(struct t4_wq *wq)486{487 return wq->rq.in_use == 0;488}489 490static inline u32 t4_rq_avail(struct t4_wq *wq)491{492 return wq->rq.size - 1 - wq->rq.in_use;493}494 495static inline void t4_rq_produce(struct t4_wq *wq, u8 len16)496{497 wq->rq.in_use++;498 if (++wq->rq.pidx == wq->rq.size)499 wq->rq.pidx = 0;500 wq->rq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);501 if (wq->rq.wq_pidx >= wq->rq.size * T4_RQ_NUM_SLOTS)502 wq->rq.wq_pidx %= wq->rq.size * T4_RQ_NUM_SLOTS;503}504 505static inline void t4_rq_consume(struct t4_wq *wq)506{507 wq->rq.in_use--;508 if (++wq->rq.cidx == wq->rq.size)509 wq->rq.cidx = 0;510}511 512static inline u16 t4_rq_host_wq_pidx(struct t4_wq *wq)513{514 return wq->rq.queue[wq->rq.size].status.host_wq_pidx;515}516 517static inline u16 t4_rq_wq_size(struct t4_wq *wq)518{519 return wq->rq.size * T4_RQ_NUM_SLOTS;520}521 522static inline int t4_sq_onchip(struct t4_sq *sq)523{524 return sq->flags & T4_SQ_ONCHIP;525}526 527static inline int t4_sq_empty(struct t4_wq *wq)528{529 return wq->sq.in_use == 0;530}531 532static inline u32 t4_sq_avail(struct t4_wq *wq)533{534 return wq->sq.size - 1 - wq->sq.in_use;535}536 537static inline void t4_sq_produce(struct t4_wq *wq, u8 len16)538{539 wq->sq.in_use++;540 if (++wq->sq.pidx == wq->sq.size)541 wq->sq.pidx = 0;542 wq->sq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);543 if (wq->sq.wq_pidx >= wq->sq.size * T4_SQ_NUM_SLOTS)544 wq->sq.wq_pidx %= wq->sq.size * T4_SQ_NUM_SLOTS;545}546 547static inline void t4_sq_consume(struct t4_wq *wq)548{549 if (wq->sq.cidx == wq->sq.flush_cidx)550 wq->sq.flush_cidx = -1;551 wq->sq.in_use--;552 if (++wq->sq.cidx == wq->sq.size)553 wq->sq.cidx = 0;554}555 556static inline u16 t4_sq_host_wq_pidx(struct t4_wq *wq)557{558 return wq->sq.queue[wq->sq.size].status.host_wq_pidx;559}560 561static inline u16 t4_sq_wq_size(struct t4_wq *wq)562{563 return wq->sq.size * T4_SQ_NUM_SLOTS;564}565 566/* This function copies 64 byte coalesced work request to memory567 * mapped BAR2 space. For coalesced WRs, the SGE fetches data568 * from the FIFO instead of from Host.569 */570static inline void pio_copy(u64 __iomem *dst, u64 *src)571{572 int count = 8;573 574 while (count) {575 writeq(*src, dst);576 src++;577 dst++;578 count--;579 }580}581 582static inline void t4_ring_srq_db(struct t4_srq *srq, u16 inc, u8 len16,583 union t4_recv_wr *wqe)584{585 /* Flush host queue memory writes. */586 wmb();587 if (inc == 1 && srq->bar2_qid == 0 && wqe) {588 pr_debug("%s : WC srq->pidx = %d; len16=%d\n",589 __func__, srq->pidx, len16);590 pio_copy(srq->bar2_va + SGE_UDB_WCDOORBELL, (u64 *)wqe);591 } else {592 pr_debug("%s: DB srq->pidx = %d; len16=%d\n",593 __func__, srq->pidx, len16);594 writel(PIDX_T5_V(inc) | QID_V(srq->bar2_qid),595 srq->bar2_va + SGE_UDB_KDOORBELL);596 }597 /* Flush user doorbell area writes. */598 wmb();599}600 601static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc, union t4_wr *wqe)602{603 604 /* Flush host queue memory writes. */605 wmb();606 if (wq->sq.bar2_va) {607 if (inc == 1 && wq->sq.bar2_qid == 0 && wqe) {608 pr_debug("WC wq->sq.pidx = %d\n", wq->sq.pidx);609 pio_copy((u64 __iomem *)610 (wq->sq.bar2_va + SGE_UDB_WCDOORBELL),611 (u64 *)wqe);612 } else {613 pr_debug("DB wq->sq.pidx = %d\n", wq->sq.pidx);614 writel(PIDX_T5_V(inc) | QID_V(wq->sq.bar2_qid),615 wq->sq.bar2_va + SGE_UDB_KDOORBELL);616 }617 618 /* Flush user doorbell area writes. */619 wmb();620 return;621 }622 writel(QID_V(wq->sq.qid) | PIDX_V(inc), wq->db);623}624 625static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc,626 union t4_recv_wr *wqe)627{628 629 /* Flush host queue memory writes. */630 wmb();631 if (wq->rq.bar2_va) {632 if (inc == 1 && wq->rq.bar2_qid == 0 && wqe) {633 pr_debug("WC wq->rq.pidx = %d\n", wq->rq.pidx);634 pio_copy((u64 __iomem *)635 (wq->rq.bar2_va + SGE_UDB_WCDOORBELL),636 (void *)wqe);637 } else {638 pr_debug("DB wq->rq.pidx = %d\n", wq->rq.pidx);639 writel(PIDX_T5_V(inc) | QID_V(wq->rq.bar2_qid),640 wq->rq.bar2_va + SGE_UDB_KDOORBELL);641 }642 643 /* Flush user doorbell area writes. */644 wmb();645 return;646 }647 writel(QID_V(wq->rq.qid) | PIDX_V(inc), wq->db);648}649 650static inline int t4_wq_in_error(struct t4_wq *wq)651{652 return *wq->qp_errp;653}654 655static inline void t4_set_wq_in_error(struct t4_wq *wq, u32 srqidx)656{657 if (srqidx)658 *wq->srqidxp = srqidx;659 *wq->qp_errp = 1;660}661 662static inline void t4_disable_wq_db(struct t4_wq *wq)663{664 wq->rq.queue[wq->rq.size].status.db_off = 1;665}666 667static inline void t4_enable_wq_db(struct t4_wq *wq)668{669 wq->rq.queue[wq->rq.size].status.db_off = 0;670}671 672enum t4_cq_flags {673 CQ_ARMED = 1,674};675 676struct t4_cq {677 struct t4_cqe *queue;678 dma_addr_t dma_addr;679 DEFINE_DMA_UNMAP_ADDR(mapping);680 struct t4_cqe *sw_queue;681 void __iomem *gts;682 void __iomem *bar2_va;683 u64 bar2_pa;684 u32 bar2_qid;685 struct c4iw_rdev *rdev;686 size_t memsize;687 __be64 bits_type_ts;688 u32 cqid;689 u32 qid_mask;690 int vector;691 u16 size; /* including status page */692 u16 cidx;693 u16 sw_pidx;694 u16 sw_cidx;695 u16 sw_in_use;696 u16 cidx_inc;697 u8 gen;698 u8 error;699 u8 *qp_errp;700 unsigned long flags;701};702 703static inline void write_gts(struct t4_cq *cq, u32 val)704{705 if (cq->bar2_va)706 writel(val | INGRESSQID_V(cq->bar2_qid),707 cq->bar2_va + SGE_UDB_GTS);708 else709 writel(val | INGRESSQID_V(cq->cqid), cq->gts);710}711 712static inline int t4_clear_cq_armed(struct t4_cq *cq)713{714 return test_and_clear_bit(CQ_ARMED, &cq->flags);715}716 717static inline int t4_arm_cq(struct t4_cq *cq, int se)718{719 u32 val;720 721 set_bit(CQ_ARMED, &cq->flags);722 while (cq->cidx_inc > CIDXINC_M) {723 val = SEINTARM_V(0) | CIDXINC_V(CIDXINC_M) | TIMERREG_V(7);724 write_gts(cq, val);725 cq->cidx_inc -= CIDXINC_M;726 }727 val = SEINTARM_V(se) | CIDXINC_V(cq->cidx_inc) | TIMERREG_V(6);728 write_gts(cq, val);729 cq->cidx_inc = 0;730 return 0;731}732 733static inline void t4_swcq_produce(struct t4_cq *cq)734{735 cq->sw_in_use++;736 if (cq->sw_in_use == cq->size) {737 pr_warn("%s cxgb4 sw cq overflow cqid %u\n",738 __func__, cq->cqid);739 cq->error = 1;740 cq->sw_in_use--;741 return;742 }743 if (++cq->sw_pidx == cq->size)744 cq->sw_pidx = 0;745}746 747static inline void t4_swcq_consume(struct t4_cq *cq)748{749 cq->sw_in_use--;750 if (++cq->sw_cidx == cq->size)751 cq->sw_cidx = 0;752}753 754static inline void t4_hwcq_consume(struct t4_cq *cq)755{756 cq->bits_type_ts = cq->queue[cq->cidx].bits_type_ts;757 if (++cq->cidx_inc == (cq->size >> 4) || cq->cidx_inc == CIDXINC_M) {758 u32 val;759 760 val = SEINTARM_V(0) | CIDXINC_V(cq->cidx_inc) | TIMERREG_V(7);761 write_gts(cq, val);762 cq->cidx_inc = 0;763 }764 if (++cq->cidx == cq->size) {765 cq->cidx = 0;766 cq->gen ^= 1;767 }768}769 770static inline int t4_valid_cqe(struct t4_cq *cq, struct t4_cqe *cqe)771{772 return (CQE_GENBIT(cqe) == cq->gen);773}774 775static inline int t4_cq_notempty(struct t4_cq *cq)776{777 return cq->sw_in_use || t4_valid_cqe(cq, &cq->queue[cq->cidx]);778}779 780static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)781{782 int ret;783 u16 prev_cidx;784 785 if (cq->cidx == 0)786 prev_cidx = cq->size - 1;787 else788 prev_cidx = cq->cidx - 1;789 790 if (cq->queue[prev_cidx].bits_type_ts != cq->bits_type_ts) {791 ret = -EOVERFLOW;792 cq->error = 1;793 pr_err("cq overflow cqid %u\n", cq->cqid);794 } else if (t4_valid_cqe(cq, &cq->queue[cq->cidx])) {795 796 /* Ensure CQE is flushed to memory */797 rmb();798 *cqe = &cq->queue[cq->cidx];799 ret = 0;800 } else801 ret = -ENODATA;802 return ret;803}804 805static inline int t4_next_cqe(struct t4_cq *cq, struct t4_cqe **cqe)806{807 int ret = 0;808 809 if (cq->error)810 ret = -ENODATA;811 else if (cq->sw_in_use)812 *cqe = &cq->sw_queue[cq->sw_cidx];813 else814 ret = t4_next_hw_cqe(cq, cqe);815 return ret;816}817 818static inline void t4_set_cq_in_error(struct t4_cq *cq)819{820 *cq->qp_errp = 1;821}822#endif823 824struct t4_dev_status_page {825 u8 db_off;826 u8 write_cmpl_supported;827 u16 pad2;828 u32 pad3;829 u64 qp_start;830 u64 qp_size;831 u64 cq_start;832 u64 cq_size;833};834