137 lines · c
1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */2/*3 * Copyright (c) 2020, Mellanox Technologies inc. All rights reserved.4 */5 6#ifndef _MLX5_IB_WR_H7#define _MLX5_IB_WR_H8 9#include "mlx5_ib.h"10 11enum {12 MLX5_IB_SQ_UMR_INLINE_THRESHOLD = 64,13};14 15struct mlx5_wqe_eth_pad {16 u8 rsvd0[16];17};18 19 20/* get_sq_edge - Get the next nearby edge.21 *22 * An 'edge' is defined as the first following address after the end23 * of the fragment or the SQ. Accordingly, during the WQE construction24 * which repetitively increases the pointer to write the next data, it25 * simply should check if it gets to an edge.26 *27 * @sq - SQ buffer.28 * @idx - Stride index in the SQ buffer.29 *30 * Return:31 * The new edge.32 */33static inline void *get_sq_edge(struct mlx5_ib_wq *sq, u32 idx)34{35 void *fragment_end;36 37 fragment_end = mlx5_frag_buf_get_wqe38 (&sq->fbc,39 mlx5_frag_buf_get_idx_last_contig_stride(&sq->fbc, idx));40 41 return fragment_end + MLX5_SEND_WQE_BB;42}43 44/* handle_post_send_edge - Check if we get to SQ edge. If yes, update to the45 * next nearby edge and get new address translation for current WQE position.46 * @sq: SQ buffer.47 * @seg: Current WQE position (16B aligned).48 * @wqe_sz: Total current WQE size [16B].49 * @cur_edge: Updated current edge.50 */51static inline void handle_post_send_edge(struct mlx5_ib_wq *sq, void **seg,52 u32 wqe_sz, void **cur_edge)53{54 u32 idx;55 56 if (likely(*seg != *cur_edge))57 return;58 59 idx = (sq->cur_post + (wqe_sz >> 2)) & (sq->wqe_cnt - 1);60 *cur_edge = get_sq_edge(sq, idx);61 62 *seg = mlx5_frag_buf_get_wqe(&sq->fbc, idx);63}64 65/* mlx5r_memcpy_send_wqe - copy data from src to WQE and update the relevant66 * WQ's pointers. At the end @seg is aligned to 16B regardless the copied size.67 * @sq: SQ buffer.68 * @cur_edge: Updated current edge.69 * @seg: Current WQE position (16B aligned).70 * @wqe_sz: Total current WQE size [16B].71 * @src: Pointer to copy from.72 * @n: Number of bytes to copy.73 */74static inline void mlx5r_memcpy_send_wqe(struct mlx5_ib_wq *sq, void **cur_edge,75 void **seg, u32 *wqe_sz,76 const void *src, size_t n)77{78 while (likely(n)) {79 size_t leftlen = *cur_edge - *seg;80 size_t copysz = min_t(size_t, leftlen, n);81 size_t stride;82 83 memcpy(*seg, src, copysz);84 85 n -= copysz;86 src += copysz;87 stride = !n ? ALIGN(copysz, 16) : copysz;88 *seg += stride;89 *wqe_sz += stride >> 4;90 handle_post_send_edge(sq, seg, *wqe_sz, cur_edge);91 }92}93 94int mlx5r_wq_overflow(struct mlx5_ib_wq *wq, int nreq, struct ib_cq *ib_cq);95int mlx5r_begin_wqe(struct mlx5_ib_qp *qp, void **seg,96 struct mlx5_wqe_ctrl_seg **ctrl, unsigned int *idx,97 int *size, void **cur_edge, int nreq, __be32 general_id,98 bool send_signaled, bool solicited);99void mlx5r_finish_wqe(struct mlx5_ib_qp *qp, struct mlx5_wqe_ctrl_seg *ctrl,100 void *seg, u8 size, void *cur_edge, unsigned int idx,101 u64 wr_id, int nreq, u8 fence, u32 mlx5_opcode);102void mlx5r_ring_db(struct mlx5_ib_qp *qp, unsigned int nreq,103 struct mlx5_wqe_ctrl_seg *ctrl);104int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,105 const struct ib_send_wr **bad_wr, bool drain);106int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,107 const struct ib_recv_wr **bad_wr, bool drain);108 109static inline int mlx5_ib_post_send_nodrain(struct ib_qp *ibqp,110 const struct ib_send_wr *wr,111 const struct ib_send_wr **bad_wr)112{113 return mlx5_ib_post_send(ibqp, wr, bad_wr, false);114}115 116static inline int mlx5_ib_post_send_drain(struct ib_qp *ibqp,117 const struct ib_send_wr *wr,118 const struct ib_send_wr **bad_wr)119{120 return mlx5_ib_post_send(ibqp, wr, bad_wr, true);121}122 123static inline int mlx5_ib_post_recv_nodrain(struct ib_qp *ibqp,124 const struct ib_recv_wr *wr,125 const struct ib_recv_wr **bad_wr)126{127 return mlx5_ib_post_recv(ibqp, wr, bad_wr, false);128}129 130static inline int mlx5_ib_post_recv_drain(struct ib_qp *ibqp,131 const struct ib_recv_wr *wr,132 const struct ib_recv_wr **bad_wr)133{134 return mlx5_ib_post_recv(ibqp, wr, bad_wr, true);135}136#endif /* _MLX5_IB_WR_H */137