213 lines · c
1/* SPDX-License-Identifier: GPL-2.0 */2#include <linux/socket.h>3#include <linux/in.h>4#include <linux/in6.h>5#include <rdma/ib_verbs.h>6#include <rdma/rdma_cm.h>7#include <rdma/rw.h>8#include <scsi/iser.h>9 10 11#define DRV_NAME "isert"12#define PFX DRV_NAME ": "13 14#define isert_dbg(fmt, arg...) \15 do { \16 if (unlikely(isert_debug_level > 2)) \17 printk(KERN_DEBUG PFX "%s: " fmt,\18 __func__ , ## arg); \19 } while (0)20 21#define isert_warn(fmt, arg...) \22 do { \23 if (unlikely(isert_debug_level > 0)) \24 pr_warn(PFX "%s: " fmt, \25 __func__ , ## arg); \26 } while (0)27 28#define isert_info(fmt, arg...) \29 do { \30 if (unlikely(isert_debug_level > 1)) \31 pr_info(PFX "%s: " fmt, \32 __func__ , ## arg); \33 } while (0)34 35#define isert_err(fmt, arg...) \36 pr_err(PFX "%s: " fmt, __func__ , ## arg)37 38/* Constant PDU lengths calculations */39#define ISER_HEADERS_LEN (sizeof(struct iser_ctrl) + \40 sizeof(struct iscsi_hdr))41#define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)42 43/* QP settings */44/* Maximal bounds on received asynchronous PDUs */45#define ISERT_MAX_TX_MISC_PDUS 4 /* NOOP_IN(2) , ASYNC_EVENT(2) */46 47#define ISERT_MAX_RX_MISC_PDUS 6 /*48 * NOOP_OUT(2), TEXT(1),49 * SCSI_TMFUNC(2), LOGOUT(1)50 */51 52#define ISCSI_DEF_XMIT_CMDS_MAX 128 /* from libiscsi.h, must be power of 2 */53 54#define ISERT_QP_MAX_RECV_DTOS (ISCSI_DEF_XMIT_CMDS_MAX)55 56#define ISERT_MIN_POSTED_RX (ISCSI_DEF_XMIT_CMDS_MAX >> 2)57 58#define ISERT_QP_MAX_REQ_DTOS (ISCSI_DEF_XMIT_CMDS_MAX + \59 ISERT_MAX_TX_MISC_PDUS + \60 ISERT_MAX_RX_MISC_PDUS)61 62/*63 * RX size is default of 8k plus headers, but data needs to align to64 * 512 boundary, so use 1024 to have the extra space for alignment.65 */66#define ISER_RX_SIZE (ISCSI_DEF_MAX_RECV_SEG_LEN + 1024)67 68/* Minimum I/O size is 512KB */69#define ISCSI_ISER_MIN_SG_TABLESIZE 12870 71/* Maximum support is 16MB I/O size */72#define ISCSI_ISER_MAX_SG_TABLESIZE 409673 74enum isert_desc_type {75 ISCSI_TX_CONTROL,76 ISCSI_TX_DATAIN77};78 79enum iser_conn_state {80 ISER_CONN_INIT,81 ISER_CONN_UP,82 ISER_CONN_BOUND,83 ISER_CONN_FULL_FEATURE,84 ISER_CONN_TERMINATING,85 ISER_CONN_DOWN,86};87 88struct iser_rx_desc {89 char buf[ISER_RX_SIZE];90 u64 dma_addr;91 struct ib_sge rx_sg;92 struct ib_cqe rx_cqe;93 bool in_use;94};95 96static inline struct iser_rx_desc *cqe_to_rx_desc(struct ib_cqe *cqe)97{98 return container_of(cqe, struct iser_rx_desc, rx_cqe);99}100 101static void *isert_get_iser_hdr(struct iser_rx_desc *desc)102{103 return PTR_ALIGN(desc->buf + ISER_HEADERS_LEN, 512) - ISER_HEADERS_LEN;104}105 106static size_t isert_get_hdr_offset(struct iser_rx_desc *desc)107{108 return isert_get_iser_hdr(desc) - (void *)desc->buf;109}110 111static void *isert_get_iscsi_hdr(struct iser_rx_desc *desc)112{113 return isert_get_iser_hdr(desc) + sizeof(struct iser_ctrl);114}115 116static void *isert_get_data(struct iser_rx_desc *desc)117{118 void *data = isert_get_iser_hdr(desc) + ISER_HEADERS_LEN;119 120 WARN_ON((uintptr_t)data & 511);121 return data;122}123 124struct iser_tx_desc {125 struct iser_ctrl iser_header;126 struct iscsi_hdr iscsi_header;127 enum isert_desc_type type;128 u64 dma_addr;129 struct ib_sge tx_sg[2];130 struct ib_cqe tx_cqe;131 int num_sge;132 struct ib_send_wr send_wr;133} __packed;134 135static inline struct iser_tx_desc *cqe_to_tx_desc(struct ib_cqe *cqe)136{137 return container_of(cqe, struct iser_tx_desc, tx_cqe);138}139 140struct isert_cmd {141 uint32_t read_stag;142 uint32_t write_stag;143 uint64_t read_va;144 uint64_t write_va;145 uint32_t inv_rkey;146 u64 pdu_buf_dma;147 u32 pdu_buf_len;148 struct isert_conn *conn;149 struct iscsit_cmd *iscsit_cmd;150 struct iser_tx_desc tx_desc;151 struct iser_rx_desc *rx_desc;152 struct rdma_rw_ctx rw;153 struct work_struct comp_work;154 struct scatterlist sg;155 bool ctx_init_done;156};157 158static inline struct isert_cmd *tx_desc_to_cmd(struct iser_tx_desc *desc)159{160 return container_of(desc, struct isert_cmd, tx_desc);161}162 163struct isert_device;164 165struct isert_conn {166 enum iser_conn_state state;167 u32 responder_resources;168 u32 initiator_depth;169 bool pi_support;170 struct iser_rx_desc *login_desc;171 char *login_rsp_buf;172 int login_req_len;173 u64 login_rsp_dma;174 struct iser_rx_desc *rx_descs;175 struct ib_recv_wr rx_wr[ISERT_QP_MAX_RECV_DTOS];176 struct iscsit_conn *conn;177 struct list_head node;178 struct completion login_comp;179 struct completion login_req_comp;180 struct iser_tx_desc login_tx_desc;181 struct rdma_cm_id *cm_id;182 struct ib_qp *qp;183 struct ib_cq *cq;184 u32 cq_size;185 struct isert_device *device;186 struct mutex mutex;187 struct kref kref;188 struct work_struct release_work;189 bool logout_posted;190 bool snd_w_inv;191 wait_queue_head_t rem_wait;192 bool dev_removed;193};194 195struct isert_device {196 bool pi_capable;197 int refcount;198 struct ib_device *ib_device;199 struct ib_pd *pd;200 struct isert_comp *comps;201 int comps_used;202 struct list_head dev_node;203};204 205struct isert_np {206 struct iscsi_np *np;207 struct semaphore sem;208 struct rdma_cm_id *cm_id;209 struct mutex mutex;210 struct list_head accepted;211 struct list_head pending;212};213