1416 lines · c
1/*2 * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management3 *4 * Copyright (C) 2003-2015 Chelsio Communications. All rights reserved.5 *6 * This program is distributed in the hope that it will be useful, but WITHOUT7 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or8 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this9 * release for licensing terms and conditions.10 *11 * Written by: Dimitris Michailidis (dm@chelsio.com)12 * Karen Xie (kxie@chelsio.com)13 */14 15#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__16 17#include <linux/module.h>18#include <linux/moduleparam.h>19#include <scsi/scsi_host.h>20 21#include "common.h"22#include "t3_cpl.h"23#include "t3cdev.h"24#include "cxgb3_defs.h"25#include "cxgb3_ctl_defs.h"26#include "cxgb3_offload.h"27#include "firmware_exports.h"28#include "cxgb3i.h"29 30static unsigned int dbg_level;31#include "../libcxgbi.h"32 33#define DRV_MODULE_NAME "cxgb3i"34#define DRV_MODULE_DESC "Chelsio T3 iSCSI Driver"35#define DRV_MODULE_VERSION "2.0.1-ko"36#define DRV_MODULE_RELDATE "Apr. 2015"37 38static char version[] =39 DRV_MODULE_DESC " " DRV_MODULE_NAME40 " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";41 42MODULE_AUTHOR("Chelsio Communications, Inc.");43MODULE_DESCRIPTION(DRV_MODULE_DESC);44MODULE_VERSION(DRV_MODULE_VERSION);45MODULE_LICENSE("GPL");46 47module_param(dbg_level, uint, 0644);48MODULE_PARM_DESC(dbg_level, "debug flag (default=0)");49 50static int cxgb3i_rcv_win = 256 * 1024;51module_param(cxgb3i_rcv_win, int, 0644);52MODULE_PARM_DESC(cxgb3i_rcv_win, "TCP receive window in bytes (default=256KB)");53 54static int cxgb3i_snd_win = 128 * 1024;55module_param(cxgb3i_snd_win, int, 0644);56MODULE_PARM_DESC(cxgb3i_snd_win, "TCP send window in bytes (default=128KB)");57 58static int cxgb3i_rx_credit_thres = 10 * 1024;59module_param(cxgb3i_rx_credit_thres, int, 0644);60MODULE_PARM_DESC(cxgb3i_rx_credit_thres,61 "RX credits return threshold in bytes (default=10KB)");62 63static unsigned int cxgb3i_max_connect = 8 * 1024;64module_param(cxgb3i_max_connect, uint, 0644);65MODULE_PARM_DESC(cxgb3i_max_connect, "Max. # of connections (default=8092)");66 67static unsigned int cxgb3i_sport_base = 20000;68module_param(cxgb3i_sport_base, uint, 0644);69MODULE_PARM_DESC(cxgb3i_sport_base, "starting port number (default=20000)");70 71static void cxgb3i_dev_open(struct t3cdev *);72static void cxgb3i_dev_close(struct t3cdev *);73static void cxgb3i_dev_event_handler(struct t3cdev *, u32, u32);74 75static struct cxgb3_client t3_client = {76 .name = DRV_MODULE_NAME,77 .handlers = cxgb3i_cpl_handlers,78 .add = cxgb3i_dev_open,79 .remove = cxgb3i_dev_close,80 .event_handler = cxgb3i_dev_event_handler,81};82 83static const struct scsi_host_template cxgb3i_host_template = {84 .module = THIS_MODULE,85 .name = DRV_MODULE_NAME,86 .proc_name = DRV_MODULE_NAME,87 .can_queue = CXGB3I_SCSI_HOST_QDEPTH,88 .queuecommand = iscsi_queuecommand,89 .change_queue_depth = scsi_change_queue_depth,90 .sg_tablesize = SG_ALL,91 .max_sectors = 0xFFFF,92 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,93 .eh_timed_out = iscsi_eh_cmd_timed_out,94 .eh_abort_handler = iscsi_eh_abort,95 .eh_device_reset_handler = iscsi_eh_device_reset,96 .eh_target_reset_handler = iscsi_eh_recover_target,97 .target_alloc = iscsi_target_alloc,98 .dma_boundary = PAGE_SIZE - 1,99 .this_id = -1,100 .track_queue_depth = 1,101 .cmd_size = sizeof(struct iscsi_cmd),102};103 104static struct iscsi_transport cxgb3i_iscsi_transport = {105 .owner = THIS_MODULE,106 .name = DRV_MODULE_NAME,107 /* owner and name should be set already */108 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST109 | CAP_DATADGST | CAP_DIGEST_OFFLOAD |110 CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO,111 .attr_is_visible = cxgbi_attr_is_visible,112 .get_host_param = cxgbi_get_host_param,113 .set_host_param = cxgbi_set_host_param,114 /* session management */115 .create_session = cxgbi_create_session,116 .destroy_session = cxgbi_destroy_session,117 .get_session_param = iscsi_session_get_param,118 /* connection management */119 .create_conn = cxgbi_create_conn,120 .bind_conn = cxgbi_bind_conn,121 .unbind_conn = iscsi_conn_unbind,122 .destroy_conn = iscsi_tcp_conn_teardown,123 .start_conn = iscsi_conn_start,124 .stop_conn = iscsi_conn_stop,125 .get_conn_param = iscsi_conn_get_param,126 .set_param = cxgbi_set_conn_param,127 .get_stats = cxgbi_get_conn_stats,128 /* pdu xmit req from user space */129 .send_pdu = iscsi_conn_send_pdu,130 /* task */131 .init_task = iscsi_tcp_task_init,132 .xmit_task = iscsi_tcp_task_xmit,133 .cleanup_task = cxgbi_cleanup_task,134 /* pdu */135 .alloc_pdu = cxgbi_conn_alloc_pdu,136 .init_pdu = cxgbi_conn_init_pdu,137 .xmit_pdu = cxgbi_conn_xmit_pdu,138 .parse_pdu_itt = cxgbi_parse_pdu_itt,139 /* TCP connect/disconnect */140 .get_ep_param = cxgbi_get_ep_param,141 .ep_connect = cxgbi_ep_connect,142 .ep_poll = cxgbi_ep_poll,143 .ep_disconnect = cxgbi_ep_disconnect,144 /* Error recovery timeout call */145 .session_recovery_timedout = iscsi_session_recovery_timedout,146};147 148static struct scsi_transport_template *cxgb3i_stt;149 150/*151 * CPL (Chelsio Protocol Language) defines a message passing interface between152 * the host driver and Chelsio asic.153 * The section below implments CPLs that related to iscsi tcp connection154 * open/close/abort and data send/receive.155 */156 157static int push_tx_frames(struct cxgbi_sock *csk, int req_completion);158 159static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,160 const struct l2t_entry *e)161{162 unsigned int wscale = cxgbi_sock_compute_wscale(csk->rcv_win);163 struct cpl_act_open_req *req = (struct cpl_act_open_req *)skb->head;164 165 skb->priority = CPL_PRIORITY_SETUP;166 167 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));168 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, csk->atid));169 req->local_port = csk->saddr.sin_port;170 req->peer_port = csk->daddr.sin_port;171 req->local_ip = csk->saddr.sin_addr.s_addr;172 req->peer_ip = csk->daddr.sin_addr.s_addr;173 174 req->opt0h = htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS |175 V_WND_SCALE(wscale) | V_MSS_IDX(csk->mss_idx) |176 V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx));177 req->opt0l = htonl(V_ULP_MODE(ULP2_MODE_ISCSI) |178 V_RCV_BUFSIZ(csk->rcv_win >> 10));179 180 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,181 "csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n",182 csk, csk->state, csk->flags, csk->atid,183 &req->local_ip, ntohs(req->local_port),184 &req->peer_ip, ntohs(req->peer_port),185 csk->mss_idx, e->idx, e->smt_idx);186 187 l2t_send(csk->cdev->lldev, skb, csk->l2t);188}189 190static inline void act_open_arp_failure(struct t3cdev *dev, struct sk_buff *skb)191{192 cxgbi_sock_act_open_req_arp_failure(NULL, skb);193}194 195/*196 * CPL connection close request: host ->197 *198 * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to199 * the write queue (i.e., after any unsent txt data).200 */201static void send_close_req(struct cxgbi_sock *csk)202{203 struct sk_buff *skb = csk->cpl_close;204 struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;205 unsigned int tid = csk->tid;206 207 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,208 "csk 0x%p,%u,0x%lx,%u.\n",209 csk, csk->state, csk->flags, csk->tid);210 211 csk->cpl_close = NULL;212 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));213 req->wr.wr_lo = htonl(V_WR_TID(tid));214 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));215 req->rsvd = htonl(csk->write_seq);216 217 cxgbi_sock_skb_entail(csk, skb);218 if (csk->state >= CTP_ESTABLISHED)219 push_tx_frames(csk, 1);220}221 222/*223 * CPL connection abort request: host ->224 *225 * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs226 * for the same connection and also that we do not try to send a message227 * after the connection has closed.228 */229static void abort_arp_failure(struct t3cdev *tdev, struct sk_buff *skb)230{231 struct cpl_abort_req *req = cplhdr(skb);232 233 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,234 "t3dev 0x%p, tid %u, skb 0x%p.\n",235 tdev, GET_TID(req), skb);236 req->cmd = CPL_ABORT_NO_RST;237 cxgb3_ofld_send(tdev, skb);238}239 240static void send_abort_req(struct cxgbi_sock *csk)241{242 struct sk_buff *skb = csk->cpl_abort_req;243 struct cpl_abort_req *req;244 245 if (unlikely(csk->state == CTP_ABORTING || !skb))246 return;247 cxgbi_sock_set_state(csk, CTP_ABORTING);248 cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING);249 /* Purge the send queue so we don't send anything after an abort. */250 cxgbi_sock_purge_write_queue(csk);251 252 csk->cpl_abort_req = NULL;253 req = (struct cpl_abort_req *)skb->head;254 skb->priority = CPL_PRIORITY_DATA;255 set_arp_failure_handler(skb, abort_arp_failure);256 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));257 req->wr.wr_lo = htonl(V_WR_TID(csk->tid));258 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid));259 req->rsvd0 = htonl(csk->snd_nxt);260 req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT);261 req->cmd = CPL_ABORT_SEND_RST;262 263 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,264 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",265 csk, csk->state, csk->flags, csk->tid, csk->snd_nxt,266 req->rsvd1);267 268 l2t_send(csk->cdev->lldev, skb, csk->l2t);269}270 271/*272 * CPL connection abort reply: host ->273 *274 * Send an ABORT_RPL message in response of the ABORT_REQ received.275 */276static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status)277{278 struct sk_buff *skb = csk->cpl_abort_rpl;279 struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;280 281 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,282 "csk 0x%p,%u,0x%lx,%u, status %d.\n",283 csk, csk->state, csk->flags, csk->tid, rst_status);284 285 csk->cpl_abort_rpl = NULL;286 skb->priority = CPL_PRIORITY_DATA;287 rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));288 rpl->wr.wr_lo = htonl(V_WR_TID(csk->tid));289 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid));290 rpl->cmd = rst_status;291 cxgb3_ofld_send(csk->cdev->lldev, skb);292}293 294/*295 * CPL connection rx data ack: host ->296 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of297 * credits sent.298 */299static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits)300{301 struct sk_buff *skb;302 struct cpl_rx_data_ack *req;303 u32 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);304 305 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,306 "csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n",307 csk, csk->state, csk->flags, csk->tid, credits, dack);308 309 skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC);310 if (!skb) {311 pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits);312 return 0;313 }314 req = (struct cpl_rx_data_ack *)skb->head;315 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));316 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, csk->tid));317 req->credit_dack = htonl(F_RX_DACK_CHANGE | V_RX_DACK_MODE(1) |318 V_RX_CREDITS(credits));319 skb->priority = CPL_PRIORITY_ACK;320 cxgb3_ofld_send(csk->cdev->lldev, skb);321 return credits;322}323 324/*325 * CPL connection tx data: host ->326 *327 * Send iscsi PDU via TX_DATA CPL message. Returns the number of328 * credits sent.329 * Each TX_DATA consumes work request credit (wrs), so we need to keep track of330 * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).331 */332 333static unsigned int wrlen __read_mostly;334static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;335 336static void init_wr_tab(unsigned int wr_len)337{338 int i;339 340 if (skb_wrs[1]) /* already initialized */341 return;342 for (i = 1; i < SKB_WR_LIST_SIZE; i++) {343 int sgl_len = (3 * i) / 2 + (i & 1);344 345 sgl_len += 3;346 skb_wrs[i] = (sgl_len <= wr_len347 ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));348 }349 wrlen = wr_len * 8;350}351 352static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb,353 int len, int req_completion)354{355 struct tx_data_wr *req;356 struct l2t_entry *l2t = csk->l2t;357 358 skb_reset_transport_header(skb);359 req = __skb_push(skb, sizeof(*req));360 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) |361 (req_completion ? F_WR_COMPL : 0));362 req->wr_lo = htonl(V_WR_TID(csk->tid));363 /* len includes the length of any HW ULP additions */364 req->len = htonl(len);365 /* V_TX_ULP_SUBMODE sets both the mode and submode */366 req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_tx_ulp_mode(skb)) |367 V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1)));368 req->sndseq = htonl(csk->snd_nxt);369 req->param = htonl(V_TX_PORT(l2t->smt_idx));370 371 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {372 req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |373 V_TX_CPU_IDX(csk->rss_qid));374 /* sendbuffer is in units of 32KB. */375 req->param |= htonl(V_TX_SNDBUF(csk->snd_win >> 15));376 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);377 }378}379 380/*381 * push_tx_frames -- start transmit382 *383 * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a384 * connection's send queue and sends them on to T3. Must be called with the385 * connection's lock held. Returns the amount of send buffer space that was386 * freed as a result of sending queued data to T3.387 */388 389static void arp_failure_skb_discard(struct t3cdev *dev, struct sk_buff *skb)390{391 kfree_skb(skb);392}393 394static int push_tx_frames(struct cxgbi_sock *csk, int req_completion)395{396 int total_size = 0;397 struct sk_buff *skb;398 399 if (unlikely(csk->state < CTP_ESTABLISHED ||400 csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) {401 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,402 "csk 0x%p,%u,0x%lx,%u, in closing state.\n",403 csk, csk->state, csk->flags, csk->tid);404 return 0;405 }406 407 while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) {408 int len = skb->len; /* length before skb_push */409 int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);410 int wrs_needed = skb_wrs[frags];411 412 if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)413 wrs_needed = 1;414 415 WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);416 417 if (csk->wr_cred < wrs_needed) {418 log_debug(1 << CXGBI_DBG_PDU_TX,419 "csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n",420 csk, skb->len, skb->data_len, frags,421 wrs_needed, csk->wr_cred);422 break;423 }424 425 __skb_unlink(skb, &csk->write_queue);426 skb->priority = CPL_PRIORITY_DATA;427 skb->csum = wrs_needed; /* remember this until the WR_ACK */428 csk->wr_cred -= wrs_needed;429 csk->wr_una_cred += wrs_needed;430 cxgbi_sock_enqueue_wr(csk, skb);431 432 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,433 "csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, "434 "left %u, unack %u.\n",435 csk, skb->len, skb->data_len, frags, skb->csum,436 csk->wr_cred, csk->wr_una_cred);437 438 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {439 if ((req_completion &&440 csk->wr_una_cred == wrs_needed) ||441 csk->wr_una_cred >= csk->wr_max_cred / 2) {442 req_completion = 1;443 csk->wr_una_cred = 0;444 }445 len += cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb));446 make_tx_data_wr(csk, skb, len, req_completion);447 csk->snd_nxt += len;448 cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);449 }450 total_size += skb->truesize;451 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,452 "csk 0x%p, tid 0x%x, send skb 0x%p.\n",453 csk, csk->tid, skb);454 set_arp_failure_handler(skb, arp_failure_skb_discard);455 l2t_send(csk->cdev->lldev, skb, csk->l2t);456 }457 return total_size;458}459 460/*461 * Process a CPL_ACT_ESTABLISH message: -> host462 * Updates connection state from an active establish CPL message. Runs with463 * the connection lock held.464 */465 466static inline void free_atid(struct cxgbi_sock *csk)467{468 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) {469 cxgb3_free_atid(csk->cdev->lldev, csk->atid);470 cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID);471 cxgbi_sock_put(csk);472 }473}474 475static int do_act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)476{477 struct cxgbi_sock *csk = ctx;478 struct cpl_act_establish *req = cplhdr(skb);479 unsigned int tid = GET_TID(req);480 unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));481 u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */482 483 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,484 "atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",485 atid, atid, csk, csk->state, csk->flags, rcv_isn);486 487 cxgbi_sock_get(csk);488 cxgbi_sock_set_flag(csk, CTPF_HAS_TID);489 csk->tid = tid;490 cxgb3_insert_tid(csk->cdev->lldev, &t3_client, csk, tid);491 492 free_atid(csk);493 494 csk->rss_qid = G_QNUM(ntohs(skb->csum));495 496 spin_lock_bh(&csk->lock);497 if (csk->retry_timer.function) {498 del_timer(&csk->retry_timer);499 csk->retry_timer.function = NULL;500 }501 502 if (unlikely(csk->state != CTP_ACTIVE_OPEN))503 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",504 csk, csk->state, csk->flags, csk->tid);505 506 csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn;507 if (csk->rcv_win > (M_RCV_BUFSIZ << 10))508 csk->rcv_wup -= csk->rcv_win - (M_RCV_BUFSIZ << 10);509 510 cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt));511 512 if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED)))513 /* upper layer has requested closing */514 send_abort_req(csk);515 else {516 if (skb_queue_len(&csk->write_queue))517 push_tx_frames(csk, 1);518 cxgbi_conn_tx_open(csk);519 }520 521 spin_unlock_bh(&csk->lock);522 __kfree_skb(skb);523 return 0;524}525 526/*527 * Process a CPL_ACT_OPEN_RPL message: -> host528 * Handle active open failures.529 */530static int act_open_rpl_status_to_errno(int status)531{532 switch (status) {533 case CPL_ERR_CONN_RESET:534 return -ECONNREFUSED;535 case CPL_ERR_ARP_MISS:536 return -EHOSTUNREACH;537 case CPL_ERR_CONN_TIMEDOUT:538 return -ETIMEDOUT;539 case CPL_ERR_TCAM_FULL:540 return -ENOMEM;541 case CPL_ERR_CONN_EXIST:542 return -EADDRINUSE;543 default:544 return -EIO;545 }546}547 548static void act_open_retry_timer(struct timer_list *t)549{550 struct cxgbi_sock *csk = from_timer(csk, t, retry_timer);551 struct sk_buff *skb;552 553 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,554 "csk 0x%p,%u,0x%lx,%u.\n",555 csk, csk->state, csk->flags, csk->tid);556 557 cxgbi_sock_get(csk);558 spin_lock_bh(&csk->lock);559 skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_ATOMIC);560 if (!skb)561 cxgbi_sock_fail_act_open(csk, -ENOMEM);562 else {563 skb->sk = (struct sock *)csk;564 set_arp_failure_handler(skb, act_open_arp_failure);565 send_act_open_req(csk, skb, csk->l2t);566 }567 spin_unlock_bh(&csk->lock);568 cxgbi_sock_put(csk);569}570 571static int do_act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)572{573 struct cxgbi_sock *csk = ctx;574 struct cpl_act_open_rpl *rpl = cplhdr(skb);575 576 pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n",577 csk, csk->state, csk->flags, csk->atid, rpl->status,578 &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),579 &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));580 581 if (rpl->status != CPL_ERR_TCAM_FULL &&582 rpl->status != CPL_ERR_CONN_EXIST &&583 rpl->status != CPL_ERR_ARP_MISS)584 cxgb3_queue_tid_release(tdev, GET_TID(rpl));585 586 cxgbi_sock_get(csk);587 spin_lock_bh(&csk->lock);588 if (rpl->status == CPL_ERR_CONN_EXIST &&589 csk->retry_timer.function != act_open_retry_timer) {590 csk->retry_timer.function = act_open_retry_timer;591 mod_timer(&csk->retry_timer, jiffies + HZ / 2);592 } else593 cxgbi_sock_fail_act_open(csk,594 act_open_rpl_status_to_errno(rpl->status));595 596 spin_unlock_bh(&csk->lock);597 cxgbi_sock_put(csk);598 __kfree_skb(skb);599 return 0;600}601 602/*603 * Process PEER_CLOSE CPL messages: -> host604 * Handle peer FIN.605 */606static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)607{608 struct cxgbi_sock *csk = ctx;609 610 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,611 "csk 0x%p,%u,0x%lx,%u.\n",612 csk, csk->state, csk->flags, csk->tid);613 614 cxgbi_sock_rcv_peer_close(csk);615 __kfree_skb(skb);616 return 0;617}618 619/*620 * Process CLOSE_CONN_RPL CPL message: -> host621 * Process a peer ACK to our FIN.622 */623static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,624 void *ctx)625{626 struct cxgbi_sock *csk = ctx;627 struct cpl_close_con_rpl *rpl = cplhdr(skb);628 629 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,630 "csk 0x%p,%u,0x%lx,%u, snxt %u.\n",631 csk, csk->state, csk->flags, csk->tid, ntohl(rpl->snd_nxt));632 633 cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt));634 __kfree_skb(skb);635 return 0;636}637 638/*639 * Process ABORT_REQ_RSS CPL message: -> host640 * Process abort requests. If we are waiting for an ABORT_RPL we ignore this641 * request except that we need to reply to it.642 */643 644static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason,645 int *need_rst)646{647 switch (abort_reason) {648 case CPL_ERR_BAD_SYN:649 case CPL_ERR_CONN_RESET:650 return csk->state > CTP_ESTABLISHED ? -EPIPE : -ECONNRESET;651 case CPL_ERR_XMIT_TIMEDOUT:652 case CPL_ERR_PERSIST_TIMEDOUT:653 case CPL_ERR_FINWAIT2_TIMEDOUT:654 case CPL_ERR_KEEPALIVE_TIMEDOUT:655 return -ETIMEDOUT;656 default:657 return -EIO;658 }659}660 661static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)662{663 const struct cpl_abort_req_rss *req = cplhdr(skb);664 struct cxgbi_sock *csk = ctx;665 int rst_status = CPL_ABORT_NO_RST;666 667 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,668 "csk 0x%p,%u,0x%lx,%u.\n",669 csk, csk->state, csk->flags, csk->tid);670 671 if (req->status == CPL_ERR_RTX_NEG_ADVICE ||672 req->status == CPL_ERR_PERSIST_NEG_ADVICE) {673 goto done;674 }675 676 cxgbi_sock_get(csk);677 spin_lock_bh(&csk->lock);678 679 if (!cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) {680 cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD);681 cxgbi_sock_set_state(csk, CTP_ABORTING);682 goto out;683 }684 685 cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD);686 send_abort_rpl(csk, rst_status);687 688 if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {689 csk->err = abort_status_to_errno(csk, req->status, &rst_status);690 cxgbi_sock_closed(csk);691 }692 693out:694 spin_unlock_bh(&csk->lock);695 cxgbi_sock_put(csk);696done:697 __kfree_skb(skb);698 return 0;699}700 701/*702 * Process ABORT_RPL_RSS CPL message: -> host703 * Process abort replies. We only process these messages if we anticipate704 * them as the coordination between SW and HW in this area is somewhat lacking705 * and sometimes we get ABORT_RPLs after we are done with the connection that706 * originated the ABORT_REQ.707 */708static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)709{710 struct cpl_abort_rpl_rss *rpl = cplhdr(skb);711 struct cxgbi_sock *csk = ctx;712 713 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,714 "status 0x%x, csk 0x%p, s %u, 0x%lx.\n",715 rpl->status, csk, csk ? csk->state : 0,716 csk ? csk->flags : 0UL);717 /*718 * Ignore replies to post-close aborts indicating that the abort was719 * requested too late. These connections are terminated when we get720 * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss721 * arrives the TID is either no longer used or it has been recycled.722 */723 if (rpl->status == CPL_ERR_ABORT_FAILED)724 goto rel_skb;725 /*726 * Sometimes we've already closed the connection, e.g., a post-close727 * abort races with ABORT_REQ_RSS, the latter frees the connection728 * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,729 * but FW turns the ABORT_REQ into a regular one and so we get730 * ABORT_RPL_RSS with status 0 and no connection.731 */732 if (csk)733 cxgbi_sock_rcv_abort_rpl(csk);734rel_skb:735 __kfree_skb(skb);736 return 0;737}738 739/*740 * Process RX_ISCSI_HDR CPL message: -> host741 * Handle received PDUs, the payload could be DDP'ed. If not, the payload742 * follow after the bhs.743 */744static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)745{746 struct cxgbi_sock *csk = ctx;747 struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);748 struct cpl_iscsi_hdr_norss data_cpl;749 struct cpl_rx_data_ddp_norss ddp_cpl;750 unsigned int hdr_len, data_len, status;751 unsigned int len;752 int err;753 754 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,755 "csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n",756 csk, csk->state, csk->flags, csk->tid, skb, skb->len);757 758 spin_lock_bh(&csk->lock);759 760 if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {761 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,762 "csk 0x%p,%u,0x%lx,%u, bad state.\n",763 csk, csk->state, csk->flags, csk->tid);764 if (csk->state != CTP_ABORTING)765 goto abort_conn;766 else767 goto discard;768 }769 770 cxgbi_skcb_tcp_seq(skb) = ntohl(hdr_cpl->seq);771 cxgbi_skcb_flags(skb) = 0;772 773 skb_reset_transport_header(skb);774 __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));775 776 len = hdr_len = ntohs(hdr_cpl->len);777 /* msg coalesce is off or not enough data received */778 if (skb->len <= hdr_len) {779 pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n",780 csk->cdev->ports[csk->port_id]->name, csk->tid,781 skb->len, hdr_len);782 goto abort_conn;783 }784 cxgbi_skcb_set_flag(skb, SKCBF_RX_COALESCED);785 786 err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,787 sizeof(ddp_cpl));788 if (err < 0) {789 pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n",790 csk->cdev->ports[csk->port_id]->name, csk->tid,791 skb->len, sizeof(ddp_cpl), err);792 goto abort_conn;793 }794 795 cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS);796 cxgbi_skcb_rx_pdulen(skb) = ntohs(ddp_cpl.len);797 cxgbi_skcb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);798 status = ntohl(ddp_cpl.ddp_status);799 800 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,801 "csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n",802 csk, skb, skb->len, cxgbi_skcb_rx_pdulen(skb), status);803 804 if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT))805 cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR);806 if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT))807 cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR);808 if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT))809 cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR);810 811 if (skb->len > (hdr_len + sizeof(ddp_cpl))) {812 err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));813 if (err < 0) {814 pr_err("%s: tid %u, cp %zu/%u failed %d.\n",815 csk->cdev->ports[csk->port_id]->name,816 csk->tid, sizeof(data_cpl), skb->len, err);817 goto abort_conn;818 }819 data_len = ntohs(data_cpl.len);820 log_debug(1 << CXGBI_DBG_DDP | 1 << CXGBI_DBG_PDU_RX,821 "skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n",822 skb, data_len, cxgbi_skcb_rx_pdulen(skb), status);823 len += sizeof(data_cpl) + data_len;824 } else if (status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT))825 cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD);826 827 csk->rcv_nxt = ntohl(ddp_cpl.seq) + cxgbi_skcb_rx_pdulen(skb);828 __pskb_trim(skb, len);829 __skb_queue_tail(&csk->receive_queue, skb);830 cxgbi_conn_pdu_ready(csk);831 832 spin_unlock_bh(&csk->lock);833 return 0;834 835abort_conn:836 send_abort_req(csk);837discard:838 spin_unlock_bh(&csk->lock);839 __kfree_skb(skb);840 return 0;841}842 843/*844 * Process TX_DATA_ACK CPL messages: -> host845 * Process an acknowledgment of WR completion. Advance snd_una and send the846 * next batch of work requests from the write queue.847 */848static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)849{850 struct cxgbi_sock *csk = ctx;851 struct cpl_wr_ack *hdr = cplhdr(skb);852 853 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,854 "csk 0x%p,%u,0x%lx,%u, cr %u.\n",855 csk, csk->state, csk->flags, csk->tid, ntohs(hdr->credits));856 857 cxgbi_sock_rcv_wr_ack(csk, ntohs(hdr->credits), ntohl(hdr->snd_una), 1);858 __kfree_skb(skb);859 return 0;860}861 862/*863 * for each connection, pre-allocate skbs needed for close/abort requests. So864 * that we can service the request right away.865 */866static int alloc_cpls(struct cxgbi_sock *csk)867{868 csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 0,869 GFP_KERNEL);870 if (!csk->cpl_close)871 return -ENOMEM;872 csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 0,873 GFP_KERNEL);874 if (!csk->cpl_abort_req)875 goto free_cpl_skbs;876 877 csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 0,878 GFP_KERNEL);879 if (!csk->cpl_abort_rpl)880 goto free_cpl_skbs;881 882 return 0;883 884free_cpl_skbs:885 cxgbi_sock_free_cpl_skbs(csk);886 return -ENOMEM;887}888 889static void l2t_put(struct cxgbi_sock *csk)890{891 struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;892 893 if (csk->l2t) {894 l2t_release(t3dev, csk->l2t);895 csk->l2t = NULL;896 cxgbi_sock_put(csk);897 }898}899 900/*901 * release_offload_resources - release offload resource902 * Release resources held by an offload connection (TID, L2T entry, etc.)903 */904static void release_offload_resources(struct cxgbi_sock *csk)905{906 struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;907 908 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,909 "csk 0x%p,%u,0x%lx,%u.\n",910 csk, csk->state, csk->flags, csk->tid);911 912 csk->rss_qid = 0;913 cxgbi_sock_free_cpl_skbs(csk);914 915 if (csk->wr_cred != csk->wr_max_cred) {916 cxgbi_sock_purge_wr_queue(csk);917 cxgbi_sock_reset_wr_list(csk);918 }919 l2t_put(csk);920 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID))921 free_atid(csk);922 else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) {923 cxgb3_remove_tid(t3dev, (void *)csk, csk->tid);924 cxgbi_sock_clear_flag(csk, CTPF_HAS_TID);925 cxgbi_sock_put(csk);926 }927 csk->dst = NULL;928 csk->cdev = NULL;929}930 931static void update_address(struct cxgbi_hba *chba)932{933 if (chba->ipv4addr) {934 if (chba->vdev &&935 chba->ipv4addr != cxgb3i_get_private_ipv4addr(chba->vdev)) {936 cxgb3i_set_private_ipv4addr(chba->vdev, chba->ipv4addr);937 cxgb3i_set_private_ipv4addr(chba->ndev, 0);938 pr_info("%s set %pI4.\n",939 chba->vdev->name, &chba->ipv4addr);940 } else if (chba->ipv4addr !=941 cxgb3i_get_private_ipv4addr(chba->ndev)) {942 cxgb3i_set_private_ipv4addr(chba->ndev, chba->ipv4addr);943 pr_info("%s set %pI4.\n",944 chba->ndev->name, &chba->ipv4addr);945 }946 } else if (cxgb3i_get_private_ipv4addr(chba->ndev)) {947 if (chba->vdev)948 cxgb3i_set_private_ipv4addr(chba->vdev, 0);949 cxgb3i_set_private_ipv4addr(chba->ndev, 0);950 }951}952 953static int init_act_open(struct cxgbi_sock *csk)954{955 struct dst_entry *dst = csk->dst;956 struct cxgbi_device *cdev = csk->cdev;957 struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;958 struct net_device *ndev = cdev->ports[csk->port_id];959 struct cxgbi_hba *chba = cdev->hbas[csk->port_id];960 struct sk_buff *skb = NULL;961 int ret;962 963 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,964 "csk 0x%p,%u,0x%lx.\n", csk, csk->state, csk->flags);965 966 update_address(chba);967 if (chba->ipv4addr)968 csk->saddr.sin_addr.s_addr = chba->ipv4addr;969 970 csk->rss_qid = 0;971 csk->l2t = t3_l2t_get(t3dev, dst, ndev,972 &csk->daddr.sin_addr.s_addr);973 if (!csk->l2t) {974 pr_err("NO l2t available.\n");975 return -EINVAL;976 }977 cxgbi_sock_get(csk);978 979 csk->atid = cxgb3_alloc_atid(t3dev, &t3_client, csk);980 if (csk->atid < 0) {981 pr_err("NO atid available.\n");982 ret = -EINVAL;983 goto put_sock;984 }985 cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);986 cxgbi_sock_get(csk);987 988 skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL);989 if (!skb) {990 ret = -ENOMEM;991 goto free_atid;992 }993 skb->sk = (struct sock *)csk;994 set_arp_failure_handler(skb, act_open_arp_failure);995 csk->snd_win = cxgb3i_snd_win;996 csk->rcv_win = cxgb3i_rcv_win;997 998 csk->wr_max_cred = csk->wr_cred = T3C_DATA(t3dev)->max_wrs - 1;999 csk->wr_una_cred = 0;1000 csk->mss_idx = cxgbi_sock_select_mss(csk, dst_mtu(dst));1001 cxgbi_sock_reset_wr_list(csk);1002 csk->err = 0;1003 1004 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,1005 "csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n",1006 csk, csk->state, csk->flags,1007 &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),1008 &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));1009 1010 cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN);1011 send_act_open_req(csk, skb, csk->l2t);1012 return 0;1013 1014free_atid:1015 cxgb3_free_atid(t3dev, csk->atid);1016put_sock:1017 cxgbi_sock_put(csk);1018 l2t_release(t3dev, csk->l2t);1019 csk->l2t = NULL;1020 1021 return ret;1022}1023 1024cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = {1025 [CPL_ACT_ESTABLISH] = do_act_establish,1026 [CPL_ACT_OPEN_RPL] = do_act_open_rpl,1027 [CPL_PEER_CLOSE] = do_peer_close,1028 [CPL_ABORT_REQ_RSS] = do_abort_req,1029 [CPL_ABORT_RPL_RSS] = do_abort_rpl,1030 [CPL_CLOSE_CON_RPL] = do_close_con_rpl,1031 [CPL_TX_DMA_ACK] = do_wr_ack,1032 [CPL_ISCSI_HDR] = do_iscsi_hdr,1033};1034 1035/**1036 * cxgb3i_ofld_init - allocate and initialize resources for each adapter found1037 * @cdev: cxgbi adapter1038 */1039static int cxgb3i_ofld_init(struct cxgbi_device *cdev)1040{1041 struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;1042 struct adap_ports port;1043 struct ofld_page_info rx_page_info;1044 unsigned int wr_len;1045 int rc;1046 1047 if (t3dev->ctl(t3dev, GET_WR_LEN, &wr_len) < 0 ||1048 t3dev->ctl(t3dev, GET_PORTS, &port) < 0 ||1049 t3dev->ctl(t3dev, GET_RX_PAGE_INFO, &rx_page_info) < 0) {1050 pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev);1051 return -EINVAL;1052 }1053 1054 if (cxgb3i_max_connect > CXGBI_MAX_CONN)1055 cxgb3i_max_connect = CXGBI_MAX_CONN;1056 1057 rc = cxgbi_device_portmap_create(cdev, cxgb3i_sport_base,1058 cxgb3i_max_connect);1059 if (rc < 0)1060 return rc;1061 1062 init_wr_tab(wr_len);1063 cdev->csk_release_offload_resources = release_offload_resources;1064 cdev->csk_push_tx_frames = push_tx_frames;1065 cdev->csk_send_abort_req = send_abort_req;1066 cdev->csk_send_close_req = send_close_req;1067 cdev->csk_send_rx_credits = send_rx_credits;1068 cdev->csk_alloc_cpls = alloc_cpls;1069 cdev->csk_init_act_open = init_act_open;1070 1071 pr_info("cdev 0x%p, offload up, added.\n", cdev);1072 return 0;1073}1074 1075/*1076 * functions to program the pagepod in h/w1077 */1078static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)1079{1080 struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head;1081 1082 memset(req, 0, sizeof(*req));1083 1084 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));1085 req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) |1086 V_ULPTX_CMD(ULP_MEM_WRITE));1087 req->len = htonl(V_ULP_MEMIO_DATA_LEN(IPPOD_SIZE >> 5) |1088 V_ULPTX_NFLITS((IPPOD_SIZE >> 3) + 1));1089}1090 1091static struct cxgbi_ppm *cdev2ppm(struct cxgbi_device *cdev)1092{1093 return ((struct t3cdev *)cdev->lldev)->ulp_iscsi;1094}1095 1096static int ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk,1097 struct cxgbi_task_tag_info *ttinfo)1098{1099 unsigned int idx = ttinfo->idx;1100 unsigned int npods = ttinfo->npods;1101 struct scatterlist *sg = ttinfo->sgl;1102 struct cxgbi_pagepod *ppod;1103 struct ulp_mem_io *req;1104 unsigned int sg_off;1105 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;1106 int i;1107 1108 for (i = 0; i < npods; i++, idx++, pm_addr += IPPOD_SIZE) {1109 struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +1110 IPPOD_SIZE, 0, GFP_ATOMIC);1111 1112 if (!skb)1113 return -ENOMEM;1114 ulp_mem_io_set_hdr(skb, pm_addr);1115 req = (struct ulp_mem_io *)skb->head;1116 ppod = (struct cxgbi_pagepod *)(req + 1);1117 sg_off = i * PPOD_PAGES_MAX;1118 cxgbi_ddp_set_one_ppod(ppod, ttinfo, &sg,1119 &sg_off);1120 skb->priority = CPL_PRIORITY_CONTROL;1121 cxgb3_ofld_send(ppm->lldev, skb);1122 }1123 return 0;1124}1125 1126static void ddp_clear_map(struct cxgbi_device *cdev, struct cxgbi_ppm *ppm,1127 struct cxgbi_task_tag_info *ttinfo)1128{1129 unsigned int idx = ttinfo->idx;1130 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;1131 unsigned int npods = ttinfo->npods;1132 int i;1133 1134 log_debug(1 << CXGBI_DBG_DDP,1135 "cdev 0x%p, clear idx %u, npods %u.\n",1136 cdev, idx, npods);1137 1138 for (i = 0; i < npods; i++, idx++, pm_addr += IPPOD_SIZE) {1139 struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +1140 IPPOD_SIZE, 0, GFP_ATOMIC);1141 1142 if (!skb) {1143 pr_err("cdev 0x%p, clear ddp, %u,%d/%u, skb OOM.\n",1144 cdev, idx, i, npods);1145 continue;1146 }1147 ulp_mem_io_set_hdr(skb, pm_addr);1148 skb->priority = CPL_PRIORITY_CONTROL;1149 cxgb3_ofld_send(ppm->lldev, skb);1150 }1151}1152 1153static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk,1154 unsigned int tid, int pg_idx)1155{1156 struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,1157 GFP_KERNEL);1158 struct cpl_set_tcb_field *req;1159 u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0;1160 1161 log_debug(1 << CXGBI_DBG_DDP,1162 "csk 0x%p, tid %u, pg_idx %d.\n", csk, tid, pg_idx);1163 if (!skb)1164 return -ENOMEM;1165 1166 /* set up ulp submode and page size */1167 req = (struct cpl_set_tcb_field *)skb->head;1168 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));1169 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));1170 req->reply = V_NO_REPLY(1);1171 req->cpu_idx = 0;1172 req->word = htons(31);1173 req->mask = cpu_to_be64(0xF0000000);1174 req->val = cpu_to_be64(val << 28);1175 skb->priority = CPL_PRIORITY_CONTROL;1176 1177 cxgb3_ofld_send(csk->cdev->lldev, skb);1178 return 0;1179}1180 1181/**1182 * ddp_setup_conn_digest - setup conn. digest setting1183 * @csk: cxgb tcp socket1184 * @tid: connection id1185 * @hcrc: header digest enabled1186 * @dcrc: data digest enabled1187 * set up the iscsi digest settings for a connection identified by tid1188 */1189static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid,1190 int hcrc, int dcrc)1191{1192 struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,1193 GFP_KERNEL);1194 struct cpl_set_tcb_field *req;1195 u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);1196 1197 log_debug(1 << CXGBI_DBG_DDP,1198 "csk 0x%p, tid %u, crc %d,%d.\n", csk, tid, hcrc, dcrc);1199 if (!skb)1200 return -ENOMEM;1201 1202 /* set up ulp submode and page size */1203 req = (struct cpl_set_tcb_field *)skb->head;1204 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));1205 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));1206 req->reply = V_NO_REPLY(1);1207 req->cpu_idx = 0;1208 req->word = htons(31);1209 req->mask = cpu_to_be64(0x0F000000);1210 req->val = cpu_to_be64(val << 24);1211 skb->priority = CPL_PRIORITY_CONTROL;1212 1213 cxgb3_ofld_send(csk->cdev->lldev, skb);1214 return 0;1215}1216 1217/**1218 * cxgb3i_ddp_init - initialize the cxgb3 adapter's ddp resource1219 * @cdev: cxgb3i adapter1220 * initialize the ddp pagepod manager for a given adapter1221 */1222static int cxgb3i_ddp_init(struct cxgbi_device *cdev)1223{1224 struct t3cdev *tdev = (struct t3cdev *)cdev->lldev;1225 struct net_device *ndev = cdev->ports[0];1226 struct cxgbi_tag_format tformat;1227 unsigned int ppmax, tagmask = 0;1228 struct ulp_iscsi_info uinfo;1229 int i, err;1230 1231 err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);1232 if (err < 0) {1233 pr_err("%s, failed to get iscsi param %d.\n",1234 ndev->name, err);1235 return err;1236 }1237 if (uinfo.llimit >= uinfo.ulimit) {1238 pr_warn("T3 %s, iscsi NOT enabled %u ~ %u!\n",1239 ndev->name, uinfo.llimit, uinfo.ulimit);1240 return -EACCES;1241 }1242 1243 ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT;1244 tagmask = cxgbi_tagmask_set(ppmax);1245 1246 pr_info("T3 %s: 0x%x~0x%x, 0x%x, tagmask 0x%x -> 0x%x.\n",1247 ndev->name, uinfo.llimit, uinfo.ulimit, ppmax, uinfo.tagmask,1248 tagmask);1249 1250 memset(&tformat, 0, sizeof(struct cxgbi_tag_format));1251 for (i = 0; i < 4; i++)1252 tformat.pgsz_order[i] = uinfo.pgsz_factor[i];1253 cxgbi_tagmask_check(tagmask, &tformat);1254 1255 err = cxgbi_ddp_ppm_setup(&tdev->ulp_iscsi, cdev, &tformat,1256 (uinfo.ulimit - uinfo.llimit + 1),1257 uinfo.llimit, uinfo.llimit, 0, 0, 0);1258 if (err)1259 return err;1260 1261 if (!(cdev->flags & CXGBI_FLAG_DDP_OFF)) {1262 uinfo.tagmask = tagmask;1263 uinfo.ulimit = uinfo.llimit + (ppmax << PPOD_SIZE_SHIFT);1264 1265 err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);1266 if (err < 0) {1267 pr_err("T3 %s fail to set iscsi param %d.\n",1268 ndev->name, err);1269 cdev->flags |= CXGBI_FLAG_DDP_OFF;1270 }1271 err = 0;1272 }1273 1274 cdev->csk_ddp_setup_digest = ddp_setup_conn_digest;1275 cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx;1276 cdev->csk_ddp_set_map = ddp_set_map;1277 cdev->csk_ddp_clear_map = ddp_clear_map;1278 cdev->cdev2ppm = cdev2ppm;1279 cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,1280 uinfo.max_txsz - ISCSI_PDU_NONPAYLOAD_LEN);1281 cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,1282 uinfo.max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN);1283 1284 return 0;1285}1286 1287static void cxgb3i_dev_close(struct t3cdev *t3dev)1288{1289 struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);1290 1291 if (!cdev || cdev->flags & CXGBI_FLAG_ADAPTER_RESET) {1292 pr_info("0x%p close, f 0x%x.\n", cdev, cdev ? cdev->flags : 0);1293 return;1294 }1295 1296 cxgbi_device_unregister(cdev);1297}1298 1299/**1300 * cxgb3i_dev_open - init a t3 adapter structure and any h/w settings1301 * @t3dev: t3cdev adapter1302 */1303static void cxgb3i_dev_open(struct t3cdev *t3dev)1304{1305 struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);1306 struct adapter *adapter = tdev2adap(t3dev);1307 int i, err;1308 1309 if (cdev) {1310 pr_info("0x%p, updating.\n", cdev);1311 return;1312 }1313 1314 cdev = cxgbi_device_register(0, adapter->params.nports);1315 if (!cdev) {1316 pr_warn("device 0x%p register failed.\n", t3dev);1317 return;1318 }1319 1320 cdev->flags = CXGBI_FLAG_DEV_T3 | CXGBI_FLAG_IPV4_SET;1321 cdev->lldev = t3dev;1322 cdev->pdev = adapter->pdev;1323 cdev->ports = adapter->port;1324 cdev->nports = adapter->params.nports;1325 cdev->mtus = adapter->params.mtus;1326 cdev->nmtus = NMTUS;1327 cdev->rx_credit_thres = cxgb3i_rx_credit_thres;1328 cdev->skb_tx_rsvd = CXGB3I_TX_HEADER_LEN;1329 cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr_norss);1330 cdev->itp = &cxgb3i_iscsi_transport;1331 1332 err = cxgb3i_ddp_init(cdev);1333 if (err) {1334 pr_info("0x%p ddp init failed %d\n", cdev, err);1335 goto err_out;1336 }1337 1338 err = cxgb3i_ofld_init(cdev);1339 if (err) {1340 pr_info("0x%p offload init failed\n", cdev);1341 goto err_out;1342 }1343 1344 err = cxgbi_hbas_add(cdev, CXGB3I_MAX_LUN, CXGBI_MAX_CONN,1345 &cxgb3i_host_template, cxgb3i_stt);1346 if (err)1347 goto err_out;1348 1349 for (i = 0; i < cdev->nports; i++)1350 cdev->hbas[i]->ipv4addr =1351 cxgb3i_get_private_ipv4addr(cdev->ports[i]);1352 1353 pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n",1354 cdev, cdev ? cdev->flags : 0, t3dev, err);1355 return;1356 1357err_out:1358 cxgbi_device_unregister(cdev);1359}1360 1361static void cxgb3i_dev_event_handler(struct t3cdev *t3dev, u32 event, u32 port)1362{1363 struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);1364 1365 log_debug(1 << CXGBI_DBG_TOE,1366 "0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n",1367 t3dev, cdev, event, port);1368 if (!cdev)1369 return;1370 1371 switch (event) {1372 case OFFLOAD_STATUS_DOWN:1373 cdev->flags |= CXGBI_FLAG_ADAPTER_RESET;1374 break;1375 case OFFLOAD_STATUS_UP:1376 cdev->flags &= ~CXGBI_FLAG_ADAPTER_RESET;1377 break;1378 }1379}1380 1381/**1382 * cxgb3i_init_module - module init entry point1383 *1384 * initialize any driver wide global data structures and register itself1385 * with the cxgb3 module1386 */1387static int __init cxgb3i_init_module(void)1388{1389 int rc;1390 1391 printk(KERN_INFO "%s", version);1392 1393 rc = cxgbi_iscsi_init(&cxgb3i_iscsi_transport, &cxgb3i_stt);1394 if (rc < 0)1395 return rc;1396 1397 cxgb3_register_client(&t3_client);1398 return 0;1399}1400 1401/**1402 * cxgb3i_exit_module - module cleanup/exit entry point1403 *1404 * go through the driver hba list and for each hba, release any resource held.1405 * and unregisters iscsi transport and the cxgb3 module1406 */1407static void __exit cxgb3i_exit_module(void)1408{1409 cxgb3_unregister_client(&t3_client);1410 cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3);1411 cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport, &cxgb3i_stt);1412}1413 1414module_init(cxgb3i_init_module);1415module_exit(cxgb3i_exit_module);1416