1251 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * iSCSI over TCP/IP Data-Path lib4 *5 * Copyright (C) 2004 Dmitry Yusupov6 * Copyright (C) 2004 Alex Aizman7 * Copyright (C) 2005 - 2006 Mike Christie8 * Copyright (C) 2006 Red Hat, Inc. All rights reserved.9 * maintained by open-iscsi@googlegroups.com10 *11 * Credits:12 * Christoph Hellwig13 * FUJITA Tomonori14 * Arne Redlich15 * Zhenyu Wang16 */17 18#include <crypto/hash.h>19#include <linux/types.h>20#include <linux/list.h>21#include <linux/inet.h>22#include <linux/slab.h>23#include <linux/file.h>24#include <linux/blkdev.h>25#include <linux/delay.h>26#include <linux/kfifo.h>27#include <linux/scatterlist.h>28#include <linux/module.h>29#include <net/tcp.h>30#include <scsi/scsi_cmnd.h>31#include <scsi/scsi_device.h>32#include <scsi/scsi_host.h>33#include <scsi/scsi.h>34#include <scsi/scsi_transport_iscsi.h>35#include <trace/events/iscsi.h>36 37#include "iscsi_tcp.h"38 39MODULE_AUTHOR("Mike Christie <michaelc@cs.wisc.edu>, "40 "Dmitry Yusupov <dmitry_yus@yahoo.com>, "41 "Alex Aizman <itn780@yahoo.com>");42MODULE_DESCRIPTION("iSCSI/TCP data-path");43MODULE_LICENSE("GPL");44 45static int iscsi_dbg_libtcp;46module_param_named(debug_libiscsi_tcp, iscsi_dbg_libtcp, int,47 S_IRUGO | S_IWUSR);48MODULE_PARM_DESC(debug_libiscsi_tcp, "Turn on debugging for libiscsi_tcp "49 "module. Set to 1 to turn on, and zero to turn off. Default "50 "is off.");51 52#define ISCSI_DBG_TCP(_conn, dbg_fmt, arg...) \53 do { \54 if (iscsi_dbg_libtcp) \55 iscsi_conn_printk(KERN_INFO, _conn, \56 "%s " dbg_fmt, \57 __func__, ##arg); \58 iscsi_dbg_trace(trace_iscsi_dbg_tcp, \59 &(_conn)->cls_conn->dev, \60 "%s " dbg_fmt, __func__, ##arg);\61 } while (0);62 63static int iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn,64 struct iscsi_segment *segment);65 66/*67 * Scatterlist handling: inside the iscsi_segment, we68 * remember an index into the scatterlist, and set data/size69 * to the current scatterlist entry. For highmem pages, we70 * kmap as needed.71 *72 * Note that the page is unmapped when we return from73 * TCP's data_ready handler, so we may end up mapping and74 * unmapping the same page repeatedly. The whole reason75 * for this is that we shouldn't keep the page mapped76 * outside the softirq.77 */78 79/**80 * iscsi_tcp_segment_init_sg - init indicated scatterlist entry81 * @segment: the buffer object82 * @sg: scatterlist83 * @offset: byte offset into that sg entry84 *85 * This function sets up the segment so that subsequent86 * data is copied to the indicated sg entry, at the given87 * offset.88 */89static inline void90iscsi_tcp_segment_init_sg(struct iscsi_segment *segment,91 struct scatterlist *sg, unsigned int offset)92{93 segment->sg = sg;94 segment->sg_offset = offset;95 segment->size = min(sg->length - offset,96 segment->total_size - segment->total_copied);97 segment->data = NULL;98}99 100/**101 * iscsi_tcp_segment_map - map the current S/G page102 * @segment: iscsi_segment103 * @recv: 1 if called from recv path104 *105 * We only need to possibly kmap data if scatter lists are being used,106 * because the iscsi passthrough and internal IO paths will never use high107 * mem pages.108 */109static void iscsi_tcp_segment_map(struct iscsi_segment *segment, int recv)110{111 struct scatterlist *sg;112 113 if (segment->data != NULL || !segment->sg)114 return;115 116 sg = segment->sg;117 BUG_ON(segment->sg_mapped);118 BUG_ON(sg->length == 0);119 120 /*121 * We always map for the recv path.122 *123 * If the page count is greater than one it is ok to send124 * to the network layer's zero copy send path. If not we125 * have to go the slow sendmsg path.126 *127 * Same goes for slab pages: skb_can_coalesce() allows128 * coalescing neighboring slab objects into a single frag which129 * triggers one of hardened usercopy checks.130 */131 if (!recv && sendpage_ok(sg_page(sg)))132 return;133 134 if (recv) {135 segment->atomic_mapped = true;136 segment->sg_mapped = kmap_atomic(sg_page(sg));137 } else {138 segment->atomic_mapped = false;139 /* the xmit path can sleep with the page mapped so use kmap */140 segment->sg_mapped = kmap(sg_page(sg));141 }142 143 segment->data = segment->sg_mapped + sg->offset + segment->sg_offset;144}145 146void iscsi_tcp_segment_unmap(struct iscsi_segment *segment)147{148 if (segment->sg_mapped) {149 if (segment->atomic_mapped)150 kunmap_atomic(segment->sg_mapped);151 else152 kunmap(sg_page(segment->sg));153 segment->sg_mapped = NULL;154 segment->data = NULL;155 }156}157EXPORT_SYMBOL_GPL(iscsi_tcp_segment_unmap);158 159/*160 * Splice the digest buffer into the buffer161 */162static inline void163iscsi_tcp_segment_splice_digest(struct iscsi_segment *segment, void *digest)164{165 segment->data = digest;166 segment->digest_len = ISCSI_DIGEST_SIZE;167 segment->total_size += ISCSI_DIGEST_SIZE;168 segment->size = ISCSI_DIGEST_SIZE;169 segment->copied = 0;170 segment->sg = NULL;171 segment->hash = NULL;172}173 174/**175 * iscsi_tcp_segment_done - check whether the segment is complete176 * @tcp_conn: iscsi tcp connection177 * @segment: iscsi segment to check178 * @recv: set to one of this is called from the recv path179 * @copied: number of bytes copied180 *181 * Check if we're done receiving this segment. If the receive182 * buffer is full but we expect more data, move on to the183 * next entry in the scatterlist.184 *185 * If the amount of data we received isn't a multiple of 4,186 * we will transparently receive the pad bytes, too.187 *188 * This function must be re-entrant.189 */190int iscsi_tcp_segment_done(struct iscsi_tcp_conn *tcp_conn,191 struct iscsi_segment *segment, int recv,192 unsigned copied)193{194 struct scatterlist sg;195 unsigned int pad;196 197 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "copied %u %u size %u %s\n",198 segment->copied, copied, segment->size,199 recv ? "recv" : "xmit");200 if (segment->hash && copied) {201 /*202 * If a segment is kmapd we must unmap it before sending203 * to the crypto layer since that will try to kmap it again.204 */205 iscsi_tcp_segment_unmap(segment);206 207 if (!segment->data) {208 sg_init_table(&sg, 1);209 sg_set_page(&sg, sg_page(segment->sg), copied,210 segment->copied + segment->sg_offset +211 segment->sg->offset);212 } else213 sg_init_one(&sg, segment->data + segment->copied,214 copied);215 ahash_request_set_crypt(segment->hash, &sg, NULL, copied);216 crypto_ahash_update(segment->hash);217 }218 219 segment->copied += copied;220 if (segment->copied < segment->size) {221 iscsi_tcp_segment_map(segment, recv);222 return 0;223 }224 225 segment->total_copied += segment->copied;226 segment->copied = 0;227 segment->size = 0;228 229 /* Unmap the current scatterlist page, if there is one. */230 iscsi_tcp_segment_unmap(segment);231 232 /* Do we have more scatterlist entries? */233 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "total copied %u total size %u\n",234 segment->total_copied, segment->total_size);235 if (segment->total_copied < segment->total_size) {236 /* Proceed to the next entry in the scatterlist. */237 iscsi_tcp_segment_init_sg(segment, sg_next(segment->sg),238 0);239 iscsi_tcp_segment_map(segment, recv);240 BUG_ON(segment->size == 0);241 return 0;242 }243 244 /* Do we need to handle padding? */245 if (!(tcp_conn->iscsi_conn->session->tt->caps & CAP_PADDING_OFFLOAD)) {246 pad = iscsi_padding(segment->total_copied);247 if (pad != 0) {248 ISCSI_DBG_TCP(tcp_conn->iscsi_conn,249 "consume %d pad bytes\n", pad);250 segment->total_size += pad;251 segment->size = pad;252 segment->data = segment->padbuf;253 return 0;254 }255 }256 257 /*258 * Set us up for transferring the data digest. hdr digest259 * is completely handled in hdr done function.260 */261 if (segment->hash) {262 ahash_request_set_crypt(segment->hash, NULL,263 segment->digest, 0);264 crypto_ahash_final(segment->hash);265 iscsi_tcp_segment_splice_digest(segment,266 recv ? segment->recv_digest : segment->digest);267 return 0;268 }269 270 return 1;271}272EXPORT_SYMBOL_GPL(iscsi_tcp_segment_done);273 274/**275 * iscsi_tcp_segment_recv - copy data to segment276 * @tcp_conn: the iSCSI TCP connection277 * @segment: the buffer to copy to278 * @ptr: data pointer279 * @len: amount of data available280 *281 * This function copies up to @len bytes to the282 * given buffer, and returns the number of bytes283 * consumed, which can actually be less than @len.284 *285 * If hash digest is enabled, the function will update the286 * hash while copying.287 * Combining these two operations doesn't buy us a lot (yet),288 * but in the future we could implement combined copy+crc,289 * just way we do for network layer checksums.290 */291static int292iscsi_tcp_segment_recv(struct iscsi_tcp_conn *tcp_conn,293 struct iscsi_segment *segment, const void *ptr,294 unsigned int len)295{296 unsigned int copy = 0, copied = 0;297 298 while (!iscsi_tcp_segment_done(tcp_conn, segment, 1, copy)) {299 if (copied == len) {300 ISCSI_DBG_TCP(tcp_conn->iscsi_conn,301 "copied %d bytes\n", len);302 break;303 }304 305 copy = min(len - copied, segment->size - segment->copied);306 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "copying %d\n", copy);307 memcpy(segment->data + segment->copied, ptr + copied, copy);308 copied += copy;309 }310 return copied;311}312 313inline void314iscsi_tcp_dgst_header(struct ahash_request *hash, const void *hdr,315 size_t hdrlen, unsigned char digest[ISCSI_DIGEST_SIZE])316{317 struct scatterlist sg;318 319 sg_init_one(&sg, hdr, hdrlen);320 ahash_request_set_crypt(hash, &sg, digest, hdrlen);321 crypto_ahash_digest(hash);322}323EXPORT_SYMBOL_GPL(iscsi_tcp_dgst_header);324 325static inline int326iscsi_tcp_dgst_verify(struct iscsi_tcp_conn *tcp_conn,327 struct iscsi_segment *segment)328{329 if (!segment->digest_len)330 return 1;331 332 if (memcmp(segment->recv_digest, segment->digest,333 segment->digest_len)) {334 ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "digest mismatch\n");335 return 0;336 }337 338 return 1;339}340 341/*342 * Helper function to set up segment buffer343 */344static inline void345__iscsi_segment_init(struct iscsi_segment *segment, size_t size,346 iscsi_segment_done_fn_t *done, struct ahash_request *hash)347{348 memset(segment, 0, sizeof(*segment));349 segment->total_size = size;350 segment->done = done;351 352 if (hash) {353 segment->hash = hash;354 crypto_ahash_init(hash);355 }356}357 358inline void359iscsi_segment_init_linear(struct iscsi_segment *segment, void *data,360 size_t size, iscsi_segment_done_fn_t *done,361 struct ahash_request *hash)362{363 __iscsi_segment_init(segment, size, done, hash);364 segment->data = data;365 segment->size = size;366}367EXPORT_SYMBOL_GPL(iscsi_segment_init_linear);368 369inline int370iscsi_segment_seek_sg(struct iscsi_segment *segment,371 struct scatterlist *sg_list, unsigned int sg_count,372 unsigned int offset, size_t size,373 iscsi_segment_done_fn_t *done,374 struct ahash_request *hash)375{376 struct scatterlist *sg;377 unsigned int i;378 379 __iscsi_segment_init(segment, size, done, hash);380 for_each_sg(sg_list, sg, sg_count, i) {381 if (offset < sg->length) {382 iscsi_tcp_segment_init_sg(segment, sg, offset);383 return 0;384 }385 offset -= sg->length;386 }387 388 return ISCSI_ERR_DATA_OFFSET;389}390EXPORT_SYMBOL_GPL(iscsi_segment_seek_sg);391 392/**393 * iscsi_tcp_hdr_recv_prep - prep segment for hdr reception394 * @tcp_conn: iscsi connection to prep for395 *396 * This function always passes NULL for the hash argument, because when this397 * function is called we do not yet know the final size of the header and want398 * to delay the digest processing until we know that.399 */400void iscsi_tcp_hdr_recv_prep(struct iscsi_tcp_conn *tcp_conn)401{402 ISCSI_DBG_TCP(tcp_conn->iscsi_conn,403 "(%s)\n", tcp_conn->iscsi_conn->hdrdgst_en ?404 "digest enabled" : "digest disabled");405 iscsi_segment_init_linear(&tcp_conn->in.segment,406 tcp_conn->in.hdr_buf, sizeof(struct iscsi_hdr),407 iscsi_tcp_hdr_recv_done, NULL);408}409EXPORT_SYMBOL_GPL(iscsi_tcp_hdr_recv_prep);410 411/*412 * Handle incoming reply to any other type of command413 */414static int415iscsi_tcp_data_recv_done(struct iscsi_tcp_conn *tcp_conn,416 struct iscsi_segment *segment)417{418 struct iscsi_conn *conn = tcp_conn->iscsi_conn;419 int rc = 0;420 421 if (!iscsi_tcp_dgst_verify(tcp_conn, segment))422 return ISCSI_ERR_DATA_DGST;423 424 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr,425 conn->data, tcp_conn->in.datalen);426 if (rc)427 return rc;428 429 iscsi_tcp_hdr_recv_prep(tcp_conn);430 return 0;431}432 433static void434iscsi_tcp_data_recv_prep(struct iscsi_tcp_conn *tcp_conn)435{436 struct iscsi_conn *conn = tcp_conn->iscsi_conn;437 struct ahash_request *rx_hash = NULL;438 439 if (conn->datadgst_en &&440 !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD))441 rx_hash = tcp_conn->rx_hash;442 443 iscsi_segment_init_linear(&tcp_conn->in.segment,444 conn->data, tcp_conn->in.datalen,445 iscsi_tcp_data_recv_done, rx_hash);446}447 448/**449 * iscsi_tcp_cleanup_task - free tcp_task resources450 * @task: iscsi task451 *452 * must be called with session back_lock453 */454void iscsi_tcp_cleanup_task(struct iscsi_task *task)455{456 struct iscsi_tcp_task *tcp_task = task->dd_data;457 struct iscsi_r2t_info *r2t;458 459 /* nothing to do for mgmt */460 if (!task->sc)461 return;462 463 spin_lock_bh(&tcp_task->queue2pool);464 /* flush task's r2t queues */465 while (kfifo_out(&tcp_task->r2tqueue, (void*)&r2t, sizeof(void*))) {466 kfifo_in(&tcp_task->r2tpool.queue, (void*)&r2t,467 sizeof(void*));468 ISCSI_DBG_TCP(task->conn, "pending r2t dropped\n");469 }470 471 r2t = tcp_task->r2t;472 if (r2t != NULL) {473 kfifo_in(&tcp_task->r2tpool.queue, (void*)&r2t,474 sizeof(void*));475 tcp_task->r2t = NULL;476 }477 spin_unlock_bh(&tcp_task->queue2pool);478}479EXPORT_SYMBOL_GPL(iscsi_tcp_cleanup_task);480 481/**482 * iscsi_tcp_data_in - SCSI Data-In Response processing483 * @conn: iscsi connection484 * @task: scsi command task485 */486static int iscsi_tcp_data_in(struct iscsi_conn *conn, struct iscsi_task *task)487{488 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;489 struct iscsi_tcp_task *tcp_task = task->dd_data;490 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr;491 int datasn = be32_to_cpu(rhdr->datasn);492 unsigned total_in_length = task->sc->sdb.length;493 494 /*495 * lib iscsi will update this in the completion handling if there496 * is status.497 */498 if (!(rhdr->flags & ISCSI_FLAG_DATA_STATUS))499 iscsi_update_cmdsn(conn->session, (struct iscsi_nopin*)rhdr);500 501 if (tcp_conn->in.datalen == 0)502 return 0;503 504 if (tcp_task->exp_datasn != datasn) {505 ISCSI_DBG_TCP(conn, "task->exp_datasn(%d) != rhdr->datasn(%d)"506 "\n", tcp_task->exp_datasn, datasn);507 return ISCSI_ERR_DATASN;508 }509 510 tcp_task->exp_datasn++;511 512 tcp_task->data_offset = be32_to_cpu(rhdr->offset);513 if (tcp_task->data_offset + tcp_conn->in.datalen > total_in_length) {514 ISCSI_DBG_TCP(conn, "data_offset(%d) + data_len(%d) > "515 "total_length_in(%d)\n", tcp_task->data_offset,516 tcp_conn->in.datalen, total_in_length);517 return ISCSI_ERR_DATA_OFFSET;518 }519 520 conn->datain_pdus_cnt++;521 return 0;522}523 524/**525 * iscsi_tcp_r2t_rsp - iSCSI R2T Response processing526 * @conn: iscsi connection527 * @hdr: PDU header528 */529static int iscsi_tcp_r2t_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr)530{531 struct iscsi_session *session = conn->session;532 struct iscsi_tcp_task *tcp_task;533 struct iscsi_tcp_conn *tcp_conn;534 struct iscsi_r2t_rsp *rhdr;535 struct iscsi_r2t_info *r2t;536 struct iscsi_task *task;537 u32 data_length;538 u32 data_offset;539 int r2tsn;540 int rc;541 542 spin_lock(&session->back_lock);543 task = iscsi_itt_to_ctask(conn, hdr->itt);544 if (!task) {545 spin_unlock(&session->back_lock);546 return ISCSI_ERR_BAD_ITT;547 } else if (task->sc->sc_data_direction != DMA_TO_DEVICE) {548 spin_unlock(&session->back_lock);549 return ISCSI_ERR_PROTO;550 }551 /*552 * A bad target might complete the cmd before we have handled R2Ts553 * so get a ref to the task that will be dropped in the xmit path.554 */555 if (task->state != ISCSI_TASK_RUNNING) {556 spin_unlock(&session->back_lock);557 /* Let the path that got the early rsp complete it */558 return 0;559 }560 task->last_xfer = jiffies;561 if (!iscsi_get_task(task)) {562 spin_unlock(&session->back_lock);563 /* Let the path that got the early rsp complete it */564 return 0;565 }566 567 tcp_conn = conn->dd_data;568 rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr;569 /* fill-in new R2T associated with the task */570 iscsi_update_cmdsn(session, (struct iscsi_nopin *)rhdr);571 spin_unlock(&session->back_lock);572 573 if (tcp_conn->in.datalen) {574 iscsi_conn_printk(KERN_ERR, conn,575 "invalid R2t with datalen %d\n",576 tcp_conn->in.datalen);577 rc = ISCSI_ERR_DATALEN;578 goto put_task;579 }580 581 tcp_task = task->dd_data;582 r2tsn = be32_to_cpu(rhdr->r2tsn);583 if (tcp_task->exp_datasn != r2tsn){584 ISCSI_DBG_TCP(conn, "task->exp_datasn(%d) != rhdr->r2tsn(%d)\n",585 tcp_task->exp_datasn, r2tsn);586 rc = ISCSI_ERR_R2TSN;587 goto put_task;588 }589 590 if (session->state != ISCSI_STATE_LOGGED_IN) {591 iscsi_conn_printk(KERN_INFO, conn,592 "dropping R2T itt %d in recovery.\n",593 task->itt);594 rc = 0;595 goto put_task;596 }597 598 data_length = be32_to_cpu(rhdr->data_length);599 if (data_length == 0) {600 iscsi_conn_printk(KERN_ERR, conn,601 "invalid R2T with zero data len\n");602 rc = ISCSI_ERR_DATALEN;603 goto put_task;604 }605 606 if (data_length > session->max_burst)607 ISCSI_DBG_TCP(conn, "invalid R2T with data len %u and max "608 "burst %u. Attempting to execute request.\n",609 data_length, session->max_burst);610 611 data_offset = be32_to_cpu(rhdr->data_offset);612 if (data_offset + data_length > task->sc->sdb.length) {613 iscsi_conn_printk(KERN_ERR, conn,614 "invalid R2T with data len %u at offset %u "615 "and total length %d\n", data_length,616 data_offset, task->sc->sdb.length);617 rc = ISCSI_ERR_DATALEN;618 goto put_task;619 }620 621 spin_lock(&tcp_task->pool2queue);622 rc = kfifo_out(&tcp_task->r2tpool.queue, (void *)&r2t, sizeof(void *));623 if (!rc) {624 iscsi_conn_printk(KERN_ERR, conn, "Could not allocate R2T. "625 "Target has sent more R2Ts than it "626 "negotiated for or driver has leaked.\n");627 spin_unlock(&tcp_task->pool2queue);628 rc = ISCSI_ERR_PROTO;629 goto put_task;630 }631 632 r2t->exp_statsn = rhdr->statsn;633 r2t->data_length = data_length;634 r2t->data_offset = data_offset;635 636 r2t->ttt = rhdr->ttt; /* no flip */637 r2t->datasn = 0;638 r2t->sent = 0;639 640 tcp_task->exp_datasn = r2tsn + 1;641 kfifo_in(&tcp_task->r2tqueue, (void*)&r2t, sizeof(void*));642 conn->r2t_pdus_cnt++;643 spin_unlock(&tcp_task->pool2queue);644 645 iscsi_requeue_task(task);646 return 0;647 648put_task:649 iscsi_put_task(task);650 return rc;651}652 653/*654 * Handle incoming reply to DataIn command655 */656static int657iscsi_tcp_process_data_in(struct iscsi_tcp_conn *tcp_conn,658 struct iscsi_segment *segment)659{660 struct iscsi_conn *conn = tcp_conn->iscsi_conn;661 struct iscsi_hdr *hdr = tcp_conn->in.hdr;662 int rc;663 664 if (!iscsi_tcp_dgst_verify(tcp_conn, segment))665 return ISCSI_ERR_DATA_DGST;666 667 /* check for non-exceptional status */668 if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {669 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);670 if (rc)671 return rc;672 }673 674 iscsi_tcp_hdr_recv_prep(tcp_conn);675 return 0;676}677 678/**679 * iscsi_tcp_hdr_dissect - process PDU header680 * @conn: iSCSI connection681 * @hdr: PDU header682 *683 * This function analyzes the header of the PDU received,684 * and performs several sanity checks. If the PDU is accompanied685 * by data, the receive buffer is set up to copy the incoming data686 * to the correct location.687 */688static int689iscsi_tcp_hdr_dissect(struct iscsi_conn *conn, struct iscsi_hdr *hdr)690{691 int rc = 0, opcode, ahslen;692 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;693 struct iscsi_task *task;694 695 /* verify PDU length */696 tcp_conn->in.datalen = ntoh24(hdr->dlength);697 if (tcp_conn->in.datalen > conn->max_recv_dlength) {698 iscsi_conn_printk(KERN_ERR, conn,699 "iscsi_tcp: datalen %d > %d\n",700 tcp_conn->in.datalen, conn->max_recv_dlength);701 return ISCSI_ERR_DATALEN;702 }703 704 /* Additional header segments. So far, we don't705 * process additional headers.706 */707 ahslen = hdr->hlength << 2;708 709 opcode = hdr->opcode & ISCSI_OPCODE_MASK;710 /* verify itt (itt encoding: age+cid+itt) */711 rc = iscsi_verify_itt(conn, hdr->itt);712 if (rc)713 return rc;714 715 ISCSI_DBG_TCP(conn, "opcode 0x%x ahslen %d datalen %d\n",716 opcode, ahslen, tcp_conn->in.datalen);717 718 switch(opcode) {719 case ISCSI_OP_SCSI_DATA_IN:720 spin_lock(&conn->session->back_lock);721 task = iscsi_itt_to_ctask(conn, hdr->itt);722 if (!task)723 rc = ISCSI_ERR_BAD_ITT;724 else725 rc = iscsi_tcp_data_in(conn, task);726 if (rc) {727 spin_unlock(&conn->session->back_lock);728 break;729 }730 731 if (tcp_conn->in.datalen) {732 struct iscsi_tcp_task *tcp_task = task->dd_data;733 struct ahash_request *rx_hash = NULL;734 struct scsi_data_buffer *sdb = &task->sc->sdb;735 736 /*737 * Setup copy of Data-In into the struct scsi_cmnd738 * Scatterlist case:739 * We set up the iscsi_segment to point to the next740 * scatterlist entry to copy to. As we go along,741 * we move on to the next scatterlist entry and742 * update the digest per-entry.743 */744 if (conn->datadgst_en &&745 !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD))746 rx_hash = tcp_conn->rx_hash;747 748 ISCSI_DBG_TCP(conn, "iscsi_tcp_begin_data_in( "749 "offset=%d, datalen=%d)\n",750 tcp_task->data_offset,751 tcp_conn->in.datalen);752 task->last_xfer = jiffies;753 rc = iscsi_segment_seek_sg(&tcp_conn->in.segment,754 sdb->table.sgl,755 sdb->table.nents,756 tcp_task->data_offset,757 tcp_conn->in.datalen,758 iscsi_tcp_process_data_in,759 rx_hash);760 spin_unlock(&conn->session->back_lock);761 return rc;762 }763 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);764 spin_unlock(&conn->session->back_lock);765 break;766 case ISCSI_OP_SCSI_CMD_RSP:767 if (tcp_conn->in.datalen) {768 iscsi_tcp_data_recv_prep(tcp_conn);769 return 0;770 }771 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);772 break;773 case ISCSI_OP_R2T:774 if (ahslen) {775 rc = ISCSI_ERR_AHSLEN;776 break;777 }778 rc = iscsi_tcp_r2t_rsp(conn, hdr);779 break;780 case ISCSI_OP_LOGIN_RSP:781 case ISCSI_OP_TEXT_RSP:782 case ISCSI_OP_REJECT:783 case ISCSI_OP_ASYNC_EVENT:784 /*785 * It is possible that we could get a PDU with a buffer larger786 * than 8K, but there are no targets that currently do this.787 * For now we fail until we find a vendor that needs it788 */789 if (ISCSI_DEF_MAX_RECV_SEG_LEN < tcp_conn->in.datalen) {790 iscsi_conn_printk(KERN_ERR, conn,791 "iscsi_tcp: received buffer of "792 "len %u but conn buffer is only %u "793 "(opcode %0x)\n",794 tcp_conn->in.datalen,795 ISCSI_DEF_MAX_RECV_SEG_LEN, opcode);796 rc = ISCSI_ERR_PROTO;797 break;798 }799 800 /* If there's data coming in with the response,801 * receive it to the connection's buffer.802 */803 if (tcp_conn->in.datalen) {804 iscsi_tcp_data_recv_prep(tcp_conn);805 return 0;806 }807 fallthrough;808 case ISCSI_OP_LOGOUT_RSP:809 case ISCSI_OP_NOOP_IN:810 case ISCSI_OP_SCSI_TMFUNC_RSP:811 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);812 break;813 default:814 rc = ISCSI_ERR_BAD_OPCODE;815 break;816 }817 818 if (rc == 0) {819 /* Anything that comes with data should have820 * been handled above. */821 if (tcp_conn->in.datalen)822 return ISCSI_ERR_PROTO;823 iscsi_tcp_hdr_recv_prep(tcp_conn);824 }825 826 return rc;827}828 829/**830 * iscsi_tcp_hdr_recv_done - process PDU header831 * @tcp_conn: iSCSI TCP connection832 * @segment: the buffer segment being processed833 *834 * This is the callback invoked when the PDU header has835 * been received. If the header is followed by additional836 * header segments, we go back for more data.837 */838static int839iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn,840 struct iscsi_segment *segment)841{842 struct iscsi_conn *conn = tcp_conn->iscsi_conn;843 struct iscsi_hdr *hdr;844 845 /* Check if there are additional header segments846 * *prior* to computing the digest, because we847 * may need to go back to the caller for more.848 */849 hdr = (struct iscsi_hdr *) tcp_conn->in.hdr_buf;850 if (segment->copied == sizeof(struct iscsi_hdr) && hdr->hlength) {851 /* Bump the header length - the caller will852 * just loop around and get the AHS for us, and853 * call again. */854 unsigned int ahslen = hdr->hlength << 2;855 856 /* Make sure we don't overflow */857 if (sizeof(*hdr) + ahslen > sizeof(tcp_conn->in.hdr_buf))858 return ISCSI_ERR_AHSLEN;859 860 segment->total_size += ahslen;861 segment->size += ahslen;862 return 0;863 }864 865 /* We're done processing the header. See if we're doing866 * header digests; if so, set up the recv_digest buffer867 * and go back for more. */868 if (conn->hdrdgst_en &&869 !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) {870 if (segment->digest_len == 0) {871 /*872 * Even if we offload the digest processing we873 * splice it in so we can increment the skb/segment874 * counters in preparation for the data segment.875 */876 iscsi_tcp_segment_splice_digest(segment,877 segment->recv_digest);878 return 0;879 }880 881 iscsi_tcp_dgst_header(tcp_conn->rx_hash, hdr,882 segment->total_copied - ISCSI_DIGEST_SIZE,883 segment->digest);884 885 if (!iscsi_tcp_dgst_verify(tcp_conn, segment))886 return ISCSI_ERR_HDR_DGST;887 }888 889 tcp_conn->in.hdr = hdr;890 return iscsi_tcp_hdr_dissect(conn, hdr);891}892 893/**894 * iscsi_tcp_recv_segment_is_hdr - tests if we are reading in a header895 * @tcp_conn: iscsi tcp conn896 *897 * returns non zero if we are currently processing or setup to process898 * a header.899 */900inline int iscsi_tcp_recv_segment_is_hdr(struct iscsi_tcp_conn *tcp_conn)901{902 return tcp_conn->in.segment.done == iscsi_tcp_hdr_recv_done;903}904EXPORT_SYMBOL_GPL(iscsi_tcp_recv_segment_is_hdr);905 906/**907 * iscsi_tcp_recv_skb - Process skb908 * @conn: iscsi connection909 * @skb: network buffer with header and/or data segment910 * @offset: offset in skb911 * @offloaded: bool indicating if transfer was offloaded912 * @status: iscsi TCP status result913 *914 * Will return status of transfer in @status. And will return915 * number of bytes copied.916 */917int iscsi_tcp_recv_skb(struct iscsi_conn *conn, struct sk_buff *skb,918 unsigned int offset, bool offloaded, int *status)919{920 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;921 struct iscsi_segment *segment = &tcp_conn->in.segment;922 struct skb_seq_state seq;923 unsigned int consumed = 0;924 int rc = 0;925 926 ISCSI_DBG_TCP(conn, "in %d bytes\n", skb->len - offset);927 /*928 * Update for each skb instead of pdu, because over slow networks a929 * data_in's data could take a while to read in. We also want to930 * account for r2ts.931 */932 conn->last_recv = jiffies;933 934 if (unlikely(test_bit(ISCSI_CONN_FLAG_SUSPEND_RX, &conn->flags))) {935 ISCSI_DBG_TCP(conn, "Rx suspended!\n");936 *status = ISCSI_TCP_SUSPENDED;937 return 0;938 }939 940 if (offloaded) {941 segment->total_copied = segment->total_size;942 goto segment_done;943 }944 945 skb_prepare_seq_read(skb, offset, skb->len, &seq);946 while (1) {947 unsigned int avail;948 const u8 *ptr;949 950 avail = skb_seq_read(consumed, &ptr, &seq);951 if (avail == 0) {952 ISCSI_DBG_TCP(conn, "no more data avail. Consumed %d\n",953 consumed);954 *status = ISCSI_TCP_SKB_DONE;955 goto skb_done;956 }957 BUG_ON(segment->copied >= segment->size);958 959 ISCSI_DBG_TCP(conn, "skb %p ptr=%p avail=%u\n", skb, ptr,960 avail);961 rc = iscsi_tcp_segment_recv(tcp_conn, segment, ptr, avail);962 BUG_ON(rc == 0);963 consumed += rc;964 965 if (segment->total_copied >= segment->total_size) {966 skb_abort_seq_read(&seq);967 goto segment_done;968 }969 }970 971segment_done:972 *status = ISCSI_TCP_SEGMENT_DONE;973 ISCSI_DBG_TCP(conn, "segment done\n");974 rc = segment->done(tcp_conn, segment);975 if (rc != 0) {976 *status = ISCSI_TCP_CONN_ERR;977 ISCSI_DBG_TCP(conn, "Error receiving PDU, errno=%d\n", rc);978 iscsi_conn_failure(conn, rc);979 return 0;980 }981 /* The done() functions sets up the next segment. */982 983skb_done:984 conn->rxdata_octets += consumed;985 return consumed;986}987EXPORT_SYMBOL_GPL(iscsi_tcp_recv_skb);988 989/**990 * iscsi_tcp_task_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands991 * @task: scsi command task992 */993int iscsi_tcp_task_init(struct iscsi_task *task)994{995 struct iscsi_tcp_task *tcp_task = task->dd_data;996 struct iscsi_conn *conn = task->conn;997 struct scsi_cmnd *sc = task->sc;998 int err;999 1000 if (!sc) {1001 /*1002 * mgmt tasks do not have a scatterlist since they come1003 * in from the iscsi interface.1004 */1005 ISCSI_DBG_TCP(conn, "mtask deq [itt 0x%x]\n", task->itt);1006 1007 return conn->session->tt->init_pdu(task, 0, task->data_count);1008 }1009 1010 BUG_ON(kfifo_len(&tcp_task->r2tqueue));1011 tcp_task->exp_datasn = 0;1012 1013 /* Prepare PDU, optionally w/ immediate data */1014 ISCSI_DBG_TCP(conn, "task deq [itt 0x%x imm %d unsol %d]\n",1015 task->itt, task->imm_count, task->unsol_r2t.data_length);1016 1017 err = conn->session->tt->init_pdu(task, 0, task->imm_count);1018 if (err)1019 return err;1020 task->imm_count = 0;1021 return 0;1022}1023EXPORT_SYMBOL_GPL(iscsi_tcp_task_init);1024 1025static struct iscsi_r2t_info *iscsi_tcp_get_curr_r2t(struct iscsi_task *task)1026{1027 struct iscsi_tcp_task *tcp_task = task->dd_data;1028 struct iscsi_r2t_info *r2t = NULL;1029 1030 if (iscsi_task_has_unsol_data(task))1031 r2t = &task->unsol_r2t;1032 else {1033 spin_lock_bh(&tcp_task->queue2pool);1034 if (tcp_task->r2t) {1035 r2t = tcp_task->r2t;1036 /* Continue with this R2T? */1037 if (r2t->data_length <= r2t->sent) {1038 ISCSI_DBG_TCP(task->conn,1039 " done with r2t %p\n", r2t);1040 kfifo_in(&tcp_task->r2tpool.queue,1041 (void *)&tcp_task->r2t,1042 sizeof(void *));1043 tcp_task->r2t = r2t = NULL;1044 }1045 }1046 1047 if (r2t == NULL) {1048 if (kfifo_out(&tcp_task->r2tqueue,1049 (void *)&tcp_task->r2t, sizeof(void *)) !=1050 sizeof(void *))1051 r2t = NULL;1052 else1053 r2t = tcp_task->r2t;1054 }1055 spin_unlock_bh(&tcp_task->queue2pool);1056 }1057 1058 return r2t;1059}1060 1061/**1062 * iscsi_tcp_task_xmit - xmit normal PDU task1063 * @task: iscsi command task1064 *1065 * We're expected to return 0 when everything was transmitted successfully,1066 * -EAGAIN if there's still data in the queue, or != 0 for any other kind1067 * of error.1068 */1069int iscsi_tcp_task_xmit(struct iscsi_task *task)1070{1071 struct iscsi_conn *conn = task->conn;1072 struct iscsi_session *session = conn->session;1073 struct iscsi_r2t_info *r2t;1074 int rc = 0;1075 1076flush:1077 /* Flush any pending data first. */1078 rc = session->tt->xmit_pdu(task);1079 if (rc < 0)1080 return rc;1081 1082 /* mgmt command */1083 if (!task->sc) {1084 if (task->hdr->itt == RESERVED_ITT)1085 iscsi_put_task(task);1086 return 0;1087 }1088 1089 /* Are we done already? */1090 if (task->sc->sc_data_direction != DMA_TO_DEVICE)1091 return 0;1092 1093 r2t = iscsi_tcp_get_curr_r2t(task);1094 if (r2t == NULL) {1095 /* Waiting for more R2Ts to arrive. */1096 ISCSI_DBG_TCP(conn, "no R2Ts yet\n");1097 return 0;1098 }1099 1100 rc = conn->session->tt->alloc_pdu(task, ISCSI_OP_SCSI_DATA_OUT);1101 if (rc)1102 return rc;1103 iscsi_prep_data_out_pdu(task, r2t, (struct iscsi_data *) task->hdr);1104 1105 ISCSI_DBG_TCP(conn, "sol dout %p [dsn %d itt 0x%x doff %d dlen %d]\n",1106 r2t, r2t->datasn - 1, task->hdr->itt,1107 r2t->data_offset + r2t->sent, r2t->data_count);1108 1109 rc = conn->session->tt->init_pdu(task, r2t->data_offset + r2t->sent,1110 r2t->data_count);1111 if (rc) {1112 iscsi_conn_failure(conn, ISCSI_ERR_XMIT_FAILED);1113 return rc;1114 }1115 1116 r2t->sent += r2t->data_count;1117 goto flush;1118}1119EXPORT_SYMBOL_GPL(iscsi_tcp_task_xmit);1120 1121struct iscsi_cls_conn *1122iscsi_tcp_conn_setup(struct iscsi_cls_session *cls_session, int dd_data_size,1123 uint32_t conn_idx)1124 1125{1126 struct iscsi_conn *conn;1127 struct iscsi_cls_conn *cls_conn;1128 struct iscsi_tcp_conn *tcp_conn;1129 1130 cls_conn = iscsi_conn_setup(cls_session,1131 sizeof(*tcp_conn) + dd_data_size, conn_idx);1132 if (!cls_conn)1133 return NULL;1134 conn = cls_conn->dd_data;1135 /*1136 * due to strange issues with iser these are not set1137 * in iscsi_conn_setup1138 */1139 conn->max_recv_dlength = ISCSI_DEF_MAX_RECV_SEG_LEN;1140 1141 tcp_conn = conn->dd_data;1142 tcp_conn->iscsi_conn = conn;1143 tcp_conn->dd_data = conn->dd_data + sizeof(*tcp_conn);1144 return cls_conn;1145}1146EXPORT_SYMBOL_GPL(iscsi_tcp_conn_setup);1147 1148void iscsi_tcp_conn_teardown(struct iscsi_cls_conn *cls_conn)1149{1150 iscsi_conn_teardown(cls_conn);1151}1152EXPORT_SYMBOL_GPL(iscsi_tcp_conn_teardown);1153 1154int iscsi_tcp_r2tpool_alloc(struct iscsi_session *session)1155{1156 int i;1157 int cmd_i;1158 1159 /*1160 * initialize per-task: R2T pool and xmit queue1161 */1162 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {1163 struct iscsi_task *task = session->cmds[cmd_i];1164 struct iscsi_tcp_task *tcp_task = task->dd_data;1165 1166 /*1167 * pre-allocated x2 as much r2ts to handle race when1168 * target acks DataOut faster than we data_xmit() queues1169 * could replenish r2tqueue.1170 */1171 1172 /* R2T pool */1173 if (iscsi_pool_init(&tcp_task->r2tpool,1174 session->max_r2t * 2, NULL,1175 sizeof(struct iscsi_r2t_info))) {1176 goto r2t_alloc_fail;1177 }1178 1179 /* R2T xmit queue */1180 if (kfifo_alloc(&tcp_task->r2tqueue,1181 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL)) {1182 iscsi_pool_free(&tcp_task->r2tpool);1183 goto r2t_alloc_fail;1184 }1185 spin_lock_init(&tcp_task->pool2queue);1186 spin_lock_init(&tcp_task->queue2pool);1187 }1188 1189 return 0;1190 1191r2t_alloc_fail:1192 for (i = 0; i < cmd_i; i++) {1193 struct iscsi_task *task = session->cmds[i];1194 struct iscsi_tcp_task *tcp_task = task->dd_data;1195 1196 kfifo_free(&tcp_task->r2tqueue);1197 iscsi_pool_free(&tcp_task->r2tpool);1198 }1199 return -ENOMEM;1200}1201EXPORT_SYMBOL_GPL(iscsi_tcp_r2tpool_alloc);1202 1203void iscsi_tcp_r2tpool_free(struct iscsi_session *session)1204{1205 int i;1206 1207 for (i = 0; i < session->cmds_max; i++) {1208 struct iscsi_task *task = session->cmds[i];1209 struct iscsi_tcp_task *tcp_task = task->dd_data;1210 1211 kfifo_free(&tcp_task->r2tqueue);1212 iscsi_pool_free(&tcp_task->r2tpool);1213 }1214}1215EXPORT_SYMBOL_GPL(iscsi_tcp_r2tpool_free);1216 1217int iscsi_tcp_set_max_r2t(struct iscsi_conn *conn, char *buf)1218{1219 struct iscsi_session *session = conn->session;1220 unsigned short r2ts = 0;1221 1222 sscanf(buf, "%hu", &r2ts);1223 if (session->max_r2t == r2ts)1224 return 0;1225 1226 if (!r2ts || !is_power_of_2(r2ts))1227 return -EINVAL;1228 1229 session->max_r2t = r2ts;1230 iscsi_tcp_r2tpool_free(session);1231 return iscsi_tcp_r2tpool_alloc(session);1232}1233EXPORT_SYMBOL_GPL(iscsi_tcp_set_max_r2t);1234 1235void iscsi_tcp_conn_get_stats(struct iscsi_cls_conn *cls_conn,1236 struct iscsi_stats *stats)1237{1238 struct iscsi_conn *conn = cls_conn->dd_data;1239 1240 stats->txdata_octets = conn->txdata_octets;1241 stats->rxdata_octets = conn->rxdata_octets;1242 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;1243 stats->dataout_pdus = conn->dataout_pdus_cnt;1244 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;1245 stats->datain_pdus = conn->datain_pdus_cnt;1246 stats->r2t_pdus = conn->r2t_pdus_cnt;1247 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;1248 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;1249}1250EXPORT_SYMBOL_GPL(iscsi_tcp_conn_get_stats);1251