691 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*******************************************************************************3 * This file contains main functions related to iSCSI DataSequenceInOrder=No4 * and DataPDUInOrder=No.5 *6 * (c) Copyright 2007-2013 Datera, Inc.7 *8 * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>9 *10 ******************************************************************************/11 12#include <linux/slab.h>13#include <linux/random.h>14 15#include <target/iscsi/iscsi_target_core.h>16#include "iscsi_target_util.h"17#include "iscsi_target_tpg.h"18#include "iscsi_target_seq_pdu_list.h"19 20#ifdef DEBUG21static void iscsit_dump_seq_list(struct iscsit_cmd *cmd)22{23 int i;24 struct iscsi_seq *seq;25 26 pr_debug("Dumping Sequence List for ITT: 0x%08x:\n",27 cmd->init_task_tag);28 29 for (i = 0; i < cmd->seq_count; i++) {30 seq = &cmd->seq_list[i];31 pr_debug("i: %d, pdu_start: %d, pdu_count: %d,"32 " offset: %d, xfer_len: %d, seq_send_order: %d,"33 " seq_no: %d\n", i, seq->pdu_start, seq->pdu_count,34 seq->offset, seq->xfer_len, seq->seq_send_order,35 seq->seq_no);36 }37}38 39static void iscsit_dump_pdu_list(struct iscsit_cmd *cmd)40{41 int i;42 struct iscsi_pdu *pdu;43 44 pr_debug("Dumping PDU List for ITT: 0x%08x:\n",45 cmd->init_task_tag);46 47 for (i = 0; i < cmd->pdu_count; i++) {48 pdu = &cmd->pdu_list[i];49 pr_debug("i: %d, offset: %d, length: %d,"50 " pdu_send_order: %d, seq_no: %d\n", i, pdu->offset,51 pdu->length, pdu->pdu_send_order, pdu->seq_no);52 }53}54#else55static void iscsit_dump_seq_list(struct iscsit_cmd *cmd) {}56static void iscsit_dump_pdu_list(struct iscsit_cmd *cmd) {}57#endif58 59static void iscsit_ordered_seq_lists(60 struct iscsit_cmd *cmd,61 u8 type)62{63 u32 i, seq_count = 0;64 65 for (i = 0; i < cmd->seq_count; i++) {66 if (cmd->seq_list[i].type != SEQTYPE_NORMAL)67 continue;68 cmd->seq_list[i].seq_send_order = seq_count++;69 }70}71 72static void iscsit_ordered_pdu_lists(73 struct iscsit_cmd *cmd,74 u8 type)75{76 u32 i, pdu_send_order = 0, seq_no = 0;77 78 for (i = 0; i < cmd->pdu_count; i++) {79redo:80 if (cmd->pdu_list[i].seq_no == seq_no) {81 cmd->pdu_list[i].pdu_send_order = pdu_send_order++;82 continue;83 }84 seq_no++;85 pdu_send_order = 0;86 goto redo;87 }88}89 90/*91 * Generate count random values into array.92 * Use 0x80000000 to mark generates valued in array[].93 */94static void iscsit_create_random_array(u32 *array, u32 count)95{96 int i, j, k;97 98 if (count == 1) {99 array[0] = 0;100 return;101 }102 103 for (i = 0; i < count; i++) {104redo:105 get_random_bytes(&j, sizeof(u32));106 j = (1 + (int) (9999 + 1) - j) % count;107 for (k = 0; k < i + 1; k++) {108 j |= 0x80000000;109 if ((array[k] & 0x80000000) && (array[k] == j))110 goto redo;111 }112 array[i] = j;113 }114 115 for (i = 0; i < count; i++)116 array[i] &= ~0x80000000;117}118 119static int iscsit_randomize_pdu_lists(120 struct iscsit_cmd *cmd,121 u8 type)122{123 int i = 0;124 u32 *array, pdu_count, seq_count = 0, seq_no = 0, seq_offset = 0;125 126 for (pdu_count = 0; pdu_count < cmd->pdu_count; pdu_count++) {127redo:128 if (cmd->pdu_list[pdu_count].seq_no == seq_no) {129 seq_count++;130 continue;131 }132 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);133 if (!array) {134 pr_err("Unable to allocate memory"135 " for random array.\n");136 return -ENOMEM;137 }138 iscsit_create_random_array(array, seq_count);139 140 for (i = 0; i < seq_count; i++)141 cmd->pdu_list[seq_offset+i].pdu_send_order = array[i];142 143 kfree(array);144 145 seq_offset += seq_count;146 seq_count = 0;147 seq_no++;148 goto redo;149 }150 151 if (seq_count) {152 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);153 if (!array) {154 pr_err("Unable to allocate memory for"155 " random array.\n");156 return -ENOMEM;157 }158 iscsit_create_random_array(array, seq_count);159 160 for (i = 0; i < seq_count; i++)161 cmd->pdu_list[seq_offset+i].pdu_send_order = array[i];162 163 kfree(array);164 }165 166 return 0;167}168 169static int iscsit_randomize_seq_lists(170 struct iscsit_cmd *cmd,171 u8 type)172{173 int i, j = 0;174 u32 *array, seq_count = cmd->seq_count;175 176 if ((type == PDULIST_IMMEDIATE) || (type == PDULIST_UNSOLICITED))177 seq_count--;178 else if (type == PDULIST_IMMEDIATE_AND_UNSOLICITED)179 seq_count -= 2;180 181 if (!seq_count)182 return 0;183 184 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);185 if (!array) {186 pr_err("Unable to allocate memory for random array.\n");187 return -ENOMEM;188 }189 iscsit_create_random_array(array, seq_count);190 191 for (i = 0; i < cmd->seq_count; i++) {192 if (cmd->seq_list[i].type != SEQTYPE_NORMAL)193 continue;194 cmd->seq_list[i].seq_send_order = array[j++];195 }196 197 kfree(array);198 return 0;199}200 201static void iscsit_determine_counts_for_list(202 struct iscsit_cmd *cmd,203 struct iscsi_build_list *bl,204 u32 *seq_count,205 u32 *pdu_count)206{207 int check_immediate = 0;208 u32 burstlength = 0, offset = 0;209 u32 unsolicited_data_length = 0;210 u32 mdsl;211 struct iscsit_conn *conn = cmd->conn;212 213 if (cmd->se_cmd.data_direction == DMA_TO_DEVICE)214 mdsl = cmd->conn->conn_ops->MaxXmitDataSegmentLength;215 else216 mdsl = cmd->conn->conn_ops->MaxRecvDataSegmentLength;217 218 if ((bl->type == PDULIST_IMMEDIATE) ||219 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))220 check_immediate = 1;221 222 if ((bl->type == PDULIST_UNSOLICITED) ||223 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))224 unsolicited_data_length = min(cmd->se_cmd.data_length,225 conn->sess->sess_ops->FirstBurstLength);226 227 while (offset < cmd->se_cmd.data_length) {228 *pdu_count += 1;229 230 if (check_immediate) {231 check_immediate = 0;232 offset += bl->immediate_data_length;233 *seq_count += 1;234 if (unsolicited_data_length)235 unsolicited_data_length -=236 bl->immediate_data_length;237 continue;238 }239 if (unsolicited_data_length > 0) {240 if ((offset + mdsl) >= cmd->se_cmd.data_length) {241 unsolicited_data_length -=242 (cmd->se_cmd.data_length - offset);243 offset += (cmd->se_cmd.data_length - offset);244 continue;245 }246 if ((offset + mdsl)247 >= conn->sess->sess_ops->FirstBurstLength) {248 unsolicited_data_length -=249 (conn->sess->sess_ops->FirstBurstLength -250 offset);251 offset += (conn->sess->sess_ops->FirstBurstLength -252 offset);253 burstlength = 0;254 *seq_count += 1;255 continue;256 }257 258 offset += mdsl;259 unsolicited_data_length -= mdsl;260 continue;261 }262 if ((offset + mdsl) >= cmd->se_cmd.data_length) {263 offset += (cmd->se_cmd.data_length - offset);264 continue;265 }266 if ((burstlength + mdsl) >=267 conn->sess->sess_ops->MaxBurstLength) {268 offset += (conn->sess->sess_ops->MaxBurstLength -269 burstlength);270 burstlength = 0;271 *seq_count += 1;272 continue;273 }274 275 burstlength += mdsl;276 offset += mdsl;277 }278}279 280 281/*282 * Builds PDU and/or Sequence list, called while DataSequenceInOrder=No283 * or DataPDUInOrder=No.284 */285static int iscsit_do_build_pdu_and_seq_lists(286 struct iscsit_cmd *cmd,287 struct iscsi_build_list *bl)288{289 int check_immediate = 0, datapduinorder, datasequenceinorder;290 u32 burstlength = 0, offset = 0, i = 0, mdsl;291 u32 pdu_count = 0, seq_no = 0, unsolicited_data_length = 0;292 struct iscsit_conn *conn = cmd->conn;293 struct iscsi_pdu *pdu = cmd->pdu_list;294 struct iscsi_seq *seq = cmd->seq_list;295 296 if (cmd->se_cmd.data_direction == DMA_TO_DEVICE)297 mdsl = cmd->conn->conn_ops->MaxXmitDataSegmentLength;298 else299 mdsl = cmd->conn->conn_ops->MaxRecvDataSegmentLength;300 301 datapduinorder = conn->sess->sess_ops->DataPDUInOrder;302 datasequenceinorder = conn->sess->sess_ops->DataSequenceInOrder;303 304 if ((bl->type == PDULIST_IMMEDIATE) ||305 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))306 check_immediate = 1;307 308 if ((bl->type == PDULIST_UNSOLICITED) ||309 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))310 unsolicited_data_length = min(cmd->se_cmd.data_length,311 conn->sess->sess_ops->FirstBurstLength);312 313 while (offset < cmd->se_cmd.data_length) {314 pdu_count++;315 if (!datapduinorder) {316 pdu[i].offset = offset;317 pdu[i].seq_no = seq_no;318 }319 if (!datasequenceinorder && (pdu_count == 1)) {320 seq[seq_no].pdu_start = i;321 seq[seq_no].seq_no = seq_no;322 seq[seq_no].offset = offset;323 seq[seq_no].orig_offset = offset;324 }325 326 if (check_immediate) {327 check_immediate = 0;328 if (!datapduinorder) {329 pdu[i].type = PDUTYPE_IMMEDIATE;330 pdu[i++].length = bl->immediate_data_length;331 }332 if (!datasequenceinorder) {333 seq[seq_no].type = SEQTYPE_IMMEDIATE;334 seq[seq_no].pdu_count = 1;335 seq[seq_no].xfer_len =336 bl->immediate_data_length;337 }338 offset += bl->immediate_data_length;339 pdu_count = 0;340 seq_no++;341 if (unsolicited_data_length)342 unsolicited_data_length -=343 bl->immediate_data_length;344 continue;345 }346 if (unsolicited_data_length > 0) {347 if ((offset + mdsl) >= cmd->se_cmd.data_length) {348 if (!datapduinorder) {349 pdu[i].type = PDUTYPE_UNSOLICITED;350 pdu[i].length =351 (cmd->se_cmd.data_length - offset);352 }353 if (!datasequenceinorder) {354 seq[seq_no].type = SEQTYPE_UNSOLICITED;355 seq[seq_no].pdu_count = pdu_count;356 seq[seq_no].xfer_len = (burstlength +357 (cmd->se_cmd.data_length - offset));358 }359 unsolicited_data_length -=360 (cmd->se_cmd.data_length - offset);361 offset += (cmd->se_cmd.data_length - offset);362 continue;363 }364 if ((offset + mdsl) >=365 conn->sess->sess_ops->FirstBurstLength) {366 if (!datapduinorder) {367 pdu[i].type = PDUTYPE_UNSOLICITED;368 pdu[i++].length =369 (conn->sess->sess_ops->FirstBurstLength -370 offset);371 }372 if (!datasequenceinorder) {373 seq[seq_no].type = SEQTYPE_UNSOLICITED;374 seq[seq_no].pdu_count = pdu_count;375 seq[seq_no].xfer_len = (burstlength +376 (conn->sess->sess_ops->FirstBurstLength -377 offset));378 }379 unsolicited_data_length -=380 (conn->sess->sess_ops->FirstBurstLength -381 offset);382 offset += (conn->sess->sess_ops->FirstBurstLength -383 offset);384 burstlength = 0;385 pdu_count = 0;386 seq_no++;387 continue;388 }389 390 if (!datapduinorder) {391 pdu[i].type = PDUTYPE_UNSOLICITED;392 pdu[i++].length = mdsl;393 }394 burstlength += mdsl;395 offset += mdsl;396 unsolicited_data_length -= mdsl;397 continue;398 }399 if ((offset + mdsl) >= cmd->se_cmd.data_length) {400 if (!datapduinorder) {401 pdu[i].type = PDUTYPE_NORMAL;402 pdu[i].length = (cmd->se_cmd.data_length - offset);403 }404 if (!datasequenceinorder) {405 seq[seq_no].type = SEQTYPE_NORMAL;406 seq[seq_no].pdu_count = pdu_count;407 seq[seq_no].xfer_len = (burstlength +408 (cmd->se_cmd.data_length - offset));409 }410 offset += (cmd->se_cmd.data_length - offset);411 continue;412 }413 if ((burstlength + mdsl) >=414 conn->sess->sess_ops->MaxBurstLength) {415 if (!datapduinorder) {416 pdu[i].type = PDUTYPE_NORMAL;417 pdu[i++].length =418 (conn->sess->sess_ops->MaxBurstLength -419 burstlength);420 }421 if (!datasequenceinorder) {422 seq[seq_no].type = SEQTYPE_NORMAL;423 seq[seq_no].pdu_count = pdu_count;424 seq[seq_no].xfer_len = (burstlength +425 (conn->sess->sess_ops->MaxBurstLength -426 burstlength));427 }428 offset += (conn->sess->sess_ops->MaxBurstLength -429 burstlength);430 burstlength = 0;431 pdu_count = 0;432 seq_no++;433 continue;434 }435 436 if (!datapduinorder) {437 pdu[i].type = PDUTYPE_NORMAL;438 pdu[i++].length = mdsl;439 }440 burstlength += mdsl;441 offset += mdsl;442 }443 444 if (!datasequenceinorder) {445 if (bl->data_direction & ISCSI_PDU_WRITE) {446 if (bl->randomize & RANDOM_R2T_OFFSETS) {447 if (iscsit_randomize_seq_lists(cmd, bl->type)448 < 0)449 return -1;450 } else451 iscsit_ordered_seq_lists(cmd, bl->type);452 } else if (bl->data_direction & ISCSI_PDU_READ) {453 if (bl->randomize & RANDOM_DATAIN_SEQ_OFFSETS) {454 if (iscsit_randomize_seq_lists(cmd, bl->type)455 < 0)456 return -1;457 } else458 iscsit_ordered_seq_lists(cmd, bl->type);459 }460 461 iscsit_dump_seq_list(cmd);462 }463 if (!datapduinorder) {464 if (bl->data_direction & ISCSI_PDU_WRITE) {465 if (bl->randomize & RANDOM_DATAOUT_PDU_OFFSETS) {466 if (iscsit_randomize_pdu_lists(cmd, bl->type)467 < 0)468 return -1;469 } else470 iscsit_ordered_pdu_lists(cmd, bl->type);471 } else if (bl->data_direction & ISCSI_PDU_READ) {472 if (bl->randomize & RANDOM_DATAIN_PDU_OFFSETS) {473 if (iscsit_randomize_pdu_lists(cmd, bl->type)474 < 0)475 return -1;476 } else477 iscsit_ordered_pdu_lists(cmd, bl->type);478 }479 480 iscsit_dump_pdu_list(cmd);481 }482 483 return 0;484}485 486int iscsit_build_pdu_and_seq_lists(487 struct iscsit_cmd *cmd,488 u32 immediate_data_length)489{490 struct iscsi_build_list bl;491 u32 pdu_count = 0, seq_count = 1;492 struct iscsit_conn *conn = cmd->conn;493 struct iscsi_pdu *pdu = NULL;494 struct iscsi_seq *seq = NULL;495 496 struct iscsit_session *sess = conn->sess;497 struct iscsi_node_attrib *na;498 499 /*500 * Do nothing if no OOO shenanigans501 */502 if (sess->sess_ops->DataSequenceInOrder &&503 sess->sess_ops->DataPDUInOrder)504 return 0;505 506 if (cmd->data_direction == DMA_NONE)507 return 0;508 509 na = iscsit_tpg_get_node_attrib(sess);510 memset(&bl, 0, sizeof(struct iscsi_build_list));511 512 if (cmd->data_direction == DMA_FROM_DEVICE) {513 bl.data_direction = ISCSI_PDU_READ;514 bl.type = PDULIST_NORMAL;515 if (na->random_datain_pdu_offsets)516 bl.randomize |= RANDOM_DATAIN_PDU_OFFSETS;517 if (na->random_datain_seq_offsets)518 bl.randomize |= RANDOM_DATAIN_SEQ_OFFSETS;519 } else {520 bl.data_direction = ISCSI_PDU_WRITE;521 bl.immediate_data_length = immediate_data_length;522 if (na->random_r2t_offsets)523 bl.randomize |= RANDOM_R2T_OFFSETS;524 525 if (!cmd->immediate_data && !cmd->unsolicited_data)526 bl.type = PDULIST_NORMAL;527 else if (cmd->immediate_data && !cmd->unsolicited_data)528 bl.type = PDULIST_IMMEDIATE;529 else if (!cmd->immediate_data && cmd->unsolicited_data)530 bl.type = PDULIST_UNSOLICITED;531 else if (cmd->immediate_data && cmd->unsolicited_data)532 bl.type = PDULIST_IMMEDIATE_AND_UNSOLICITED;533 }534 535 iscsit_determine_counts_for_list(cmd, &bl, &seq_count, &pdu_count);536 537 if (!conn->sess->sess_ops->DataSequenceInOrder) {538 seq = kcalloc(seq_count, sizeof(struct iscsi_seq), GFP_ATOMIC);539 if (!seq) {540 pr_err("Unable to allocate struct iscsi_seq list\n");541 return -ENOMEM;542 }543 cmd->seq_list = seq;544 cmd->seq_count = seq_count;545 }546 547 if (!conn->sess->sess_ops->DataPDUInOrder) {548 pdu = kcalloc(pdu_count, sizeof(struct iscsi_pdu), GFP_ATOMIC);549 if (!pdu) {550 pr_err("Unable to allocate struct iscsi_pdu list.\n");551 kfree(seq);552 return -ENOMEM;553 }554 cmd->pdu_list = pdu;555 cmd->pdu_count = pdu_count;556 }557 558 return iscsit_do_build_pdu_and_seq_lists(cmd, &bl);559}560 561struct iscsi_pdu *iscsit_get_pdu_holder(562 struct iscsit_cmd *cmd,563 u32 offset,564 u32 length)565{566 u32 i;567 struct iscsi_pdu *pdu = NULL;568 569 if (!cmd->pdu_list) {570 pr_err("struct iscsit_cmd->pdu_list is NULL!\n");571 return NULL;572 }573 574 pdu = &cmd->pdu_list[0];575 576 for (i = 0; i < cmd->pdu_count; i++)577 if ((pdu[i].offset == offset) && (pdu[i].length == length))578 return &pdu[i];579 580 pr_err("Unable to locate PDU holder for ITT: 0x%08x, Offset:"581 " %u, Length: %u\n", cmd->init_task_tag, offset, length);582 return NULL;583}584 585struct iscsi_pdu *iscsit_get_pdu_holder_for_seq(586 struct iscsit_cmd *cmd,587 struct iscsi_seq *seq)588{589 u32 i;590 struct iscsit_conn *conn = cmd->conn;591 struct iscsi_pdu *pdu = NULL;592 593 if (!cmd->pdu_list) {594 pr_err("struct iscsit_cmd->pdu_list is NULL!\n");595 return NULL;596 }597 598 if (conn->sess->sess_ops->DataSequenceInOrder) {599redo:600 pdu = &cmd->pdu_list[cmd->pdu_start];601 602 for (i = 0; pdu[i].seq_no != cmd->seq_no; i++) {603 pr_debug("pdu[i].seq_no: %d, pdu[i].pdu"604 "_send_order: %d, pdu[i].offset: %d,"605 " pdu[i].length: %d\n", pdu[i].seq_no,606 pdu[i].pdu_send_order, pdu[i].offset,607 pdu[i].length);608 609 if (pdu[i].pdu_send_order == cmd->pdu_send_order) {610 cmd->pdu_send_order++;611 return &pdu[i];612 }613 }614 615 cmd->pdu_start += cmd->pdu_send_order;616 cmd->pdu_send_order = 0;617 cmd->seq_no++;618 619 if (cmd->pdu_start < cmd->pdu_count)620 goto redo;621 622 pr_err("Command ITT: 0x%08x unable to locate"623 " struct iscsi_pdu for cmd->pdu_send_order: %u.\n",624 cmd->init_task_tag, cmd->pdu_send_order);625 return NULL;626 } else {627 if (!seq) {628 pr_err("struct iscsi_seq is NULL!\n");629 return NULL;630 }631 632 pr_debug("seq->pdu_start: %d, seq->pdu_count: %d,"633 " seq->seq_no: %d\n", seq->pdu_start, seq->pdu_count,634 seq->seq_no);635 636 pdu = &cmd->pdu_list[seq->pdu_start];637 638 if (seq->pdu_send_order == seq->pdu_count) {639 pr_err("Command ITT: 0x%08x seq->pdu_send"640 "_order: %u equals seq->pdu_count: %u\n",641 cmd->init_task_tag, seq->pdu_send_order,642 seq->pdu_count);643 return NULL;644 }645 646 for (i = 0; i < seq->pdu_count; i++) {647 if (pdu[i].pdu_send_order == seq->pdu_send_order) {648 seq->pdu_send_order++;649 return &pdu[i];650 }651 }652 653 pr_err("Command ITT: 0x%08x unable to locate iscsi"654 "_pdu_t for seq->pdu_send_order: %u.\n",655 cmd->init_task_tag, seq->pdu_send_order);656 return NULL;657 }658 659 return NULL;660}661 662struct iscsi_seq *iscsit_get_seq_holder(663 struct iscsit_cmd *cmd,664 u32 offset,665 u32 length)666{667 u32 i;668 669 if (!cmd->seq_list) {670 pr_err("struct iscsit_cmd->seq_list is NULL!\n");671 return NULL;672 }673 674 for (i = 0; i < cmd->seq_count; i++) {675 pr_debug("seq_list[i].orig_offset: %d, seq_list[i]."676 "xfer_len: %d, seq_list[i].seq_no %u\n",677 cmd->seq_list[i].orig_offset, cmd->seq_list[i].xfer_len,678 cmd->seq_list[i].seq_no);679 680 if ((cmd->seq_list[i].orig_offset +681 cmd->seq_list[i].xfer_len) >=682 (offset + length))683 return &cmd->seq_list[i];684 }685 686 pr_err("Unable to locate Sequence holder for ITT: 0x%08x,"687 " Offset: %u, Length: %u\n", cmd->init_task_tag, offset,688 length);689 return NULL;690}691