467 lines · c
1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)2/* QLogic qed NIC Driver3 * Copyright (c) 2015-2017 QLogic Corporation4 * Copyright (c) 2019-2020 Marvell International Ltd.5 */6 7#include <linux/types.h>8#include <linux/dma-mapping.h>9#include <linux/kernel.h>10#include <linux/list.h>11#include <linux/pci.h>12#include <linux/slab.h>13#include <linux/string.h>14#include "qed.h"15#include "qed_iscsi.h"16#include "qed_ll2.h"17#include "qed_ooo.h"18#include "qed_cxt.h"19#include "qed_nvmetcp.h"20static struct qed_ooo_archipelago21*qed_ooo_seek_archipelago(struct qed_hwfn *p_hwfn,22 struct qed_ooo_info23 *p_ooo_info,24 u32 cid)25{26 u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;27 struct qed_ooo_archipelago *p_archipelago;28 29 if (unlikely(idx >= p_ooo_info->max_num_archipelagos))30 return NULL;31 32 p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];33 34 if (unlikely(list_empty(&p_archipelago->isles_list)))35 return NULL;36 37 return p_archipelago;38}39 40static struct qed_ooo_isle *qed_ooo_seek_isle(struct qed_hwfn *p_hwfn,41 struct qed_ooo_info *p_ooo_info,42 u32 cid, u8 isle)43{44 struct qed_ooo_archipelago *p_archipelago = NULL;45 struct qed_ooo_isle *p_isle = NULL;46 u8 the_num_of_isle = 1;47 48 p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);49 if (unlikely(!p_archipelago)) {50 DP_NOTICE(p_hwfn,51 "Connection %d is not found in OOO list\n", cid);52 return NULL;53 }54 55 list_for_each_entry(p_isle, &p_archipelago->isles_list, list_entry) {56 if (the_num_of_isle == isle)57 return p_isle;58 the_num_of_isle++;59 }60 61 return NULL;62}63 64void qed_ooo_save_history_entry(struct qed_hwfn *p_hwfn,65 struct qed_ooo_info *p_ooo_info,66 struct ooo_opaque *p_cqe)67{68 struct qed_ooo_history *p_history = &p_ooo_info->ooo_history;69 70 if (p_history->head_idx == p_history->num_of_cqes)71 p_history->head_idx = 0;72 p_history->p_cqes[p_history->head_idx] = *p_cqe;73 p_history->head_idx++;74}75 76int qed_ooo_alloc(struct qed_hwfn *p_hwfn)77{78 u16 max_num_archipelagos = 0, cid_base;79 struct qed_ooo_info *p_ooo_info;80 enum protocol_type proto;81 u16 max_num_isles = 0;82 u32 i;83 84 switch (p_hwfn->hw_info.personality) {85 case QED_PCI_ISCSI:86 case QED_PCI_NVMETCP:87 proto = PROTOCOLID_TCP_ULP;88 break;89 case QED_PCI_ETH_RDMA:90 case QED_PCI_ETH_IWARP:91 proto = PROTOCOLID_IWARP;92 break;93 default:94 DP_NOTICE(p_hwfn,95 "Failed to allocate qed_ooo_info: unknown personality\n");96 return -EINVAL;97 }98 99 max_num_archipelagos = (u16)qed_cxt_get_proto_cid_count(p_hwfn, proto,100 NULL);101 max_num_isles = QED_MAX_NUM_ISLES + max_num_archipelagos;102 cid_base = (u16)qed_cxt_get_proto_cid_start(p_hwfn, proto);103 104 if (!max_num_archipelagos) {105 DP_NOTICE(p_hwfn,106 "Failed to allocate qed_ooo_info: unknown amount of connections\n");107 return -EINVAL;108 }109 110 p_ooo_info = kzalloc(sizeof(*p_ooo_info), GFP_KERNEL);111 if (!p_ooo_info)112 return -ENOMEM;113 114 p_ooo_info->cid_base = cid_base;115 p_ooo_info->max_num_archipelagos = max_num_archipelagos;116 117 INIT_LIST_HEAD(&p_ooo_info->free_buffers_list);118 INIT_LIST_HEAD(&p_ooo_info->ready_buffers_list);119 INIT_LIST_HEAD(&p_ooo_info->free_isles_list);120 121 p_ooo_info->p_isles_mem = kcalloc(max_num_isles,122 sizeof(struct qed_ooo_isle),123 GFP_KERNEL);124 if (!p_ooo_info->p_isles_mem)125 goto no_isles_mem;126 127 for (i = 0; i < max_num_isles; i++) {128 INIT_LIST_HEAD(&p_ooo_info->p_isles_mem[i].buffers_list);129 list_add_tail(&p_ooo_info->p_isles_mem[i].list_entry,130 &p_ooo_info->free_isles_list);131 }132 133 p_ooo_info->p_archipelagos_mem =134 kcalloc(max_num_archipelagos,135 sizeof(struct qed_ooo_archipelago),136 GFP_KERNEL);137 if (!p_ooo_info->p_archipelagos_mem)138 goto no_archipelagos_mem;139 140 for (i = 0; i < max_num_archipelagos; i++)141 INIT_LIST_HEAD(&p_ooo_info->p_archipelagos_mem[i].isles_list);142 143 p_ooo_info->ooo_history.p_cqes =144 kcalloc(QED_MAX_NUM_OOO_HISTORY_ENTRIES,145 sizeof(struct ooo_opaque),146 GFP_KERNEL);147 if (!p_ooo_info->ooo_history.p_cqes)148 goto no_history_mem;149 150 p_ooo_info->ooo_history.num_of_cqes = QED_MAX_NUM_OOO_HISTORY_ENTRIES;151 152 p_hwfn->p_ooo_info = p_ooo_info;153 return 0;154 155no_history_mem:156 kfree(p_ooo_info->p_archipelagos_mem);157no_archipelagos_mem:158 kfree(p_ooo_info->p_isles_mem);159no_isles_mem:160 kfree(p_ooo_info);161 return -ENOMEM;162}163 164void qed_ooo_release_connection_isles(struct qed_hwfn *p_hwfn,165 struct qed_ooo_info *p_ooo_info, u32 cid)166{167 struct qed_ooo_archipelago *p_archipelago;168 struct qed_ooo_buffer *p_buffer;169 struct qed_ooo_isle *p_isle;170 171 p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);172 if (!p_archipelago)173 return;174 175 while (!list_empty(&p_archipelago->isles_list)) {176 p_isle = list_first_entry(&p_archipelago->isles_list,177 struct qed_ooo_isle, list_entry);178 179 list_del(&p_isle->list_entry);180 181 while (!list_empty(&p_isle->buffers_list)) {182 p_buffer = list_first_entry(&p_isle->buffers_list,183 struct qed_ooo_buffer,184 list_entry);185 186 if (!p_buffer)187 break;188 189 list_move_tail(&p_buffer->list_entry,190 &p_ooo_info->free_buffers_list);191 }192 list_add_tail(&p_isle->list_entry,193 &p_ooo_info->free_isles_list);194 }195}196 197void qed_ooo_release_all_isles(struct qed_hwfn *p_hwfn,198 struct qed_ooo_info *p_ooo_info)199{200 struct qed_ooo_archipelago *p_archipelago;201 struct qed_ooo_buffer *p_buffer;202 struct qed_ooo_isle *p_isle;203 u32 i;204 205 for (i = 0; i < p_ooo_info->max_num_archipelagos; i++) {206 p_archipelago = &(p_ooo_info->p_archipelagos_mem[i]);207 208 while (!list_empty(&p_archipelago->isles_list)) {209 p_isle = list_first_entry(&p_archipelago->isles_list,210 struct qed_ooo_isle,211 list_entry);212 213 list_del(&p_isle->list_entry);214 215 while (!list_empty(&p_isle->buffers_list)) {216 p_buffer =217 list_first_entry(&p_isle->buffers_list,218 struct qed_ooo_buffer,219 list_entry);220 221 if (!p_buffer)222 break;223 224 list_move_tail(&p_buffer->list_entry,225 &p_ooo_info->free_buffers_list);226 }227 list_add_tail(&p_isle->list_entry,228 &p_ooo_info->free_isles_list);229 }230 }231 if (!list_empty(&p_ooo_info->ready_buffers_list))232 list_splice_tail_init(&p_ooo_info->ready_buffers_list,233 &p_ooo_info->free_buffers_list);234}235 236void qed_ooo_setup(struct qed_hwfn *p_hwfn)237{238 qed_ooo_release_all_isles(p_hwfn, p_hwfn->p_ooo_info);239 memset(p_hwfn->p_ooo_info->ooo_history.p_cqes, 0,240 p_hwfn->p_ooo_info->ooo_history.num_of_cqes *241 sizeof(struct ooo_opaque));242 p_hwfn->p_ooo_info->ooo_history.head_idx = 0;243}244 245void qed_ooo_free(struct qed_hwfn *p_hwfn)246{247 struct qed_ooo_info *p_ooo_info = p_hwfn->p_ooo_info;248 struct qed_ooo_buffer *p_buffer;249 250 if (!p_ooo_info)251 return;252 253 qed_ooo_release_all_isles(p_hwfn, p_ooo_info);254 while (!list_empty(&p_ooo_info->free_buffers_list)) {255 p_buffer = list_first_entry(&p_ooo_info->free_buffers_list,256 struct qed_ooo_buffer, list_entry);257 258 if (!p_buffer)259 break;260 261 list_del(&p_buffer->list_entry);262 dma_free_coherent(&p_hwfn->cdev->pdev->dev,263 p_buffer->rx_buffer_size,264 p_buffer->rx_buffer_virt_addr,265 p_buffer->rx_buffer_phys_addr);266 kfree(p_buffer);267 }268 269 kfree(p_ooo_info->p_isles_mem);270 kfree(p_ooo_info->p_archipelagos_mem);271 kfree(p_ooo_info->ooo_history.p_cqes);272 kfree(p_ooo_info);273 p_hwfn->p_ooo_info = NULL;274}275 276void qed_ooo_put_free_buffer(struct qed_hwfn *p_hwfn,277 struct qed_ooo_info *p_ooo_info,278 struct qed_ooo_buffer *p_buffer)279{280 list_add_tail(&p_buffer->list_entry, &p_ooo_info->free_buffers_list);281}282 283struct qed_ooo_buffer *qed_ooo_get_free_buffer(struct qed_hwfn *p_hwfn,284 struct qed_ooo_info *p_ooo_info)285{286 struct qed_ooo_buffer *p_buffer = NULL;287 288 if (!list_empty(&p_ooo_info->free_buffers_list)) {289 p_buffer = list_first_entry(&p_ooo_info->free_buffers_list,290 struct qed_ooo_buffer, list_entry);291 292 list_del(&p_buffer->list_entry);293 }294 295 return p_buffer;296}297 298void qed_ooo_put_ready_buffer(struct qed_hwfn *p_hwfn,299 struct qed_ooo_info *p_ooo_info,300 struct qed_ooo_buffer *p_buffer, u8 on_tail)301{302 if (on_tail)303 list_add_tail(&p_buffer->list_entry,304 &p_ooo_info->ready_buffers_list);305 else306 list_add(&p_buffer->list_entry,307 &p_ooo_info->ready_buffers_list);308}309 310struct qed_ooo_buffer *qed_ooo_get_ready_buffer(struct qed_hwfn *p_hwfn,311 struct qed_ooo_info *p_ooo_info)312{313 struct qed_ooo_buffer *p_buffer = NULL;314 315 if (!list_empty(&p_ooo_info->ready_buffers_list)) {316 p_buffer = list_first_entry(&p_ooo_info->ready_buffers_list,317 struct qed_ooo_buffer, list_entry);318 319 list_del(&p_buffer->list_entry);320 }321 322 return p_buffer;323}324 325void qed_ooo_delete_isles(struct qed_hwfn *p_hwfn,326 struct qed_ooo_info *p_ooo_info,327 u32 cid, u8 drop_isle, u8 drop_size)328{329 struct qed_ooo_isle *p_isle = NULL;330 u8 isle_idx;331 332 for (isle_idx = 0; isle_idx < drop_size; isle_idx++) {333 p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, drop_isle);334 if (!p_isle) {335 DP_NOTICE(p_hwfn,336 "Isle %d is not found(cid %d)\n",337 drop_isle, cid);338 return;339 }340 if (list_empty(&p_isle->buffers_list))341 DP_NOTICE(p_hwfn,342 "Isle %d is empty(cid %d)\n", drop_isle, cid);343 else344 list_splice_tail_init(&p_isle->buffers_list,345 &p_ooo_info->free_buffers_list);346 347 list_del(&p_isle->list_entry);348 p_ooo_info->cur_isles_number--;349 list_add(&p_isle->list_entry, &p_ooo_info->free_isles_list);350 }351}352 353void qed_ooo_add_new_isle(struct qed_hwfn *p_hwfn,354 struct qed_ooo_info *p_ooo_info,355 u32 cid, u8 ooo_isle,356 struct qed_ooo_buffer *p_buffer)357{358 struct qed_ooo_archipelago *p_archipelago = NULL;359 struct qed_ooo_isle *p_prev_isle = NULL;360 struct qed_ooo_isle *p_isle = NULL;361 362 if (ooo_isle > 1) {363 p_prev_isle = qed_ooo_seek_isle(p_hwfn,364 p_ooo_info, cid, ooo_isle - 1);365 if (unlikely(!p_prev_isle)) {366 DP_NOTICE(p_hwfn,367 "Isle %d is not found(cid %d)\n",368 ooo_isle - 1, cid);369 return;370 }371 }372 p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);373 if (unlikely(!p_archipelago && ooo_isle != 1)) {374 DP_NOTICE(p_hwfn,375 "Connection %d is not found in OOO list\n", cid);376 return;377 }378 379 if (!list_empty(&p_ooo_info->free_isles_list)) {380 p_isle = list_first_entry(&p_ooo_info->free_isles_list,381 struct qed_ooo_isle, list_entry);382 383 list_del(&p_isle->list_entry);384 if (unlikely(!list_empty(&p_isle->buffers_list))) {385 DP_NOTICE(p_hwfn, "Free isle is not empty\n");386 INIT_LIST_HEAD(&p_isle->buffers_list);387 }388 } else {389 DP_NOTICE(p_hwfn, "No more free isles\n");390 return;391 }392 393 if (!p_archipelago) {394 u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;395 396 p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];397 }398 399 list_add(&p_buffer->list_entry, &p_isle->buffers_list);400 p_ooo_info->cur_isles_number++;401 p_ooo_info->gen_isles_number++;402 403 if (p_ooo_info->cur_isles_number > p_ooo_info->max_isles_number)404 p_ooo_info->max_isles_number = p_ooo_info->cur_isles_number;405 406 if (!p_prev_isle)407 list_add(&p_isle->list_entry, &p_archipelago->isles_list);408 else409 list_add(&p_isle->list_entry, &p_prev_isle->list_entry);410}411 412void qed_ooo_add_new_buffer(struct qed_hwfn *p_hwfn,413 struct qed_ooo_info *p_ooo_info,414 u32 cid,415 u8 ooo_isle,416 struct qed_ooo_buffer *p_buffer, u8 buffer_side)417{418 struct qed_ooo_isle *p_isle = NULL;419 420 p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle);421 if (unlikely(!p_isle)) {422 DP_NOTICE(p_hwfn,423 "Isle %d is not found(cid %d)\n", ooo_isle, cid);424 return;425 }426 427 if (unlikely(buffer_side == QED_OOO_LEFT_BUF))428 list_add(&p_buffer->list_entry, &p_isle->buffers_list);429 else430 list_add_tail(&p_buffer->list_entry, &p_isle->buffers_list);431}432 433void qed_ooo_join_isles(struct qed_hwfn *p_hwfn,434 struct qed_ooo_info *p_ooo_info, u32 cid, u8 left_isle)435{436 struct qed_ooo_isle *p_right_isle = NULL;437 struct qed_ooo_isle *p_left_isle = NULL;438 439 p_right_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid,440 left_isle + 1);441 if (unlikely(!p_right_isle)) {442 DP_NOTICE(p_hwfn,443 "Right isle %d is not found(cid %d)\n",444 left_isle + 1, cid);445 return;446 }447 448 list_del(&p_right_isle->list_entry);449 p_ooo_info->cur_isles_number--;450 if (left_isle) {451 p_left_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid,452 left_isle);453 if (unlikely(!p_left_isle)) {454 DP_NOTICE(p_hwfn,455 "Left isle %d is not found(cid %d)\n",456 left_isle, cid);457 return;458 }459 list_splice_tail_init(&p_right_isle->buffers_list,460 &p_left_isle->buffers_list);461 } else {462 list_splice_tail_init(&p_right_isle->buffers_list,463 &p_ooo_info->ready_buffers_list);464 }465 list_add_tail(&p_right_isle->list_entry, &p_ooo_info->free_isles_list);466}467