2429 lines · c
1/*2 * Copyright (c) 2013-2016, Mellanox Technologies. All rights reserved.3 *4 * This software is available to you under a choice of one of two5 * licenses. You may choose to be licensed under the terms of the GNU6 * General Public License (GPL) Version 2, available from the file7 * COPYING in the main directory of this source tree, or the8 * OpenIB.org BSD license below:9 *10 * Redistribution and use in source and binary forms, with or11 * without modification, are permitted provided that the following12 * conditions are met:13 *14 * - Redistributions of source code must retain the above15 * copyright notice, this list of conditions and the following16 * disclaimer.17 *18 * - Redistributions in binary form must reproduce the above19 * copyright notice, this list of conditions and the following20 * disclaimer in the documentation and/or other materials21 * provided with the distribution.22 *23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE30 * SOFTWARE.31 */32 33#include <linux/highmem.h>34#include <linux/errno.h>35#include <linux/pci.h>36#include <linux/dma-mapping.h>37#include <linux/slab.h>38#include <linux/delay.h>39#include <linux/random.h>40#include <linux/mlx5/driver.h>41#include <linux/mlx5/eq.h>42#include <linux/debugfs.h>43 44#include "mlx5_core.h"45#include "lib/eq.h"46#include "lib/tout.h"47#define CREATE_TRACE_POINTS48#include "diag/cmd_tracepoint.h"49 50struct mlx5_ifc_mbox_out_bits {51 u8 status[0x8];52 u8 reserved_at_8[0x18];53 54 u8 syndrome[0x20];55 56 u8 reserved_at_40[0x40];57};58 59struct mlx5_ifc_mbox_in_bits {60 u8 opcode[0x10];61 u8 uid[0x10];62 63 u8 reserved_at_20[0x10];64 u8 op_mod[0x10];65 66 u8 reserved_at_40[0x40];67};68 69enum {70 CMD_IF_REV = 5,71};72 73enum {74 CMD_MODE_POLLING,75 CMD_MODE_EVENTS76};77 78enum {79 MLX5_CMD_DELIVERY_STAT_OK = 0x0,80 MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR = 0x1,81 MLX5_CMD_DELIVERY_STAT_TOK_ERR = 0x2,82 MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR = 0x3,83 MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR = 0x4,84 MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR = 0x5,85 MLX5_CMD_DELIVERY_STAT_FW_ERR = 0x6,86 MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR = 0x7,87 MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR = 0x8,88 MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR = 0x9,89 MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR = 0x10,90};91 92static u16 in_to_opcode(void *in)93{94 return MLX5_GET(mbox_in, in, opcode);95}96 97/* Returns true for opcodes that might be triggered very frequently and throttle98 * the command interface. Limit their command slots usage.99 */100static bool mlx5_cmd_is_throttle_opcode(u16 op)101{102 switch (op) {103 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:104 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT:105 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:106 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:107 case MLX5_CMD_OP_SYNC_CRYPTO:108 return true;109 }110 return false;111}112 113static struct mlx5_cmd_work_ent *114cmd_alloc_ent(struct mlx5_cmd *cmd, struct mlx5_cmd_msg *in,115 struct mlx5_cmd_msg *out, void *uout, int uout_size,116 mlx5_cmd_cbk_t cbk, void *context, int page_queue)117{118 gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL;119 struct mlx5_cmd_work_ent *ent;120 121 ent = kzalloc(sizeof(*ent), alloc_flags);122 if (!ent)123 return ERR_PTR(-ENOMEM);124 125 ent->idx = -EINVAL;126 ent->in = in;127 ent->out = out;128 ent->uout = uout;129 ent->uout_size = uout_size;130 ent->callback = cbk;131 ent->context = context;132 ent->cmd = cmd;133 ent->page_queue = page_queue;134 ent->op = in_to_opcode(in->first.data);135 refcount_set(&ent->refcnt, 1);136 137 return ent;138}139 140static void cmd_free_ent(struct mlx5_cmd_work_ent *ent)141{142 kfree(ent);143}144 145static u8 alloc_token(struct mlx5_cmd *cmd)146{147 u8 token;148 149 spin_lock(&cmd->token_lock);150 cmd->token++;151 if (cmd->token == 0)152 cmd->token++;153 token = cmd->token;154 spin_unlock(&cmd->token_lock);155 156 return token;157}158 159static int cmd_alloc_index(struct mlx5_cmd *cmd, struct mlx5_cmd_work_ent *ent)160{161 unsigned long flags;162 int ret;163 164 spin_lock_irqsave(&cmd->alloc_lock, flags);165 ret = find_first_bit(&cmd->vars.bitmask, cmd->vars.max_reg_cmds);166 if (ret < cmd->vars.max_reg_cmds) {167 clear_bit(ret, &cmd->vars.bitmask);168 ent->idx = ret;169 cmd->ent_arr[ent->idx] = ent;170 }171 spin_unlock_irqrestore(&cmd->alloc_lock, flags);172 173 return ret < cmd->vars.max_reg_cmds ? ret : -ENOMEM;174}175 176static void cmd_free_index(struct mlx5_cmd *cmd, int idx)177{178 lockdep_assert_held(&cmd->alloc_lock);179 set_bit(idx, &cmd->vars.bitmask);180}181 182static void cmd_ent_get(struct mlx5_cmd_work_ent *ent)183{184 refcount_inc(&ent->refcnt);185}186 187static void cmd_ent_put(struct mlx5_cmd_work_ent *ent)188{189 struct mlx5_cmd *cmd = ent->cmd;190 unsigned long flags;191 192 spin_lock_irqsave(&cmd->alloc_lock, flags);193 if (!refcount_dec_and_test(&ent->refcnt))194 goto out;195 196 if (ent->idx >= 0) {197 cmd_free_index(cmd, ent->idx);198 up(ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem);199 }200 201 cmd_free_ent(ent);202out:203 spin_unlock_irqrestore(&cmd->alloc_lock, flags);204}205 206static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx)207{208 return cmd->cmd_buf + (idx << cmd->vars.log_stride);209}210 211static int mlx5_calc_cmd_blocks(struct mlx5_cmd_msg *msg)212{213 int size = msg->len;214 int blen = size - min_t(int, sizeof(msg->first.data), size);215 216 return DIV_ROUND_UP(blen, MLX5_CMD_DATA_BLOCK_SIZE);217}218 219static u8 xor8_buf(void *buf, size_t offset, int len)220{221 u8 *ptr = buf;222 u8 sum = 0;223 int i;224 int end = len + offset;225 226 for (i = offset; i < end; i++)227 sum ^= ptr[i];228 229 return sum;230}231 232static int verify_block_sig(struct mlx5_cmd_prot_block *block)233{234 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);235 int xor_len = sizeof(*block) - sizeof(block->data) - 1;236 237 if (xor8_buf(block, rsvd0_off, xor_len) != 0xff)238 return -EHWPOISON;239 240 if (xor8_buf(block, 0, sizeof(*block)) != 0xff)241 return -EHWPOISON;242 243 return 0;244}245 246static void calc_block_sig(struct mlx5_cmd_prot_block *block)247{248 int ctrl_xor_len = sizeof(*block) - sizeof(block->data) - 2;249 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);250 251 block->ctrl_sig = ~xor8_buf(block, rsvd0_off, ctrl_xor_len);252 block->sig = ~xor8_buf(block, 0, sizeof(*block) - 1);253}254 255static void calc_chain_sig(struct mlx5_cmd_msg *msg)256{257 struct mlx5_cmd_mailbox *next = msg->next;258 int n = mlx5_calc_cmd_blocks(msg);259 int i = 0;260 261 for (i = 0; i < n && next; i++) {262 calc_block_sig(next->buf);263 next = next->next;264 }265}266 267static void set_signature(struct mlx5_cmd_work_ent *ent, int csum)268{269 ent->lay->sig = ~xor8_buf(ent->lay, 0, sizeof(*ent->lay));270 if (csum) {271 calc_chain_sig(ent->in);272 calc_chain_sig(ent->out);273 }274}275 276static void poll_timeout(struct mlx5_cmd_work_ent *ent)277{278 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, cmd);279 u64 cmd_to_ms = mlx5_tout_ms(dev, CMD);280 unsigned long poll_end;281 u8 own;282 283 poll_end = jiffies + msecs_to_jiffies(cmd_to_ms + 1000);284 285 do {286 own = READ_ONCE(ent->lay->status_own);287 if (!(own & CMD_OWNER_HW)) {288 ent->ret = 0;289 return;290 }291 cond_resched();292 } while (time_before(jiffies, poll_end));293 294 ent->ret = -ETIMEDOUT;295}296 297static int verify_signature(struct mlx5_cmd_work_ent *ent)298{299 struct mlx5_cmd_mailbox *next = ent->out->next;300 int n = mlx5_calc_cmd_blocks(ent->out);301 int err;302 u8 sig;303 int i = 0;304 305 sig = xor8_buf(ent->lay, 0, sizeof(*ent->lay));306 if (sig != 0xff)307 return -EHWPOISON;308 309 for (i = 0; i < n && next; i++) {310 err = verify_block_sig(next->buf);311 if (err)312 return -EHWPOISON;313 314 next = next->next;315 }316 317 return 0;318}319 320static void dump_buf(void *buf, int size, int data_only, int offset, int idx)321{322 __be32 *p = buf;323 int i;324 325 for (i = 0; i < size; i += 16) {326 pr_debug("cmd[%d]: %03x: %08x %08x %08x %08x\n", idx, offset,327 be32_to_cpu(p[0]), be32_to_cpu(p[1]),328 be32_to_cpu(p[2]), be32_to_cpu(p[3]));329 p += 4;330 offset += 16;331 }332 if (!data_only)333 pr_debug("\n");334}335 336static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,337 u32 *synd, u8 *status)338{339 *synd = 0;340 *status = 0;341 342 switch (op) {343 case MLX5_CMD_OP_TEARDOWN_HCA:344 case MLX5_CMD_OP_DISABLE_HCA:345 case MLX5_CMD_OP_MANAGE_PAGES:346 case MLX5_CMD_OP_DESTROY_MKEY:347 case MLX5_CMD_OP_DESTROY_EQ:348 case MLX5_CMD_OP_DESTROY_CQ:349 case MLX5_CMD_OP_DESTROY_QP:350 case MLX5_CMD_OP_DESTROY_PSV:351 case MLX5_CMD_OP_DESTROY_SRQ:352 case MLX5_CMD_OP_DESTROY_XRC_SRQ:353 case MLX5_CMD_OP_DESTROY_XRQ:354 case MLX5_CMD_OP_DESTROY_DCT:355 case MLX5_CMD_OP_DEALLOC_Q_COUNTER:356 case MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT:357 case MLX5_CMD_OP_DESTROY_QOS_PARA_VPORT:358 case MLX5_CMD_OP_DEALLOC_PD:359 case MLX5_CMD_OP_DEALLOC_UAR:360 case MLX5_CMD_OP_DETACH_FROM_MCG:361 case MLX5_CMD_OP_DEALLOC_XRCD:362 case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN:363 case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT:364 case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY:365 case MLX5_CMD_OP_DESTROY_LAG:366 case MLX5_CMD_OP_DESTROY_VPORT_LAG:367 case MLX5_CMD_OP_DESTROY_TIR:368 case MLX5_CMD_OP_DESTROY_SQ:369 case MLX5_CMD_OP_DESTROY_RQ:370 case MLX5_CMD_OP_DESTROY_RMP:371 case MLX5_CMD_OP_DESTROY_TIS:372 case MLX5_CMD_OP_DESTROY_RQT:373 case MLX5_CMD_OP_DESTROY_FLOW_TABLE:374 case MLX5_CMD_OP_DESTROY_FLOW_GROUP:375 case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY:376 case MLX5_CMD_OP_DEALLOC_FLOW_COUNTER:377 case MLX5_CMD_OP_2ERR_QP:378 case MLX5_CMD_OP_2RST_QP:379 case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT:380 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:381 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:382 case MLX5_CMD_OP_SET_FLOW_TABLE_ROOT:383 case MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT:384 case MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT:385 case MLX5_CMD_OP_FPGA_DESTROY_QP:386 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT:387 case MLX5_CMD_OP_DEALLOC_MEMIC:388 case MLX5_CMD_OP_PAGE_FAULT_RESUME:389 case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:390 case MLX5_CMD_OP_DEALLOC_SF:391 case MLX5_CMD_OP_DESTROY_UCTX:392 case MLX5_CMD_OP_DESTROY_UMEM:393 case MLX5_CMD_OP_MODIFY_RQT:394 return MLX5_CMD_STAT_OK;395 396 case MLX5_CMD_OP_QUERY_HCA_CAP:397 case MLX5_CMD_OP_QUERY_ADAPTER:398 case MLX5_CMD_OP_INIT_HCA:399 case MLX5_CMD_OP_ENABLE_HCA:400 case MLX5_CMD_OP_QUERY_PAGES:401 case MLX5_CMD_OP_SET_HCA_CAP:402 case MLX5_CMD_OP_QUERY_ISSI:403 case MLX5_CMD_OP_SET_ISSI:404 case MLX5_CMD_OP_CREATE_MKEY:405 case MLX5_CMD_OP_QUERY_MKEY:406 case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS:407 case MLX5_CMD_OP_CREATE_EQ:408 case MLX5_CMD_OP_QUERY_EQ:409 case MLX5_CMD_OP_GEN_EQE:410 case MLX5_CMD_OP_CREATE_CQ:411 case MLX5_CMD_OP_QUERY_CQ:412 case MLX5_CMD_OP_MODIFY_CQ:413 case MLX5_CMD_OP_CREATE_QP:414 case MLX5_CMD_OP_RST2INIT_QP:415 case MLX5_CMD_OP_INIT2RTR_QP:416 case MLX5_CMD_OP_RTR2RTS_QP:417 case MLX5_CMD_OP_RTS2RTS_QP:418 case MLX5_CMD_OP_SQERR2RTS_QP:419 case MLX5_CMD_OP_QUERY_QP:420 case MLX5_CMD_OP_SQD_RTS_QP:421 case MLX5_CMD_OP_INIT2INIT_QP:422 case MLX5_CMD_OP_CREATE_PSV:423 case MLX5_CMD_OP_CREATE_SRQ:424 case MLX5_CMD_OP_QUERY_SRQ:425 case MLX5_CMD_OP_ARM_RQ:426 case MLX5_CMD_OP_CREATE_XRC_SRQ:427 case MLX5_CMD_OP_QUERY_XRC_SRQ:428 case MLX5_CMD_OP_ARM_XRC_SRQ:429 case MLX5_CMD_OP_CREATE_XRQ:430 case MLX5_CMD_OP_QUERY_XRQ:431 case MLX5_CMD_OP_ARM_XRQ:432 case MLX5_CMD_OP_CREATE_DCT:433 case MLX5_CMD_OP_DRAIN_DCT:434 case MLX5_CMD_OP_QUERY_DCT:435 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:436 case MLX5_CMD_OP_QUERY_VPORT_STATE:437 case MLX5_CMD_OP_MODIFY_VPORT_STATE:438 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:439 case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT:440 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:441 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:442 case MLX5_CMD_OP_SET_ROCE_ADDRESS:443 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:444 case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT:445 case MLX5_CMD_OP_QUERY_HCA_VPORT_GID:446 case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY:447 case MLX5_CMD_OP_QUERY_VNIC_ENV:448 case MLX5_CMD_OP_QUERY_VPORT_COUNTER:449 case MLX5_CMD_OP_ALLOC_Q_COUNTER:450 case MLX5_CMD_OP_QUERY_Q_COUNTER:451 case MLX5_CMD_OP_SET_MONITOR_COUNTER:452 case MLX5_CMD_OP_ARM_MONITOR_COUNTER:453 case MLX5_CMD_OP_SET_PP_RATE_LIMIT:454 case MLX5_CMD_OP_QUERY_RATE_LIMIT:455 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:456 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:457 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:458 case MLX5_CMD_OP_CREATE_QOS_PARA_VPORT:459 case MLX5_CMD_OP_ALLOC_PD:460 case MLX5_CMD_OP_ALLOC_UAR:461 case MLX5_CMD_OP_CONFIG_INT_MODERATION:462 case MLX5_CMD_OP_ACCESS_REG:463 case MLX5_CMD_OP_ATTACH_TO_MCG:464 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:465 case MLX5_CMD_OP_MAD_IFC:466 case MLX5_CMD_OP_QUERY_MAD_DEMUX:467 case MLX5_CMD_OP_SET_MAD_DEMUX:468 case MLX5_CMD_OP_NOP:469 case MLX5_CMD_OP_ALLOC_XRCD:470 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:471 case MLX5_CMD_OP_QUERY_CONG_STATUS:472 case MLX5_CMD_OP_MODIFY_CONG_STATUS:473 case MLX5_CMD_OP_QUERY_CONG_PARAMS:474 case MLX5_CMD_OP_MODIFY_CONG_PARAMS:475 case MLX5_CMD_OP_QUERY_CONG_STATISTICS:476 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:477 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:478 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:479 case MLX5_CMD_OP_CREATE_LAG:480 case MLX5_CMD_OP_MODIFY_LAG:481 case MLX5_CMD_OP_QUERY_LAG:482 case MLX5_CMD_OP_CREATE_VPORT_LAG:483 case MLX5_CMD_OP_CREATE_TIR:484 case MLX5_CMD_OP_MODIFY_TIR:485 case MLX5_CMD_OP_QUERY_TIR:486 case MLX5_CMD_OP_CREATE_SQ:487 case MLX5_CMD_OP_MODIFY_SQ:488 case MLX5_CMD_OP_QUERY_SQ:489 case MLX5_CMD_OP_CREATE_RQ:490 case MLX5_CMD_OP_MODIFY_RQ:491 case MLX5_CMD_OP_QUERY_RQ:492 case MLX5_CMD_OP_CREATE_RMP:493 case MLX5_CMD_OP_MODIFY_RMP:494 case MLX5_CMD_OP_QUERY_RMP:495 case MLX5_CMD_OP_CREATE_TIS:496 case MLX5_CMD_OP_MODIFY_TIS:497 case MLX5_CMD_OP_QUERY_TIS:498 case MLX5_CMD_OP_CREATE_RQT:499 case MLX5_CMD_OP_QUERY_RQT:500 501 case MLX5_CMD_OP_CREATE_FLOW_TABLE:502 case MLX5_CMD_OP_QUERY_FLOW_TABLE:503 case MLX5_CMD_OP_CREATE_FLOW_GROUP:504 case MLX5_CMD_OP_QUERY_FLOW_GROUP:505 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:506 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:507 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:508 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:509 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:510 case MLX5_CMD_OP_FPGA_CREATE_QP:511 case MLX5_CMD_OP_FPGA_MODIFY_QP:512 case MLX5_CMD_OP_FPGA_QUERY_QP:513 case MLX5_CMD_OP_FPGA_QUERY_QP_COUNTERS:514 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:515 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:516 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:517 case MLX5_CMD_OP_CREATE_UCTX:518 case MLX5_CMD_OP_CREATE_UMEM:519 case MLX5_CMD_OP_ALLOC_MEMIC:520 case MLX5_CMD_OP_MODIFY_XRQ:521 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:522 case MLX5_CMD_OP_QUERY_VHCA_STATE:523 case MLX5_CMD_OP_MODIFY_VHCA_STATE:524 case MLX5_CMD_OP_ALLOC_SF:525 case MLX5_CMD_OP_SUSPEND_VHCA:526 case MLX5_CMD_OP_RESUME_VHCA:527 case MLX5_CMD_OP_QUERY_VHCA_MIGRATION_STATE:528 case MLX5_CMD_OP_SAVE_VHCA_STATE:529 case MLX5_CMD_OP_LOAD_VHCA_STATE:530 case MLX5_CMD_OP_SYNC_CRYPTO:531 case MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS:532 *status = MLX5_DRIVER_STATUS_ABORTED;533 *synd = MLX5_DRIVER_SYND;534 return -ENOLINK;535 default:536 mlx5_core_err(dev, "Unknown FW command (%d)\n", op);537 return -EINVAL;538 }539}540 541const char *mlx5_command_str(int command)542{543#define MLX5_COMMAND_STR_CASE(__cmd) case MLX5_CMD_OP_ ## __cmd: return #__cmd544 545 switch (command) {546 MLX5_COMMAND_STR_CASE(QUERY_HCA_CAP);547 MLX5_COMMAND_STR_CASE(QUERY_ADAPTER);548 MLX5_COMMAND_STR_CASE(INIT_HCA);549 MLX5_COMMAND_STR_CASE(TEARDOWN_HCA);550 MLX5_COMMAND_STR_CASE(ENABLE_HCA);551 MLX5_COMMAND_STR_CASE(DISABLE_HCA);552 MLX5_COMMAND_STR_CASE(QUERY_PAGES);553 MLX5_COMMAND_STR_CASE(MANAGE_PAGES);554 MLX5_COMMAND_STR_CASE(SET_HCA_CAP);555 MLX5_COMMAND_STR_CASE(QUERY_ISSI);556 MLX5_COMMAND_STR_CASE(SET_ISSI);557 MLX5_COMMAND_STR_CASE(SET_DRIVER_VERSION);558 MLX5_COMMAND_STR_CASE(CREATE_MKEY);559 MLX5_COMMAND_STR_CASE(QUERY_MKEY);560 MLX5_COMMAND_STR_CASE(DESTROY_MKEY);561 MLX5_COMMAND_STR_CASE(QUERY_SPECIAL_CONTEXTS);562 MLX5_COMMAND_STR_CASE(PAGE_FAULT_RESUME);563 MLX5_COMMAND_STR_CASE(CREATE_EQ);564 MLX5_COMMAND_STR_CASE(DESTROY_EQ);565 MLX5_COMMAND_STR_CASE(QUERY_EQ);566 MLX5_COMMAND_STR_CASE(GEN_EQE);567 MLX5_COMMAND_STR_CASE(CREATE_CQ);568 MLX5_COMMAND_STR_CASE(DESTROY_CQ);569 MLX5_COMMAND_STR_CASE(QUERY_CQ);570 MLX5_COMMAND_STR_CASE(MODIFY_CQ);571 MLX5_COMMAND_STR_CASE(CREATE_QP);572 MLX5_COMMAND_STR_CASE(DESTROY_QP);573 MLX5_COMMAND_STR_CASE(RST2INIT_QP);574 MLX5_COMMAND_STR_CASE(INIT2RTR_QP);575 MLX5_COMMAND_STR_CASE(RTR2RTS_QP);576 MLX5_COMMAND_STR_CASE(RTS2RTS_QP);577 MLX5_COMMAND_STR_CASE(SQERR2RTS_QP);578 MLX5_COMMAND_STR_CASE(2ERR_QP);579 MLX5_COMMAND_STR_CASE(2RST_QP);580 MLX5_COMMAND_STR_CASE(QUERY_QP);581 MLX5_COMMAND_STR_CASE(SQD_RTS_QP);582 MLX5_COMMAND_STR_CASE(INIT2INIT_QP);583 MLX5_COMMAND_STR_CASE(CREATE_PSV);584 MLX5_COMMAND_STR_CASE(DESTROY_PSV);585 MLX5_COMMAND_STR_CASE(CREATE_SRQ);586 MLX5_COMMAND_STR_CASE(DESTROY_SRQ);587 MLX5_COMMAND_STR_CASE(QUERY_SRQ);588 MLX5_COMMAND_STR_CASE(ARM_RQ);589 MLX5_COMMAND_STR_CASE(CREATE_XRC_SRQ);590 MLX5_COMMAND_STR_CASE(DESTROY_XRC_SRQ);591 MLX5_COMMAND_STR_CASE(QUERY_XRC_SRQ);592 MLX5_COMMAND_STR_CASE(ARM_XRC_SRQ);593 MLX5_COMMAND_STR_CASE(CREATE_DCT);594 MLX5_COMMAND_STR_CASE(DESTROY_DCT);595 MLX5_COMMAND_STR_CASE(DRAIN_DCT);596 MLX5_COMMAND_STR_CASE(QUERY_DCT);597 MLX5_COMMAND_STR_CASE(ARM_DCT_FOR_KEY_VIOLATION);598 MLX5_COMMAND_STR_CASE(QUERY_VPORT_STATE);599 MLX5_COMMAND_STR_CASE(MODIFY_VPORT_STATE);600 MLX5_COMMAND_STR_CASE(QUERY_ESW_VPORT_CONTEXT);601 MLX5_COMMAND_STR_CASE(MODIFY_ESW_VPORT_CONTEXT);602 MLX5_COMMAND_STR_CASE(QUERY_NIC_VPORT_CONTEXT);603 MLX5_COMMAND_STR_CASE(MODIFY_NIC_VPORT_CONTEXT);604 MLX5_COMMAND_STR_CASE(QUERY_ROCE_ADDRESS);605 MLX5_COMMAND_STR_CASE(SET_ROCE_ADDRESS);606 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_CONTEXT);607 MLX5_COMMAND_STR_CASE(MODIFY_HCA_VPORT_CONTEXT);608 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_GID);609 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_PKEY);610 MLX5_COMMAND_STR_CASE(QUERY_VNIC_ENV);611 MLX5_COMMAND_STR_CASE(QUERY_VPORT_COUNTER);612 MLX5_COMMAND_STR_CASE(ALLOC_Q_COUNTER);613 MLX5_COMMAND_STR_CASE(DEALLOC_Q_COUNTER);614 MLX5_COMMAND_STR_CASE(QUERY_Q_COUNTER);615 MLX5_COMMAND_STR_CASE(SET_MONITOR_COUNTER);616 MLX5_COMMAND_STR_CASE(ARM_MONITOR_COUNTER);617 MLX5_COMMAND_STR_CASE(SET_PP_RATE_LIMIT);618 MLX5_COMMAND_STR_CASE(QUERY_RATE_LIMIT);619 MLX5_COMMAND_STR_CASE(CREATE_SCHEDULING_ELEMENT);620 MLX5_COMMAND_STR_CASE(DESTROY_SCHEDULING_ELEMENT);621 MLX5_COMMAND_STR_CASE(QUERY_SCHEDULING_ELEMENT);622 MLX5_COMMAND_STR_CASE(MODIFY_SCHEDULING_ELEMENT);623 MLX5_COMMAND_STR_CASE(CREATE_QOS_PARA_VPORT);624 MLX5_COMMAND_STR_CASE(DESTROY_QOS_PARA_VPORT);625 MLX5_COMMAND_STR_CASE(ALLOC_PD);626 MLX5_COMMAND_STR_CASE(DEALLOC_PD);627 MLX5_COMMAND_STR_CASE(ALLOC_UAR);628 MLX5_COMMAND_STR_CASE(DEALLOC_UAR);629 MLX5_COMMAND_STR_CASE(CONFIG_INT_MODERATION);630 MLX5_COMMAND_STR_CASE(ACCESS_REG);631 MLX5_COMMAND_STR_CASE(ATTACH_TO_MCG);632 MLX5_COMMAND_STR_CASE(DETACH_FROM_MCG);633 MLX5_COMMAND_STR_CASE(GET_DROPPED_PACKET_LOG);634 MLX5_COMMAND_STR_CASE(MAD_IFC);635 MLX5_COMMAND_STR_CASE(QUERY_MAD_DEMUX);636 MLX5_COMMAND_STR_CASE(SET_MAD_DEMUX);637 MLX5_COMMAND_STR_CASE(NOP);638 MLX5_COMMAND_STR_CASE(ALLOC_XRCD);639 MLX5_COMMAND_STR_CASE(DEALLOC_XRCD);640 MLX5_COMMAND_STR_CASE(ALLOC_TRANSPORT_DOMAIN);641 MLX5_COMMAND_STR_CASE(DEALLOC_TRANSPORT_DOMAIN);642 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATUS);643 MLX5_COMMAND_STR_CASE(MODIFY_CONG_STATUS);644 MLX5_COMMAND_STR_CASE(QUERY_CONG_PARAMS);645 MLX5_COMMAND_STR_CASE(MODIFY_CONG_PARAMS);646 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATISTICS);647 MLX5_COMMAND_STR_CASE(ADD_VXLAN_UDP_DPORT);648 MLX5_COMMAND_STR_CASE(DELETE_VXLAN_UDP_DPORT);649 MLX5_COMMAND_STR_CASE(SET_L2_TABLE_ENTRY);650 MLX5_COMMAND_STR_CASE(QUERY_L2_TABLE_ENTRY);651 MLX5_COMMAND_STR_CASE(DELETE_L2_TABLE_ENTRY);652 MLX5_COMMAND_STR_CASE(SET_WOL_ROL);653 MLX5_COMMAND_STR_CASE(QUERY_WOL_ROL);654 MLX5_COMMAND_STR_CASE(CREATE_LAG);655 MLX5_COMMAND_STR_CASE(MODIFY_LAG);656 MLX5_COMMAND_STR_CASE(QUERY_LAG);657 MLX5_COMMAND_STR_CASE(DESTROY_LAG);658 MLX5_COMMAND_STR_CASE(CREATE_VPORT_LAG);659 MLX5_COMMAND_STR_CASE(DESTROY_VPORT_LAG);660 MLX5_COMMAND_STR_CASE(CREATE_TIR);661 MLX5_COMMAND_STR_CASE(MODIFY_TIR);662 MLX5_COMMAND_STR_CASE(DESTROY_TIR);663 MLX5_COMMAND_STR_CASE(QUERY_TIR);664 MLX5_COMMAND_STR_CASE(CREATE_SQ);665 MLX5_COMMAND_STR_CASE(MODIFY_SQ);666 MLX5_COMMAND_STR_CASE(DESTROY_SQ);667 MLX5_COMMAND_STR_CASE(QUERY_SQ);668 MLX5_COMMAND_STR_CASE(CREATE_RQ);669 MLX5_COMMAND_STR_CASE(MODIFY_RQ);670 MLX5_COMMAND_STR_CASE(DESTROY_RQ);671 MLX5_COMMAND_STR_CASE(QUERY_RQ);672 MLX5_COMMAND_STR_CASE(CREATE_RMP);673 MLX5_COMMAND_STR_CASE(MODIFY_RMP);674 MLX5_COMMAND_STR_CASE(DESTROY_RMP);675 MLX5_COMMAND_STR_CASE(QUERY_RMP);676 MLX5_COMMAND_STR_CASE(CREATE_TIS);677 MLX5_COMMAND_STR_CASE(MODIFY_TIS);678 MLX5_COMMAND_STR_CASE(DESTROY_TIS);679 MLX5_COMMAND_STR_CASE(QUERY_TIS);680 MLX5_COMMAND_STR_CASE(CREATE_RQT);681 MLX5_COMMAND_STR_CASE(MODIFY_RQT);682 MLX5_COMMAND_STR_CASE(DESTROY_RQT);683 MLX5_COMMAND_STR_CASE(QUERY_RQT);684 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ROOT);685 MLX5_COMMAND_STR_CASE(CREATE_FLOW_TABLE);686 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_TABLE);687 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE);688 MLX5_COMMAND_STR_CASE(CREATE_FLOW_GROUP);689 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_GROUP);690 MLX5_COMMAND_STR_CASE(QUERY_FLOW_GROUP);691 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ENTRY);692 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE_ENTRY);693 MLX5_COMMAND_STR_CASE(DELETE_FLOW_TABLE_ENTRY);694 MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER);695 MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER);696 MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER);697 MLX5_COMMAND_STR_CASE(MODIFY_FLOW_TABLE);698 MLX5_COMMAND_STR_CASE(ALLOC_PACKET_REFORMAT_CONTEXT);699 MLX5_COMMAND_STR_CASE(DEALLOC_PACKET_REFORMAT_CONTEXT);700 MLX5_COMMAND_STR_CASE(ALLOC_MODIFY_HEADER_CONTEXT);701 MLX5_COMMAND_STR_CASE(DEALLOC_MODIFY_HEADER_CONTEXT);702 MLX5_COMMAND_STR_CASE(FPGA_CREATE_QP);703 MLX5_COMMAND_STR_CASE(FPGA_MODIFY_QP);704 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP);705 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP_COUNTERS);706 MLX5_COMMAND_STR_CASE(FPGA_DESTROY_QP);707 MLX5_COMMAND_STR_CASE(CREATE_XRQ);708 MLX5_COMMAND_STR_CASE(DESTROY_XRQ);709 MLX5_COMMAND_STR_CASE(QUERY_XRQ);710 MLX5_COMMAND_STR_CASE(ARM_XRQ);711 MLX5_COMMAND_STR_CASE(CREATE_GENERAL_OBJECT);712 MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJECT);713 MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJECT);714 MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJECT);715 MLX5_COMMAND_STR_CASE(QUERY_MODIFY_HEADER_CONTEXT);716 MLX5_COMMAND_STR_CASE(ALLOC_MEMIC);717 MLX5_COMMAND_STR_CASE(DEALLOC_MEMIC);718 MLX5_COMMAND_STR_CASE(QUERY_ESW_FUNCTIONS);719 MLX5_COMMAND_STR_CASE(CREATE_UCTX);720 MLX5_COMMAND_STR_CASE(DESTROY_UCTX);721 MLX5_COMMAND_STR_CASE(CREATE_UMEM);722 MLX5_COMMAND_STR_CASE(DESTROY_UMEM);723 MLX5_COMMAND_STR_CASE(RELEASE_XRQ_ERROR);724 MLX5_COMMAND_STR_CASE(MODIFY_XRQ);725 MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE);726 MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE);727 MLX5_COMMAND_STR_CASE(ALLOC_SF);728 MLX5_COMMAND_STR_CASE(DEALLOC_SF);729 MLX5_COMMAND_STR_CASE(SUSPEND_VHCA);730 MLX5_COMMAND_STR_CASE(RESUME_VHCA);731 MLX5_COMMAND_STR_CASE(QUERY_VHCA_MIGRATION_STATE);732 MLX5_COMMAND_STR_CASE(SAVE_VHCA_STATE);733 MLX5_COMMAND_STR_CASE(LOAD_VHCA_STATE);734 MLX5_COMMAND_STR_CASE(SYNC_CRYPTO);735 MLX5_COMMAND_STR_CASE(ALLOW_OTHER_VHCA_ACCESS);736 default: return "unknown command opcode";737 }738}739 740static const char *cmd_status_str(u8 status)741{742 switch (status) {743 case MLX5_CMD_STAT_OK:744 return "OK";745 case MLX5_CMD_STAT_INT_ERR:746 return "internal error";747 case MLX5_CMD_STAT_BAD_OP_ERR:748 return "bad operation";749 case MLX5_CMD_STAT_BAD_PARAM_ERR:750 return "bad parameter";751 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:752 return "bad system state";753 case MLX5_CMD_STAT_BAD_RES_ERR:754 return "bad resource";755 case MLX5_CMD_STAT_RES_BUSY:756 return "resource busy";757 case MLX5_CMD_STAT_NOT_READY:758 return "FW not ready";759 case MLX5_CMD_STAT_LIM_ERR:760 return "limits exceeded";761 case MLX5_CMD_STAT_BAD_RES_STATE_ERR:762 return "bad resource state";763 case MLX5_CMD_STAT_IX_ERR:764 return "bad index";765 case MLX5_CMD_STAT_NO_RES_ERR:766 return "no resources";767 case MLX5_CMD_STAT_BAD_INP_LEN_ERR:768 return "bad input length";769 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:770 return "bad output length";771 case MLX5_CMD_STAT_BAD_QP_STATE_ERR:772 return "bad QP state";773 case MLX5_CMD_STAT_BAD_PKT_ERR:774 return "bad packet (discarded)";775 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:776 return "bad size too many outstanding CQEs";777 default:778 return "unknown status";779 }780}781 782static int cmd_status_to_err(u8 status)783{784 switch (status) {785 case MLX5_CMD_STAT_OK: return 0;786 case MLX5_CMD_STAT_INT_ERR: return -EIO;787 case MLX5_CMD_STAT_BAD_OP_ERR: return -EINVAL;788 case MLX5_CMD_STAT_BAD_PARAM_ERR: return -EINVAL;789 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: return -EIO;790 case MLX5_CMD_STAT_BAD_RES_ERR: return -EINVAL;791 case MLX5_CMD_STAT_RES_BUSY: return -EBUSY;792 case MLX5_CMD_STAT_NOT_READY: return -EAGAIN;793 case MLX5_CMD_STAT_LIM_ERR: return -ENOMEM;794 case MLX5_CMD_STAT_BAD_RES_STATE_ERR: return -EINVAL;795 case MLX5_CMD_STAT_IX_ERR: return -EINVAL;796 case MLX5_CMD_STAT_NO_RES_ERR: return -EAGAIN;797 case MLX5_CMD_STAT_BAD_INP_LEN_ERR: return -EIO;798 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: return -EIO;799 case MLX5_CMD_STAT_BAD_QP_STATE_ERR: return -EINVAL;800 case MLX5_CMD_STAT_BAD_PKT_ERR: return -EINVAL;801 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: return -EINVAL;802 default: return -EIO;803 }804}805 806void mlx5_cmd_out_err(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)807{808 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);809 u8 status = MLX5_GET(mbox_out, out, status);810 811 mlx5_core_err_rl(dev,812 "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x), err(%d)\n",813 mlx5_command_str(opcode), opcode, op_mod,814 cmd_status_str(status), status, syndrome, cmd_status_to_err(status));815}816EXPORT_SYMBOL(mlx5_cmd_out_err);817 818static void cmd_status_print(struct mlx5_core_dev *dev, void *in, void *out)819{820 u16 opcode, op_mod;821 u8 status;822 u16 uid;823 824 opcode = in_to_opcode(in);825 op_mod = MLX5_GET(mbox_in, in, op_mod);826 uid = MLX5_GET(mbox_in, in, uid);827 status = MLX5_GET(mbox_out, out, status);828 829 if (!uid && opcode != MLX5_CMD_OP_DESTROY_MKEY &&830 opcode != MLX5_CMD_OP_CREATE_UCTX && status != MLX5_CMD_STAT_NOT_READY)831 mlx5_cmd_out_err(dev, opcode, op_mod, out);832}833 834int mlx5_cmd_check(struct mlx5_core_dev *dev, int err, void *in, void *out)835{836 /* aborted due to PCI error or via reset flow mlx5_cmd_trigger_completions() */837 if (err == -ENXIO) {838 u16 opcode = in_to_opcode(in);839 u32 syndrome;840 u8 status;841 842 /* PCI Error, emulate command return status, for smooth reset */843 err = mlx5_internal_err_ret_value(dev, opcode, &syndrome, &status);844 MLX5_SET(mbox_out, out, status, status);845 MLX5_SET(mbox_out, out, syndrome, syndrome);846 if (!err)847 return 0;848 }849 850 /* driver or FW delivery error */851 if (err != -EREMOTEIO && err)852 return err;853 854 /* check outbox status */855 err = cmd_status_to_err(MLX5_GET(mbox_out, out, status));856 if (err)857 cmd_status_print(dev, in, out);858 859 return err;860}861EXPORT_SYMBOL(mlx5_cmd_check);862 863static void dump_command(struct mlx5_core_dev *dev,864 struct mlx5_cmd_work_ent *ent, int input)865{866 struct mlx5_cmd_msg *msg = input ? ent->in : ent->out;867 struct mlx5_cmd_mailbox *next = msg->next;868 int n = mlx5_calc_cmd_blocks(msg);869 u16 op = ent->op;870 int data_only;871 u32 offset = 0;872 int dump_len;873 int i;874 875 mlx5_core_dbg(dev, "cmd[%d]: start dump\n", ent->idx);876 data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA));877 878 if (data_only)879 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_DATA,880 "cmd[%d]: dump command data %s(0x%x) %s\n",881 ent->idx, mlx5_command_str(op), op,882 input ? "INPUT" : "OUTPUT");883 else884 mlx5_core_dbg(dev, "cmd[%d]: dump command %s(0x%x) %s\n",885 ent->idx, mlx5_command_str(op), op,886 input ? "INPUT" : "OUTPUT");887 888 if (data_only) {889 if (input) {890 dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset, ent->idx);891 offset += sizeof(ent->lay->in);892 } else {893 dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset, ent->idx);894 offset += sizeof(ent->lay->out);895 }896 } else {897 dump_buf(ent->lay, sizeof(*ent->lay), 0, offset, ent->idx);898 offset += sizeof(*ent->lay);899 }900 901 for (i = 0; i < n && next; i++) {902 if (data_only) {903 dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset);904 dump_buf(next->buf, dump_len, 1, offset, ent->idx);905 offset += MLX5_CMD_DATA_BLOCK_SIZE;906 } else {907 mlx5_core_dbg(dev, "cmd[%d]: command block:\n", ent->idx);908 dump_buf(next->buf, sizeof(struct mlx5_cmd_prot_block), 0, offset,909 ent->idx);910 offset += sizeof(struct mlx5_cmd_prot_block);911 }912 next = next->next;913 }914 915 if (data_only)916 pr_debug("\n");917 918 mlx5_core_dbg(dev, "cmd[%d]: end dump\n", ent->idx);919}920 921static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced);922 923static void cb_timeout_handler(struct work_struct *work)924{925 struct delayed_work *dwork = container_of(work, struct delayed_work,926 work);927 struct mlx5_cmd_work_ent *ent = container_of(dwork,928 struct mlx5_cmd_work_ent,929 cb_timeout_work);930 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev,931 cmd);932 933 mlx5_cmd_eq_recover(dev);934 935 /* Maybe got handled by eq recover ? */936 if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) {937 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx,938 mlx5_command_str(ent->op), ent->op);939 goto out; /* phew, already handled */940 }941 942 ent->ret = -ETIMEDOUT;943 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, timeout. Will cause a leak of a command resource\n",944 ent->idx, mlx5_command_str(ent->op), ent->op);945 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);946 947out:948 cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */949}950 951static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg);952static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,953 struct mlx5_cmd_msg *msg);954 955static bool opcode_allowed(struct mlx5_cmd *cmd, u16 opcode)956{957 if (cmd->allowed_opcode == CMD_ALLOWED_OPCODE_ALL)958 return true;959 960 return cmd->allowed_opcode == opcode;961}962 963bool mlx5_cmd_is_down(struct mlx5_core_dev *dev)964{965 return pci_channel_offline(dev->pdev) ||966 dev->cmd.state != MLX5_CMDIF_STATE_UP ||967 dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR;968}969 970static void cmd_work_handler(struct work_struct *work)971{972 struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work);973 struct mlx5_cmd *cmd = ent->cmd;974 bool poll_cmd = ent->polling;975 struct mlx5_cmd_layout *lay;976 struct mlx5_core_dev *dev;977 unsigned long timeout;978 unsigned long flags;979 int alloc_ret;980 int cmd_mode;981 982 complete(&ent->handling);983 984 dev = container_of(cmd, struct mlx5_core_dev, cmd);985 timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD));986 987 if (!ent->page_queue) {988 if (down_timeout(&cmd->vars.sem, timeout)) {989 mlx5_core_warn(dev, "%s(0x%x) timed out while waiting for a slot.\n",990 mlx5_command_str(ent->op), ent->op);991 if (ent->callback) {992 ent->callback(-EBUSY, ent->context);993 mlx5_free_cmd_msg(dev, ent->out);994 free_msg(dev, ent->in);995 cmd_ent_put(ent);996 } else {997 ent->ret = -EBUSY;998 complete(&ent->done);999 }1000 complete(&ent->slotted);1001 return;1002 }1003 alloc_ret = cmd_alloc_index(cmd, ent);1004 if (alloc_ret < 0) {1005 mlx5_core_err_rl(dev, "failed to allocate command entry\n");1006 if (ent->callback) {1007 ent->callback(-EAGAIN, ent->context);1008 mlx5_free_cmd_msg(dev, ent->out);1009 free_msg(dev, ent->in);1010 cmd_ent_put(ent);1011 } else {1012 ent->ret = -EAGAIN;1013 complete(&ent->done);1014 }1015 up(&cmd->vars.sem);1016 return;1017 }1018 } else {1019 down(&cmd->vars.pages_sem);1020 ent->idx = cmd->vars.max_reg_cmds;1021 spin_lock_irqsave(&cmd->alloc_lock, flags);1022 clear_bit(ent->idx, &cmd->vars.bitmask);1023 cmd->ent_arr[ent->idx] = ent;1024 spin_unlock_irqrestore(&cmd->alloc_lock, flags);1025 }1026 1027 complete(&ent->slotted);1028 1029 lay = get_inst(cmd, ent->idx);1030 ent->lay = lay;1031 memset(lay, 0, sizeof(*lay));1032 memcpy(lay->in, ent->in->first.data, sizeof(lay->in));1033 if (ent->in->next)1034 lay->in_ptr = cpu_to_be64(ent->in->next->dma);1035 lay->inlen = cpu_to_be32(ent->in->len);1036 if (ent->out->next)1037 lay->out_ptr = cpu_to_be64(ent->out->next->dma);1038 lay->outlen = cpu_to_be32(ent->out->len);1039 lay->type = MLX5_PCI_CMD_XPORT;1040 lay->token = ent->token;1041 lay->status_own = CMD_OWNER_HW;1042 set_signature(ent, !cmd->checksum_disabled);1043 dump_command(dev, ent, 1);1044 ent->ts1 = ktime_get_ns();1045 cmd_mode = cmd->mode;1046 1047 if (ent->callback && schedule_delayed_work(&ent->cb_timeout_work, timeout))1048 cmd_ent_get(ent);1049 set_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state);1050 1051 cmd_ent_get(ent); /* for the _real_ FW event on completion */1052 /* Skip sending command to fw if internal error */1053 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, ent->op)) {1054 ent->ret = -ENXIO;1055 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);1056 return;1057 }1058 1059 /* ring doorbell after the descriptor is valid */1060 mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx);1061 wmb();1062 iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell);1063 /* if not in polling don't use ent after this point */1064 if (cmd_mode == CMD_MODE_POLLING || poll_cmd) {1065 poll_timeout(ent);1066 /* make sure we read the descriptor after ownership is SW */1067 rmb();1068 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, (ent->ret == -ETIMEDOUT));1069 }1070}1071 1072static int deliv_status_to_err(u8 status)1073{1074 switch (status) {1075 case MLX5_CMD_DELIVERY_STAT_OK:1076 case MLX5_DRIVER_STATUS_ABORTED:1077 return 0;1078 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:1079 case MLX5_CMD_DELIVERY_STAT_TOK_ERR:1080 return -EBADR;1081 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:1082 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:1083 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:1084 return -EFAULT; /* Bad address */1085 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:1086 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:1087 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:1088 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:1089 return -ENOMSG;1090 case MLX5_CMD_DELIVERY_STAT_FW_ERR:1091 return -EIO;1092 default:1093 return -EINVAL;1094 }1095}1096 1097static const char *deliv_status_to_str(u8 status)1098{1099 switch (status) {1100 case MLX5_CMD_DELIVERY_STAT_OK:1101 return "no errors";1102 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:1103 return "signature error";1104 case MLX5_CMD_DELIVERY_STAT_TOK_ERR:1105 return "token error";1106 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:1107 return "bad block number";1108 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:1109 return "output pointer not aligned to block size";1110 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:1111 return "input pointer not aligned to block size";1112 case MLX5_CMD_DELIVERY_STAT_FW_ERR:1113 return "firmware internal error";1114 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:1115 return "command input length error";1116 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:1117 return "command output length error";1118 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:1119 return "reserved fields not cleared";1120 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:1121 return "bad command descriptor type";1122 default:1123 return "unknown status code";1124 }1125}1126 1127enum {1128 MLX5_CMD_TIMEOUT_RECOVER_MSEC = 5 * 1000,1129};1130 1131static void wait_func_handle_exec_timeout(struct mlx5_core_dev *dev,1132 struct mlx5_cmd_work_ent *ent)1133{1134 unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_RECOVER_MSEC);1135 1136 mlx5_cmd_eq_recover(dev);1137 1138 /* Re-wait on the ent->done after executing the recovery flow. If the1139 * recovery flow (or any other recovery flow running simultaneously)1140 * has recovered an EQE, it should cause the entry to be completed by1141 * the command interface.1142 */1143 if (wait_for_completion_timeout(&ent->done, timeout)) {1144 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx,1145 mlx5_command_str(ent->op), ent->op);1146 return;1147 }1148 1149 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx,1150 mlx5_command_str(ent->op), ent->op);1151 1152 ent->ret = -ETIMEDOUT;1153 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);1154}1155 1156static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)1157{1158 unsigned long timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD));1159 struct mlx5_cmd *cmd = &dev->cmd;1160 int err;1161 1162 if (!wait_for_completion_timeout(&ent->handling, timeout) &&1163 cancel_work_sync(&ent->work)) {1164 ent->ret = -ECANCELED;1165 goto out_err;1166 }1167 1168 wait_for_completion(&ent->slotted);1169 1170 if (cmd->mode == CMD_MODE_POLLING || ent->polling)1171 wait_for_completion(&ent->done);1172 else if (!wait_for_completion_timeout(&ent->done, timeout))1173 wait_func_handle_exec_timeout(dev, ent);1174 1175out_err:1176 err = ent->ret;1177 1178 if (err == -ETIMEDOUT) {1179 mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n",1180 mlx5_command_str(ent->op), ent->op);1181 } else if (err == -ECANCELED) {1182 mlx5_core_warn(dev, "%s(0x%x) canceled on out of queue timeout.\n",1183 mlx5_command_str(ent->op), ent->op);1184 } else if (err == -EBUSY) {1185 mlx5_core_warn(dev, "%s(0x%x) timeout while waiting for command semaphore.\n",1186 mlx5_command_str(ent->op), ent->op);1187 }1188 mlx5_core_dbg(dev, "err %d, delivery status %s(%d)\n",1189 err, deliv_status_to_str(ent->status), ent->status);1190 1191 return err;1192}1193 1194/* Notes:1195 * 1. Callback functions may not sleep1196 * 2. page queue commands do not support asynchrous completion1197 *1198 * return value in case (!callback):1199 * ret < 0 : Command execution couldn't be submitted by driver1200 * ret > 0 : Command execution couldn't be performed by firmware1201 * ret == 0: Command was executed by FW, Caller must check FW outbox status.1202 *1203 * return value in case (callback):1204 * ret < 0 : Command execution couldn't be submitted by driver1205 * ret == 0: Command will be submitted to FW for execution1206 * and the callback will be called for further status updates1207 */1208static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,1209 struct mlx5_cmd_msg *out, void *uout, int uout_size,1210 mlx5_cmd_cbk_t callback,1211 void *context, int page_queue,1212 u8 token, bool force_polling)1213{1214 struct mlx5_cmd *cmd = &dev->cmd;1215 struct mlx5_cmd_work_ent *ent;1216 struct mlx5_cmd_stats *stats;1217 u8 status = 0;1218 int err = 0;1219 s64 ds;1220 1221 if (callback && page_queue)1222 return -EINVAL;1223 1224 ent = cmd_alloc_ent(cmd, in, out, uout, uout_size,1225 callback, context, page_queue);1226 if (IS_ERR(ent))1227 return PTR_ERR(ent);1228 1229 /* put for this ent is when consumed, depending on the use case1230 * 1) (!callback) blocking flow: by caller after wait_func completes1231 * 2) (callback) flow: by mlx5_cmd_comp_handler() when ent is handled1232 */1233 1234 ent->token = token;1235 ent->polling = force_polling;1236 1237 init_completion(&ent->handling);1238 init_completion(&ent->slotted);1239 if (!callback)1240 init_completion(&ent->done);1241 1242 INIT_DELAYED_WORK(&ent->cb_timeout_work, cb_timeout_handler);1243 INIT_WORK(&ent->work, cmd_work_handler);1244 if (page_queue) {1245 cmd_work_handler(&ent->work);1246 } else if (!queue_work(cmd->wq, &ent->work)) {1247 mlx5_core_warn(dev, "failed to queue work\n");1248 err = -EALREADY;1249 goto out_free;1250 }1251 1252 if (callback)1253 return 0; /* mlx5_cmd_comp_handler() will put(ent) */1254 1255 err = wait_func(dev, ent);1256 if (err == -ETIMEDOUT || err == -ECANCELED || err == -EBUSY)1257 goto out_free;1258 1259 ds = ent->ts2 - ent->ts1;1260 stats = xa_load(&cmd->stats, ent->op);1261 if (stats) {1262 spin_lock_irq(&stats->lock);1263 stats->sum += ds;1264 ++stats->n;1265 spin_unlock_irq(&stats->lock);1266 }1267 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME,1268 "fw exec time for %s is %lld nsec\n",1269 mlx5_command_str(ent->op), ds);1270 1271out_free:1272 status = ent->status;1273 cmd_ent_put(ent);1274 return err ? : status;1275}1276 1277static ssize_t dbg_write(struct file *filp, const char __user *buf,1278 size_t count, loff_t *pos)1279{1280 struct mlx5_core_dev *dev = filp->private_data;1281 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;1282 char lbuf[3];1283 int err;1284 1285 if (!dbg->in_msg || !dbg->out_msg)1286 return -ENOMEM;1287 1288 if (count < sizeof(lbuf) - 1)1289 return -EINVAL;1290 1291 if (copy_from_user(lbuf, buf, sizeof(lbuf) - 1))1292 return -EFAULT;1293 1294 lbuf[sizeof(lbuf) - 1] = 0;1295 1296 if (strcmp(lbuf, "go"))1297 return -EINVAL;1298 1299 err = mlx5_cmd_exec(dev, dbg->in_msg, dbg->inlen, dbg->out_msg, dbg->outlen);1300 1301 return err ? err : count;1302}1303 1304static const struct file_operations fops = {1305 .owner = THIS_MODULE,1306 .open = simple_open,1307 .write = dbg_write,1308};1309 1310static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size,1311 u8 token)1312{1313 struct mlx5_cmd_prot_block *block;1314 struct mlx5_cmd_mailbox *next;1315 int copy;1316 1317 if (!to || !from)1318 return -ENOMEM;1319 1320 copy = min_t(int, size, sizeof(to->first.data));1321 memcpy(to->first.data, from, copy);1322 size -= copy;1323 from += copy;1324 1325 next = to->next;1326 while (size) {1327 if (!next) {1328 /* this is a BUG */1329 return -ENOMEM;1330 }1331 1332 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);1333 block = next->buf;1334 memcpy(block->data, from, copy);1335 from += copy;1336 size -= copy;1337 block->token = token;1338 next = next->next;1339 }1340 1341 return 0;1342}1343 1344static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size)1345{1346 struct mlx5_cmd_prot_block *block;1347 struct mlx5_cmd_mailbox *next;1348 int copy;1349 1350 if (!to || !from)1351 return -ENOMEM;1352 1353 copy = min_t(int, size, sizeof(from->first.data));1354 memcpy(to, from->first.data, copy);1355 size -= copy;1356 to += copy;1357 1358 next = from->next;1359 while (size) {1360 if (!next) {1361 /* this is a BUG */1362 return -ENOMEM;1363 }1364 1365 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);1366 block = next->buf;1367 1368 memcpy(to, block->data, copy);1369 to += copy;1370 size -= copy;1371 next = next->next;1372 }1373 1374 return 0;1375}1376 1377static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev,1378 gfp_t flags)1379{1380 struct mlx5_cmd_mailbox *mailbox;1381 1382 mailbox = kmalloc(sizeof(*mailbox), flags);1383 if (!mailbox)1384 return ERR_PTR(-ENOMEM);1385 1386 mailbox->buf = dma_pool_zalloc(dev->cmd.pool, flags,1387 &mailbox->dma);1388 if (!mailbox->buf) {1389 mlx5_core_dbg(dev, "failed allocation\n");1390 kfree(mailbox);1391 return ERR_PTR(-ENOMEM);1392 }1393 mailbox->next = NULL;1394 1395 return mailbox;1396}1397 1398static void free_cmd_box(struct mlx5_core_dev *dev,1399 struct mlx5_cmd_mailbox *mailbox)1400{1401 dma_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma);1402 kfree(mailbox);1403}1404 1405static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,1406 gfp_t flags, int size,1407 u8 token)1408{1409 struct mlx5_cmd_mailbox *tmp, *head = NULL;1410 struct mlx5_cmd_prot_block *block;1411 struct mlx5_cmd_msg *msg;1412 int err;1413 int n;1414 int i;1415 1416 msg = kzalloc(sizeof(*msg), flags);1417 if (!msg)1418 return ERR_PTR(-ENOMEM);1419 1420 msg->len = size;1421 n = mlx5_calc_cmd_blocks(msg);1422 1423 for (i = 0; i < n; i++) {1424 tmp = alloc_cmd_box(dev, flags);1425 if (IS_ERR(tmp)) {1426 mlx5_core_warn(dev, "failed allocating block\n");1427 err = PTR_ERR(tmp);1428 goto err_alloc;1429 }1430 1431 block = tmp->buf;1432 tmp->next = head;1433 block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0);1434 block->block_num = cpu_to_be32(n - i - 1);1435 block->token = token;1436 head = tmp;1437 }1438 msg->next = head;1439 return msg;1440 1441err_alloc:1442 while (head) {1443 tmp = head->next;1444 free_cmd_box(dev, head);1445 head = tmp;1446 }1447 kfree(msg);1448 1449 return ERR_PTR(err);1450}1451 1452static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,1453 struct mlx5_cmd_msg *msg)1454{1455 struct mlx5_cmd_mailbox *head = msg->next;1456 struct mlx5_cmd_mailbox *next;1457 1458 while (head) {1459 next = head->next;1460 free_cmd_box(dev, head);1461 head = next;1462 }1463 kfree(msg);1464}1465 1466static ssize_t data_write(struct file *filp, const char __user *buf,1467 size_t count, loff_t *pos)1468{1469 struct mlx5_core_dev *dev = filp->private_data;1470 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;1471 void *ptr;1472 1473 if (*pos != 0)1474 return -EINVAL;1475 1476 kfree(dbg->in_msg);1477 dbg->in_msg = NULL;1478 dbg->inlen = 0;1479 ptr = memdup_user(buf, count);1480 if (IS_ERR(ptr))1481 return PTR_ERR(ptr);1482 dbg->in_msg = ptr;1483 dbg->inlen = count;1484 1485 *pos = count;1486 1487 return count;1488}1489 1490static ssize_t data_read(struct file *filp, char __user *buf, size_t count,1491 loff_t *pos)1492{1493 struct mlx5_core_dev *dev = filp->private_data;1494 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;1495 1496 if (!dbg->out_msg)1497 return -ENOMEM;1498 1499 return simple_read_from_buffer(buf, count, pos, dbg->out_msg,1500 dbg->outlen);1501}1502 1503static const struct file_operations dfops = {1504 .owner = THIS_MODULE,1505 .open = simple_open,1506 .write = data_write,1507 .read = data_read,1508};1509 1510static ssize_t outlen_read(struct file *filp, char __user *buf, size_t count,1511 loff_t *pos)1512{1513 struct mlx5_core_dev *dev = filp->private_data;1514 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;1515 char outlen[8];1516 int err;1517 1518 err = snprintf(outlen, sizeof(outlen), "%d", dbg->outlen);1519 if (err < 0)1520 return err;1521 1522 return simple_read_from_buffer(buf, count, pos, outlen, err);1523}1524 1525static ssize_t outlen_write(struct file *filp, const char __user *buf,1526 size_t count, loff_t *pos)1527{1528 struct mlx5_core_dev *dev = filp->private_data;1529 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;1530 char outlen_str[8] = {0};1531 int outlen;1532 void *ptr;1533 int err;1534 1535 if (*pos != 0 || count > 6)1536 return -EINVAL;1537 1538 kfree(dbg->out_msg);1539 dbg->out_msg = NULL;1540 dbg->outlen = 0;1541 1542 if (copy_from_user(outlen_str, buf, count))1543 return -EFAULT;1544 1545 err = sscanf(outlen_str, "%d", &outlen);1546 if (err != 1)1547 return -EINVAL;1548 1549 ptr = kzalloc(outlen, GFP_KERNEL);1550 if (!ptr)1551 return -ENOMEM;1552 1553 dbg->out_msg = ptr;1554 dbg->outlen = outlen;1555 1556 *pos = count;1557 1558 return count;1559}1560 1561static const struct file_operations olfops = {1562 .owner = THIS_MODULE,1563 .open = simple_open,1564 .write = outlen_write,1565 .read = outlen_read,1566};1567 1568static void set_wqname(struct mlx5_core_dev *dev)1569{1570 struct mlx5_cmd *cmd = &dev->cmd;1571 1572 snprintf(cmd->wq_name, sizeof(cmd->wq_name), "mlx5_cmd_%s",1573 dev_name(dev->device));1574}1575 1576static void clean_debug_files(struct mlx5_core_dev *dev)1577{1578 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;1579 1580 if (!mlx5_debugfs_root)1581 return;1582 1583 debugfs_remove_recursive(dbg->dbg_root);1584}1585 1586static void create_debugfs_files(struct mlx5_core_dev *dev)1587{1588 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;1589 1590 dbg->dbg_root = debugfs_create_dir("cmd", mlx5_debugfs_get_dev_root(dev));1591 1592 debugfs_create_file("in", 0400, dbg->dbg_root, dev, &dfops);1593 debugfs_create_file("out", 0200, dbg->dbg_root, dev, &dfops);1594 debugfs_create_file("out_len", 0600, dbg->dbg_root, dev, &olfops);1595 debugfs_create_u8("status", 0600, dbg->dbg_root, &dbg->status);1596 debugfs_create_file("run", 0200, dbg->dbg_root, dev, &fops);1597}1598 1599void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode)1600{1601 struct mlx5_cmd *cmd = &dev->cmd;1602 int i;1603 1604 for (i = 0; i < cmd->vars.max_reg_cmds; i++)1605 down(&cmd->vars.sem);1606 down(&cmd->vars.pages_sem);1607 1608 cmd->allowed_opcode = opcode;1609 1610 up(&cmd->vars.pages_sem);1611 for (i = 0; i < cmd->vars.max_reg_cmds; i++)1612 up(&cmd->vars.sem);1613}1614 1615static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode)1616{1617 struct mlx5_cmd *cmd = &dev->cmd;1618 int i;1619 1620 for (i = 0; i < cmd->vars.max_reg_cmds; i++)1621 down(&cmd->vars.sem);1622 down(&cmd->vars.pages_sem);1623 1624 cmd->mode = mode;1625 1626 up(&cmd->vars.pages_sem);1627 for (i = 0; i < cmd->vars.max_reg_cmds; i++)1628 up(&cmd->vars.sem);1629}1630 1631static int cmd_comp_notifier(struct notifier_block *nb,1632 unsigned long type, void *data)1633{1634 struct mlx5_core_dev *dev;1635 struct mlx5_cmd *cmd;1636 struct mlx5_eqe *eqe;1637 1638 cmd = mlx5_nb_cof(nb, struct mlx5_cmd, nb);1639 dev = container_of(cmd, struct mlx5_core_dev, cmd);1640 eqe = data;1641 1642 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)1643 return NOTIFY_DONE;1644 1645 mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector), false);1646 1647 return NOTIFY_OK;1648}1649void mlx5_cmd_use_events(struct mlx5_core_dev *dev)1650{1651 MLX5_NB_INIT(&dev->cmd.nb, cmd_comp_notifier, CMD);1652 mlx5_eq_notifier_register(dev, &dev->cmd.nb);1653 mlx5_cmd_change_mod(dev, CMD_MODE_EVENTS);1654}1655 1656void mlx5_cmd_use_polling(struct mlx5_core_dev *dev)1657{1658 mlx5_cmd_change_mod(dev, CMD_MODE_POLLING);1659 mlx5_eq_notifier_unregister(dev, &dev->cmd.nb);1660}1661 1662static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg)1663{1664 unsigned long flags;1665 1666 if (msg->parent) {1667 spin_lock_irqsave(&msg->parent->lock, flags);1668 list_add_tail(&msg->list, &msg->parent->head);1669 spin_unlock_irqrestore(&msg->parent->lock, flags);1670 } else {1671 mlx5_free_cmd_msg(dev, msg);1672 }1673}1674 1675static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced)1676{1677 struct mlx5_cmd *cmd = &dev->cmd;1678 struct mlx5_cmd_work_ent *ent;1679 mlx5_cmd_cbk_t callback;1680 void *context;1681 int err;1682 int i;1683 s64 ds;1684 struct mlx5_cmd_stats *stats;1685 unsigned long flags;1686 unsigned long vector;1687 1688 /* there can be at most 32 command queues */1689 vector = vec & 0xffffffff;1690 for (i = 0; i < (1 << cmd->vars.log_sz); i++) {1691 if (test_bit(i, &vector)) {1692 ent = cmd->ent_arr[i];1693 1694 /* if we already completed the command, ignore it */1695 if (!test_and_clear_bit(MLX5_CMD_ENT_STATE_PENDING_COMP,1696 &ent->state)) {1697 /* only real completion can free the cmd slot */1698 if (!forced) {1699 mlx5_core_err(dev, "Command completion arrived after timeout (entry idx = %d).\n",1700 ent->idx);1701 cmd_ent_put(ent);1702 }1703 continue;1704 }1705 1706 if (ent->callback && cancel_delayed_work(&ent->cb_timeout_work))1707 cmd_ent_put(ent); /* timeout work was canceled */1708 1709 if (!forced || /* Real FW completion */1710 mlx5_cmd_is_down(dev) || /* No real FW completion is expected */1711 !opcode_allowed(cmd, ent->op))1712 cmd_ent_put(ent);1713 1714 ent->ts2 = ktime_get_ns();1715 memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out));1716 dump_command(dev, ent, 0);1717 1718 if (vec & MLX5_TRIGGERED_CMD_COMP)1719 ent->ret = -ENXIO;1720 1721 if (!ent->ret) { /* Command completed by FW */1722 if (!cmd->checksum_disabled)1723 ent->ret = verify_signature(ent);1724 1725 ent->status = ent->lay->status_own >> 1;1726 1727 mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n",1728 ent->ret, deliv_status_to_str(ent->status), ent->status);1729 }1730 1731 if (ent->callback) {1732 ds = ent->ts2 - ent->ts1;1733 stats = xa_load(&cmd->stats, ent->op);1734 if (stats) {1735 spin_lock_irqsave(&stats->lock, flags);1736 stats->sum += ds;1737 ++stats->n;1738 spin_unlock_irqrestore(&stats->lock, flags);1739 }1740 1741 callback = ent->callback;1742 context = ent->context;1743 err = ent->ret ? : ent->status;1744 if (err > 0) /* Failed in FW, command didn't execute */1745 err = deliv_status_to_err(err);1746 1747 if (!err)1748 err = mlx5_copy_from_msg(ent->uout,1749 ent->out,1750 ent->uout_size);1751 1752 mlx5_free_cmd_msg(dev, ent->out);1753 free_msg(dev, ent->in);1754 1755 /* final consumer is done, release ent */1756 cmd_ent_put(ent);1757 callback(err, context);1758 } else {1759 /* release wait_func() so mlx5_cmd_invoke()1760 * can make the final ent_put()1761 */1762 complete(&ent->done);1763 }1764 }1765 }1766}1767 1768#define MLX5_MAX_MANAGE_PAGES_CMD_ENT 11769#define MLX5_CMD_MASK ((1UL << (cmd->vars.max_reg_cmds + \1770 MLX5_MAX_MANAGE_PAGES_CMD_ENT)) - 1)1771 1772static void mlx5_cmd_trigger_completions(struct mlx5_core_dev *dev)1773{1774 struct mlx5_cmd *cmd = &dev->cmd;1775 unsigned long bitmask;1776 unsigned long flags;1777 u64 vector;1778 int i;1779 1780 /* wait for pending handlers to complete */1781 mlx5_eq_synchronize_cmd_irq(dev);1782 spin_lock_irqsave(&dev->cmd.alloc_lock, flags);1783 vector = ~dev->cmd.vars.bitmask & MLX5_CMD_MASK;1784 if (!vector)1785 goto no_trig;1786 1787 bitmask = vector;1788 /* we must increment the allocated entries refcount before triggering the completions1789 * to guarantee pending commands will not get freed in the meanwhile.1790 * For that reason, it also has to be done inside the alloc_lock.1791 */1792 for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz))1793 cmd_ent_get(cmd->ent_arr[i]);1794 vector |= MLX5_TRIGGERED_CMD_COMP;1795 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);1796 1797 mlx5_core_dbg(dev, "vector 0x%llx\n", vector);1798 mlx5_cmd_comp_handler(dev, vector, true);1799 for_each_set_bit(i, &bitmask, (1 << cmd->vars.log_sz))1800 cmd_ent_put(cmd->ent_arr[i]);1801 return;1802 1803no_trig:1804 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);1805}1806 1807void mlx5_cmd_flush(struct mlx5_core_dev *dev)1808{1809 struct mlx5_cmd *cmd = &dev->cmd;1810 int i;1811 1812 for (i = 0; i < cmd->vars.max_reg_cmds; i++) {1813 while (down_trylock(&cmd->vars.sem)) {1814 mlx5_cmd_trigger_completions(dev);1815 cond_resched();1816 }1817 }1818 1819 while (down_trylock(&cmd->vars.pages_sem)) {1820 mlx5_cmd_trigger_completions(dev);1821 cond_resched();1822 }1823 1824 /* Unlock cmdif */1825 up(&cmd->vars.pages_sem);1826 for (i = 0; i < cmd->vars.max_reg_cmds; i++)1827 up(&cmd->vars.sem);1828}1829 1830static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size,1831 gfp_t gfp)1832{1833 struct mlx5_cmd_msg *msg = ERR_PTR(-ENOMEM);1834 struct cmd_msg_cache *ch = NULL;1835 struct mlx5_cmd *cmd = &dev->cmd;1836 int i;1837 1838 if (in_size <= 16)1839 goto cache_miss;1840 1841 for (i = 0; i < dev->profile.num_cmd_caches; i++) {1842 ch = &cmd->cache[i];1843 if (in_size > ch->max_inbox_size)1844 continue;1845 spin_lock_irq(&ch->lock);1846 if (list_empty(&ch->head)) {1847 spin_unlock_irq(&ch->lock);1848 continue;1849 }1850 msg = list_entry(ch->head.next, typeof(*msg), list);1851 /* For cached lists, we must explicitly state what is1852 * the real size1853 */1854 msg->len = in_size;1855 list_del(&msg->list);1856 spin_unlock_irq(&ch->lock);1857 break;1858 }1859 1860 if (!IS_ERR(msg))1861 return msg;1862 1863cache_miss:1864 msg = mlx5_alloc_cmd_msg(dev, gfp, in_size, 0);1865 return msg;1866}1867 1868static int is_manage_pages(void *in)1869{1870 return in_to_opcode(in) == MLX5_CMD_OP_MANAGE_PAGES;1871}1872 1873/* Notes:1874 * 1. Callback functions may not sleep1875 * 2. Page queue commands do not support asynchrous completion1876 */1877static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,1878 int out_size, mlx5_cmd_cbk_t callback, void *context,1879 bool force_polling)1880{1881 struct mlx5_cmd_msg *inb, *outb;1882 u16 opcode = in_to_opcode(in);1883 bool throttle_op;1884 int pages_queue;1885 gfp_t gfp;1886 u8 token;1887 int err;1888 1889 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, opcode))1890 return -ENXIO;1891 1892 throttle_op = mlx5_cmd_is_throttle_opcode(opcode);1893 if (throttle_op) {1894 if (callback) {1895 if (down_trylock(&dev->cmd.vars.throttle_sem))1896 return -EBUSY;1897 } else {1898 down(&dev->cmd.vars.throttle_sem);1899 }1900 }1901 1902 pages_queue = is_manage_pages(in);1903 gfp = callback ? GFP_ATOMIC : GFP_KERNEL;1904 1905 inb = alloc_msg(dev, in_size, gfp);1906 if (IS_ERR(inb)) {1907 err = PTR_ERR(inb);1908 goto out_up;1909 }1910 1911 token = alloc_token(&dev->cmd);1912 1913 err = mlx5_copy_to_msg(inb, in, in_size, token);1914 if (err) {1915 mlx5_core_warn(dev, "err %d\n", err);1916 goto out_in;1917 }1918 1919 outb = mlx5_alloc_cmd_msg(dev, gfp, out_size, token);1920 if (IS_ERR(outb)) {1921 err = PTR_ERR(outb);1922 goto out_in;1923 }1924 1925 err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context,1926 pages_queue, token, force_polling);1927 if (callback)1928 return err;1929 1930 if (err > 0) /* Failed in FW, command didn't execute */1931 err = deliv_status_to_err(err);1932 1933 if (err)1934 goto out_out;1935 1936 /* command completed by FW */1937 err = mlx5_copy_from_msg(out, outb, out_size);1938out_out:1939 mlx5_free_cmd_msg(dev, outb);1940out_in:1941 free_msg(dev, inb);1942out_up:1943 if (throttle_op)1944 up(&dev->cmd.vars.throttle_sem);1945 return err;1946}1947 1948static void mlx5_cmd_err_trace(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)1949{1950 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);1951 u8 status = MLX5_GET(mbox_out, out, status);1952 1953 trace_mlx5_cmd(mlx5_command_str(opcode), opcode, op_mod,1954 cmd_status_str(status), status, syndrome,1955 cmd_status_to_err(status));1956}1957 1958static void cmd_status_log(struct mlx5_core_dev *dev, u16 opcode, u8 status,1959 u32 syndrome, int err)1960{1961 const char *namep = mlx5_command_str(opcode);1962 struct mlx5_cmd_stats *stats;1963 unsigned long flags;1964 1965 if (!err || !(strcmp(namep, "unknown command opcode")))1966 return;1967 1968 stats = xa_load(&dev->cmd.stats, opcode);1969 if (!stats)1970 return;1971 spin_lock_irqsave(&stats->lock, flags);1972 stats->failed++;1973 if (err < 0)1974 stats->last_failed_errno = -err;1975 if (err == -EREMOTEIO) {1976 stats->failed_mbox_status++;1977 stats->last_failed_mbox_status = status;1978 stats->last_failed_syndrome = syndrome;1979 }1980 spin_unlock_irqrestore(&stats->lock, flags);1981}1982 1983/* preserve -EREMOTEIO for outbox.status != OK, otherwise return err as is */1984static int cmd_status_err(struct mlx5_core_dev *dev, int err, u16 opcode, u16 op_mod, void *out)1985{1986 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);1987 u8 status = MLX5_GET(mbox_out, out, status);1988 1989 if (err == -EREMOTEIO) /* -EREMOTEIO is preserved */1990 err = -EIO;1991 1992 if (!err && status != MLX5_CMD_STAT_OK) {1993 err = -EREMOTEIO;1994 mlx5_cmd_err_trace(dev, opcode, op_mod, out);1995 }1996 1997 cmd_status_log(dev, opcode, status, syndrome, err);1998 return err;1999}2000 2001/**2002 * mlx5_cmd_do - Executes a fw command, wait for completion.2003 * Unlike mlx5_cmd_exec, this function will not translate or intercept2004 * outbox.status and will return -EREMOTEIO when2005 * outbox.status != MLX5_CMD_STAT_OK2006 *2007 * @dev: mlx5 core device2008 * @in: inbox mlx5_ifc command buffer2009 * @in_size: inbox buffer size2010 * @out: outbox mlx5_ifc buffer2011 * @out_size: outbox size2012 *2013 * @return:2014 * -EREMOTEIO : Command executed by FW, outbox.status != MLX5_CMD_STAT_OK.2015 * Caller must check FW outbox status.2016 * 0 : Command execution successful, outbox.status == MLX5_CMD_STAT_OK.2017 * < 0 : Command execution couldn't be performed by firmware or driver2018 */2019int mlx5_cmd_do(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size)2020{2021 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, false);2022 u16 op_mod = MLX5_GET(mbox_in, in, op_mod);2023 u16 opcode = in_to_opcode(in);2024 2025 return cmd_status_err(dev, err, opcode, op_mod, out);2026}2027EXPORT_SYMBOL(mlx5_cmd_do);2028 2029/**2030 * mlx5_cmd_exec - Executes a fw command, wait for completion2031 *2032 * @dev: mlx5 core device2033 * @in: inbox mlx5_ifc command buffer2034 * @in_size: inbox buffer size2035 * @out: outbox mlx5_ifc buffer2036 * @out_size: outbox size2037 *2038 * @return: 0 if no error, FW command execution was successful2039 * and outbox status is ok.2040 */2041int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,2042 int out_size)2043{2044 int err = mlx5_cmd_do(dev, in, in_size, out, out_size);2045 2046 return mlx5_cmd_check(dev, err, in, out);2047}2048EXPORT_SYMBOL(mlx5_cmd_exec);2049 2050/**2051 * mlx5_cmd_exec_polling - Executes a fw command, poll for completion2052 * Needed for driver force teardown, when command completion EQ2053 * will not be available to complete the command2054 *2055 * @dev: mlx5 core device2056 * @in: inbox mlx5_ifc command buffer2057 * @in_size: inbox buffer size2058 * @out: outbox mlx5_ifc buffer2059 * @out_size: outbox size2060 *2061 * @return: 0 if no error, FW command execution was successful2062 * and outbox status is ok.2063 */2064int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size,2065 void *out, int out_size)2066{2067 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, true);2068 u16 op_mod = MLX5_GET(mbox_in, in, op_mod);2069 u16 opcode = in_to_opcode(in);2070 2071 err = cmd_status_err(dev, err, opcode, op_mod, out);2072 return mlx5_cmd_check(dev, err, in, out);2073}2074EXPORT_SYMBOL(mlx5_cmd_exec_polling);2075 2076void mlx5_cmd_init_async_ctx(struct mlx5_core_dev *dev,2077 struct mlx5_async_ctx *ctx)2078{2079 ctx->dev = dev;2080 /* Starts at 1 to avoid doing wake_up if we are not cleaning up */2081 atomic_set(&ctx->num_inflight, 1);2082 init_completion(&ctx->inflight_done);2083}2084EXPORT_SYMBOL(mlx5_cmd_init_async_ctx);2085 2086/**2087 * mlx5_cmd_cleanup_async_ctx - Clean up an async_ctx2088 * @ctx: The ctx to clean2089 *2090 * Upon return all callbacks given to mlx5_cmd_exec_cb() have been called. The2091 * caller must ensure that mlx5_cmd_exec_cb() is not called during or after2092 * the call mlx5_cleanup_async_ctx().2093 */2094void mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx *ctx)2095{2096 if (!atomic_dec_and_test(&ctx->num_inflight))2097 wait_for_completion(&ctx->inflight_done);2098}2099EXPORT_SYMBOL(mlx5_cmd_cleanup_async_ctx);2100 2101static void mlx5_cmd_exec_cb_handler(int status, void *_work)2102{2103 struct mlx5_async_work *work = _work;2104 struct mlx5_async_ctx *ctx;2105 struct mlx5_core_dev *dev;2106 u16 opcode;2107 2108 ctx = work->ctx;2109 dev = ctx->dev;2110 opcode = work->opcode;2111 status = cmd_status_err(dev, status, work->opcode, work->op_mod, work->out);2112 work->user_callback(status, work);2113 /* Can't access "work" from this point on. It could have been freed in2114 * the callback.2115 */2116 if (mlx5_cmd_is_throttle_opcode(opcode))2117 up(&dev->cmd.vars.throttle_sem);2118 if (atomic_dec_and_test(&ctx->num_inflight))2119 complete(&ctx->inflight_done);2120}2121 2122int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size,2123 void *out, int out_size, mlx5_async_cbk_t callback,2124 struct mlx5_async_work *work)2125{2126 int ret;2127 2128 work->ctx = ctx;2129 work->user_callback = callback;2130 work->opcode = in_to_opcode(in);2131 work->op_mod = MLX5_GET(mbox_in, in, op_mod);2132 work->out = out;2133 if (WARN_ON(!atomic_inc_not_zero(&ctx->num_inflight)))2134 return -EIO;2135 ret = cmd_exec(ctx->dev, in, in_size, out, out_size,2136 mlx5_cmd_exec_cb_handler, work, false);2137 if (ret && atomic_dec_and_test(&ctx->num_inflight))2138 complete(&ctx->inflight_done);2139 2140 return ret;2141}2142EXPORT_SYMBOL(mlx5_cmd_exec_cb);2143 2144int mlx5_cmd_allow_other_vhca_access(struct mlx5_core_dev *dev,2145 struct mlx5_cmd_allow_other_vhca_access_attr *attr)2146{2147 u32 out[MLX5_ST_SZ_DW(allow_other_vhca_access_out)] = {};2148 u32 in[MLX5_ST_SZ_DW(allow_other_vhca_access_in)] = {};2149 void *key;2150 2151 MLX5_SET(allow_other_vhca_access_in,2152 in, opcode, MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS);2153 MLX5_SET(allow_other_vhca_access_in,2154 in, object_type_to_be_accessed, attr->obj_type);2155 MLX5_SET(allow_other_vhca_access_in,2156 in, object_id_to_be_accessed, attr->obj_id);2157 2158 key = MLX5_ADDR_OF(allow_other_vhca_access_in, in, access_key);2159 memcpy(key, attr->access_key, sizeof(attr->access_key));2160 2161 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));2162}2163 2164int mlx5_cmd_alias_obj_create(struct mlx5_core_dev *dev,2165 struct mlx5_cmd_alias_obj_create_attr *alias_attr,2166 u32 *obj_id)2167{2168 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};2169 u32 in[MLX5_ST_SZ_DW(create_alias_obj_in)] = {};2170 void *param;2171 void *attr;2172 void *key;2173 int ret;2174 2175 attr = MLX5_ADDR_OF(create_alias_obj_in, in, hdr);2176 MLX5_SET(general_obj_in_cmd_hdr,2177 attr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);2178 MLX5_SET(general_obj_in_cmd_hdr,2179 attr, obj_type, alias_attr->obj_type);2180 param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param);2181 MLX5_SET(general_obj_create_param, param, alias_object, 1);2182 2183 attr = MLX5_ADDR_OF(create_alias_obj_in, in, alias_ctx);2184 MLX5_SET(alias_context, attr, vhca_id_to_be_accessed, alias_attr->vhca_id);2185 MLX5_SET(alias_context, attr, object_id_to_be_accessed, alias_attr->obj_id);2186 2187 key = MLX5_ADDR_OF(alias_context, attr, access_key);2188 memcpy(key, alias_attr->access_key, sizeof(alias_attr->access_key));2189 2190 ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));2191 if (ret)2192 return ret;2193 2194 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);2195 2196 return 0;2197}2198 2199int mlx5_cmd_alias_obj_destroy(struct mlx5_core_dev *dev, u32 obj_id,2200 u16 obj_type)2201{2202 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};2203 u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};2204 2205 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);2206 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, obj_type);2207 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id);2208 2209 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));2210}2211 2212static void destroy_msg_cache(struct mlx5_core_dev *dev)2213{2214 struct cmd_msg_cache *ch;2215 struct mlx5_cmd_msg *msg;2216 struct mlx5_cmd_msg *n;2217 int i;2218 2219 for (i = 0; i < dev->profile.num_cmd_caches; i++) {2220 ch = &dev->cmd.cache[i];2221 list_for_each_entry_safe(msg, n, &ch->head, list) {2222 list_del(&msg->list);2223 mlx5_free_cmd_msg(dev, msg);2224 }2225 }2226}2227 2228static unsigned cmd_cache_num_ent[MLX5_NUM_COMMAND_CACHES] = {2229 512, 32, 16, 8, 22230};2231 2232static unsigned cmd_cache_ent_size[MLX5_NUM_COMMAND_CACHES] = {2233 16 + MLX5_CMD_DATA_BLOCK_SIZE,2234 16 + MLX5_CMD_DATA_BLOCK_SIZE * 2,2235 16 + MLX5_CMD_DATA_BLOCK_SIZE * 16,2236 16 + MLX5_CMD_DATA_BLOCK_SIZE * 256,2237 16 + MLX5_CMD_DATA_BLOCK_SIZE * 512,2238};2239 2240static void create_msg_cache(struct mlx5_core_dev *dev)2241{2242 struct mlx5_cmd *cmd = &dev->cmd;2243 struct cmd_msg_cache *ch;2244 struct mlx5_cmd_msg *msg;2245 int i;2246 int k;2247 2248 /* Initialize and fill the caches with initial entries */2249 for (k = 0; k < dev->profile.num_cmd_caches; k++) {2250 ch = &cmd->cache[k];2251 spin_lock_init(&ch->lock);2252 INIT_LIST_HEAD(&ch->head);2253 ch->num_ent = cmd_cache_num_ent[k];2254 ch->max_inbox_size = cmd_cache_ent_size[k];2255 for (i = 0; i < ch->num_ent; i++) {2256 msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL | __GFP_NOWARN,2257 ch->max_inbox_size, 0);2258 if (IS_ERR(msg))2259 break;2260 msg->parent = ch;2261 list_add_tail(&msg->list, &ch->head);2262 }2263 }2264}2265 2266static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)2267{2268 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE,2269 &cmd->alloc_dma, GFP_KERNEL);2270 if (!cmd->cmd_alloc_buf)2271 return -ENOMEM;2272 2273 /* make sure it is aligned to 4K */2274 if (!((uintptr_t)cmd->cmd_alloc_buf & (MLX5_ADAPTER_PAGE_SIZE - 1))) {2275 cmd->cmd_buf = cmd->cmd_alloc_buf;2276 cmd->dma = cmd->alloc_dma;2277 cmd->alloc_size = MLX5_ADAPTER_PAGE_SIZE;2278 return 0;2279 }2280 2281 dma_free_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, cmd->cmd_alloc_buf,2282 cmd->alloc_dma);2283 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev),2284 2 * MLX5_ADAPTER_PAGE_SIZE - 1,2285 &cmd->alloc_dma, GFP_KERNEL);2286 if (!cmd->cmd_alloc_buf)2287 return -ENOMEM;2288 2289 cmd->cmd_buf = PTR_ALIGN(cmd->cmd_alloc_buf, MLX5_ADAPTER_PAGE_SIZE);2290 cmd->dma = ALIGN(cmd->alloc_dma, MLX5_ADAPTER_PAGE_SIZE);2291 cmd->alloc_size = 2 * MLX5_ADAPTER_PAGE_SIZE - 1;2292 return 0;2293}2294 2295static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)2296{2297 dma_free_coherent(mlx5_core_dma_dev(dev), cmd->alloc_size, cmd->cmd_alloc_buf,2298 cmd->alloc_dma);2299}2300 2301static u16 cmdif_rev(struct mlx5_core_dev *dev)2302{2303 return ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16;2304}2305 2306int mlx5_cmd_init(struct mlx5_core_dev *dev)2307{2308 struct mlx5_cmd *cmd = &dev->cmd;2309 2310 cmd->checksum_disabled = 1;2311 2312 spin_lock_init(&cmd->alloc_lock);2313 spin_lock_init(&cmd->token_lock);2314 2315 set_wqname(dev);2316 cmd->wq = create_singlethread_workqueue(cmd->wq_name);2317 if (!cmd->wq) {2318 mlx5_core_err(dev, "failed to create command workqueue\n");2319 return -ENOMEM;2320 }2321 2322 mlx5_cmdif_debugfs_init(dev);2323 2324 return 0;2325}2326 2327void mlx5_cmd_cleanup(struct mlx5_core_dev *dev)2328{2329 struct mlx5_cmd *cmd = &dev->cmd;2330 2331 mlx5_cmdif_debugfs_cleanup(dev);2332 destroy_workqueue(cmd->wq);2333}2334 2335int mlx5_cmd_enable(struct mlx5_core_dev *dev)2336{2337 int size = sizeof(struct mlx5_cmd_prot_block);2338 int align = roundup_pow_of_two(size);2339 struct mlx5_cmd *cmd = &dev->cmd;2340 u32 cmd_h, cmd_l;2341 int err;2342 2343 memset(&cmd->vars, 0, sizeof(cmd->vars));2344 cmd->vars.cmdif_rev = cmdif_rev(dev);2345 if (cmd->vars.cmdif_rev != CMD_IF_REV) {2346 mlx5_core_err(dev,2347 "Driver cmdif rev(%d) differs from firmware's(%d)\n",2348 CMD_IF_REV, cmd->vars.cmdif_rev);2349 return -EINVAL;2350 }2351 2352 cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff;2353 cmd->vars.log_sz = cmd_l >> 4 & 0xf;2354 cmd->vars.log_stride = cmd_l & 0xf;2355 if (1 << cmd->vars.log_sz > MLX5_MAX_COMMANDS) {2356 mlx5_core_err(dev, "firmware reports too many outstanding commands %d\n",2357 1 << cmd->vars.log_sz);2358 return -EINVAL;2359 }2360 2361 if (cmd->vars.log_sz + cmd->vars.log_stride > MLX5_ADAPTER_PAGE_SHIFT) {2362 mlx5_core_err(dev, "command queue size overflow\n");2363 return -EINVAL;2364 }2365 2366 cmd->state = MLX5_CMDIF_STATE_DOWN;2367 cmd->vars.max_reg_cmds = (1 << cmd->vars.log_sz) - 1;2368 cmd->vars.bitmask = MLX5_CMD_MASK;2369 2370 sema_init(&cmd->vars.sem, cmd->vars.max_reg_cmds);2371 sema_init(&cmd->vars.pages_sem, 1);2372 sema_init(&cmd->vars.throttle_sem, DIV_ROUND_UP(cmd->vars.max_reg_cmds, 2));2373 2374 cmd->pool = dma_pool_create("mlx5_cmd", mlx5_core_dma_dev(dev), size, align, 0);2375 if (!cmd->pool)2376 return -ENOMEM;2377 2378 err = alloc_cmd_page(dev, cmd);2379 if (err)2380 goto err_free_pool;2381 2382 cmd_h = (u32)((u64)(cmd->dma) >> 32);2383 cmd_l = (u32)(cmd->dma);2384 if (cmd_l & 0xfff) {2385 mlx5_core_err(dev, "invalid command queue address\n");2386 err = -ENOMEM;2387 goto err_cmd_page;2388 }2389 2390 iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h);2391 iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz);2392 2393 /* Make sure firmware sees the complete address before we proceed */2394 wmb();2395 2396 mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma));2397 2398 cmd->mode = CMD_MODE_POLLING;2399 cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL;2400 2401 create_msg_cache(dev);2402 create_debugfs_files(dev);2403 2404 return 0;2405 2406err_cmd_page:2407 free_cmd_page(dev, cmd);2408err_free_pool:2409 dma_pool_destroy(cmd->pool);2410 return err;2411}2412 2413void mlx5_cmd_disable(struct mlx5_core_dev *dev)2414{2415 struct mlx5_cmd *cmd = &dev->cmd;2416 2417 flush_workqueue(cmd->wq);2418 clean_debug_files(dev);2419 destroy_msg_cache(dev);2420 free_cmd_page(dev, cmd);2421 dma_pool_destroy(cmd->pool);2422}2423 2424void mlx5_cmd_set_state(struct mlx5_core_dev *dev,2425 enum mlx5_cmdif_state cmdif_state)2426{2427 dev->cmd.state = cmdif_state;2428}2429