506 lines · c
1/*2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.3 *4 * This software is available to you under a choice of one of two5 * licenses. You may choose to be licensed under the terms of the GNU6 * General Public License (GPL) Version 2, available from the file7 * COPYING in the main directory of this source tree, or the8 * OpenIB.org BSD license below:9 *10 * Redistribution and use in source and binary forms, with or11 * without modification, are permitted provided that the following12 * conditions are met:13 *14 * - Redistributions of source code must retain the above15 * copyright notice, this list of conditions and the following16 * disclaimer.17 *18 * - Redistributions in binary form must reproduce the above19 * copyright notice, this list of conditions and the following20 * disclaimer in the documentation and/or other materials21 * provided with the distribution.22 *23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE30 * SOFTWARE.31 */32 33#include <linux/mlx5/driver.h>34#include "mlx5_core.h"35#include <linux/mlx5/transobj.h>36 37int mlx5_core_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)38{39 u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {};40 u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {};41 int err;42 43 MLX5_SET(alloc_transport_domain_in, in, opcode,44 MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);45 46 err = mlx5_cmd_exec_inout(dev, alloc_transport_domain, in, out);47 if (!err)48 *tdn = MLX5_GET(alloc_transport_domain_out, out,49 transport_domain);50 51 return err;52}53EXPORT_SYMBOL(mlx5_core_alloc_transport_domain);54 55void mlx5_core_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)56{57 u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {};58 59 MLX5_SET(dealloc_transport_domain_in, in, opcode,60 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);61 MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);62 mlx5_cmd_exec_in(dev, dealloc_transport_domain, in);63}64EXPORT_SYMBOL(mlx5_core_dealloc_transport_domain);65 66int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rqn)67{68 u32 out[MLX5_ST_SZ_DW(create_rq_out)] = {};69 int err;70 71 MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ);72 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));73 if (!err)74 *rqn = MLX5_GET(create_rq_out, out, rqn);75 76 return err;77}78EXPORT_SYMBOL(mlx5_core_create_rq);79 80int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *in)81{82 MLX5_SET(modify_rq_in, in, rqn, rqn);83 MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ);84 85 return mlx5_cmd_exec_in(dev, modify_rq, in);86}87EXPORT_SYMBOL(mlx5_core_modify_rq);88 89void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn)90{91 u32 in[MLX5_ST_SZ_DW(destroy_rq_in)] = {};92 93 MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ);94 MLX5_SET(destroy_rq_in, in, rqn, rqn);95 mlx5_cmd_exec_in(dev, destroy_rq, in);96}97EXPORT_SYMBOL(mlx5_core_destroy_rq);98 99int mlx5_core_query_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *out)100{101 u32 in[MLX5_ST_SZ_DW(query_rq_in)] = {};102 103 MLX5_SET(query_rq_in, in, opcode, MLX5_CMD_OP_QUERY_RQ);104 MLX5_SET(query_rq_in, in, rqn, rqn);105 106 return mlx5_cmd_exec_inout(dev, query_rq, in, out);107}108EXPORT_SYMBOL(mlx5_core_query_rq);109 110int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *sqn)111{112 u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {};113 int err;114 115 MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);116 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));117 if (!err)118 *sqn = MLX5_GET(create_sq_out, out, sqn);119 120 return err;121}122 123int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *in)124{125 MLX5_SET(modify_sq_in, in, sqn, sqn);126 MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ);127 return mlx5_cmd_exec_in(dev, modify_sq, in);128}129EXPORT_SYMBOL(mlx5_core_modify_sq);130 131void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn)132{133 u32 in[MLX5_ST_SZ_DW(destroy_sq_in)] = {};134 135 MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ);136 MLX5_SET(destroy_sq_in, in, sqn, sqn);137 mlx5_cmd_exec_in(dev, destroy_sq, in);138}139 140int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out)141{142 u32 in[MLX5_ST_SZ_DW(query_sq_in)] = {};143 144 MLX5_SET(query_sq_in, in, opcode, MLX5_CMD_OP_QUERY_SQ);145 MLX5_SET(query_sq_in, in, sqn, sqn);146 return mlx5_cmd_exec_inout(dev, query_sq, in, out);147}148EXPORT_SYMBOL(mlx5_core_query_sq);149 150int mlx5_core_query_sq_state(struct mlx5_core_dev *dev, u32 sqn, u8 *state)151{152 void *out;153 void *sqc;154 int inlen;155 int err;156 157 inlen = MLX5_ST_SZ_BYTES(query_sq_out);158 out = kvzalloc(inlen, GFP_KERNEL);159 if (!out)160 return -ENOMEM;161 162 err = mlx5_core_query_sq(dev, sqn, out);163 if (err)164 goto out;165 166 sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context);167 *state = MLX5_GET(sqc, sqc, state);168 169out:170 kvfree(out);171 return err;172}173EXPORT_SYMBOL_GPL(mlx5_core_query_sq_state);174 175int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, u32 *tirn)176{177 u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};178 int err;179 180 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);181 err = mlx5_cmd_exec_inout(dev, create_tir, in, out);182 if (!err)183 *tirn = MLX5_GET(create_tir_out, out, tirn);184 185 return err;186}187EXPORT_SYMBOL(mlx5_core_create_tir);188 189int mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 tirn, u32 *in)190{191 MLX5_SET(modify_tir_in, in, tirn, tirn);192 MLX5_SET(modify_tir_in, in, opcode, MLX5_CMD_OP_MODIFY_TIR);193 return mlx5_cmd_exec_in(dev, modify_tir, in);194}195 196void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn)197{198 u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};199 200 MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);201 MLX5_SET(destroy_tir_in, in, tirn, tirn);202 mlx5_cmd_exec_in(dev, destroy_tir, in);203}204EXPORT_SYMBOL(mlx5_core_destroy_tir);205 206int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, u32 *tisn)207{208 u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {};209 int err;210 211 MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);212 err = mlx5_cmd_exec_inout(dev, create_tis, in, out);213 if (!err)214 *tisn = MLX5_GET(create_tis_out, out, tisn);215 216 return err;217}218EXPORT_SYMBOL(mlx5_core_create_tis);219 220int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in)221{222 MLX5_SET(modify_tis_in, in, tisn, tisn);223 MLX5_SET(modify_tis_in, in, opcode, MLX5_CMD_OP_MODIFY_TIS);224 225 return mlx5_cmd_exec_in(dev, modify_tis, in);226}227EXPORT_SYMBOL(mlx5_core_modify_tis);228 229void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn)230{231 u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {};232 233 MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);234 MLX5_SET(destroy_tis_in, in, tisn, tisn);235 mlx5_cmd_exec_in(dev, destroy_tis, in);236}237EXPORT_SYMBOL(mlx5_core_destroy_tis);238 239int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,240 u32 *rqtn)241{242 u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {};243 int err;244 245 MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);246 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));247 if (!err)248 *rqtn = MLX5_GET(create_rqt_out, out, rqtn);249 250 return err;251}252EXPORT_SYMBOL(mlx5_core_create_rqt);253 254int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in,255 int inlen)256{257 u32 out[MLX5_ST_SZ_DW(modify_rqt_out)] = {};258 259 MLX5_SET(modify_rqt_in, in, rqtn, rqtn);260 MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT);261 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));262}263 264void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn)265{266 u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {};267 268 MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);269 MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);270 mlx5_cmd_exec_in(dev, destroy_rqt, in);271}272EXPORT_SYMBOL(mlx5_core_destroy_rqt);273 274static int mlx5_hairpin_create_rq(struct mlx5_core_dev *mdev,275 struct mlx5_hairpin_params *params, u32 *rqn)276{277 u32 in[MLX5_ST_SZ_DW(create_rq_in)] = {0};278 void *rqc, *wq;279 280 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);281 wq = MLX5_ADDR_OF(rqc, rqc, wq);282 283 MLX5_SET(rqc, rqc, hairpin, 1);284 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);285 MLX5_SET(rqc, rqc, counter_set_id, params->q_counter);286 287 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);288 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);289 290 return mlx5_core_create_rq(mdev, in, MLX5_ST_SZ_BYTES(create_rq_in), rqn);291}292 293static int mlx5_hairpin_create_sq(struct mlx5_core_dev *mdev,294 struct mlx5_hairpin_params *params, u32 *sqn)295{296 u32 in[MLX5_ST_SZ_DW(create_sq_in)] = {0};297 void *sqc, *wq;298 299 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);300 wq = MLX5_ADDR_OF(sqc, sqc, wq);301 302 MLX5_SET(sqc, sqc, hairpin, 1);303 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);304 305 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);306 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);307 308 return mlx5_core_create_sq(mdev, in, MLX5_ST_SZ_BYTES(create_sq_in), sqn);309}310 311static int mlx5_hairpin_create_queues(struct mlx5_hairpin *hp,312 struct mlx5_hairpin_params *params)313{314 int i, j, err;315 316 for (i = 0; i < hp->num_channels; i++) {317 err = mlx5_hairpin_create_rq(hp->func_mdev, params, &hp->rqn[i]);318 if (err)319 goto out_err_rq;320 }321 322 for (i = 0; i < hp->num_channels; i++) {323 err = mlx5_hairpin_create_sq(hp->peer_mdev, params, &hp->sqn[i]);324 if (err)325 goto out_err_sq;326 }327 328 return 0;329 330out_err_sq:331 for (j = 0; j < i; j++)332 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[j]);333 i = hp->num_channels;334out_err_rq:335 for (j = 0; j < i; j++)336 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[j]);337 return err;338}339 340static void mlx5_hairpin_destroy_queues(struct mlx5_hairpin *hp)341{342 int i;343 344 for (i = 0; i < hp->num_channels; i++) {345 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[i]);346 if (!hp->peer_gone)347 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]);348 }349}350 351static int mlx5_hairpin_modify_rq(struct mlx5_core_dev *func_mdev, u32 rqn,352 int curr_state, int next_state,353 u16 peer_vhca, u32 peer_sq)354{355 u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {};356 void *rqc;357 358 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);359 360 if (next_state == MLX5_RQC_STATE_RDY) {361 MLX5_SET(rqc, rqc, hairpin_peer_sq, peer_sq);362 MLX5_SET(rqc, rqc, hairpin_peer_vhca, peer_vhca);363 }364 365 MLX5_SET(modify_rq_in, in, rq_state, curr_state);366 MLX5_SET(rqc, rqc, state, next_state);367 368 return mlx5_core_modify_rq(func_mdev, rqn, in);369}370 371static int mlx5_hairpin_modify_sq(struct mlx5_core_dev *peer_mdev, u32 sqn,372 int curr_state, int next_state,373 u16 peer_vhca, u32 peer_rq)374{375 u32 in[MLX5_ST_SZ_DW(modify_sq_in)] = {0};376 void *sqc;377 378 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);379 380 if (next_state == MLX5_SQC_STATE_RDY) {381 MLX5_SET(sqc, sqc, hairpin_peer_rq, peer_rq);382 MLX5_SET(sqc, sqc, hairpin_peer_vhca, peer_vhca);383 }384 385 MLX5_SET(modify_sq_in, in, sq_state, curr_state);386 MLX5_SET(sqc, sqc, state, next_state);387 388 return mlx5_core_modify_sq(peer_mdev, sqn, in);389}390 391static int mlx5_hairpin_pair_queues(struct mlx5_hairpin *hp)392{393 int i, j, err;394 395 /* set peer SQs */396 for (i = 0; i < hp->num_channels; i++) {397 err = mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i],398 MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,399 MLX5_CAP_GEN(hp->func_mdev, vhca_id), hp->rqn[i]);400 if (err)401 goto err_modify_sq;402 }403 404 /* set func RQs */405 for (i = 0; i < hp->num_channels; i++) {406 err = mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i],407 MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY,408 MLX5_CAP_GEN(hp->peer_mdev, vhca_id), hp->sqn[i]);409 if (err)410 goto err_modify_rq;411 }412 413 return 0;414 415err_modify_rq:416 for (j = 0; j < i; j++)417 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[j], MLX5_RQC_STATE_RDY,418 MLX5_RQC_STATE_RST, 0, 0);419 i = hp->num_channels;420err_modify_sq:421 for (j = 0; j < i; j++)422 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[j], MLX5_SQC_STATE_RDY,423 MLX5_SQC_STATE_RST, 0, 0);424 return err;425}426 427static void mlx5_hairpin_unpair_peer_sq(struct mlx5_hairpin *hp)428{429 int i;430 431 for (i = 0; i < hp->num_channels; i++)432 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], MLX5_SQC_STATE_RDY,433 MLX5_SQC_STATE_RST, 0, 0);434}435 436static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp)437{438 int i;439 440 /* unset func RQs */441 for (i = 0; i < hp->num_channels; i++)442 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], MLX5_RQC_STATE_RDY,443 MLX5_RQC_STATE_RST, 0, 0);444 /* unset peer SQs */445 if (!hp->peer_gone)446 mlx5_hairpin_unpair_peer_sq(hp);447}448 449struct mlx5_hairpin *450mlx5_core_hairpin_create(struct mlx5_core_dev *func_mdev,451 struct mlx5_core_dev *peer_mdev,452 struct mlx5_hairpin_params *params)453{454 struct mlx5_hairpin *hp;455 int size, err;456 457 size = sizeof(*hp) + params->num_channels * 2 * sizeof(u32);458 hp = kzalloc(size, GFP_KERNEL);459 if (!hp)460 return ERR_PTR(-ENOMEM);461 462 hp->func_mdev = func_mdev;463 hp->peer_mdev = peer_mdev;464 hp->num_channels = params->num_channels;465 466 hp->rqn = (void *)hp + sizeof(*hp);467 hp->sqn = hp->rqn + params->num_channels;468 469 /* alloc and pair func --> peer hairpin */470 err = mlx5_hairpin_create_queues(hp, params);471 if (err)472 goto err_create_queues;473 474 err = mlx5_hairpin_pair_queues(hp);475 if (err)476 goto err_pair_queues;477 478 return hp;479 480err_pair_queues:481 mlx5_hairpin_destroy_queues(hp);482err_create_queues:483 kfree(hp);484 return ERR_PTR(err);485}486 487void mlx5_core_hairpin_destroy(struct mlx5_hairpin *hp)488{489 mlx5_hairpin_unpair_queues(hp);490 mlx5_hairpin_destroy_queues(hp);491 kfree(hp);492}493 494void mlx5_core_hairpin_clear_dead_peer(struct mlx5_hairpin *hp)495{496 int i;497 498 mlx5_hairpin_unpair_peer_sq(hp);499 500 /* destroy peer SQ */501 for (i = 0; i < hp->num_channels; i++)502 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]);503 504 hp->peer_gone = true;505}506