339 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB2/*3 * Copyright (c) 2020 Hewlett Packard Enterprise, Inc. All rights reserved.4 */5 6/*7 * The rdma_rxe driver supports type 1 or type 2B memory windows.8 * Type 1 MWs are created by ibv_alloc_mw() verbs calls and bound by9 * ibv_bind_mw() calls. Type 2 MWs are also created by ibv_alloc_mw()10 * but bound by bind_mw work requests. The ibv_bind_mw() call is converted11 * by libibverbs to a bind_mw work request.12 */13 14#include "rxe.h"15 16int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)17{18 struct rxe_mw *mw = to_rmw(ibmw);19 struct rxe_pd *pd = to_rpd(ibmw->pd);20 struct rxe_dev *rxe = to_rdev(ibmw->device);21 int ret;22 23 rxe_get(pd);24 25 ret = rxe_add_to_pool(&rxe->mw_pool, mw);26 if (ret) {27 rxe_put(pd);28 return ret;29 }30 31 mw->rkey = ibmw->rkey = (mw->elem.index << 8) | rxe_get_next_key(-1);32 mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ?33 RXE_MW_STATE_FREE : RXE_MW_STATE_VALID;34 spin_lock_init(&mw->lock);35 36 rxe_finalize(mw);37 38 return 0;39}40 41int rxe_dealloc_mw(struct ib_mw *ibmw)42{43 struct rxe_mw *mw = to_rmw(ibmw);44 45 rxe_cleanup(mw);46 47 return 0;48}49 50static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,51 struct rxe_mw *mw, struct rxe_mr *mr, int access)52{53 if (mw->ibmw.type == IB_MW_TYPE_1) {54 if (unlikely(mw->state != RXE_MW_STATE_VALID)) {55 rxe_dbg_mw(mw,56 "attempt to bind a type 1 MW not in the valid state\n");57 return -EINVAL;58 }59 60 /* o10-36.2.2 */61 if (unlikely((access & IB_ZERO_BASED))) {62 rxe_dbg_mw(mw, "attempt to bind a zero based type 1 MW\n");63 return -EINVAL;64 }65 }66 67 if (mw->ibmw.type == IB_MW_TYPE_2) {68 /* o10-37.2.30 */69 if (unlikely(mw->state != RXE_MW_STATE_FREE)) {70 rxe_dbg_mw(mw,71 "attempt to bind a type 2 MW not in the free state\n");72 return -EINVAL;73 }74 75 /* C10-72 */76 if (unlikely(qp->pd != to_rpd(mw->ibmw.pd))) {77 rxe_dbg_mw(mw,78 "attempt to bind type 2 MW with qp with different PD\n");79 return -EINVAL;80 }81 82 /* o10-37.2.40 */83 if (unlikely(!mr || wqe->wr.wr.mw.length == 0)) {84 rxe_dbg_mw(mw,85 "attempt to invalidate type 2 MW by binding with NULL or zero length MR\n");86 return -EINVAL;87 }88 }89 90 /* remaining checks only apply to a nonzero MR */91 if (!mr)92 return 0;93 94 if (unlikely(mr->access & IB_ZERO_BASED)) {95 rxe_dbg_mw(mw, "attempt to bind MW to zero based MR\n");96 return -EINVAL;97 }98 99 /* C10-73 */100 if (unlikely(!(mr->access & IB_ACCESS_MW_BIND))) {101 rxe_dbg_mw(mw,102 "attempt to bind an MW to an MR without bind access\n");103 return -EINVAL;104 }105 106 /* C10-74 */107 if (unlikely((access &108 (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_ATOMIC)) &&109 !(mr->access & IB_ACCESS_LOCAL_WRITE))) {110 rxe_dbg_mw(mw,111 "attempt to bind an Writable MW to an MR without local write access\n");112 return -EINVAL;113 }114 115 /* C10-75 */116 if (access & IB_ZERO_BASED) {117 if (unlikely(wqe->wr.wr.mw.length > mr->ibmr.length)) {118 rxe_dbg_mw(mw,119 "attempt to bind a ZB MW outside of the MR\n");120 return -EINVAL;121 }122 } else {123 if (unlikely((wqe->wr.wr.mw.addr < mr->ibmr.iova) ||124 ((wqe->wr.wr.mw.addr + wqe->wr.wr.mw.length) >125 (mr->ibmr.iova + mr->ibmr.length)))) {126 rxe_dbg_mw(mw,127 "attempt to bind a VA MW outside of the MR\n");128 return -EINVAL;129 }130 }131 132 return 0;133}134 135static void rxe_do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,136 struct rxe_mw *mw, struct rxe_mr *mr, int access)137{138 u32 key = wqe->wr.wr.mw.rkey & 0xff;139 140 mw->rkey = (mw->rkey & ~0xff) | key;141 mw->access = access;142 mw->state = RXE_MW_STATE_VALID;143 mw->addr = wqe->wr.wr.mw.addr;144 mw->length = wqe->wr.wr.mw.length;145 146 if (mw->mr) {147 rxe_put(mw->mr);148 atomic_dec(&mw->mr->num_mw);149 mw->mr = NULL;150 }151 152 if (mw->length) {153 mw->mr = mr;154 atomic_inc(&mr->num_mw);155 rxe_get(mr);156 }157 158 if (mw->ibmw.type == IB_MW_TYPE_2) {159 rxe_get(qp);160 mw->qp = qp;161 }162}163 164int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe)165{166 int ret;167 struct rxe_mw *mw;168 struct rxe_mr *mr;169 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);170 u32 mw_rkey = wqe->wr.wr.mw.mw_rkey;171 u32 mr_lkey = wqe->wr.wr.mw.mr_lkey;172 int access = wqe->wr.wr.mw.access;173 174 mw = rxe_pool_get_index(&rxe->mw_pool, mw_rkey >> 8);175 if (unlikely(!mw)) {176 ret = -EINVAL;177 goto err;178 }179 180 if (unlikely(mw->rkey != mw_rkey)) {181 ret = -EINVAL;182 goto err_drop_mw;183 }184 185 if (likely(wqe->wr.wr.mw.length)) {186 mr = rxe_pool_get_index(&rxe->mr_pool, mr_lkey >> 8);187 if (unlikely(!mr)) {188 ret = -EINVAL;189 goto err_drop_mw;190 }191 192 if (unlikely(mr->lkey != mr_lkey)) {193 ret = -EINVAL;194 goto err_drop_mr;195 }196 } else {197 mr = NULL;198 }199 200 if (access & ~RXE_ACCESS_SUPPORTED_MW) {201 rxe_err_mw(mw, "access %#x not supported\n", access);202 ret = -EOPNOTSUPP;203 goto err_drop_mr;204 }205 206 spin_lock_bh(&mw->lock);207 208 ret = rxe_check_bind_mw(qp, wqe, mw, mr, access);209 if (ret)210 goto err_unlock;211 212 rxe_do_bind_mw(qp, wqe, mw, mr, access);213err_unlock:214 spin_unlock_bh(&mw->lock);215err_drop_mr:216 if (mr)217 rxe_put(mr);218err_drop_mw:219 rxe_put(mw);220err:221 return ret;222}223 224static int rxe_check_invalidate_mw(struct rxe_qp *qp, struct rxe_mw *mw)225{226 if (unlikely(mw->state == RXE_MW_STATE_INVALID))227 return -EINVAL;228 229 /* o10-37.2.26 */230 if (unlikely(mw->ibmw.type == IB_MW_TYPE_1))231 return -EINVAL;232 233 return 0;234}235 236static void rxe_do_invalidate_mw(struct rxe_mw *mw)237{238 struct rxe_qp *qp;239 struct rxe_mr *mr;240 241 /* valid type 2 MW will always have a QP pointer */242 qp = mw->qp;243 mw->qp = NULL;244 rxe_put(qp);245 246 /* valid type 2 MW will always have an MR pointer */247 mr = mw->mr;248 mw->mr = NULL;249 atomic_dec(&mr->num_mw);250 rxe_put(mr);251 252 mw->access = 0;253 mw->addr = 0;254 mw->length = 0;255 mw->state = RXE_MW_STATE_FREE;256}257 258int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey)259{260 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);261 struct rxe_mw *mw;262 int ret;263 264 mw = rxe_pool_get_index(&rxe->mw_pool, rkey >> 8);265 if (!mw) {266 ret = -EINVAL;267 goto err;268 }269 270 if (rkey != mw->rkey) {271 ret = -EINVAL;272 goto err_drop_ref;273 }274 275 spin_lock_bh(&mw->lock);276 277 ret = rxe_check_invalidate_mw(qp, mw);278 if (ret)279 goto err_unlock;280 281 rxe_do_invalidate_mw(mw);282err_unlock:283 spin_unlock_bh(&mw->lock);284err_drop_ref:285 rxe_put(mw);286err:287 return ret;288}289 290struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey)291{292 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);293 struct rxe_pd *pd = to_rpd(qp->ibqp.pd);294 struct rxe_mw *mw;295 int index = rkey >> 8;296 297 mw = rxe_pool_get_index(&rxe->mw_pool, index);298 if (!mw)299 return NULL;300 301 if (unlikely((mw->rkey != rkey) || rxe_mw_pd(mw) != pd ||302 (mw->ibmw.type == IB_MW_TYPE_2 && mw->qp != qp) ||303 (mw->length == 0) || ((access & mw->access) != access) ||304 mw->state != RXE_MW_STATE_VALID)) {305 rxe_put(mw);306 return NULL;307 }308 309 return mw;310}311 312void rxe_mw_cleanup(struct rxe_pool_elem *elem)313{314 struct rxe_mw *mw = container_of(elem, typeof(*mw), elem);315 struct rxe_pd *pd = to_rpd(mw->ibmw.pd);316 317 rxe_put(pd);318 319 if (mw->mr) {320 struct rxe_mr *mr = mw->mr;321 322 mw->mr = NULL;323 atomic_dec(&mr->num_mw);324 rxe_put(mr);325 }326 327 if (mw->qp) {328 struct rxe_qp *qp = mw->qp;329 330 mw->qp = NULL;331 rxe_put(qp);332 }333 334 mw->access = 0;335 mw->addr = 0;336 mw->length = 0;337 mw->state = RXE_MW_STATE_INVALID;338}339