475 lines · c
1// SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause2/*3 * Copyright(c) 2023 - Cornelis Networks, Inc.4 */5 6#include <linux/types.h>7 8#include "hfi.h"9#include "common.h"10#include "device.h"11#include "pinning.h"12#include "mmu_rb.h"13#include "user_sdma.h"14#include "trace.h"15 16struct sdma_mmu_node {17 struct mmu_rb_node rb;18 struct hfi1_user_sdma_pkt_q *pq;19 struct page **pages;20 unsigned int npages;21};22 23static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,24 unsigned long len);25static int sdma_rb_evict(void *arg, struct mmu_rb_node *mnode, void *arg2,26 bool *stop);27static void sdma_rb_remove(void *arg, struct mmu_rb_node *mnode);28 29static const struct mmu_rb_ops sdma_rb_ops = {30 .filter = sdma_rb_filter,31 .evict = sdma_rb_evict,32 .remove = sdma_rb_remove,33};34 35int hfi1_init_system_pinning(struct hfi1_user_sdma_pkt_q *pq)36{37 struct hfi1_devdata *dd = pq->dd;38 int ret;39 40 ret = hfi1_mmu_rb_register(pq, &sdma_rb_ops, dd->pport->hfi1_wq,41 &pq->handler);42 if (ret)43 dd_dev_err(dd,44 "[%u:%u] Failed to register system memory DMA support with MMU: %d\n",45 pq->ctxt, pq->subctxt, ret);46 return ret;47}48 49void hfi1_free_system_pinning(struct hfi1_user_sdma_pkt_q *pq)50{51 if (pq->handler)52 hfi1_mmu_rb_unregister(pq->handler);53}54 55static u32 sdma_cache_evict(struct hfi1_user_sdma_pkt_q *pq, u32 npages)56{57 struct evict_data evict_data;58 59 evict_data.cleared = 0;60 evict_data.target = npages;61 hfi1_mmu_rb_evict(pq->handler, &evict_data);62 return evict_data.cleared;63}64 65static void unpin_vector_pages(struct mm_struct *mm, struct page **pages,66 unsigned int start, unsigned int npages)67{68 hfi1_release_user_pages(mm, pages + start, npages, false);69 kfree(pages);70}71 72static inline struct mm_struct *mm_from_sdma_node(struct sdma_mmu_node *node)73{74 return node->rb.handler->mn.mm;75}76 77static void free_system_node(struct sdma_mmu_node *node)78{79 if (node->npages) {80 unpin_vector_pages(mm_from_sdma_node(node), node->pages, 0,81 node->npages);82 atomic_sub(node->npages, &node->pq->n_locked);83 }84 kfree(node);85}86 87/*88 * kref_get()'s an additional kref on the returned rb_node to prevent rb_node89 * from being released until after rb_node is assigned to an SDMA descriptor90 * (struct sdma_desc) under add_system_iovec_to_sdma_packet(), even if the91 * virtual address range for rb_node is invalidated between now and then.92 */93static struct sdma_mmu_node *find_system_node(struct mmu_rb_handler *handler,94 unsigned long start,95 unsigned long end)96{97 struct mmu_rb_node *rb_node;98 unsigned long flags;99 100 spin_lock_irqsave(&handler->lock, flags);101 rb_node = hfi1_mmu_rb_get_first(handler, start, (end - start));102 if (!rb_node) {103 spin_unlock_irqrestore(&handler->lock, flags);104 return NULL;105 }106 107 /* "safety" kref to prevent release before add_system_iovec_to_sdma_packet() */108 kref_get(&rb_node->refcount);109 spin_unlock_irqrestore(&handler->lock, flags);110 111 return container_of(rb_node, struct sdma_mmu_node, rb);112}113 114static int pin_system_pages(struct user_sdma_request *req,115 uintptr_t start_address, size_t length,116 struct sdma_mmu_node *node, int npages)117{118 struct hfi1_user_sdma_pkt_q *pq = req->pq;119 int pinned, cleared;120 struct page **pages;121 122 pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL);123 if (!pages)124 return -ENOMEM;125 126retry:127 if (!hfi1_can_pin_pages(pq->dd, current->mm, atomic_read(&pq->n_locked),128 npages)) {129 SDMA_DBG(req, "Evicting: nlocked %u npages %u",130 atomic_read(&pq->n_locked), npages);131 cleared = sdma_cache_evict(pq, npages);132 if (cleared >= npages)133 goto retry;134 }135 136 SDMA_DBG(req, "Acquire user pages start_address %lx node->npages %u npages %u",137 start_address, node->npages, npages);138 pinned = hfi1_acquire_user_pages(current->mm, start_address, npages, 0,139 pages);140 141 if (pinned < 0) {142 kfree(pages);143 SDMA_DBG(req, "pinned %d", pinned);144 return pinned;145 }146 if (pinned != npages) {147 unpin_vector_pages(current->mm, pages, node->npages, pinned);148 SDMA_DBG(req, "npages %u pinned %d", npages, pinned);149 return -EFAULT;150 }151 node->rb.addr = start_address;152 node->rb.len = length;153 node->pages = pages;154 node->npages = npages;155 atomic_add(pinned, &pq->n_locked);156 SDMA_DBG(req, "done. pinned %d", pinned);157 return 0;158}159 160/*161 * kref refcount on *node_p will be 2 on successful addition: one kref from162 * kref_init() for mmu_rb_handler and one kref to prevent *node_p from being163 * released until after *node_p is assigned to an SDMA descriptor (struct164 * sdma_desc) under add_system_iovec_to_sdma_packet(), even if the virtual165 * address range for *node_p is invalidated between now and then.166 */167static int add_system_pinning(struct user_sdma_request *req,168 struct sdma_mmu_node **node_p,169 unsigned long start, unsigned long len)170 171{172 struct hfi1_user_sdma_pkt_q *pq = req->pq;173 struct sdma_mmu_node *node;174 int ret;175 176 node = kzalloc(sizeof(*node), GFP_KERNEL);177 if (!node)178 return -ENOMEM;179 180 /* First kref "moves" to mmu_rb_handler */181 kref_init(&node->rb.refcount);182 183 /* "safety" kref to prevent release before add_system_iovec_to_sdma_packet() */184 kref_get(&node->rb.refcount);185 186 node->pq = pq;187 ret = pin_system_pages(req, start, len, node, PFN_DOWN(len));188 if (ret == 0) {189 ret = hfi1_mmu_rb_insert(pq->handler, &node->rb);190 if (ret)191 free_system_node(node);192 else193 *node_p = node;194 195 return ret;196 }197 198 kfree(node);199 return ret;200}201 202static int get_system_cache_entry(struct user_sdma_request *req,203 struct sdma_mmu_node **node_p,204 size_t req_start, size_t req_len)205{206 struct hfi1_user_sdma_pkt_q *pq = req->pq;207 u64 start = ALIGN_DOWN(req_start, PAGE_SIZE);208 u64 end = PFN_ALIGN(req_start + req_len);209 int ret;210 211 if ((end - start) == 0) {212 SDMA_DBG(req,213 "Request for empty cache entry req_start %lx req_len %lx start %llx end %llx",214 req_start, req_len, start, end);215 return -EINVAL;216 }217 218 SDMA_DBG(req, "req_start %lx req_len %lu", req_start, req_len);219 220 while (1) {221 struct sdma_mmu_node *node =222 find_system_node(pq->handler, start, end);223 u64 prepend_len = 0;224 225 SDMA_DBG(req, "node %p start %llx end %llu", node, start, end);226 if (!node) {227 ret = add_system_pinning(req, node_p, start,228 end - start);229 if (ret == -EEXIST) {230 /*231 * Another execution context has inserted a232 * conficting entry first.233 */234 continue;235 }236 return ret;237 }238 239 if (node->rb.addr <= start) {240 /*241 * This entry covers at least part of the region. If it doesn't extend242 * to the end, then this will be called again for the next segment.243 */244 *node_p = node;245 return 0;246 }247 248 SDMA_DBG(req, "prepend: node->rb.addr %lx, node->rb.refcount %d",249 node->rb.addr, kref_read(&node->rb.refcount));250 prepend_len = node->rb.addr - start;251 252 /*253 * This node will not be returned, instead a new node254 * will be. So release the reference.255 */256 kref_put(&node->rb.refcount, hfi1_mmu_rb_release);257 258 /* Prepend a node to cover the beginning of the allocation */259 ret = add_system_pinning(req, node_p, start, prepend_len);260 if (ret == -EEXIST) {261 /* Another execution context has inserted a conficting entry first. */262 continue;263 }264 return ret;265 }266}267 268static void sdma_mmu_rb_node_get(void *ctx)269{270 struct mmu_rb_node *node = ctx;271 272 kref_get(&node->refcount);273}274 275static void sdma_mmu_rb_node_put(void *ctx)276{277 struct sdma_mmu_node *node = ctx;278 279 kref_put(&node->rb.refcount, hfi1_mmu_rb_release);280}281 282static int add_mapping_to_sdma_packet(struct user_sdma_request *req,283 struct user_sdma_txreq *tx,284 struct sdma_mmu_node *cache_entry,285 size_t start,286 size_t from_this_cache_entry)287{288 struct hfi1_user_sdma_pkt_q *pq = req->pq;289 unsigned int page_offset;290 unsigned int from_this_page;291 size_t page_index;292 void *ctx;293 int ret;294 295 /*296 * Because the cache may be more fragmented than the memory that is being accessed,297 * it's not strictly necessary to have a descriptor per cache entry.298 */299 300 while (from_this_cache_entry) {301 page_index = PFN_DOWN(start - cache_entry->rb.addr);302 303 if (page_index >= cache_entry->npages) {304 SDMA_DBG(req,305 "Request for page_index %zu >= cache_entry->npages %u",306 page_index, cache_entry->npages);307 return -EINVAL;308 }309 310 page_offset = start - ALIGN_DOWN(start, PAGE_SIZE);311 from_this_page = PAGE_SIZE - page_offset;312 313 if (from_this_page < from_this_cache_entry) {314 ctx = NULL;315 } else {316 /*317 * In the case they are equal the next line has no practical effect,318 * but it's better to do a register to register copy than a conditional319 * branch.320 */321 from_this_page = from_this_cache_entry;322 ctx = cache_entry;323 }324 325 ret = sdma_txadd_page(pq->dd, &tx->txreq,326 cache_entry->pages[page_index],327 page_offset, from_this_page,328 ctx,329 sdma_mmu_rb_node_get,330 sdma_mmu_rb_node_put);331 if (ret) {332 /*333 * When there's a failure, the entire request is freed by334 * user_sdma_send_pkts().335 */336 SDMA_DBG(req,337 "sdma_txadd_page failed %d page_index %lu page_offset %u from_this_page %u",338 ret, page_index, page_offset, from_this_page);339 return ret;340 }341 start += from_this_page;342 from_this_cache_entry -= from_this_page;343 }344 return 0;345}346 347static int add_system_iovec_to_sdma_packet(struct user_sdma_request *req,348 struct user_sdma_txreq *tx,349 struct user_sdma_iovec *iovec,350 size_t from_this_iovec)351{352 while (from_this_iovec > 0) {353 struct sdma_mmu_node *cache_entry;354 size_t from_this_cache_entry;355 size_t start;356 int ret;357 358 start = (uintptr_t)iovec->iov.iov_base + iovec->offset;359 ret = get_system_cache_entry(req, &cache_entry, start,360 from_this_iovec);361 if (ret) {362 SDMA_DBG(req, "pin system segment failed %d", ret);363 return ret;364 }365 366 from_this_cache_entry = cache_entry->rb.len - (start - cache_entry->rb.addr);367 if (from_this_cache_entry > from_this_iovec)368 from_this_cache_entry = from_this_iovec;369 370 ret = add_mapping_to_sdma_packet(req, tx, cache_entry, start,371 from_this_cache_entry);372 373 /*374 * Done adding cache_entry to zero or more sdma_desc. Can375 * kref_put() the "safety" kref taken under376 * get_system_cache_entry().377 */378 kref_put(&cache_entry->rb.refcount, hfi1_mmu_rb_release);379 380 if (ret) {381 SDMA_DBG(req, "add system segment failed %d", ret);382 return ret;383 }384 385 iovec->offset += from_this_cache_entry;386 from_this_iovec -= from_this_cache_entry;387 }388 389 return 0;390}391 392/*393 * Add up to pkt_data_remaining bytes to the txreq, starting at the current394 * offset in the given iovec entry and continuing until all data has been added395 * to the iovec or the iovec entry type changes.396 *397 * On success, prior to returning, adjust pkt_data_remaining, req->iov_idx, and398 * the offset value in req->iov[req->iov_idx] to reflect the data that has been399 * consumed.400 */401int hfi1_add_pages_to_sdma_packet(struct user_sdma_request *req,402 struct user_sdma_txreq *tx,403 struct user_sdma_iovec *iovec,404 u32 *pkt_data_remaining)405{406 size_t remaining_to_add = *pkt_data_remaining;407 /*408 * Walk through iovec entries, ensure the associated pages409 * are pinned and mapped, add data to the packet until no more410 * data remains to be added or the iovec entry type changes.411 */412 while (remaining_to_add > 0) {413 struct user_sdma_iovec *cur_iovec;414 size_t from_this_iovec;415 int ret;416 417 cur_iovec = iovec;418 from_this_iovec = iovec->iov.iov_len - iovec->offset;419 420 if (from_this_iovec > remaining_to_add) {421 from_this_iovec = remaining_to_add;422 } else {423 /* The current iovec entry will be consumed by this pass. */424 req->iov_idx++;425 iovec++;426 }427 428 ret = add_system_iovec_to_sdma_packet(req, tx, cur_iovec,429 from_this_iovec);430 if (ret)431 return ret;432 433 remaining_to_add -= from_this_iovec;434 }435 *pkt_data_remaining = remaining_to_add;436 437 return 0;438}439 440static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,441 unsigned long len)442{443 return (bool)(node->addr == addr);444}445 446/*447 * Return 1 to remove the node from the rb tree and call the remove op.448 *449 * Called with the rb tree lock held.450 */451static int sdma_rb_evict(void *arg, struct mmu_rb_node *mnode,452 void *evict_arg, bool *stop)453{454 struct sdma_mmu_node *node =455 container_of(mnode, struct sdma_mmu_node, rb);456 struct evict_data *evict_data = evict_arg;457 458 /* this node will be evicted, add its pages to our count */459 evict_data->cleared += node->npages;460 461 /* have enough pages been cleared? */462 if (evict_data->cleared >= evict_data->target)463 *stop = true;464 465 return 1; /* remove this node */466}467 468static void sdma_rb_remove(void *arg, struct mmu_rb_node *mnode)469{470 struct sdma_mmu_node *node =471 container_of(mnode, struct sdma_mmu_node, rb);472 473 free_system_node(node);474}475