226 lines · c
1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */2/*3 * Copyright (c) 2007 Cisco Systems. All rights reserved.4 * Copyright (c) 2020 Intel Corporation. All rights reserved.5 */6 7#ifndef IB_UMEM_H8#define IB_UMEM_H9 10#include <linux/list.h>11#include <linux/scatterlist.h>12#include <linux/workqueue.h>13#include <rdma/ib_verbs.h>14 15struct ib_ucontext;16struct ib_umem_odp;17struct dma_buf_attach_ops;18 19struct ib_umem {20 struct ib_device *ibdev;21 struct mm_struct *owning_mm;22 u64 iova;23 size_t length;24 unsigned long address;25 u32 writable : 1;26 u32 is_odp : 1;27 u32 is_dmabuf : 1;28 struct sg_append_table sgt_append;29};30 31struct ib_umem_dmabuf {32 struct ib_umem umem;33 struct dma_buf_attachment *attach;34 struct sg_table *sgt;35 struct scatterlist *first_sg;36 struct scatterlist *last_sg;37 unsigned long first_sg_offset;38 unsigned long last_sg_trim;39 void *private;40 u8 pinned : 1;41 u8 revoked : 1;42};43 44static inline struct ib_umem_dmabuf *to_ib_umem_dmabuf(struct ib_umem *umem)45{46 return container_of(umem, struct ib_umem_dmabuf, umem);47}48 49/* Returns the offset of the umem start relative to the first page. */50static inline int ib_umem_offset(struct ib_umem *umem)51{52 return umem->address & ~PAGE_MASK;53}54 55static inline unsigned long ib_umem_dma_offset(struct ib_umem *umem,56 unsigned long pgsz)57{58 return (sg_dma_address(umem->sgt_append.sgt.sgl) + ib_umem_offset(umem)) &59 (pgsz - 1);60}61 62static inline size_t ib_umem_num_dma_blocks(struct ib_umem *umem,63 unsigned long pgsz)64{65 return (size_t)((ALIGN(umem->iova + umem->length, pgsz) -66 ALIGN_DOWN(umem->iova, pgsz))) /67 pgsz;68}69 70static inline size_t ib_umem_num_pages(struct ib_umem *umem)71{72 return ib_umem_num_dma_blocks(umem, PAGE_SIZE);73}74 75static inline void __rdma_umem_block_iter_start(struct ib_block_iter *biter,76 struct ib_umem *umem,77 unsigned long pgsz)78{79 __rdma_block_iter_start(biter, umem->sgt_append.sgt.sgl,80 umem->sgt_append.sgt.nents, pgsz);81 biter->__sg_advance = ib_umem_offset(umem) & ~(pgsz - 1);82 biter->__sg_numblocks = ib_umem_num_dma_blocks(umem, pgsz);83}84 85static inline bool __rdma_umem_block_iter_next(struct ib_block_iter *biter)86{87 return __rdma_block_iter_next(biter) && biter->__sg_numblocks--;88}89 90/**91 * rdma_umem_for_each_dma_block - iterate over contiguous DMA blocks of the umem92 * @umem: umem to iterate over93 * @pgsz: Page size to split the list into94 *95 * pgsz must be <= PAGE_SIZE or computed by ib_umem_find_best_pgsz(). The96 * returned DMA blocks will be aligned to pgsz and span the range:97 * ALIGN_DOWN(umem->address, pgsz) to ALIGN(umem->address + umem->length, pgsz)98 *99 * Performs exactly ib_umem_num_dma_blocks() iterations.100 */101#define rdma_umem_for_each_dma_block(umem, biter, pgsz) \102 for (__rdma_umem_block_iter_start(biter, umem, pgsz); \103 __rdma_umem_block_iter_next(biter);)104 105#ifdef CONFIG_INFINIBAND_USER_MEM106 107struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr,108 size_t size, int access);109void ib_umem_release(struct ib_umem *umem);110int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,111 size_t length);112unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,113 unsigned long pgsz_bitmap,114 unsigned long virt);115 116/**117 * ib_umem_find_best_pgoff - Find best HW page size118 *119 * @umem: umem struct120 * @pgsz_bitmap bitmap of HW supported page sizes121 * @pgoff_bitmask: Mask of bits that can be represented with an offset122 *123 * This is very similar to ib_umem_find_best_pgsz() except instead of accepting124 * an IOVA it accepts a bitmask specifying what address bits can be represented125 * with a page offset.126 *127 * For instance if the HW has multiple page sizes, requires 64 byte alignemnt,128 * and can support aligned offsets up to 4032 then pgoff_bitmask would be129 * "111111000000".130 *131 * If the pgoff_bitmask requires either alignment in the low bit or an132 * unavailable page size for the high bits, this function returns 0.133 */134static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem,135 unsigned long pgsz_bitmap,136 u64 pgoff_bitmask)137{138 struct scatterlist *sg = umem->sgt_append.sgt.sgl;139 dma_addr_t dma_addr;140 141 dma_addr = sg_dma_address(sg) + (umem->address & ~PAGE_MASK);142 return ib_umem_find_best_pgsz(umem, pgsz_bitmap,143 dma_addr & pgoff_bitmask);144}145 146struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device,147 unsigned long offset, size_t size,148 int fd, int access,149 const struct dma_buf_attach_ops *ops);150struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device,151 unsigned long offset,152 size_t size, int fd,153 int access);154struct ib_umem_dmabuf *155ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device,156 struct device *dma_device,157 unsigned long offset, size_t size,158 int fd, int access);159int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf);160void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf);161void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf);162void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf);163 164#else /* CONFIG_INFINIBAND_USER_MEM */165 166#include <linux/err.h>167 168static inline struct ib_umem *ib_umem_get(struct ib_device *device,169 unsigned long addr, size_t size,170 int access)171{172 return ERR_PTR(-EOPNOTSUPP);173}174static inline void ib_umem_release(struct ib_umem *umem) { }175static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,176 size_t length) {177 return -EOPNOTSUPP;178}179static inline unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,180 unsigned long pgsz_bitmap,181 unsigned long virt)182{183 return 0;184}185static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem,186 unsigned long pgsz_bitmap,187 u64 pgoff_bitmask)188{189 return 0;190}191static inline192struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device,193 unsigned long offset,194 size_t size, int fd,195 int access,196 struct dma_buf_attach_ops *ops)197{198 return ERR_PTR(-EOPNOTSUPP);199}200static inline struct ib_umem_dmabuf *201ib_umem_dmabuf_get_pinned(struct ib_device *device, unsigned long offset,202 size_t size, int fd, int access)203{204 return ERR_PTR(-EOPNOTSUPP);205}206 207static inline struct ib_umem_dmabuf *208ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device,209 struct device *dma_device,210 unsigned long offset, size_t size,211 int fd, int access)212{213 return ERR_PTR(-EOPNOTSUPP);214}215 216static inline int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf)217{218 return -EOPNOTSUPP;219}220static inline void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf) { }221static inline void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf) { }222static inline void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf) {}223 224#endif /* CONFIG_INFINIBAND_USER_MEM */225#endif /* IB_UMEM_H */226