500 lines · c
1/*2 * Copyright 2020 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 * Authors: Christian König23 */24 25#ifndef _TTM_RESOURCE_H_26#define _TTM_RESOURCE_H_27 28#include <linux/types.h>29#include <linux/list.h>30#include <linux/mutex.h>31#include <linux/iosys-map.h>32#include <linux/dma-fence.h>33 34#include <drm/drm_print.h>35#include <drm/ttm/ttm_caching.h>36#include <drm/ttm/ttm_kmap_iter.h>37 38#define TTM_MAX_BO_PRIORITY 4U39#define TTM_NUM_MEM_TYPES 840 41struct ttm_device;42struct ttm_resource_manager;43struct ttm_resource;44struct ttm_place;45struct ttm_buffer_object;46struct ttm_placement;47struct iosys_map;48struct io_mapping;49struct sg_table;50struct scatterlist;51 52/**53 * enum ttm_lru_item_type - enumerate ttm_lru_item subclasses54 */55enum ttm_lru_item_type {56 /** @TTM_LRU_RESOURCE: The resource subclass */57 TTM_LRU_RESOURCE,58 /** @TTM_LRU_HITCH: The iterator hitch subclass */59 TTM_LRU_HITCH60};61 62/**63 * struct ttm_lru_item - The TTM lru list node base class64 * @link: The list link65 * @type: The subclass type66 */67struct ttm_lru_item {68 struct list_head link;69 enum ttm_lru_item_type type;70};71 72/**73 * ttm_lru_item_init() - initialize a struct ttm_lru_item74 * @item: The item to initialize75 * @type: The subclass type76 */77static inline void ttm_lru_item_init(struct ttm_lru_item *item,78 enum ttm_lru_item_type type)79{80 item->type = type;81 INIT_LIST_HEAD(&item->link);82}83 84static inline bool ttm_lru_item_is_res(const struct ttm_lru_item *item)85{86 return item->type == TTM_LRU_RESOURCE;87}88 89struct ttm_resource_manager_func {90 /**91 * struct ttm_resource_manager_func member alloc92 *93 * @man: Pointer to a memory type manager.94 * @bo: Pointer to the buffer object we're allocating space for.95 * @place: Placement details.96 * @res: Resulting pointer to the ttm_resource.97 *98 * This function should allocate space in the memory type managed99 * by @man. Placement details if applicable are given by @place. If100 * successful, a filled in ttm_resource object should be returned in101 * @res. @res::start should be set to a value identifying the beginning102 * of the range allocated, and the function should return zero.103 * If the manager can't fulfill the request -ENOSPC should be returned.104 * If a system error occurred, preventing the request to be fulfilled,105 * the function should return a negative error code.106 *107 * This function may not be called from within atomic context and needs108 * to take care of its own locking to protect any data structures109 * managing the space.110 */111 int (*alloc)(struct ttm_resource_manager *man,112 struct ttm_buffer_object *bo,113 const struct ttm_place *place,114 struct ttm_resource **res);115 116 /**117 * struct ttm_resource_manager_func member free118 *119 * @man: Pointer to a memory type manager.120 * @res: Pointer to a struct ttm_resource to be freed.121 *122 * This function frees memory type resources previously allocated.123 * May not be called from within atomic context.124 */125 void (*free)(struct ttm_resource_manager *man,126 struct ttm_resource *res);127 128 /**129 * struct ttm_resource_manager_func member intersects130 *131 * @man: Pointer to a memory type manager.132 * @res: Pointer to a struct ttm_resource to be checked.133 * @place: Placement to check against.134 * @size: Size of the check.135 *136 * Test if @res intersects with @place + @size. Used to judge if137 * evictions are valueable or not.138 */139 bool (*intersects)(struct ttm_resource_manager *man,140 struct ttm_resource *res,141 const struct ttm_place *place,142 size_t size);143 144 /**145 * struct ttm_resource_manager_func member compatible146 *147 * @man: Pointer to a memory type manager.148 * @res: Pointer to a struct ttm_resource to be checked.149 * @place: Placement to check against.150 * @size: Size of the check.151 *152 * Test if @res compatible with @place + @size. Used to check of153 * the need to move the backing store or not.154 */155 bool (*compatible)(struct ttm_resource_manager *man,156 struct ttm_resource *res,157 const struct ttm_place *place,158 size_t size);159 160 /**161 * struct ttm_resource_manager_func member debug162 *163 * @man: Pointer to a memory type manager.164 * @printer: Prefix to be used in printout to identify the caller.165 *166 * This function is called to print out the state of the memory167 * type manager to aid debugging of out-of-memory conditions.168 * It may not be called from within atomic context.169 */170 void (*debug)(struct ttm_resource_manager *man,171 struct drm_printer *printer);172};173 174/**175 * struct ttm_resource_manager176 *177 * @use_type: The memory type is enabled.178 * @use_tt: If a TT object should be used for the backing store.179 * @size: Size of the managed region.180 * @bdev: ttm device this manager belongs to181 * @func: structure pointer implementing the range manager. See above182 * @move_lock: lock for move fence183 * @move: The fence of the last pipelined move operation.184 * @lru: The lru list for this memory type.185 *186 * This structure is used to identify and manage memory types for a device.187 */188struct ttm_resource_manager {189 /*190 * No protection. Constant from start.191 */192 bool use_type;193 bool use_tt;194 struct ttm_device *bdev;195 uint64_t size;196 const struct ttm_resource_manager_func *func;197 spinlock_t move_lock;198 199 /*200 * Protected by @move_lock.201 */202 struct dma_fence *move;203 204 /*205 * Protected by the bdev->lru_lock.206 */207 struct list_head lru[TTM_MAX_BO_PRIORITY];208 209 /**210 * @usage: How much of the resources are used, protected by the211 * bdev->lru_lock.212 */213 uint64_t usage;214};215 216/**217 * struct ttm_bus_placement218 *219 * @addr: mapped virtual address220 * @offset: physical addr221 * @is_iomem: is this io memory ?222 * @caching: See enum ttm_caching223 *224 * Structure indicating the bus placement of an object.225 */226struct ttm_bus_placement {227 void *addr;228 phys_addr_t offset;229 bool is_iomem;230 enum ttm_caching caching;231};232 233/**234 * struct ttm_resource235 *236 * @start: Start of the allocation.237 * @size: Actual size of resource in bytes.238 * @mem_type: Resource type of the allocation.239 * @placement: Placement flags.240 * @bus: Placement on io bus accessible to the CPU241 * @bo: weak reference to the BO, protected by ttm_device::lru_lock242 *243 * Structure indicating the placement and space resources used by a244 * buffer object.245 */246struct ttm_resource {247 unsigned long start;248 size_t size;249 uint32_t mem_type;250 uint32_t placement;251 struct ttm_bus_placement bus;252 struct ttm_buffer_object *bo;253 254 /**255 * @lru: Least recently used list, see &ttm_resource_manager.lru256 */257 struct ttm_lru_item lru;258};259 260/**261 * ttm_lru_item_to_res() - Downcast a struct ttm_lru_item to a struct ttm_resource262 * @item: The struct ttm_lru_item to downcast263 *264 * Return: Pointer to the embedding struct ttm_resource265 */266static inline struct ttm_resource *267ttm_lru_item_to_res(struct ttm_lru_item *item)268{269 return container_of(item, struct ttm_resource, lru);270}271 272/**273 * struct ttm_lru_bulk_move_pos274 *275 * @first: first res in the bulk move range276 * @last: last res in the bulk move range277 *278 * Range of resources for a lru bulk move.279 */280struct ttm_lru_bulk_move_pos {281 struct ttm_resource *first;282 struct ttm_resource *last;283};284 285/**286 * struct ttm_lru_bulk_move287 * @pos: first/last lru entry for resources in the each domain/priority288 * @cursor_list: The list of cursors currently traversing any of289 * the sublists of @pos. Protected by the ttm device's lru_lock.290 *291 * Container for the current bulk move state. Should be used with292 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move().293 * All BOs in a bulk_move structure need to share the same reservation object to294 * ensure that the bulk as a whole is locked for eviction even if only one BO of295 * the bulk is evicted.296 */297struct ttm_lru_bulk_move {298 struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY];299 struct list_head cursor_list;300};301 302/**303 * struct ttm_resource_cursor304 * @man: The resource manager currently being iterated over305 * @hitch: A hitch list node inserted before the next resource306 * to iterate over.307 * @bulk_link: A list link for the list of cursors traversing the308 * bulk sublist of @bulk. Protected by the ttm device's lru_lock.309 * @bulk: Pointer to struct ttm_lru_bulk_move whose subrange @hitch is310 * inserted to. NULL if none. Never dereference this pointer since311 * the struct ttm_lru_bulk_move object pointed to might have been312 * freed. The pointer is only for comparison.313 * @mem_type: The memory type of the LRU list being traversed.314 * This field is valid iff @bulk != NULL.315 * @priority: the current priority316 *317 * Cursor to iterate over the resources in a manager.318 */319struct ttm_resource_cursor {320 struct ttm_resource_manager *man;321 struct ttm_lru_item hitch;322 struct list_head bulk_link;323 struct ttm_lru_bulk_move *bulk;324 unsigned int mem_type;325 unsigned int priority;326};327 328void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor);329 330/**331 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping +332 * struct sg_table backed struct ttm_resource.333 * @base: Embedded struct ttm_kmap_iter providing the usage interface.334 * @iomap: struct io_mapping representing the underlying linear io_memory.335 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource.336 * @start: Offset that needs to be subtracted from @st to make337 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.338 * @cache: Scatterlist traversal cache for fast lookups.339 * @cache.sg: Pointer to the currently cached scatterlist segment.340 * @cache.i: First index of @sg. PAGE_SIZE granularity.341 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity.342 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity.343 */344struct ttm_kmap_iter_iomap {345 struct ttm_kmap_iter base;346 struct io_mapping *iomap;347 struct sg_table *st;348 resource_size_t start;349 struct {350 struct scatterlist *sg;351 pgoff_t i;352 pgoff_t end;353 pgoff_t offs;354 } cache;355};356 357/**358 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io359 * @base: The base iterator360 * @dmap: Points to the starting address of the region361 * @needs_unmap: Whether we need to unmap on fini362 */363struct ttm_kmap_iter_linear_io {364 struct ttm_kmap_iter base;365 struct iosys_map dmap;366 bool needs_unmap;367};368 369/**370 * ttm_resource_manager_set_used371 *372 * @man: A memory manager object.373 * @used: usage state to set.374 *375 * Set the manager in use flag. If disabled the manager is no longer376 * used for object placement.377 */378static inline void379ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used)380{381 int i;382 383 for (i = 0; i < TTM_MAX_BO_PRIORITY; i++)384 WARN_ON(!list_empty(&man->lru[i]));385 man->use_type = used;386}387 388/**389 * ttm_resource_manager_used390 *391 * @man: Manager to get used state for392 *393 * Get the in use flag for a manager.394 * Returns:395 * true is used, false if not.396 */397static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man)398{399 return man->use_type;400}401 402/**403 * ttm_resource_manager_cleanup404 *405 * @man: A memory manager object.406 *407 * Cleanup the move fences from the memory manager object.408 */409static inline void410ttm_resource_manager_cleanup(struct ttm_resource_manager *man)411{412 dma_fence_put(man->move);413 man->move = NULL;414}415 416void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk);417void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk);418void ttm_lru_bulk_move_fini(struct ttm_device *bdev,419 struct ttm_lru_bulk_move *bulk);420 421void ttm_resource_add_bulk_move(struct ttm_resource *res,422 struct ttm_buffer_object *bo);423void ttm_resource_del_bulk_move(struct ttm_resource *res,424 struct ttm_buffer_object *bo);425void ttm_resource_move_to_lru_tail(struct ttm_resource *res);426 427void ttm_resource_init(struct ttm_buffer_object *bo,428 const struct ttm_place *place,429 struct ttm_resource *res);430void ttm_resource_fini(struct ttm_resource_manager *man,431 struct ttm_resource *res);432 433int ttm_resource_alloc(struct ttm_buffer_object *bo,434 const struct ttm_place *place,435 struct ttm_resource **res);436void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res);437bool ttm_resource_intersects(struct ttm_device *bdev,438 struct ttm_resource *res,439 const struct ttm_place *place,440 size_t size);441bool ttm_resource_compatible(struct ttm_resource *res,442 struct ttm_placement *placement,443 bool evicting);444void ttm_resource_set_bo(struct ttm_resource *res,445 struct ttm_buffer_object *bo);446 447void ttm_resource_manager_init(struct ttm_resource_manager *man,448 struct ttm_device *bdev,449 uint64_t size);450 451int ttm_resource_manager_evict_all(struct ttm_device *bdev,452 struct ttm_resource_manager *man);453 454uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man);455void ttm_resource_manager_debug(struct ttm_resource_manager *man,456 struct drm_printer *p);457 458struct ttm_resource *459ttm_resource_manager_first(struct ttm_resource_manager *man,460 struct ttm_resource_cursor *cursor);461struct ttm_resource *462ttm_resource_manager_next(struct ttm_resource_cursor *cursor);463 464struct ttm_resource *465ttm_lru_first_res_or_null(struct list_head *head);466 467/**468 * ttm_resource_manager_for_each_res - iterate over all resources469 * @man: the resource manager470 * @cursor: struct ttm_resource_cursor for the current position471 * @res: the current resource472 *473 * Iterate over all the evictable resources in a resource manager.474 */475#define ttm_resource_manager_for_each_res(man, cursor, res) \476 for (res = ttm_resource_manager_first(man, cursor); res; \477 res = ttm_resource_manager_next(cursor))478 479struct ttm_kmap_iter *480ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,481 struct io_mapping *iomap,482 struct sg_table *st,483 resource_size_t start);484 485struct ttm_kmap_iter_linear_io;486 487struct ttm_kmap_iter *488ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,489 struct ttm_device *bdev,490 struct ttm_resource *mem);491 492void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,493 struct ttm_device *bdev,494 struct ttm_resource *mem);495 496void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,497 struct dentry * parent,498 const char *name);499#endif500