brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-only OR MIT2/*3 * Copyright (c) 2022 Red Hat.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice shall be included in13 * all copies or substantial portions of the Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR21 * OTHER DEALINGS IN THE SOFTWARE.22 *23 * Authors:24 *     Danilo Krummrich <dakr@redhat.com>25 *26 */27 28#include <drm/drm_gpuvm.h>29 30#include <linux/interval_tree_generic.h>31#include <linux/mm.h>32 33/**34 * DOC: Overview35 *36 * The DRM GPU VA Manager, represented by struct drm_gpuvm keeps track of a37 * GPU's virtual address (VA) space and manages the corresponding virtual38 * mappings represented by &drm_gpuva objects. It also keeps track of the39 * mapping's backing &drm_gem_object buffers.40 *41 * &drm_gem_object buffers maintain a list of &drm_gpuva objects representing42 * all existent GPU VA mappings using this &drm_gem_object as backing buffer.43 *44 * GPU VAs can be flagged as sparse, such that drivers may use GPU VAs to also45 * keep track of sparse PTEs in order to support Vulkan 'Sparse Resources'.46 *47 * The GPU VA manager internally uses a rb-tree to manage the48 * &drm_gpuva mappings within a GPU's virtual address space.49 *50 * The &drm_gpuvm structure contains a special &drm_gpuva representing the51 * portion of VA space reserved by the kernel. This node is initialized together52 * with the GPU VA manager instance and removed when the GPU VA manager is53 * destroyed.54 *55 * In a typical application drivers would embed struct drm_gpuvm and56 * struct drm_gpuva within their own driver specific structures, there won't be57 * any memory allocations of its own nor memory allocations of &drm_gpuva58 * entries.59 *60 * The data structures needed to store &drm_gpuvas within the &drm_gpuvm are61 * contained within struct drm_gpuva already. Hence, for inserting &drm_gpuva62 * entries from within dma-fence signalling critical sections it is enough to63 * pre-allocate the &drm_gpuva structures.64 *65 * &drm_gem_objects which are private to a single VM can share a common66 * &dma_resv in order to improve locking efficiency (e.g. with &drm_exec).67 * For this purpose drivers must pass a &drm_gem_object to drm_gpuvm_init(), in68 * the following called 'resv object', which serves as the container of the69 * GPUVM's shared &dma_resv. This resv object can be a driver specific70 * &drm_gem_object, such as the &drm_gem_object containing the root page table,71 * but it can also be a 'dummy' object, which can be allocated with72 * drm_gpuvm_resv_object_alloc().73 *74 * In order to connect a struct drm_gpuva its backing &drm_gem_object each75 * &drm_gem_object maintains a list of &drm_gpuvm_bo structures, and each76 * &drm_gpuvm_bo contains a list of &drm_gpuva structures.77 *78 * A &drm_gpuvm_bo is an abstraction that represents a combination of a79 * &drm_gpuvm and a &drm_gem_object. Every such combination should be unique.80 * This is ensured by the API through drm_gpuvm_bo_obtain() and81 * drm_gpuvm_bo_obtain_prealloc() which first look into the corresponding82 * &drm_gem_object list of &drm_gpuvm_bos for an existing instance of this83 * particular combination. If not existent a new instance is created and linked84 * to the &drm_gem_object.85 *86 * &drm_gpuvm_bo structures, since unique for a given &drm_gpuvm, are also used87 * as entry for the &drm_gpuvm's lists of external and evicted objects. Those88 * lists are maintained in order to accelerate locking of dma-resv locks and89 * validation of evicted objects bound in a &drm_gpuvm. For instance, all90 * &drm_gem_object's &dma_resv of a given &drm_gpuvm can be locked by calling91 * drm_gpuvm_exec_lock(). Once locked drivers can call drm_gpuvm_validate() in92 * order to validate all evicted &drm_gem_objects. It is also possible to lock93 * additional &drm_gem_objects by providing the corresponding parameters to94 * drm_gpuvm_exec_lock() as well as open code the &drm_exec loop while making95 * use of helper functions such as drm_gpuvm_prepare_range() or96 * drm_gpuvm_prepare_objects().97 *98 * Every bound &drm_gem_object is treated as external object when its &dma_resv99 * structure is different than the &drm_gpuvm's common &dma_resv structure.100 */101 102/**103 * DOC: Split and Merge104 *105 * Besides its capability to manage and represent a GPU VA space, the106 * GPU VA manager also provides functions to let the &drm_gpuvm calculate a107 * sequence of operations to satisfy a given map or unmap request.108 *109 * Therefore the DRM GPU VA manager provides an algorithm implementing splitting110 * and merging of existent GPU VA mappings with the ones that are requested to111 * be mapped or unmapped. This feature is required by the Vulkan API to112 * implement Vulkan 'Sparse Memory Bindings' - drivers UAPIs often refer to this113 * as VM BIND.114 *115 * Drivers can call drm_gpuvm_sm_map() to receive a sequence of callbacks116 * containing map, unmap and remap operations for a given newly requested117 * mapping. The sequence of callbacks represents the set of operations to118 * execute in order to integrate the new mapping cleanly into the current state119 * of the GPU VA space.120 *121 * Depending on how the new GPU VA mapping intersects with the existent mappings122 * of the GPU VA space the &drm_gpuvm_ops callbacks contain an arbitrary amount123 * of unmap operations, a maximum of two remap operations and a single map124 * operation. The caller might receive no callback at all if no operation is125 * required, e.g. if the requested mapping already exists in the exact same way.126 *127 * The single map operation represents the original map operation requested by128 * the caller.129 *130 * &drm_gpuva_op_unmap contains a 'keep' field, which indicates whether the131 * &drm_gpuva to unmap is physically contiguous with the original mapping132 * request. Optionally, if 'keep' is set, drivers may keep the actual page table133 * entries for this &drm_gpuva, adding the missing page table entries only and134 * update the &drm_gpuvm's view of things accordingly.135 *136 * Drivers may do the same optimization, namely delta page table updates, also137 * for remap operations. This is possible since &drm_gpuva_op_remap consists of138 * one unmap operation and one or two map operations, such that drivers can139 * derive the page table update delta accordingly.140 *141 * Note that there can't be more than two existent mappings to split up, one at142 * the beginning and one at the end of the new mapping, hence there is a143 * maximum of two remap operations.144 *145 * Analogous to drm_gpuvm_sm_map() drm_gpuvm_sm_unmap() uses &drm_gpuvm_ops to146 * call back into the driver in order to unmap a range of GPU VA space. The147 * logic behind this function is way simpler though: For all existent mappings148 * enclosed by the given range unmap operations are created. For mappings which149 * are only partically located within the given range, remap operations are150 * created such that those mappings are split up and re-mapped partically.151 *152 * As an alternative to drm_gpuvm_sm_map() and drm_gpuvm_sm_unmap(),153 * drm_gpuvm_sm_map_ops_create() and drm_gpuvm_sm_unmap_ops_create() can be used154 * to directly obtain an instance of struct drm_gpuva_ops containing a list of155 * &drm_gpuva_op, which can be iterated with drm_gpuva_for_each_op(). This list156 * contains the &drm_gpuva_ops analogous to the callbacks one would receive when157 * calling drm_gpuvm_sm_map() or drm_gpuvm_sm_unmap(). While this way requires158 * more memory (to allocate the &drm_gpuva_ops), it provides drivers a way to159 * iterate the &drm_gpuva_op multiple times, e.g. once in a context where memory160 * allocations are possible (e.g. to allocate GPU page tables) and once in the161 * dma-fence signalling critical path.162 *163 * To update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert() and164 * drm_gpuva_remove() may be used. These functions can safely be used from165 * &drm_gpuvm_ops callbacks originating from drm_gpuvm_sm_map() or166 * drm_gpuvm_sm_unmap(). However, it might be more convenient to use the167 * provided helper functions drm_gpuva_map(), drm_gpuva_remap() and168 * drm_gpuva_unmap() instead.169 *170 * The following diagram depicts the basic relationships of existent GPU VA171 * mappings, a newly requested mapping and the resulting mappings as implemented172 * by drm_gpuvm_sm_map() - it doesn't cover any arbitrary combinations of these.173 *174 * 1) Requested mapping is identical. Replace it, but indicate the backing PTEs175 *    could be kept.176 *177 *    ::178 *179 *	     0     a     1180 *	old: |-----------| (bo_offset=n)181 *182 *	     0     a     1183 *	req: |-----------| (bo_offset=n)184 *185 *	     0     a     1186 *	new: |-----------| (bo_offset=n)187 *188 *189 * 2) Requested mapping is identical, except for the BO offset, hence replace190 *    the mapping.191 *192 *    ::193 *194 *	     0     a     1195 *	old: |-----------| (bo_offset=n)196 *197 *	     0     a     1198 *	req: |-----------| (bo_offset=m)199 *200 *	     0     a     1201 *	new: |-----------| (bo_offset=m)202 *203 *204 * 3) Requested mapping is identical, except for the backing BO, hence replace205 *    the mapping.206 *207 *    ::208 *209 *	     0     a     1210 *	old: |-----------| (bo_offset=n)211 *212 *	     0     b     1213 *	req: |-----------| (bo_offset=n)214 *215 *	     0     b     1216 *	new: |-----------| (bo_offset=n)217 *218 *219 * 4) Existent mapping is a left aligned subset of the requested one, hence220 *    replace the existent one.221 *222 *    ::223 *224 *	     0  a  1225 *	old: |-----|       (bo_offset=n)226 *227 *	     0     a     2228 *	req: |-----------| (bo_offset=n)229 *230 *	     0     a     2231 *	new: |-----------| (bo_offset=n)232 *233 *    .. note::234 *       We expect to see the same result for a request with a different BO235 *       and/or non-contiguous BO offset.236 *237 *238 * 5) Requested mapping's range is a left aligned subset of the existent one,239 *    but backed by a different BO. Hence, map the requested mapping and split240 *    the existent one adjusting its BO offset.241 *242 *    ::243 *244 *	     0     a     2245 *	old: |-----------| (bo_offset=n)246 *247 *	     0  b  1248 *	req: |-----|       (bo_offset=n)249 *250 *	     0  b  1  a' 2251 *	new: |-----|-----| (b.bo_offset=n, a.bo_offset=n+1)252 *253 *    .. note::254 *       We expect to see the same result for a request with a different BO255 *       and/or non-contiguous BO offset.256 *257 *258 * 6) Existent mapping is a superset of the requested mapping. Split it up, but259 *    indicate that the backing PTEs could be kept.260 *261 *    ::262 *263 *	     0     a     2264 *	old: |-----------| (bo_offset=n)265 *266 *	     0  a  1267 *	req: |-----|       (bo_offset=n)268 *269 *	     0  a  1  a' 2270 *	new: |-----|-----| (a.bo_offset=n, a'.bo_offset=n+1)271 *272 *273 * 7) Requested mapping's range is a right aligned subset of the existent one,274 *    but backed by a different BO. Hence, map the requested mapping and split275 *    the existent one, without adjusting the BO offset.276 *277 *    ::278 *279 *	     0     a     2280 *	old: |-----------| (bo_offset=n)281 *282 *	           1  b  2283 *	req:       |-----| (bo_offset=m)284 *285 *	     0  a  1  b  2286 *	new: |-----|-----| (a.bo_offset=n,b.bo_offset=m)287 *288 *289 * 8) Existent mapping is a superset of the requested mapping. Split it up, but290 *    indicate that the backing PTEs could be kept.291 *292 *    ::293 *294 *	      0     a     2295 *	old: |-----------| (bo_offset=n)296 *297 *	           1  a  2298 *	req:       |-----| (bo_offset=n+1)299 *300 *	     0  a' 1  a  2301 *	new: |-----|-----| (a'.bo_offset=n, a.bo_offset=n+1)302 *303 *304 * 9) Existent mapping is overlapped at the end by the requested mapping backed305 *    by a different BO. Hence, map the requested mapping and split up the306 *    existent one, without adjusting the BO offset.307 *308 *    ::309 *310 *	     0     a     2311 *	old: |-----------|       (bo_offset=n)312 *313 *	           1     b     3314 *	req:       |-----------| (bo_offset=m)315 *316 *	     0  a  1     b     3317 *	new: |-----|-----------| (a.bo_offset=n,b.bo_offset=m)318 *319 *320 * 10) Existent mapping is overlapped by the requested mapping, both having the321 *     same backing BO with a contiguous offset. Indicate the backing PTEs of322 *     the old mapping could be kept.323 *324 *     ::325 *326 *	      0     a     2327 *	 old: |-----------|       (bo_offset=n)328 *329 *	            1     a     3330 *	 req:       |-----------| (bo_offset=n+1)331 *332 *	      0  a' 1     a     3333 *	 new: |-----|-----------| (a'.bo_offset=n, a.bo_offset=n+1)334 *335 *336 * 11) Requested mapping's range is a centered subset of the existent one337 *     having a different backing BO. Hence, map the requested mapping and split338 *     up the existent one in two mappings, adjusting the BO offset of the right339 *     one accordingly.340 *341 *     ::342 *343 *	      0        a        3344 *	 old: |-----------------| (bo_offset=n)345 *346 *	            1  b  2347 *	 req:       |-----|       (bo_offset=m)348 *349 *	      0  a  1  b  2  a' 3350 *	 new: |-----|-----|-----| (a.bo_offset=n,b.bo_offset=m,a'.bo_offset=n+2)351 *352 *353 * 12) Requested mapping is a contiguous subset of the existent one. Split it354 *     up, but indicate that the backing PTEs could be kept.355 *356 *     ::357 *358 *	      0        a        3359 *	 old: |-----------------| (bo_offset=n)360 *361 *	            1  a  2362 *	 req:       |-----|       (bo_offset=n+1)363 *364 *	      0  a' 1  a  2 a'' 3365 *	 old: |-----|-----|-----| (a'.bo_offset=n, a.bo_offset=n+1, a''.bo_offset=n+2)366 *367 *368 * 13) Existent mapping is a right aligned subset of the requested one, hence369 *     replace the existent one.370 *371 *     ::372 *373 *	            1  a  2374 *	 old:       |-----| (bo_offset=n+1)375 *376 *	      0     a     2377 *	 req: |-----------| (bo_offset=n)378 *379 *	      0     a     2380 *	 new: |-----------| (bo_offset=n)381 *382 *     .. note::383 *        We expect to see the same result for a request with a different bo384 *        and/or non-contiguous bo_offset.385 *386 *387 * 14) Existent mapping is a centered subset of the requested one, hence388 *     replace the existent one.389 *390 *     ::391 *392 *	            1  a  2393 *	 old:       |-----| (bo_offset=n+1)394 *395 *	      0        a       3396 *	 req: |----------------| (bo_offset=n)397 *398 *	      0        a       3399 *	 new: |----------------| (bo_offset=n)400 *401 *     .. note::402 *        We expect to see the same result for a request with a different bo403 *        and/or non-contiguous bo_offset.404 *405 *406 * 15) Existent mappings is overlapped at the beginning by the requested mapping407 *     backed by a different BO. Hence, map the requested mapping and split up408 *     the existent one, adjusting its BO offset accordingly.409 *410 *     ::411 *412 *	            1     a     3413 *	 old:       |-----------| (bo_offset=n)414 *415 *	      0     b     2416 *	 req: |-----------|       (bo_offset=m)417 *418 *	      0     b     2  a' 3419 *	 new: |-----------|-----| (b.bo_offset=m,a.bo_offset=n+2)420 */421 422/**423 * DOC: Locking424 *425 * In terms of managing &drm_gpuva entries DRM GPUVM does not take care of426 * locking itself, it is the drivers responsibility to take care about locking.427 * Drivers might want to protect the following operations: inserting, removing428 * and iterating &drm_gpuva objects as well as generating all kinds of429 * operations, such as split / merge or prefetch.430 *431 * DRM GPUVM also does not take care of the locking of the backing432 * &drm_gem_object buffers GPU VA lists and &drm_gpuvm_bo abstractions by433 * itself; drivers are responsible to enforce mutual exclusion using either the434 * GEMs dma_resv lock or alternatively a driver specific external lock. For the435 * latter see also drm_gem_gpuva_set_lock().436 *437 * However, DRM GPUVM contains lockdep checks to ensure callers of its API hold438 * the corresponding lock whenever the &drm_gem_objects GPU VA list is accessed439 * by functions such as drm_gpuva_link() or drm_gpuva_unlink(), but also440 * drm_gpuvm_bo_obtain() and drm_gpuvm_bo_put().441 *442 * The latter is required since on creation and destruction of a &drm_gpuvm_bo443 * the &drm_gpuvm_bo is attached / removed from the &drm_gem_objects gpuva list.444 * Subsequent calls to drm_gpuvm_bo_obtain() for the same &drm_gpuvm and445 * &drm_gem_object must be able to observe previous creations and destructions446 * of &drm_gpuvm_bos in order to keep instances unique.447 *448 * The &drm_gpuvm's lists for keeping track of external and evicted objects are449 * protected against concurrent insertion / removal and iteration internally.450 *451 * However, drivers still need ensure to protect concurrent calls to functions452 * iterating those lists, namely drm_gpuvm_prepare_objects() and453 * drm_gpuvm_validate().454 *455 * Alternatively, drivers can set the &DRM_GPUVM_RESV_PROTECTED flag to indicate456 * that the corresponding &dma_resv locks are held in order to protect the457 * lists. If &DRM_GPUVM_RESV_PROTECTED is set, internal locking is disabled and458 * the corresponding lockdep checks are enabled. This is an optimization for459 * drivers which are capable of taking the corresponding &dma_resv locks and460 * hence do not require internal locking.461 */462 463/**464 * DOC: Examples465 *466 * This section gives two examples on how to let the DRM GPUVA Manager generate467 * &drm_gpuva_op in order to satisfy a given map or unmap request and how to468 * make use of them.469 *470 * The below code is strictly limited to illustrate the generic usage pattern.471 * To maintain simplicitly, it doesn't make use of any abstractions for common472 * code, different (asyncronous) stages with fence signalling critical paths,473 * any other helpers or error handling in terms of freeing memory and dropping474 * previously taken locks.475 *476 * 1) Obtain a list of &drm_gpuva_op to create a new mapping::477 *478 *	// Allocates a new &drm_gpuva.479 *	struct drm_gpuva * driver_gpuva_alloc(void);480 *481 *	// Typically drivers would embedd the &drm_gpuvm and &drm_gpuva482 *	// structure in individual driver structures and lock the dma-resv with483 *	// drm_exec or similar helpers.484 *	int driver_mapping_create(struct drm_gpuvm *gpuvm,485 *				  u64 addr, u64 range,486 *				  struct drm_gem_object *obj, u64 offset)487 *	{488 *		struct drm_gpuva_ops *ops;489 *		struct drm_gpuva_op *op490 *		struct drm_gpuvm_bo *vm_bo;491 *492 *		driver_lock_va_space();493 *		ops = drm_gpuvm_sm_map_ops_create(gpuvm, addr, range,494 *						  obj, offset);495 *		if (IS_ERR(ops))496 *			return PTR_ERR(ops);497 *498 *		vm_bo = drm_gpuvm_bo_obtain(gpuvm, obj);499 *		if (IS_ERR(vm_bo))500 *			return PTR_ERR(vm_bo);501 *502 *		drm_gpuva_for_each_op(op, ops) {503 *			struct drm_gpuva *va;504 *505 *			switch (op->op) {506 *			case DRM_GPUVA_OP_MAP:507 *				va = driver_gpuva_alloc();508 *				if (!va)509 *					; // unwind previous VA space updates,510 *					  // free memory and unlock511 *512 *				driver_vm_map();513 *				drm_gpuva_map(gpuvm, va, &op->map);514 *				drm_gpuva_link(va, vm_bo);515 *516 *				break;517 *			case DRM_GPUVA_OP_REMAP: {518 *				struct drm_gpuva *prev = NULL, *next = NULL;519 *520 *				va = op->remap.unmap->va;521 *522 *				if (op->remap.prev) {523 *					prev = driver_gpuva_alloc();524 *					if (!prev)525 *						; // unwind previous VA space526 *						  // updates, free memory and527 *						  // unlock528 *				}529 *530 *				if (op->remap.next) {531 *					next = driver_gpuva_alloc();532 *					if (!next)533 *						; // unwind previous VA space534 *						  // updates, free memory and535 *						  // unlock536 *				}537 *538 *				driver_vm_remap();539 *				drm_gpuva_remap(prev, next, &op->remap);540 *541 *				if (prev)542 *					drm_gpuva_link(prev, va->vm_bo);543 *				if (next)544 *					drm_gpuva_link(next, va->vm_bo);545 *				drm_gpuva_unlink(va);546 *547 *				break;548 *			}549 *			case DRM_GPUVA_OP_UNMAP:550 *				va = op->unmap->va;551 *552 *				driver_vm_unmap();553 *				drm_gpuva_unlink(va);554 *				drm_gpuva_unmap(&op->unmap);555 *556 *				break;557 *			default:558 *				break;559 *			}560 *		}561 *		drm_gpuvm_bo_put(vm_bo);562 *		driver_unlock_va_space();563 *564 *		return 0;565 *	}566 *567 * 2) Receive a callback for each &drm_gpuva_op to create a new mapping::568 *569 *	struct driver_context {570 *		struct drm_gpuvm *gpuvm;571 *		struct drm_gpuvm_bo *vm_bo;572 *		struct drm_gpuva *new_va;573 *		struct drm_gpuva *prev_va;574 *		struct drm_gpuva *next_va;575 *	};576 *577 *	// ops to pass to drm_gpuvm_init()578 *	static const struct drm_gpuvm_ops driver_gpuvm_ops = {579 *		.sm_step_map = driver_gpuva_map,580 *		.sm_step_remap = driver_gpuva_remap,581 *		.sm_step_unmap = driver_gpuva_unmap,582 *	};583 *584 *	// Typically drivers would embedd the &drm_gpuvm and &drm_gpuva585 *	// structure in individual driver structures and lock the dma-resv with586 *	// drm_exec or similar helpers.587 *	int driver_mapping_create(struct drm_gpuvm *gpuvm,588 *				  u64 addr, u64 range,589 *				  struct drm_gem_object *obj, u64 offset)590 *	{591 *		struct driver_context ctx;592 *		struct drm_gpuvm_bo *vm_bo;593 *		struct drm_gpuva_ops *ops;594 *		struct drm_gpuva_op *op;595 *		int ret = 0;596 *597 *		ctx.gpuvm = gpuvm;598 *599 *		ctx.new_va = kzalloc(sizeof(*ctx.new_va), GFP_KERNEL);600 *		ctx.prev_va = kzalloc(sizeof(*ctx.prev_va), GFP_KERNEL);601 *		ctx.next_va = kzalloc(sizeof(*ctx.next_va), GFP_KERNEL);602 *		ctx.vm_bo = drm_gpuvm_bo_create(gpuvm, obj);603 *		if (!ctx.new_va || !ctx.prev_va || !ctx.next_va || !vm_bo) {604 *			ret = -ENOMEM;605 *			goto out;606 *		}607 *608 *		// Typically protected with a driver specific GEM gpuva lock609 *		// used in the fence signaling path for drm_gpuva_link() and610 *		// drm_gpuva_unlink(), hence pre-allocate.611 *		ctx.vm_bo = drm_gpuvm_bo_obtain_prealloc(ctx.vm_bo);612 *613 *		driver_lock_va_space();614 *		ret = drm_gpuvm_sm_map(gpuvm, &ctx, addr, range, obj, offset);615 *		driver_unlock_va_space();616 *617 *	out:618 *		drm_gpuvm_bo_put(ctx.vm_bo);619 *		kfree(ctx.new_va);620 *		kfree(ctx.prev_va);621 *		kfree(ctx.next_va);622 *		return ret;623 *	}624 *625 *	int driver_gpuva_map(struct drm_gpuva_op *op, void *__ctx)626 *	{627 *		struct driver_context *ctx = __ctx;628 *629 *		drm_gpuva_map(ctx->vm, ctx->new_va, &op->map);630 *631 *		drm_gpuva_link(ctx->new_va, ctx->vm_bo);632 *633 *		// prevent the new GPUVA from being freed in634 *		// driver_mapping_create()635 *		ctx->new_va = NULL;636 *637 *		return 0;638 *	}639 *640 *	int driver_gpuva_remap(struct drm_gpuva_op *op, void *__ctx)641 *	{642 *		struct driver_context *ctx = __ctx;643 *		struct drm_gpuva *va = op->remap.unmap->va;644 *645 *		drm_gpuva_remap(ctx->prev_va, ctx->next_va, &op->remap);646 *647 *		if (op->remap.prev) {648 *			drm_gpuva_link(ctx->prev_va, va->vm_bo);649 *			ctx->prev_va = NULL;650 *		}651 *652 *		if (op->remap.next) {653 *			drm_gpuva_link(ctx->next_va, va->vm_bo);654 *			ctx->next_va = NULL;655 *		}656 *657 *		drm_gpuva_unlink(va);658 *		kfree(va);659 *660 *		return 0;661 *	}662 *663 *	int driver_gpuva_unmap(struct drm_gpuva_op *op, void *__ctx)664 *	{665 *		drm_gpuva_unlink(op->unmap.va);666 *		drm_gpuva_unmap(&op->unmap);667 *		kfree(op->unmap.va);668 *669 *		return 0;670 *	}671 */672 673/**674 * get_next_vm_bo_from_list() - get the next vm_bo element675 * @__gpuvm: the &drm_gpuvm676 * @__list_name: the name of the list we're iterating on677 * @__local_list: a pointer to the local list used to store already iterated items678 * @__prev_vm_bo: the previous element we got from get_next_vm_bo_from_list()679 *680 * This helper is here to provide lockless list iteration. Lockless as in, the681 * iterator releases the lock immediately after picking the first element from682 * the list, so list insertion deletion can happen concurrently.683 *684 * Elements popped from the original list are kept in a local list, so removal685 * and is_empty checks can still happen while we're iterating the list.686 */687#define get_next_vm_bo_from_list(__gpuvm, __list_name, __local_list, __prev_vm_bo)	\688	({										\689		struct drm_gpuvm_bo *__vm_bo = NULL;					\690											\691		drm_gpuvm_bo_put(__prev_vm_bo);						\692											\693		spin_lock(&(__gpuvm)->__list_name.lock);				\694		if (!(__gpuvm)->__list_name.local_list)					\695			(__gpuvm)->__list_name.local_list = __local_list;		\696		else									\697			drm_WARN_ON((__gpuvm)->drm,					\698				    (__gpuvm)->__list_name.local_list != __local_list);	\699											\700		while (!list_empty(&(__gpuvm)->__list_name.list)) {			\701			__vm_bo = list_first_entry(&(__gpuvm)->__list_name.list,	\702						   struct drm_gpuvm_bo,			\703						   list.entry.__list_name);		\704			if (kref_get_unless_zero(&__vm_bo->kref)) {			\705				list_move_tail(&(__vm_bo)->list.entry.__list_name,	\706					       __local_list);				\707				break;							\708			} else {							\709				list_del_init(&(__vm_bo)->list.entry.__list_name);	\710				__vm_bo = NULL;						\711			}								\712		}									\713		spin_unlock(&(__gpuvm)->__list_name.lock);				\714											\715		__vm_bo;								\716	})717 718/**719 * for_each_vm_bo_in_list() - internal vm_bo list iterator720 * @__gpuvm: the &drm_gpuvm721 * @__list_name: the name of the list we're iterating on722 * @__local_list: a pointer to the local list used to store already iterated items723 * @__vm_bo: the struct drm_gpuvm_bo to assign in each iteration step724 *725 * This helper is here to provide lockless list iteration. Lockless as in, the726 * iterator releases the lock immediately after picking the first element from the727 * list, hence list insertion and deletion can happen concurrently.728 *729 * It is not allowed to re-assign the vm_bo pointer from inside this loop.730 *731 * Typical use:732 *733 *	struct drm_gpuvm_bo *vm_bo;734 *	LIST_HEAD(my_local_list);735 *736 *	ret = 0;737 *	for_each_vm_bo_in_list(gpuvm, <list_name>, &my_local_list, vm_bo) {738 *		ret = do_something_with_vm_bo(..., vm_bo);739 *		if (ret)740 *			break;741 *	}742 *	// Drop ref in case we break out of the loop.743 *	drm_gpuvm_bo_put(vm_bo);744 *	restore_vm_bo_list(gpuvm, <list_name>, &my_local_list);745 *746 *747 * Only used for internal list iterations, not meant to be exposed to the outside748 * world.749 */750#define for_each_vm_bo_in_list(__gpuvm, __list_name, __local_list, __vm_bo)	\751	for (__vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name,		\752						__local_list, NULL);		\753	     __vm_bo;								\754	     __vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name,		\755						__local_list, __vm_bo))756 757static void758__restore_vm_bo_list(struct drm_gpuvm *gpuvm, spinlock_t *lock,759		     struct list_head *list, struct list_head **local_list)760{761	/* Merge back the two lists, moving local list elements to the762	 * head to preserve previous ordering, in case it matters.763	 */764	spin_lock(lock);765	if (*local_list) {766		list_splice(*local_list, list);767		*local_list = NULL;768	}769	spin_unlock(lock);770}771 772/**773 * restore_vm_bo_list() - move vm_bo elements back to their original list774 * @__gpuvm: the &drm_gpuvm775 * @__list_name: the name of the list we're iterating on776 *777 * When we're done iterating a vm_bo list, we should call restore_vm_bo_list()778 * to restore the original state and let new iterations take place.779 */780#define restore_vm_bo_list(__gpuvm, __list_name)			\781	__restore_vm_bo_list((__gpuvm), &(__gpuvm)->__list_name.lock,	\782			     &(__gpuvm)->__list_name.list,		\783			     &(__gpuvm)->__list_name.local_list)784 785static void786cond_spin_lock(spinlock_t *lock, bool cond)787{788	if (cond)789		spin_lock(lock);790}791 792static void793cond_spin_unlock(spinlock_t *lock, bool cond)794{795	if (cond)796		spin_unlock(lock);797}798 799static void800__drm_gpuvm_bo_list_add(struct drm_gpuvm *gpuvm, spinlock_t *lock,801			struct list_head *entry, struct list_head *list)802{803	cond_spin_lock(lock, !!lock);804	if (list_empty(entry))805		list_add_tail(entry, list);806	cond_spin_unlock(lock, !!lock);807}808 809/**810 * drm_gpuvm_bo_list_add() - insert a vm_bo into the given list811 * @__vm_bo: the &drm_gpuvm_bo812 * @__list_name: the name of the list to insert into813 * @__lock: whether to lock with the internal spinlock814 *815 * Inserts the given @__vm_bo into the list specified by @__list_name.816 */817#define drm_gpuvm_bo_list_add(__vm_bo, __list_name, __lock)			\818	__drm_gpuvm_bo_list_add((__vm_bo)->vm,					\819				__lock ? &(__vm_bo)->vm->__list_name.lock :	\820					 NULL,					\821				&(__vm_bo)->list.entry.__list_name,		\822				&(__vm_bo)->vm->__list_name.list)823 824static void825__drm_gpuvm_bo_list_del(struct drm_gpuvm *gpuvm, spinlock_t *lock,826			struct list_head *entry, bool init)827{828	cond_spin_lock(lock, !!lock);829	if (init) {830		if (!list_empty(entry))831			list_del_init(entry);832	} else {833		list_del(entry);834	}835	cond_spin_unlock(lock, !!lock);836}837 838/**839 * drm_gpuvm_bo_list_del_init() - remove a vm_bo from the given list840 * @__vm_bo: the &drm_gpuvm_bo841 * @__list_name: the name of the list to insert into842 * @__lock: whether to lock with the internal spinlock843 *844 * Removes the given @__vm_bo from the list specified by @__list_name.845 */846#define drm_gpuvm_bo_list_del_init(__vm_bo, __list_name, __lock)		\847	__drm_gpuvm_bo_list_del((__vm_bo)->vm,					\848				__lock ? &(__vm_bo)->vm->__list_name.lock :	\849					 NULL,					\850				&(__vm_bo)->list.entry.__list_name,		\851				true)852 853/**854 * drm_gpuvm_bo_list_del() - remove a vm_bo from the given list855 * @__vm_bo: the &drm_gpuvm_bo856 * @__list_name: the name of the list to insert into857 * @__lock: whether to lock with the internal spinlock858 *859 * Removes the given @__vm_bo from the list specified by @__list_name.860 */861#define drm_gpuvm_bo_list_del(__vm_bo, __list_name, __lock)			\862	__drm_gpuvm_bo_list_del((__vm_bo)->vm,					\863				__lock ? &(__vm_bo)->vm->__list_name.lock :	\864					 NULL,					\865				&(__vm_bo)->list.entry.__list_name,		\866				false)867 868#define to_drm_gpuva(__node)	container_of((__node), struct drm_gpuva, rb.node)869 870#define GPUVA_START(node) ((node)->va.addr)871#define GPUVA_LAST(node) ((node)->va.addr + (node)->va.range - 1)872 873/* We do not actually use drm_gpuva_it_next(), tell the compiler to not complain874 * about this.875 */876INTERVAL_TREE_DEFINE(struct drm_gpuva, rb.node, u64, rb.__subtree_last,877		     GPUVA_START, GPUVA_LAST, static __maybe_unused,878		     drm_gpuva_it)879 880static int __drm_gpuva_insert(struct drm_gpuvm *gpuvm,881			      struct drm_gpuva *va);882static void __drm_gpuva_remove(struct drm_gpuva *va);883 884static bool885drm_gpuvm_check_overflow(u64 addr, u64 range)886{887	u64 end;888 889	return check_add_overflow(addr, range, &end);890}891 892static bool893drm_gpuvm_warn_check_overflow(struct drm_gpuvm *gpuvm, u64 addr, u64 range)894{895	return drm_WARN(gpuvm->drm, drm_gpuvm_check_overflow(addr, range),896			"GPUVA address limited to %zu bytes.\n", sizeof(addr));897}898 899static bool900drm_gpuvm_in_mm_range(struct drm_gpuvm *gpuvm, u64 addr, u64 range)901{902	u64 end = addr + range;903	u64 mm_start = gpuvm->mm_start;904	u64 mm_end = mm_start + gpuvm->mm_range;905 906	return addr >= mm_start && end <= mm_end;907}908 909static bool910drm_gpuvm_in_kernel_node(struct drm_gpuvm *gpuvm, u64 addr, u64 range)911{912	u64 end = addr + range;913	u64 kstart = gpuvm->kernel_alloc_node.va.addr;914	u64 krange = gpuvm->kernel_alloc_node.va.range;915	u64 kend = kstart + krange;916 917	return krange && addr < kend && kstart < end;918}919 920/**921 * drm_gpuvm_range_valid() - checks whether the given range is valid for the922 * given &drm_gpuvm923 * @gpuvm: the GPUVM to check the range for924 * @addr: the base address925 * @range: the range starting from the base address926 *927 * Checks whether the range is within the GPUVM's managed boundaries.928 *929 * Returns: true for a valid range, false otherwise930 */931bool932drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm,933		      u64 addr, u64 range)934{935	return !drm_gpuvm_check_overflow(addr, range) &&936	       drm_gpuvm_in_mm_range(gpuvm, addr, range) &&937	       !drm_gpuvm_in_kernel_node(gpuvm, addr, range);938}939EXPORT_SYMBOL_GPL(drm_gpuvm_range_valid);940 941static void942drm_gpuvm_gem_object_free(struct drm_gem_object *obj)943{944	drm_gem_object_release(obj);945	kfree(obj);946}947 948static const struct drm_gem_object_funcs drm_gpuvm_object_funcs = {949	.free = drm_gpuvm_gem_object_free,950};951 952/**953 * drm_gpuvm_resv_object_alloc() - allocate a dummy &drm_gem_object954 * @drm: the drivers &drm_device955 *956 * Allocates a dummy &drm_gem_object which can be passed to drm_gpuvm_init() in957 * order to serve as root GEM object providing the &drm_resv shared across958 * &drm_gem_objects local to a single GPUVM.959 *960 * Returns: the &drm_gem_object on success, NULL on failure961 */962struct drm_gem_object *963drm_gpuvm_resv_object_alloc(struct drm_device *drm)964{965	struct drm_gem_object *obj;966 967	obj = kzalloc(sizeof(*obj), GFP_KERNEL);968	if (!obj)969		return NULL;970 971	obj->funcs = &drm_gpuvm_object_funcs;972	drm_gem_private_object_init(drm, obj, 0);973 974	return obj;975}976EXPORT_SYMBOL_GPL(drm_gpuvm_resv_object_alloc);977 978/**979 * drm_gpuvm_init() - initialize a &drm_gpuvm980 * @gpuvm: pointer to the &drm_gpuvm to initialize981 * @name: the name of the GPU VA space982 * @flags: the &drm_gpuvm_flags for this GPUVM983 * @drm: the &drm_device this VM resides in984 * @r_obj: the resv &drm_gem_object providing the GPUVM's common &dma_resv985 * @start_offset: the start offset of the GPU VA space986 * @range: the size of the GPU VA space987 * @reserve_offset: the start of the kernel reserved GPU VA area988 * @reserve_range: the size of the kernel reserved GPU VA area989 * @ops: &drm_gpuvm_ops called on &drm_gpuvm_sm_map / &drm_gpuvm_sm_unmap990 *991 * The &drm_gpuvm must be initialized with this function before use.992 *993 * Note that @gpuvm must be cleared to 0 before calling this function. The given994 * &name is expected to be managed by the surrounding driver structures.995 */996void997drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,998	       enum drm_gpuvm_flags flags,999	       struct drm_device *drm,1000	       struct drm_gem_object *r_obj,1001	       u64 start_offset, u64 range,1002	       u64 reserve_offset, u64 reserve_range,1003	       const struct drm_gpuvm_ops *ops)1004{1005	gpuvm->rb.tree = RB_ROOT_CACHED;1006	INIT_LIST_HEAD(&gpuvm->rb.list);1007 1008	INIT_LIST_HEAD(&gpuvm->extobj.list);1009	spin_lock_init(&gpuvm->extobj.lock);1010 1011	INIT_LIST_HEAD(&gpuvm->evict.list);1012	spin_lock_init(&gpuvm->evict.lock);1013 1014	kref_init(&gpuvm->kref);1015 1016	gpuvm->name = name ? name : "unknown";1017	gpuvm->flags = flags;1018	gpuvm->ops = ops;1019	gpuvm->drm = drm;1020	gpuvm->r_obj = r_obj;1021 1022	drm_gem_object_get(r_obj);1023 1024	drm_gpuvm_warn_check_overflow(gpuvm, start_offset, range);1025	gpuvm->mm_start = start_offset;1026	gpuvm->mm_range = range;1027 1028	memset(&gpuvm->kernel_alloc_node, 0, sizeof(struct drm_gpuva));1029	if (reserve_range) {1030		gpuvm->kernel_alloc_node.va.addr = reserve_offset;1031		gpuvm->kernel_alloc_node.va.range = reserve_range;1032 1033		if (likely(!drm_gpuvm_warn_check_overflow(gpuvm, reserve_offset,1034							  reserve_range)))1035			__drm_gpuva_insert(gpuvm, &gpuvm->kernel_alloc_node);1036	}1037}1038EXPORT_SYMBOL_GPL(drm_gpuvm_init);1039 1040static void1041drm_gpuvm_fini(struct drm_gpuvm *gpuvm)1042{1043	gpuvm->name = NULL;1044 1045	if (gpuvm->kernel_alloc_node.va.range)1046		__drm_gpuva_remove(&gpuvm->kernel_alloc_node);1047 1048	drm_WARN(gpuvm->drm, !RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root),1049		 "GPUVA tree is not empty, potentially leaking memory.\n");1050 1051	drm_WARN(gpuvm->drm, !list_empty(&gpuvm->extobj.list),1052		 "Extobj list should be empty.\n");1053	drm_WARN(gpuvm->drm, !list_empty(&gpuvm->evict.list),1054		 "Evict list should be empty.\n");1055 1056	drm_gem_object_put(gpuvm->r_obj);1057}1058 1059static void1060drm_gpuvm_free(struct kref *kref)1061{1062	struct drm_gpuvm *gpuvm = container_of(kref, struct drm_gpuvm, kref);1063 1064	drm_gpuvm_fini(gpuvm);1065 1066	if (drm_WARN_ON(gpuvm->drm, !gpuvm->ops->vm_free))1067		return;1068 1069	gpuvm->ops->vm_free(gpuvm);1070}1071 1072/**1073 * drm_gpuvm_put() - drop a struct drm_gpuvm reference1074 * @gpuvm: the &drm_gpuvm to release the reference of1075 *1076 * This releases a reference to @gpuvm.1077 *1078 * This function may be called from atomic context.1079 */1080void1081drm_gpuvm_put(struct drm_gpuvm *gpuvm)1082{1083	if (gpuvm)1084		kref_put(&gpuvm->kref, drm_gpuvm_free);1085}1086EXPORT_SYMBOL_GPL(drm_gpuvm_put);1087 1088static int1089exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj,1090		 unsigned int num_fences)1091{1092	return num_fences ? drm_exec_prepare_obj(exec, obj, num_fences) :1093			    drm_exec_lock_obj(exec, obj);1094}1095 1096/**1097 * drm_gpuvm_prepare_vm() - prepare the GPUVMs common dma-resv1098 * @gpuvm: the &drm_gpuvm1099 * @exec: the &drm_exec context1100 * @num_fences: the amount of &dma_fences to reserve1101 *1102 * Calls drm_exec_prepare_obj() for the GPUVMs dummy &drm_gem_object; if1103 * @num_fences is zero drm_exec_lock_obj() is called instead.1104 *1105 * Using this function directly, it is the drivers responsibility to call1106 * drm_exec_init() and drm_exec_fini() accordingly.1107 *1108 * Returns: 0 on success, negative error code on failure.1109 */1110int1111drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm,1112		     struct drm_exec *exec,1113		     unsigned int num_fences)1114{1115	return exec_prepare_obj(exec, gpuvm->r_obj, num_fences);1116}1117EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_vm);1118 1119static int1120__drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,1121			    struct drm_exec *exec,1122			    unsigned int num_fences)1123{1124	struct drm_gpuvm_bo *vm_bo;1125	LIST_HEAD(extobjs);1126	int ret = 0;1127 1128	for_each_vm_bo_in_list(gpuvm, extobj, &extobjs, vm_bo) {1129		ret = exec_prepare_obj(exec, vm_bo->obj, num_fences);1130		if (ret)1131			break;1132	}1133	/* Drop ref in case we break out of the loop. */1134	drm_gpuvm_bo_put(vm_bo);1135	restore_vm_bo_list(gpuvm, extobj);1136 1137	return ret;1138}1139 1140static int1141drm_gpuvm_prepare_objects_locked(struct drm_gpuvm *gpuvm,1142				 struct drm_exec *exec,1143				 unsigned int num_fences)1144{1145	struct drm_gpuvm_bo *vm_bo;1146	int ret = 0;1147 1148	drm_gpuvm_resv_assert_held(gpuvm);1149	list_for_each_entry(vm_bo, &gpuvm->extobj.list, list.entry.extobj) {1150		ret = exec_prepare_obj(exec, vm_bo->obj, num_fences);1151		if (ret)1152			break;1153 1154		if (vm_bo->evicted)1155			drm_gpuvm_bo_list_add(vm_bo, evict, false);1156	}1157 1158	return ret;1159}1160 1161/**1162 * drm_gpuvm_prepare_objects() - prepare all assoiciated BOs1163 * @gpuvm: the &drm_gpuvm1164 * @exec: the &drm_exec locking context1165 * @num_fences: the amount of &dma_fences to reserve1166 *1167 * Calls drm_exec_prepare_obj() for all &drm_gem_objects the given1168 * &drm_gpuvm contains mappings of; if @num_fences is zero drm_exec_lock_obj()1169 * is called instead.1170 *1171 * Using this function directly, it is the drivers responsibility to call1172 * drm_exec_init() and drm_exec_fini() accordingly.1173 *1174 * Note: This function is safe against concurrent insertion and removal of1175 * external objects, however it is not safe against concurrent usage itself.1176 *1177 * Drivers need to make sure to protect this case with either an outer VM lock1178 * or by calling drm_gpuvm_prepare_vm() before this function within the1179 * drm_exec_until_all_locked() loop, such that the GPUVM's dma-resv lock ensures1180 * mutual exclusion.1181 *1182 * Returns: 0 on success, negative error code on failure.1183 */1184int1185drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,1186			  struct drm_exec *exec,1187			  unsigned int num_fences)1188{1189	if (drm_gpuvm_resv_protected(gpuvm))1190		return drm_gpuvm_prepare_objects_locked(gpuvm, exec,1191							num_fences);1192	else1193		return __drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);1194}1195EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_objects);1196 1197/**1198 * drm_gpuvm_prepare_range() - prepare all BOs mapped within a given range1199 * @gpuvm: the &drm_gpuvm1200 * @exec: the &drm_exec locking context1201 * @addr: the start address within the VA space1202 * @range: the range to iterate within the VA space1203 * @num_fences: the amount of &dma_fences to reserve1204 *1205 * Calls drm_exec_prepare_obj() for all &drm_gem_objects mapped between @addr1206 * and @addr + @range; if @num_fences is zero drm_exec_lock_obj() is called1207 * instead.1208 *1209 * Returns: 0 on success, negative error code on failure.1210 */1211int1212drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec,1213			u64 addr, u64 range, unsigned int num_fences)1214{1215	struct drm_gpuva *va;1216	u64 end = addr + range;1217	int ret;1218 1219	drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) {1220		struct drm_gem_object *obj = va->gem.obj;1221 1222		ret = exec_prepare_obj(exec, obj, num_fences);1223		if (ret)1224			return ret;1225	}1226 1227	return 0;1228}1229EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_range);1230 1231/**1232 * drm_gpuvm_exec_lock() - lock all dma-resv of all assoiciated BOs1233 * @vm_exec: the &drm_gpuvm_exec wrapper1234 *1235 * Acquires all dma-resv locks of all &drm_gem_objects the given1236 * &drm_gpuvm contains mappings of.1237 *1238 * Addionally, when calling this function with struct drm_gpuvm_exec::extra1239 * being set the driver receives the given @fn callback to lock additional1240 * dma-resv in the context of the &drm_gpuvm_exec instance. Typically, drivers1241 * would call drm_exec_prepare_obj() from within this callback.1242 *1243 * Returns: 0 on success, negative error code on failure.1244 */1245int1246drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec)1247{1248	struct drm_gpuvm *gpuvm = vm_exec->vm;1249	struct drm_exec *exec = &vm_exec->exec;1250	unsigned int num_fences = vm_exec->num_fences;1251	int ret;1252 1253	drm_exec_init(exec, vm_exec->flags, 0);1254 1255	drm_exec_until_all_locked(exec) {1256		ret = drm_gpuvm_prepare_vm(gpuvm, exec, num_fences);1257		drm_exec_retry_on_contention(exec);1258		if (ret)1259			goto err;1260 1261		ret = drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);1262		drm_exec_retry_on_contention(exec);1263		if (ret)1264			goto err;1265 1266		if (vm_exec->extra.fn) {1267			ret = vm_exec->extra.fn(vm_exec);1268			drm_exec_retry_on_contention(exec);1269			if (ret)1270				goto err;1271		}1272	}1273 1274	return 0;1275 1276err:1277	drm_exec_fini(exec);1278	return ret;1279}1280EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock);1281 1282static int1283fn_lock_array(struct drm_gpuvm_exec *vm_exec)1284{1285	struct {1286		struct drm_gem_object **objs;1287		unsigned int num_objs;1288	} *args = vm_exec->extra.priv;1289 1290	return drm_exec_prepare_array(&vm_exec->exec, args->objs,1291				      args->num_objs, vm_exec->num_fences);1292}1293 1294/**1295 * drm_gpuvm_exec_lock_array() - lock all dma-resv of all assoiciated BOs1296 * @vm_exec: the &drm_gpuvm_exec wrapper1297 * @objs: additional &drm_gem_objects to lock1298 * @num_objs: the number of additional &drm_gem_objects to lock1299 *1300 * Acquires all dma-resv locks of all &drm_gem_objects the given &drm_gpuvm1301 * contains mappings of, plus the ones given through @objs.1302 *1303 * Returns: 0 on success, negative error code on failure.1304 */1305int1306drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec,1307			  struct drm_gem_object **objs,1308			  unsigned int num_objs)1309{1310	struct {1311		struct drm_gem_object **objs;1312		unsigned int num_objs;1313	} args;1314 1315	args.objs = objs;1316	args.num_objs = num_objs;1317 1318	vm_exec->extra.fn = fn_lock_array;1319	vm_exec->extra.priv = &args;1320 1321	return drm_gpuvm_exec_lock(vm_exec);1322}1323EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_array);1324 1325/**1326 * drm_gpuvm_exec_lock_range() - prepare all BOs mapped within a given range1327 * @vm_exec: the &drm_gpuvm_exec wrapper1328 * @addr: the start address within the VA space1329 * @range: the range to iterate within the VA space1330 *1331 * Acquires all dma-resv locks of all &drm_gem_objects mapped between @addr and1332 * @addr + @range.1333 *1334 * Returns: 0 on success, negative error code on failure.1335 */1336int1337drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec,1338			  u64 addr, u64 range)1339{1340	struct drm_gpuvm *gpuvm = vm_exec->vm;1341	struct drm_exec *exec = &vm_exec->exec;1342	int ret;1343 1344	drm_exec_init(exec, vm_exec->flags, 0);1345 1346	drm_exec_until_all_locked(exec) {1347		ret = drm_gpuvm_prepare_range(gpuvm, exec, addr, range,1348					      vm_exec->num_fences);1349		drm_exec_retry_on_contention(exec);1350		if (ret)1351			goto err;1352	}1353 1354	return ret;1355 1356err:1357	drm_exec_fini(exec);1358	return ret;1359}1360EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_range);1361 1362static int1363__drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec)1364{1365	const struct drm_gpuvm_ops *ops = gpuvm->ops;1366	struct drm_gpuvm_bo *vm_bo;1367	LIST_HEAD(evict);1368	int ret = 0;1369 1370	for_each_vm_bo_in_list(gpuvm, evict, &evict, vm_bo) {1371		ret = ops->vm_bo_validate(vm_bo, exec);1372		if (ret)1373			break;1374	}1375	/* Drop ref in case we break out of the loop. */1376	drm_gpuvm_bo_put(vm_bo);1377	restore_vm_bo_list(gpuvm, evict);1378 1379	return ret;1380}1381 1382static int1383drm_gpuvm_validate_locked(struct drm_gpuvm *gpuvm, struct drm_exec *exec)1384{1385	const struct drm_gpuvm_ops *ops = gpuvm->ops;1386	struct drm_gpuvm_bo *vm_bo, *next;1387	int ret = 0;1388 1389	drm_gpuvm_resv_assert_held(gpuvm);1390 1391	list_for_each_entry_safe(vm_bo, next, &gpuvm->evict.list,1392				 list.entry.evict) {1393		ret = ops->vm_bo_validate(vm_bo, exec);1394		if (ret)1395			break;1396 1397		dma_resv_assert_held(vm_bo->obj->resv);1398		if (!vm_bo->evicted)1399			drm_gpuvm_bo_list_del_init(vm_bo, evict, false);1400	}1401 1402	return ret;1403}1404 1405/**1406 * drm_gpuvm_validate() - validate all BOs marked as evicted1407 * @gpuvm: the &drm_gpuvm to validate evicted BOs1408 * @exec: the &drm_exec instance used for locking the GPUVM1409 *1410 * Calls the &drm_gpuvm_ops::vm_bo_validate callback for all evicted buffer1411 * objects being mapped in the given &drm_gpuvm.1412 *1413 * Returns: 0 on success, negative error code on failure.1414 */1415int1416drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec)1417{1418	const struct drm_gpuvm_ops *ops = gpuvm->ops;1419 1420	if (unlikely(!ops || !ops->vm_bo_validate))1421		return -EOPNOTSUPP;1422 1423	if (drm_gpuvm_resv_protected(gpuvm))1424		return drm_gpuvm_validate_locked(gpuvm, exec);1425	else1426		return __drm_gpuvm_validate(gpuvm, exec);1427}1428EXPORT_SYMBOL_GPL(drm_gpuvm_validate);1429 1430/**1431 * drm_gpuvm_resv_add_fence - add fence to private and all extobj1432 * dma-resv1433 * @gpuvm: the &drm_gpuvm to add a fence to1434 * @exec: the &drm_exec locking context1435 * @fence: fence to add1436 * @private_usage: private dma-resv usage1437 * @extobj_usage: extobj dma-resv usage1438 */1439void1440drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm,1441			 struct drm_exec *exec,1442			 struct dma_fence *fence,1443			 enum dma_resv_usage private_usage,1444			 enum dma_resv_usage extobj_usage)1445{1446	struct drm_gem_object *obj;1447	unsigned long index;1448 1449	drm_exec_for_each_locked_object(exec, index, obj) {1450		dma_resv_assert_held(obj->resv);1451		dma_resv_add_fence(obj->resv, fence,1452				   drm_gpuvm_is_extobj(gpuvm, obj) ?1453				   extobj_usage : private_usage);1454	}1455}1456EXPORT_SYMBOL_GPL(drm_gpuvm_resv_add_fence);1457 1458/**1459 * drm_gpuvm_bo_create() - create a new instance of struct drm_gpuvm_bo1460 * @gpuvm: The &drm_gpuvm the @obj is mapped in.1461 * @obj: The &drm_gem_object being mapped in the @gpuvm.1462 *1463 * If provided by the driver, this function uses the &drm_gpuvm_ops1464 * vm_bo_alloc() callback to allocate.1465 *1466 * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure1467 */1468struct drm_gpuvm_bo *1469drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm,1470		    struct drm_gem_object *obj)1471{1472	const struct drm_gpuvm_ops *ops = gpuvm->ops;1473	struct drm_gpuvm_bo *vm_bo;1474 1475	if (ops && ops->vm_bo_alloc)1476		vm_bo = ops->vm_bo_alloc();1477	else1478		vm_bo = kzalloc(sizeof(*vm_bo), GFP_KERNEL);1479 1480	if (unlikely(!vm_bo))1481		return NULL;1482 1483	vm_bo->vm = drm_gpuvm_get(gpuvm);1484	vm_bo->obj = obj;1485	drm_gem_object_get(obj);1486 1487	kref_init(&vm_bo->kref);1488	INIT_LIST_HEAD(&vm_bo->list.gpuva);1489	INIT_LIST_HEAD(&vm_bo->list.entry.gem);1490 1491	INIT_LIST_HEAD(&vm_bo->list.entry.extobj);1492	INIT_LIST_HEAD(&vm_bo->list.entry.evict);1493 1494	return vm_bo;1495}1496EXPORT_SYMBOL_GPL(drm_gpuvm_bo_create);1497 1498static void1499drm_gpuvm_bo_destroy(struct kref *kref)1500{1501	struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo,1502						  kref);1503	struct drm_gpuvm *gpuvm = vm_bo->vm;1504	const struct drm_gpuvm_ops *ops = gpuvm->ops;1505	struct drm_gem_object *obj = vm_bo->obj;1506	bool lock = !drm_gpuvm_resv_protected(gpuvm);1507 1508	if (!lock)1509		drm_gpuvm_resv_assert_held(gpuvm);1510 1511	drm_gpuvm_bo_list_del(vm_bo, extobj, lock);1512	drm_gpuvm_bo_list_del(vm_bo, evict, lock);1513 1514	drm_gem_gpuva_assert_lock_held(obj);1515	list_del(&vm_bo->list.entry.gem);1516 1517	if (ops && ops->vm_bo_free)1518		ops->vm_bo_free(vm_bo);1519	else1520		kfree(vm_bo);1521 1522	drm_gpuvm_put(gpuvm);1523	drm_gem_object_put(obj);1524}1525 1526/**1527 * drm_gpuvm_bo_put() - drop a struct drm_gpuvm_bo reference1528 * @vm_bo: the &drm_gpuvm_bo to release the reference of1529 *1530 * This releases a reference to @vm_bo.1531 *1532 * If the reference count drops to zero, the &gpuvm_bo is destroyed, which1533 * includes removing it from the GEMs gpuva list. Hence, if a call to this1534 * function can potentially let the reference count drop to zero the caller must1535 * hold the dma-resv or driver specific GEM gpuva lock.1536 *1537 * This function may only be called from non-atomic context.1538 *1539 * Returns: true if vm_bo was destroyed, false otherwise.1540 */1541bool1542drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo)1543{1544	might_sleep();1545 1546	if (vm_bo)1547		return !!kref_put(&vm_bo->kref, drm_gpuvm_bo_destroy);1548 1549	return false;1550}1551EXPORT_SYMBOL_GPL(drm_gpuvm_bo_put);1552 1553static struct drm_gpuvm_bo *1554__drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,1555		    struct drm_gem_object *obj)1556{1557	struct drm_gpuvm_bo *vm_bo;1558 1559	drm_gem_gpuva_assert_lock_held(obj);1560	drm_gem_for_each_gpuvm_bo(vm_bo, obj)1561		if (vm_bo->vm == gpuvm)1562			return vm_bo;1563 1564	return NULL;1565}1566 1567/**1568 * drm_gpuvm_bo_find() - find the &drm_gpuvm_bo for the given1569 * &drm_gpuvm and &drm_gem_object1570 * @gpuvm: The &drm_gpuvm the @obj is mapped in.1571 * @obj: The &drm_gem_object being mapped in the @gpuvm.1572 *1573 * Find the &drm_gpuvm_bo representing the combination of the given1574 * &drm_gpuvm and &drm_gem_object. If found, increases the reference1575 * count of the &drm_gpuvm_bo accordingly.1576 *1577 * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure1578 */1579struct drm_gpuvm_bo *1580drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,1581		  struct drm_gem_object *obj)1582{1583	struct drm_gpuvm_bo *vm_bo = __drm_gpuvm_bo_find(gpuvm, obj);1584 1585	return vm_bo ? drm_gpuvm_bo_get(vm_bo) : NULL;1586}1587EXPORT_SYMBOL_GPL(drm_gpuvm_bo_find);1588 1589/**1590 * drm_gpuvm_bo_obtain() - obtains and instance of the &drm_gpuvm_bo for the1591 * given &drm_gpuvm and &drm_gem_object1592 * @gpuvm: The &drm_gpuvm the @obj is mapped in.1593 * @obj: The &drm_gem_object being mapped in the @gpuvm.1594 *1595 * Find the &drm_gpuvm_bo representing the combination of the given1596 * &drm_gpuvm and &drm_gem_object. If found, increases the reference1597 * count of the &drm_gpuvm_bo accordingly. If not found, allocates a new1598 * &drm_gpuvm_bo.1599 *1600 * A new &drm_gpuvm_bo is added to the GEMs gpuva list.1601 *1602 * Returns: a pointer to the &drm_gpuvm_bo on success, an ERR_PTR on failure1603 */1604struct drm_gpuvm_bo *1605drm_gpuvm_bo_obtain(struct drm_gpuvm *gpuvm,1606		    struct drm_gem_object *obj)1607{1608	struct drm_gpuvm_bo *vm_bo;1609 1610	vm_bo = drm_gpuvm_bo_find(gpuvm, obj);1611	if (vm_bo)1612		return vm_bo;1613 1614	vm_bo = drm_gpuvm_bo_create(gpuvm, obj);1615	if (!vm_bo)1616		return ERR_PTR(-ENOMEM);1617 1618	drm_gem_gpuva_assert_lock_held(obj);1619	list_add_tail(&vm_bo->list.entry.gem, &obj->gpuva.list);1620 1621	return vm_bo;1622}1623EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain);1624 1625/**1626 * drm_gpuvm_bo_obtain_prealloc() - obtains and instance of the &drm_gpuvm_bo1627 * for the given &drm_gpuvm and &drm_gem_object1628 * @__vm_bo: A pre-allocated struct drm_gpuvm_bo.1629 *1630 * Find the &drm_gpuvm_bo representing the combination of the given1631 * &drm_gpuvm and &drm_gem_object. If found, increases the reference1632 * count of the found &drm_gpuvm_bo accordingly, while the @__vm_bo reference1633 * count is decreased. If not found @__vm_bo is returned without further1634 * increase of the reference count.1635 *1636 * A new &drm_gpuvm_bo is added to the GEMs gpuva list.1637 *1638 * Returns: a pointer to the found &drm_gpuvm_bo or @__vm_bo if no existing1639 * &drm_gpuvm_bo was found1640 */1641struct drm_gpuvm_bo *1642drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *__vm_bo)1643{1644	struct drm_gpuvm *gpuvm = __vm_bo->vm;1645	struct drm_gem_object *obj = __vm_bo->obj;1646	struct drm_gpuvm_bo *vm_bo;1647 1648	vm_bo = drm_gpuvm_bo_find(gpuvm, obj);1649	if (vm_bo) {1650		drm_gpuvm_bo_put(__vm_bo);1651		return vm_bo;1652	}1653 1654	drm_gem_gpuva_assert_lock_held(obj);1655	list_add_tail(&__vm_bo->list.entry.gem, &obj->gpuva.list);1656 1657	return __vm_bo;1658}1659EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain_prealloc);1660 1661/**1662 * drm_gpuvm_bo_extobj_add() - adds the &drm_gpuvm_bo to its &drm_gpuvm's1663 * extobj list1664 * @vm_bo: The &drm_gpuvm_bo to add to its &drm_gpuvm's the extobj list.1665 *1666 * Adds the given @vm_bo to its &drm_gpuvm's extobj list if not on the list1667 * already and if the corresponding &drm_gem_object is an external object,1668 * actually.1669 */1670void1671drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo)1672{1673	struct drm_gpuvm *gpuvm = vm_bo->vm;1674	bool lock = !drm_gpuvm_resv_protected(gpuvm);1675 1676	if (!lock)1677		drm_gpuvm_resv_assert_held(gpuvm);1678 1679	if (drm_gpuvm_is_extobj(gpuvm, vm_bo->obj))1680		drm_gpuvm_bo_list_add(vm_bo, extobj, lock);1681}1682EXPORT_SYMBOL_GPL(drm_gpuvm_bo_extobj_add);1683 1684/**1685 * drm_gpuvm_bo_evict() - add / remove a &drm_gpuvm_bo to / from the &drm_gpuvms1686 * evicted list1687 * @vm_bo: the &drm_gpuvm_bo to add or remove1688 * @evict: indicates whether the object is evicted1689 *1690 * Adds a &drm_gpuvm_bo to or removes it from the &drm_gpuvms evicted list.1691 */1692void1693drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict)1694{1695	struct drm_gpuvm *gpuvm = vm_bo->vm;1696	struct drm_gem_object *obj = vm_bo->obj;1697	bool lock = !drm_gpuvm_resv_protected(gpuvm);1698 1699	dma_resv_assert_held(obj->resv);1700	vm_bo->evicted = evict;1701 1702	/* Can't add external objects to the evicted list directly if not using1703	 * internal spinlocks, since in this case the evicted list is protected1704	 * with the VM's common dma-resv lock.1705	 */1706	if (drm_gpuvm_is_extobj(gpuvm, obj) && !lock)1707		return;1708 1709	if (evict)1710		drm_gpuvm_bo_list_add(vm_bo, evict, lock);1711	else1712		drm_gpuvm_bo_list_del_init(vm_bo, evict, lock);1713}1714EXPORT_SYMBOL_GPL(drm_gpuvm_bo_evict);1715 1716static int1717__drm_gpuva_insert(struct drm_gpuvm *gpuvm,1718		   struct drm_gpuva *va)1719{1720	struct rb_node *node;1721	struct list_head *head;1722 1723	if (drm_gpuva_it_iter_first(&gpuvm->rb.tree,1724				    GPUVA_START(va),1725				    GPUVA_LAST(va)))1726		return -EEXIST;1727 1728	va->vm = gpuvm;1729 1730	drm_gpuva_it_insert(va, &gpuvm->rb.tree);1731 1732	node = rb_prev(&va->rb.node);1733	if (node)1734		head = &(to_drm_gpuva(node))->rb.entry;1735	else1736		head = &gpuvm->rb.list;1737 1738	list_add(&va->rb.entry, head);1739 1740	return 0;1741}1742 1743/**1744 * drm_gpuva_insert() - insert a &drm_gpuva1745 * @gpuvm: the &drm_gpuvm to insert the &drm_gpuva in1746 * @va: the &drm_gpuva to insert1747 *1748 * Insert a &drm_gpuva with a given address and range into a1749 * &drm_gpuvm.1750 *1751 * It is safe to use this function using the safe versions of iterating the GPU1752 * VA space, such as drm_gpuvm_for_each_va_safe() and1753 * drm_gpuvm_for_each_va_range_safe().1754 *1755 * Returns: 0 on success, negative error code on failure.1756 */1757int1758drm_gpuva_insert(struct drm_gpuvm *gpuvm,1759		 struct drm_gpuva *va)1760{1761	u64 addr = va->va.addr;1762	u64 range = va->va.range;1763	int ret;1764 1765	if (unlikely(!drm_gpuvm_range_valid(gpuvm, addr, range)))1766		return -EINVAL;1767 1768	ret = __drm_gpuva_insert(gpuvm, va);1769	if (likely(!ret))1770		/* Take a reference of the GPUVM for the successfully inserted1771		 * drm_gpuva. We can't take the reference in1772		 * __drm_gpuva_insert() itself, since we don't want to increse1773		 * the reference count for the GPUVM's kernel_alloc_node.1774		 */1775		drm_gpuvm_get(gpuvm);1776 1777	return ret;1778}1779EXPORT_SYMBOL_GPL(drm_gpuva_insert);1780 1781static void1782__drm_gpuva_remove(struct drm_gpuva *va)1783{1784	drm_gpuva_it_remove(va, &va->vm->rb.tree);1785	list_del_init(&va->rb.entry);1786}1787 1788/**1789 * drm_gpuva_remove() - remove a &drm_gpuva1790 * @va: the &drm_gpuva to remove1791 *1792 * This removes the given &va from the underlaying tree.1793 *1794 * It is safe to use this function using the safe versions of iterating the GPU1795 * VA space, such as drm_gpuvm_for_each_va_safe() and1796 * drm_gpuvm_for_each_va_range_safe().1797 */1798void1799drm_gpuva_remove(struct drm_gpuva *va)1800{1801	struct drm_gpuvm *gpuvm = va->vm;1802 1803	if (unlikely(va == &gpuvm->kernel_alloc_node)) {1804		drm_WARN(gpuvm->drm, 1,1805			 "Can't destroy kernel reserved node.\n");1806		return;1807	}1808 1809	__drm_gpuva_remove(va);1810	drm_gpuvm_put(va->vm);1811}1812EXPORT_SYMBOL_GPL(drm_gpuva_remove);1813 1814/**1815 * drm_gpuva_link() - link a &drm_gpuva1816 * @va: the &drm_gpuva to link1817 * @vm_bo: the &drm_gpuvm_bo to add the &drm_gpuva to1818 *1819 * This adds the given &va to the GPU VA list of the &drm_gpuvm_bo and the1820 * &drm_gpuvm_bo to the &drm_gem_object it is associated with.1821 *1822 * For every &drm_gpuva entry added to the &drm_gpuvm_bo an additional1823 * reference of the latter is taken.1824 *1825 * This function expects the caller to protect the GEM's GPUVA list against1826 * concurrent access using either the GEMs dma_resv lock or a driver specific1827 * lock set through drm_gem_gpuva_set_lock().1828 */1829void1830drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo)1831{1832	struct drm_gem_object *obj = va->gem.obj;1833	struct drm_gpuvm *gpuvm = va->vm;1834 1835	if (unlikely(!obj))1836		return;1837 1838	drm_WARN_ON(gpuvm->drm, obj != vm_bo->obj);1839 1840	va->vm_bo = drm_gpuvm_bo_get(vm_bo);1841 1842	drm_gem_gpuva_assert_lock_held(obj);1843	list_add_tail(&va->gem.entry, &vm_bo->list.gpuva);1844}1845EXPORT_SYMBOL_GPL(drm_gpuva_link);1846 1847/**1848 * drm_gpuva_unlink() - unlink a &drm_gpuva1849 * @va: the &drm_gpuva to unlink1850 *1851 * This removes the given &va from the GPU VA list of the &drm_gem_object it is1852 * associated with.1853 *1854 * This removes the given &va from the GPU VA list of the &drm_gpuvm_bo and1855 * the &drm_gpuvm_bo from the &drm_gem_object it is associated with in case1856 * this call unlinks the last &drm_gpuva from the &drm_gpuvm_bo.1857 *1858 * For every &drm_gpuva entry removed from the &drm_gpuvm_bo a reference of1859 * the latter is dropped.1860 *1861 * This function expects the caller to protect the GEM's GPUVA list against1862 * concurrent access using either the GEMs dma_resv lock or a driver specific1863 * lock set through drm_gem_gpuva_set_lock().1864 */1865void1866drm_gpuva_unlink(struct drm_gpuva *va)1867{1868	struct drm_gem_object *obj = va->gem.obj;1869	struct drm_gpuvm_bo *vm_bo = va->vm_bo;1870 1871	if (unlikely(!obj))1872		return;1873 1874	drm_gem_gpuva_assert_lock_held(obj);1875	list_del_init(&va->gem.entry);1876 1877	va->vm_bo = NULL;1878	drm_gpuvm_bo_put(vm_bo);1879}1880EXPORT_SYMBOL_GPL(drm_gpuva_unlink);1881 1882/**1883 * drm_gpuva_find_first() - find the first &drm_gpuva in the given range1884 * @gpuvm: the &drm_gpuvm to search in1885 * @addr: the &drm_gpuvas address1886 * @range: the &drm_gpuvas range1887 *1888 * Returns: the first &drm_gpuva within the given range1889 */1890struct drm_gpuva *1891drm_gpuva_find_first(struct drm_gpuvm *gpuvm,1892		     u64 addr, u64 range)1893{1894	u64 last = addr + range - 1;1895 1896	return drm_gpuva_it_iter_first(&gpuvm->rb.tree, addr, last);1897}1898EXPORT_SYMBOL_GPL(drm_gpuva_find_first);1899 1900/**1901 * drm_gpuva_find() - find a &drm_gpuva1902 * @gpuvm: the &drm_gpuvm to search in1903 * @addr: the &drm_gpuvas address1904 * @range: the &drm_gpuvas range1905 *1906 * Returns: the &drm_gpuva at a given &addr and with a given &range1907 */1908struct drm_gpuva *1909drm_gpuva_find(struct drm_gpuvm *gpuvm,1910	       u64 addr, u64 range)1911{1912	struct drm_gpuva *va;1913 1914	va = drm_gpuva_find_first(gpuvm, addr, range);1915	if (!va)1916		goto out;1917 1918	if (va->va.addr != addr ||1919	    va->va.range != range)1920		goto out;1921 1922	return va;1923 1924out:1925	return NULL;1926}1927EXPORT_SYMBOL_GPL(drm_gpuva_find);1928 1929/**1930 * drm_gpuva_find_prev() - find the &drm_gpuva before the given address1931 * @gpuvm: the &drm_gpuvm to search in1932 * @start: the given GPU VA's start address1933 *1934 * Find the adjacent &drm_gpuva before the GPU VA with given &start address.1935 *1936 * Note that if there is any free space between the GPU VA mappings no mapping1937 * is returned.1938 *1939 * Returns: a pointer to the found &drm_gpuva or NULL if none was found1940 */1941struct drm_gpuva *1942drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start)1943{1944	if (!drm_gpuvm_range_valid(gpuvm, start - 1, 1))1945		return NULL;1946 1947	return drm_gpuva_it_iter_first(&gpuvm->rb.tree, start - 1, start);1948}1949EXPORT_SYMBOL_GPL(drm_gpuva_find_prev);1950 1951/**1952 * drm_gpuva_find_next() - find the &drm_gpuva after the given address1953 * @gpuvm: the &drm_gpuvm to search in1954 * @end: the given GPU VA's end address1955 *1956 * Find the adjacent &drm_gpuva after the GPU VA with given &end address.1957 *1958 * Note that if there is any free space between the GPU VA mappings no mapping1959 * is returned.1960 *1961 * Returns: a pointer to the found &drm_gpuva or NULL if none was found1962 */1963struct drm_gpuva *1964drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end)1965{1966	if (!drm_gpuvm_range_valid(gpuvm, end, 1))1967		return NULL;1968 1969	return drm_gpuva_it_iter_first(&gpuvm->rb.tree, end, end + 1);1970}1971EXPORT_SYMBOL_GPL(drm_gpuva_find_next);1972 1973/**1974 * drm_gpuvm_interval_empty() - indicate whether a given interval of the VA space1975 * is empty1976 * @gpuvm: the &drm_gpuvm to check the range for1977 * @addr: the start address of the range1978 * @range: the range of the interval1979 *1980 * Returns: true if the interval is empty, false otherwise1981 */1982bool1983drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range)1984{1985	return !drm_gpuva_find_first(gpuvm, addr, range);1986}1987EXPORT_SYMBOL_GPL(drm_gpuvm_interval_empty);1988 1989/**1990 * drm_gpuva_map() - helper to insert a &drm_gpuva according to a1991 * &drm_gpuva_op_map1992 * @gpuvm: the &drm_gpuvm1993 * @va: the &drm_gpuva to insert1994 * @op: the &drm_gpuva_op_map to initialize @va with1995 *1996 * Initializes the @va from the @op and inserts it into the given @gpuvm.1997 */1998void1999drm_gpuva_map(struct drm_gpuvm *gpuvm,2000	      struct drm_gpuva *va,2001	      struct drm_gpuva_op_map *op)2002{2003	drm_gpuva_init_from_op(va, op);2004	drm_gpuva_insert(gpuvm, va);2005}2006EXPORT_SYMBOL_GPL(drm_gpuva_map);2007 2008/**2009 * drm_gpuva_remap() - helper to remap a &drm_gpuva according to a2010 * &drm_gpuva_op_remap2011 * @prev: the &drm_gpuva to remap when keeping the start of a mapping2012 * @next: the &drm_gpuva to remap when keeping the end of a mapping2013 * @op: the &drm_gpuva_op_remap to initialize @prev and @next with2014 *2015 * Removes the currently mapped &drm_gpuva and remaps it using @prev and/or2016 * @next.2017 */2018void2019drm_gpuva_remap(struct drm_gpuva *prev,2020		struct drm_gpuva *next,2021		struct drm_gpuva_op_remap *op)2022{2023	struct drm_gpuva *va = op->unmap->va;2024	struct drm_gpuvm *gpuvm = va->vm;2025 2026	drm_gpuva_remove(va);2027 2028	if (op->prev) {2029		drm_gpuva_init_from_op(prev, op->prev);2030		drm_gpuva_insert(gpuvm, prev);2031	}2032 2033	if (op->next) {2034		drm_gpuva_init_from_op(next, op->next);2035		drm_gpuva_insert(gpuvm, next);2036	}2037}2038EXPORT_SYMBOL_GPL(drm_gpuva_remap);2039 2040/**2041 * drm_gpuva_unmap() - helper to remove a &drm_gpuva according to a2042 * &drm_gpuva_op_unmap2043 * @op: the &drm_gpuva_op_unmap specifying the &drm_gpuva to remove2044 *2045 * Removes the &drm_gpuva associated with the &drm_gpuva_op_unmap.2046 */2047void2048drm_gpuva_unmap(struct drm_gpuva_op_unmap *op)2049{2050	drm_gpuva_remove(op->va);2051}2052EXPORT_SYMBOL_GPL(drm_gpuva_unmap);2053 2054static int2055op_map_cb(const struct drm_gpuvm_ops *fn, void *priv,2056	  u64 addr, u64 range,2057	  struct drm_gem_object *obj, u64 offset)2058{2059	struct drm_gpuva_op op = {};2060 2061	op.op = DRM_GPUVA_OP_MAP;2062	op.map.va.addr = addr;2063	op.map.va.range = range;2064	op.map.gem.obj = obj;2065	op.map.gem.offset = offset;2066 2067	return fn->sm_step_map(&op, priv);2068}2069 2070static int2071op_remap_cb(const struct drm_gpuvm_ops *fn, void *priv,2072	    struct drm_gpuva_op_map *prev,2073	    struct drm_gpuva_op_map *next,2074	    struct drm_gpuva_op_unmap *unmap)2075{2076	struct drm_gpuva_op op = {};2077	struct drm_gpuva_op_remap *r;2078 2079	op.op = DRM_GPUVA_OP_REMAP;2080	r = &op.remap;2081	r->prev = prev;2082	r->next = next;2083	r->unmap = unmap;2084 2085	return fn->sm_step_remap(&op, priv);2086}2087 2088static int2089op_unmap_cb(const struct drm_gpuvm_ops *fn, void *priv,2090	    struct drm_gpuva *va, bool merge)2091{2092	struct drm_gpuva_op op = {};2093 2094	op.op = DRM_GPUVA_OP_UNMAP;2095	op.unmap.va = va;2096	op.unmap.keep = merge;2097 2098	return fn->sm_step_unmap(&op, priv);2099}2100 2101static int2102__drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm,2103		   const struct drm_gpuvm_ops *ops, void *priv,2104		   u64 req_addr, u64 req_range,2105		   struct drm_gem_object *req_obj, u64 req_offset)2106{2107	struct drm_gpuva *va, *next;2108	u64 req_end = req_addr + req_range;2109	int ret;2110 2111	if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range)))2112		return -EINVAL;2113 2114	drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) {2115		struct drm_gem_object *obj = va->gem.obj;2116		u64 offset = va->gem.offset;2117		u64 addr = va->va.addr;2118		u64 range = va->va.range;2119		u64 end = addr + range;2120		bool merge = !!va->gem.obj;2121 2122		if (addr == req_addr) {2123			merge &= obj == req_obj &&2124				 offset == req_offset;2125 2126			if (end == req_end) {2127				ret = op_unmap_cb(ops, priv, va, merge);2128				if (ret)2129					return ret;2130				break;2131			}2132 2133			if (end < req_end) {2134				ret = op_unmap_cb(ops, priv, va, merge);2135				if (ret)2136					return ret;2137				continue;2138			}2139 2140			if (end > req_end) {2141				struct drm_gpuva_op_map n = {2142					.va.addr = req_end,2143					.va.range = range - req_range,2144					.gem.obj = obj,2145					.gem.offset = offset + req_range,2146				};2147				struct drm_gpuva_op_unmap u = {2148					.va = va,2149					.keep = merge,2150				};2151 2152				ret = op_remap_cb(ops, priv, NULL, &n, &u);2153				if (ret)2154					return ret;2155				break;2156			}2157		} else if (addr < req_addr) {2158			u64 ls_range = req_addr - addr;2159			struct drm_gpuva_op_map p = {2160				.va.addr = addr,2161				.va.range = ls_range,2162				.gem.obj = obj,2163				.gem.offset = offset,2164			};2165			struct drm_gpuva_op_unmap u = { .va = va };2166 2167			merge &= obj == req_obj &&2168				 offset + ls_range == req_offset;2169			u.keep = merge;2170 2171			if (end == req_end) {2172				ret = op_remap_cb(ops, priv, &p, NULL, &u);2173				if (ret)2174					return ret;2175				break;2176			}2177 2178			if (end < req_end) {2179				ret = op_remap_cb(ops, priv, &p, NULL, &u);2180				if (ret)2181					return ret;2182				continue;2183			}2184 2185			if (end > req_end) {2186				struct drm_gpuva_op_map n = {2187					.va.addr = req_end,2188					.va.range = end - req_end,2189					.gem.obj = obj,2190					.gem.offset = offset + ls_range +2191						      req_range,2192				};2193 2194				ret = op_remap_cb(ops, priv, &p, &n, &u);2195				if (ret)2196					return ret;2197				break;2198			}2199		} else if (addr > req_addr) {2200			merge &= obj == req_obj &&2201				 offset == req_offset +2202					   (addr - req_addr);2203 2204			if (end == req_end) {2205				ret = op_unmap_cb(ops, priv, va, merge);2206				if (ret)2207					return ret;2208				break;2209			}2210 2211			if (end < req_end) {2212				ret = op_unmap_cb(ops, priv, va, merge);2213				if (ret)2214					return ret;2215				continue;2216			}2217 2218			if (end > req_end) {2219				struct drm_gpuva_op_map n = {2220					.va.addr = req_end,2221					.va.range = end - req_end,2222					.gem.obj = obj,2223					.gem.offset = offset + req_end - addr,2224				};2225				struct drm_gpuva_op_unmap u = {2226					.va = va,2227					.keep = merge,2228				};2229 2230				ret = op_remap_cb(ops, priv, NULL, &n, &u);2231				if (ret)2232					return ret;2233				break;2234			}2235		}2236	}2237 2238	return op_map_cb(ops, priv,2239			 req_addr, req_range,2240			 req_obj, req_offset);2241}2242 2243static int2244__drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm,2245		     const struct drm_gpuvm_ops *ops, void *priv,2246		     u64 req_addr, u64 req_range)2247{2248	struct drm_gpuva *va, *next;2249	u64 req_end = req_addr + req_range;2250	int ret;2251 2252	if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range)))2253		return -EINVAL;2254 2255	drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) {2256		struct drm_gpuva_op_map prev = {}, next = {};2257		bool prev_split = false, next_split = false;2258		struct drm_gem_object *obj = va->gem.obj;2259		u64 offset = va->gem.offset;2260		u64 addr = va->va.addr;2261		u64 range = va->va.range;2262		u64 end = addr + range;2263 2264		if (addr < req_addr) {2265			prev.va.addr = addr;2266			prev.va.range = req_addr - addr;2267			prev.gem.obj = obj;2268			prev.gem.offset = offset;2269 2270			prev_split = true;2271		}2272 2273		if (end > req_end) {2274			next.va.addr = req_end;2275			next.va.range = end - req_end;2276			next.gem.obj = obj;2277			next.gem.offset = offset + (req_end - addr);2278 2279			next_split = true;2280		}2281 2282		if (prev_split || next_split) {2283			struct drm_gpuva_op_unmap unmap = { .va = va };2284 2285			ret = op_remap_cb(ops, priv,2286					  prev_split ? &prev : NULL,2287					  next_split ? &next : NULL,2288					  &unmap);2289			if (ret)2290				return ret;2291		} else {2292			ret = op_unmap_cb(ops, priv, va, false);2293			if (ret)2294				return ret;2295		}2296	}2297 2298	return 0;2299}2300 2301/**2302 * drm_gpuvm_sm_map() - creates the &drm_gpuva_op split/merge steps2303 * @gpuvm: the &drm_gpuvm representing the GPU VA space2304 * @req_addr: the start address of the new mapping2305 * @req_range: the range of the new mapping2306 * @req_obj: the &drm_gem_object to map2307 * @req_offset: the offset within the &drm_gem_object2308 * @priv: pointer to a driver private data structure2309 *2310 * This function iterates the given range of the GPU VA space. It utilizes the2311 * &drm_gpuvm_ops to call back into the driver providing the split and merge2312 * steps.2313 *2314 * Drivers may use these callbacks to update the GPU VA space right away within2315 * the callback. In case the driver decides to copy and store the operations for2316 * later processing neither this function nor &drm_gpuvm_sm_unmap is allowed to2317 * be called before the &drm_gpuvm's view of the GPU VA space was2318 * updated with the previous set of operations. To update the2319 * &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(),2320 * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be2321 * used.2322 *2323 * A sequence of callbacks can contain map, unmap and remap operations, but2324 * the sequence of callbacks might also be empty if no operation is required,2325 * e.g. if the requested mapping already exists in the exact same way.2326 *2327 * There can be an arbitrary amount of unmap operations, a maximum of two remap2328 * operations and a single map operation. The latter one represents the original2329 * map operation requested by the caller.2330 *2331 * Returns: 0 on success or a negative error code2332 */2333int2334drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv,2335		 u64 req_addr, u64 req_range,2336		 struct drm_gem_object *req_obj, u64 req_offset)2337{2338	const struct drm_gpuvm_ops *ops = gpuvm->ops;2339 2340	if (unlikely(!(ops && ops->sm_step_map &&2341		       ops->sm_step_remap &&2342		       ops->sm_step_unmap)))2343		return -EINVAL;2344 2345	return __drm_gpuvm_sm_map(gpuvm, ops, priv,2346				  req_addr, req_range,2347				  req_obj, req_offset);2348}2349EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map);2350 2351/**2352 * drm_gpuvm_sm_unmap() - creates the &drm_gpuva_ops to split on unmap2353 * @gpuvm: the &drm_gpuvm representing the GPU VA space2354 * @priv: pointer to a driver private data structure2355 * @req_addr: the start address of the range to unmap2356 * @req_range: the range of the mappings to unmap2357 *2358 * This function iterates the given range of the GPU VA space. It utilizes the2359 * &drm_gpuvm_ops to call back into the driver providing the operations to2360 * unmap and, if required, split existent mappings.2361 *2362 * Drivers may use these callbacks to update the GPU VA space right away within2363 * the callback. In case the driver decides to copy and store the operations for2364 * later processing neither this function nor &drm_gpuvm_sm_map is allowed to be2365 * called before the &drm_gpuvm's view of the GPU VA space was updated2366 * with the previous set of operations. To update the &drm_gpuvm's view2367 * of the GPU VA space drm_gpuva_insert(), drm_gpuva_destroy_locked() and/or2368 * drm_gpuva_destroy_unlocked() should be used.2369 *2370 * A sequence of callbacks can contain unmap and remap operations, depending on2371 * whether there are actual overlapping mappings to split.2372 *2373 * There can be an arbitrary amount of unmap operations and a maximum of two2374 * remap operations.2375 *2376 * Returns: 0 on success or a negative error code2377 */2378int2379drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv,2380		   u64 req_addr, u64 req_range)2381{2382	const struct drm_gpuvm_ops *ops = gpuvm->ops;2383 2384	if (unlikely(!(ops && ops->sm_step_remap &&2385		       ops->sm_step_unmap)))2386		return -EINVAL;2387 2388	return __drm_gpuvm_sm_unmap(gpuvm, ops, priv,2389				    req_addr, req_range);2390}2391EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap);2392 2393static struct drm_gpuva_op *2394gpuva_op_alloc(struct drm_gpuvm *gpuvm)2395{2396	const struct drm_gpuvm_ops *fn = gpuvm->ops;2397	struct drm_gpuva_op *op;2398 2399	if (fn && fn->op_alloc)2400		op = fn->op_alloc();2401	else2402		op = kzalloc(sizeof(*op), GFP_KERNEL);2403 2404	if (unlikely(!op))2405		return NULL;2406 2407	return op;2408}2409 2410static void2411gpuva_op_free(struct drm_gpuvm *gpuvm,2412	      struct drm_gpuva_op *op)2413{2414	const struct drm_gpuvm_ops *fn = gpuvm->ops;2415 2416	if (fn && fn->op_free)2417		fn->op_free(op);2418	else2419		kfree(op);2420}2421 2422static int2423drm_gpuva_sm_step(struct drm_gpuva_op *__op,2424		  void *priv)2425{2426	struct {2427		struct drm_gpuvm *vm;2428		struct drm_gpuva_ops *ops;2429	} *args = priv;2430	struct drm_gpuvm *gpuvm = args->vm;2431	struct drm_gpuva_ops *ops = args->ops;2432	struct drm_gpuva_op *op;2433 2434	op = gpuva_op_alloc(gpuvm);2435	if (unlikely(!op))2436		goto err;2437 2438	memcpy(op, __op, sizeof(*op));2439 2440	if (op->op == DRM_GPUVA_OP_REMAP) {2441		struct drm_gpuva_op_remap *__r = &__op->remap;2442		struct drm_gpuva_op_remap *r = &op->remap;2443 2444		r->unmap = kmemdup(__r->unmap, sizeof(*r->unmap),2445				   GFP_KERNEL);2446		if (unlikely(!r->unmap))2447			goto err_free_op;2448 2449		if (__r->prev) {2450			r->prev = kmemdup(__r->prev, sizeof(*r->prev),2451					  GFP_KERNEL);2452			if (unlikely(!r->prev))2453				goto err_free_unmap;2454		}2455 2456		if (__r->next) {2457			r->next = kmemdup(__r->next, sizeof(*r->next),2458					  GFP_KERNEL);2459			if (unlikely(!r->next))2460				goto err_free_prev;2461		}2462	}2463 2464	list_add_tail(&op->entry, &ops->list);2465 2466	return 0;2467 2468err_free_unmap:2469	kfree(op->remap.unmap);2470err_free_prev:2471	kfree(op->remap.prev);2472err_free_op:2473	gpuva_op_free(gpuvm, op);2474err:2475	return -ENOMEM;2476}2477 2478static const struct drm_gpuvm_ops gpuvm_list_ops = {2479	.sm_step_map = drm_gpuva_sm_step,2480	.sm_step_remap = drm_gpuva_sm_step,2481	.sm_step_unmap = drm_gpuva_sm_step,2482};2483 2484/**2485 * drm_gpuvm_sm_map_ops_create() - creates the &drm_gpuva_ops to split and merge2486 * @gpuvm: the &drm_gpuvm representing the GPU VA space2487 * @req_addr: the start address of the new mapping2488 * @req_range: the range of the new mapping2489 * @req_obj: the &drm_gem_object to map2490 * @req_offset: the offset within the &drm_gem_object2491 *2492 * This function creates a list of operations to perform splitting and merging2493 * of existent mapping(s) with the newly requested one.2494 *2495 * The list can be iterated with &drm_gpuva_for_each_op and must be processed2496 * in the given order. It can contain map, unmap and remap operations, but it2497 * also can be empty if no operation is required, e.g. if the requested mapping2498 * already exists is the exact same way.2499 *2500 * There can be an arbitrary amount of unmap operations, a maximum of two remap2501 * operations and a single map operation. The latter one represents the original2502 * map operation requested by the caller.2503 *2504 * Note that before calling this function again with another mapping request it2505 * is necessary to update the &drm_gpuvm's view of the GPU VA space. The2506 * previously obtained operations must be either processed or abandoned. To2507 * update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(),2508 * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be2509 * used.2510 *2511 * After the caller finished processing the returned &drm_gpuva_ops, they must2512 * be freed with &drm_gpuva_ops_free.2513 *2514 * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure2515 */2516struct drm_gpuva_ops *2517drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm,2518			    u64 req_addr, u64 req_range,2519			    struct drm_gem_object *req_obj, u64 req_offset)2520{2521	struct drm_gpuva_ops *ops;2522	struct {2523		struct drm_gpuvm *vm;2524		struct drm_gpuva_ops *ops;2525	} args;2526	int ret;2527 2528	ops = kzalloc(sizeof(*ops), GFP_KERNEL);2529	if (unlikely(!ops))2530		return ERR_PTR(-ENOMEM);2531 2532	INIT_LIST_HEAD(&ops->list);2533 2534	args.vm = gpuvm;2535	args.ops = ops;2536 2537	ret = __drm_gpuvm_sm_map(gpuvm, &gpuvm_list_ops, &args,2538				 req_addr, req_range,2539				 req_obj, req_offset);2540	if (ret)2541		goto err_free_ops;2542 2543	return ops;2544 2545err_free_ops:2546	drm_gpuva_ops_free(gpuvm, ops);2547	return ERR_PTR(ret);2548}2549EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map_ops_create);2550 2551/**2552 * drm_gpuvm_sm_unmap_ops_create() - creates the &drm_gpuva_ops to split on2553 * unmap2554 * @gpuvm: the &drm_gpuvm representing the GPU VA space2555 * @req_addr: the start address of the range to unmap2556 * @req_range: the range of the mappings to unmap2557 *2558 * This function creates a list of operations to perform unmapping and, if2559 * required, splitting of the mappings overlapping the unmap range.2560 *2561 * The list can be iterated with &drm_gpuva_for_each_op and must be processed2562 * in the given order. It can contain unmap and remap operations, depending on2563 * whether there are actual overlapping mappings to split.2564 *2565 * There can be an arbitrary amount of unmap operations and a maximum of two2566 * remap operations.2567 *2568 * Note that before calling this function again with another range to unmap it2569 * is necessary to update the &drm_gpuvm's view of the GPU VA space. The2570 * previously obtained operations must be processed or abandoned. To update the2571 * &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(),2572 * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be2573 * used.2574 *2575 * After the caller finished processing the returned &drm_gpuva_ops, they must2576 * be freed with &drm_gpuva_ops_free.2577 *2578 * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure2579 */2580struct drm_gpuva_ops *2581drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm,2582			      u64 req_addr, u64 req_range)2583{2584	struct drm_gpuva_ops *ops;2585	struct {2586		struct drm_gpuvm *vm;2587		struct drm_gpuva_ops *ops;2588	} args;2589	int ret;2590 2591	ops = kzalloc(sizeof(*ops), GFP_KERNEL);2592	if (unlikely(!ops))2593		return ERR_PTR(-ENOMEM);2594 2595	INIT_LIST_HEAD(&ops->list);2596 2597	args.vm = gpuvm;2598	args.ops = ops;2599 2600	ret = __drm_gpuvm_sm_unmap(gpuvm, &gpuvm_list_ops, &args,2601				   req_addr, req_range);2602	if (ret)2603		goto err_free_ops;2604 2605	return ops;2606 2607err_free_ops:2608	drm_gpuva_ops_free(gpuvm, ops);2609	return ERR_PTR(ret);2610}2611EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap_ops_create);2612 2613/**2614 * drm_gpuvm_prefetch_ops_create() - creates the &drm_gpuva_ops to prefetch2615 * @gpuvm: the &drm_gpuvm representing the GPU VA space2616 * @addr: the start address of the range to prefetch2617 * @range: the range of the mappings to prefetch2618 *2619 * This function creates a list of operations to perform prefetching.2620 *2621 * The list can be iterated with &drm_gpuva_for_each_op and must be processed2622 * in the given order. It can contain prefetch operations.2623 *2624 * There can be an arbitrary amount of prefetch operations.2625 *2626 * After the caller finished processing the returned &drm_gpuva_ops, they must2627 * be freed with &drm_gpuva_ops_free.2628 *2629 * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure2630 */2631struct drm_gpuva_ops *2632drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm,2633			      u64 addr, u64 range)2634{2635	struct drm_gpuva_ops *ops;2636	struct drm_gpuva_op *op;2637	struct drm_gpuva *va;2638	u64 end = addr + range;2639	int ret;2640 2641	ops = kzalloc(sizeof(*ops), GFP_KERNEL);2642	if (!ops)2643		return ERR_PTR(-ENOMEM);2644 2645	INIT_LIST_HEAD(&ops->list);2646 2647	drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) {2648		op = gpuva_op_alloc(gpuvm);2649		if (!op) {2650			ret = -ENOMEM;2651			goto err_free_ops;2652		}2653 2654		op->op = DRM_GPUVA_OP_PREFETCH;2655		op->prefetch.va = va;2656		list_add_tail(&op->entry, &ops->list);2657	}2658 2659	return ops;2660 2661err_free_ops:2662	drm_gpuva_ops_free(gpuvm, ops);2663	return ERR_PTR(ret);2664}2665EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create);2666 2667/**2668 * drm_gpuvm_bo_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM2669 * @vm_bo: the &drm_gpuvm_bo abstraction2670 *2671 * This function creates a list of operations to perform unmapping for every2672 * GPUVA attached to a GEM.2673 *2674 * The list can be iterated with &drm_gpuva_for_each_op and consists out of an2675 * arbitrary amount of unmap operations.2676 *2677 * After the caller finished processing the returned &drm_gpuva_ops, they must2678 * be freed with &drm_gpuva_ops_free.2679 *2680 * It is the callers responsibility to protect the GEMs GPUVA list against2681 * concurrent access using the GEMs dma_resv lock.2682 *2683 * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure2684 */2685struct drm_gpuva_ops *2686drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo)2687{2688	struct drm_gpuva_ops *ops;2689	struct drm_gpuva_op *op;2690	struct drm_gpuva *va;2691	int ret;2692 2693	drm_gem_gpuva_assert_lock_held(vm_bo->obj);2694 2695	ops = kzalloc(sizeof(*ops), GFP_KERNEL);2696	if (!ops)2697		return ERR_PTR(-ENOMEM);2698 2699	INIT_LIST_HEAD(&ops->list);2700 2701	drm_gpuvm_bo_for_each_va(va, vm_bo) {2702		op = gpuva_op_alloc(vm_bo->vm);2703		if (!op) {2704			ret = -ENOMEM;2705			goto err_free_ops;2706		}2707 2708		op->op = DRM_GPUVA_OP_UNMAP;2709		op->unmap.va = va;2710		list_add_tail(&op->entry, &ops->list);2711	}2712 2713	return ops;2714 2715err_free_ops:2716	drm_gpuva_ops_free(vm_bo->vm, ops);2717	return ERR_PTR(ret);2718}2719EXPORT_SYMBOL_GPL(drm_gpuvm_bo_unmap_ops_create);2720 2721/**2722 * drm_gpuva_ops_free() - free the given &drm_gpuva_ops2723 * @gpuvm: the &drm_gpuvm the ops were created for2724 * @ops: the &drm_gpuva_ops to free2725 *2726 * Frees the given &drm_gpuva_ops structure including all the ops associated2727 * with it.2728 */2729void2730drm_gpuva_ops_free(struct drm_gpuvm *gpuvm,2731		   struct drm_gpuva_ops *ops)2732{2733	struct drm_gpuva_op *op, *next;2734 2735	drm_gpuva_for_each_op_safe(op, next, ops) {2736		list_del(&op->entry);2737 2738		if (op->op == DRM_GPUVA_OP_REMAP) {2739			kfree(op->remap.prev);2740			kfree(op->remap.next);2741			kfree(op->remap.unmap);2742		}2743 2744		gpuva_op_free(gpuvm, op);2745	}2746 2747	kfree(ops);2748}2749EXPORT_SYMBOL_GPL(drm_gpuva_ops_free);2750 2751MODULE_DESCRIPTION("DRM GPUVM");2752MODULE_LICENSE("GPL");2753