415 lines · c
1// SPDX-License-Identifier: GPL-2.0-only OR MIT2/* Copyright (c) 2023 Imagination Technologies Ltd. */3 4#include "pvr_device.h"5#include "pvr_gem.h"6#include "pvr_vm.h"7 8#include <drm/drm_gem.h>9#include <drm/drm_prime.h>10 11#include <linux/compiler.h>12#include <linux/compiler_attributes.h>13#include <linux/dma-buf.h>14#include <linux/dma-direction.h>15#include <linux/dma-mapping.h>16#include <linux/err.h>17#include <linux/gfp.h>18#include <linux/iosys-map.h>19#include <linux/log2.h>20#include <linux/mutex.h>21#include <linux/pagemap.h>22#include <linux/refcount.h>23#include <linux/scatterlist.h>24 25static void pvr_gem_object_free(struct drm_gem_object *obj)26{27 drm_gem_shmem_object_free(obj);28}29 30static int pvr_gem_mmap(struct drm_gem_object *gem_obj, struct vm_area_struct *vma)31{32 struct pvr_gem_object *pvr_obj = gem_to_pvr_gem(gem_obj);33 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj);34 35 if (!(pvr_obj->flags & DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS))36 return -EINVAL;37 38 return drm_gem_shmem_mmap(shmem_obj, vma);39}40 41static const struct drm_gem_object_funcs pvr_gem_object_funcs = {42 .free = pvr_gem_object_free,43 .print_info = drm_gem_shmem_object_print_info,44 .pin = drm_gem_shmem_object_pin,45 .unpin = drm_gem_shmem_object_unpin,46 .get_sg_table = drm_gem_shmem_object_get_sg_table,47 .vmap = drm_gem_shmem_object_vmap,48 .vunmap = drm_gem_shmem_object_vunmap,49 .mmap = pvr_gem_mmap,50 .vm_ops = &drm_gem_shmem_vm_ops,51};52 53/**54 * pvr_gem_object_flags_validate() - Verify that a collection of PowerVR GEM55 * mapping and/or creation flags form a valid combination.56 * @flags: PowerVR GEM mapping/creation flags to validate.57 *58 * This function explicitly allows kernel-only flags. All ioctl entrypoints59 * should do their own validation as well as relying on this function.60 *61 * Return:62 * * %true if @flags contains valid mapping and/or creation flags, or63 * * %false otherwise.64 */65static bool66pvr_gem_object_flags_validate(u64 flags)67{68 static const u64 invalid_combinations[] = {69 /*70 * Memory flagged as PM/FW-protected cannot be mapped to71 * userspace. To make this explicit, we require that the two72 * flags allowing each of these respective features are never73 * specified together.74 */75 (DRM_PVR_BO_PM_FW_PROTECT |76 DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS),77 };78 79 int i;80 81 /*82 * Check for bits set in undefined regions. Reserved regions refer to83 * options that can only be set by the kernel. These are explicitly84 * allowed in most cases, and must be checked specifically in IOCTL85 * callback code.86 */87 if ((flags & PVR_BO_UNDEFINED_MASK) != 0)88 return false;89 90 /*91 * Check for all combinations of flags marked as invalid in the array92 * above.93 */94 for (i = 0; i < ARRAY_SIZE(invalid_combinations); ++i) {95 u64 combo = invalid_combinations[i];96 97 if ((flags & combo) == combo)98 return false;99 }100 101 return true;102}103 104/**105 * pvr_gem_object_into_handle() - Convert a reference to an object into a106 * userspace-accessible handle.107 * @pvr_obj: [IN] Target PowerVR-specific object.108 * @pvr_file: [IN] File to associate the handle with.109 * @handle: [OUT] Pointer to store the created handle in. Remains unmodified if110 * an error is encountered.111 *112 * If an error is encountered, ownership of @pvr_obj will not have been113 * transferred. If this function succeeds, however, further use of @pvr_obj is114 * considered undefined behaviour unless another reference to it is explicitly115 * held.116 *117 * Return:118 * * 0 on success, or119 * * Any error encountered while attempting to allocate a handle on @pvr_file.120 */121int122pvr_gem_object_into_handle(struct pvr_gem_object *pvr_obj,123 struct pvr_file *pvr_file, u32 *handle)124{125 struct drm_gem_object *gem_obj = gem_from_pvr_gem(pvr_obj);126 struct drm_file *file = from_pvr_file(pvr_file);127 128 u32 new_handle;129 int err;130 131 err = drm_gem_handle_create(file, gem_obj, &new_handle);132 if (err)133 return err;134 135 /*136 * Release our reference to @pvr_obj, effectively transferring137 * ownership to the handle.138 */139 pvr_gem_object_put(pvr_obj);140 141 /*142 * Do not store the new handle in @handle until no more errors can143 * occur.144 */145 *handle = new_handle;146 147 return 0;148}149 150/**151 * pvr_gem_object_from_handle() - Obtain a reference to an object from a152 * userspace handle.153 * @pvr_file: PowerVR-specific file to which @handle is associated.154 * @handle: Userspace handle referencing the target object.155 *156 * On return, @handle always maintains its reference to the requested object157 * (if it had one in the first place). If this function succeeds, the returned158 * object will hold an additional reference. When the caller is finished with159 * the returned object, they should call pvr_gem_object_put() on it to release160 * this reference.161 *162 * Return:163 * * A pointer to the requested PowerVR-specific object on success, or164 * * %NULL otherwise.165 */166struct pvr_gem_object *167pvr_gem_object_from_handle(struct pvr_file *pvr_file, u32 handle)168{169 struct drm_file *file = from_pvr_file(pvr_file);170 struct drm_gem_object *gem_obj;171 172 gem_obj = drm_gem_object_lookup(file, handle);173 if (!gem_obj)174 return NULL;175 176 return gem_to_pvr_gem(gem_obj);177}178 179/**180 * pvr_gem_object_vmap() - Map a PowerVR GEM object into CPU virtual address181 * space.182 * @pvr_obj: Target PowerVR GEM object.183 *184 * Once the caller is finished with the CPU mapping, they must call185 * pvr_gem_object_vunmap() on @pvr_obj.186 *187 * If @pvr_obj is CPU-cached, dma_sync_sgtable_for_cpu() is called to make188 * sure the CPU mapping is consistent.189 *190 * Return:191 * * A pointer to the CPU mapping on success,192 * * -%ENOMEM if the mapping fails, or193 * * Any error encountered while attempting to acquire a reference to the194 * backing pages for @pvr_obj.195 */196void *197pvr_gem_object_vmap(struct pvr_gem_object *pvr_obj)198{199 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj);200 struct drm_gem_object *obj = gem_from_pvr_gem(pvr_obj);201 struct iosys_map map;202 int err;203 204 dma_resv_lock(obj->resv, NULL);205 206 err = drm_gem_shmem_vmap(shmem_obj, &map);207 if (err)208 goto err_unlock;209 210 if (pvr_obj->flags & PVR_BO_CPU_CACHED) {211 struct device *dev = shmem_obj->base.dev->dev;212 213 /* If shmem_obj->sgt is NULL, that means the buffer hasn't been mapped214 * in GPU space yet.215 */216 if (shmem_obj->sgt)217 dma_sync_sgtable_for_cpu(dev, shmem_obj->sgt, DMA_BIDIRECTIONAL);218 }219 220 dma_resv_unlock(obj->resv);221 222 return map.vaddr;223 224err_unlock:225 dma_resv_unlock(obj->resv);226 227 return ERR_PTR(err);228}229 230/**231 * pvr_gem_object_vunmap() - Unmap a PowerVR memory object from CPU virtual232 * address space.233 * @pvr_obj: Target PowerVR GEM object.234 *235 * If @pvr_obj is CPU-cached, dma_sync_sgtable_for_device() is called to make236 * sure the GPU mapping is consistent.237 */238void239pvr_gem_object_vunmap(struct pvr_gem_object *pvr_obj)240{241 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj);242 struct iosys_map map = IOSYS_MAP_INIT_VADDR(shmem_obj->vaddr);243 struct drm_gem_object *obj = gem_from_pvr_gem(pvr_obj);244 245 if (WARN_ON(!map.vaddr))246 return;247 248 dma_resv_lock(obj->resv, NULL);249 250 if (pvr_obj->flags & PVR_BO_CPU_CACHED) {251 struct device *dev = shmem_obj->base.dev->dev;252 253 /* If shmem_obj->sgt is NULL, that means the buffer hasn't been mapped254 * in GPU space yet.255 */256 if (shmem_obj->sgt)257 dma_sync_sgtable_for_device(dev, shmem_obj->sgt, DMA_BIDIRECTIONAL);258 }259 260 drm_gem_shmem_vunmap(shmem_obj, &map);261 262 dma_resv_unlock(obj->resv);263}264 265/**266 * pvr_gem_object_zero() - Zeroes the physical memory behind an object.267 * @pvr_obj: Target PowerVR GEM object.268 *269 * Return:270 * * 0 on success, or271 * * Any error encountered while attempting to map @pvr_obj to the CPU (see272 * pvr_gem_object_vmap()).273 */274static int275pvr_gem_object_zero(struct pvr_gem_object *pvr_obj)276{277 void *cpu_ptr;278 279 cpu_ptr = pvr_gem_object_vmap(pvr_obj);280 if (IS_ERR(cpu_ptr))281 return PTR_ERR(cpu_ptr);282 283 memset(cpu_ptr, 0, pvr_gem_object_size(pvr_obj));284 285 /* Make sure the zero-ing is done before vumap-ing the object. */286 wmb();287 288 pvr_gem_object_vunmap(pvr_obj);289 290 return 0;291}292 293/**294 * pvr_gem_create_object() - Allocate and pre-initializes a pvr_gem_object295 * @drm_dev: DRM device creating this object.296 * @size: Size of the object to allocate in bytes.297 *298 * Return:299 * * The new pre-initialized GEM object on success,300 * * -ENOMEM if the allocation failed.301 */302struct drm_gem_object *pvr_gem_create_object(struct drm_device *drm_dev, size_t size)303{304 struct drm_gem_object *gem_obj;305 struct pvr_gem_object *pvr_obj;306 307 pvr_obj = kzalloc(sizeof(*pvr_obj), GFP_KERNEL);308 if (!pvr_obj)309 return ERR_PTR(-ENOMEM);310 311 gem_obj = gem_from_pvr_gem(pvr_obj);312 gem_obj->funcs = &pvr_gem_object_funcs;313 314 return gem_obj;315}316 317/**318 * pvr_gem_object_create() - Creates a PowerVR-specific buffer object.319 * @pvr_dev: Target PowerVR device.320 * @size: Size of the object to allocate in bytes. Must be greater than zero.321 * Any value which is not an exact multiple of the system page size will be322 * rounded up to satisfy this condition.323 * @flags: Options which affect both this operation and future mapping324 * operations performed on the returned object. Must be a combination of325 * DRM_PVR_BO_* and/or PVR_BO_* flags.326 *327 * The created object may be larger than @size, but can never be smaller. To328 * get the exact size, call pvr_gem_object_size() on the returned pointer.329 *330 * Return:331 * * The newly-minted PowerVR-specific buffer object on success,332 * * -%EINVAL if @size is zero or @flags is not valid,333 * * -%ENOMEM if sufficient physical memory cannot be allocated, or334 * * Any other error returned by drm_gem_create_mmap_offset().335 */336struct pvr_gem_object *337pvr_gem_object_create(struct pvr_device *pvr_dev, size_t size, u64 flags)338{339 struct drm_gem_shmem_object *shmem_obj;340 struct pvr_gem_object *pvr_obj;341 struct sg_table *sgt;342 int err;343 344 /* Verify @size and @flags before continuing. */345 if (size == 0 || !pvr_gem_object_flags_validate(flags))346 return ERR_PTR(-EINVAL);347 348 shmem_obj = drm_gem_shmem_create(from_pvr_device(pvr_dev), size);349 if (IS_ERR(shmem_obj))350 return ERR_CAST(shmem_obj);351 352 shmem_obj->pages_mark_dirty_on_put = true;353 shmem_obj->map_wc = !(flags & PVR_BO_CPU_CACHED);354 pvr_obj = shmem_gem_to_pvr_gem(shmem_obj);355 pvr_obj->flags = flags;356 357 sgt = drm_gem_shmem_get_pages_sgt(shmem_obj);358 if (IS_ERR(sgt)) {359 err = PTR_ERR(sgt);360 goto err_shmem_object_free;361 }362 363 dma_sync_sgtable_for_device(shmem_obj->base.dev->dev, sgt,364 DMA_BIDIRECTIONAL);365 366 /*367 * Do this last because pvr_gem_object_zero() requires a fully368 * configured instance of struct pvr_gem_object.369 */370 pvr_gem_object_zero(pvr_obj);371 372 return pvr_obj;373 374err_shmem_object_free:375 drm_gem_shmem_free(shmem_obj);376 377 return ERR_PTR(err);378}379 380/**381 * pvr_gem_get_dma_addr() - Get DMA address for given offset in object382 * @pvr_obj: Pointer to object to lookup address in.383 * @offset: Offset within object to lookup address at.384 * @dma_addr_out: Pointer to location to store DMA address.385 *386 * Returns:387 * * 0 on success, or388 * * -%EINVAL if object is not currently backed, or if @offset is out of valid389 * range for this object.390 */391int392pvr_gem_get_dma_addr(struct pvr_gem_object *pvr_obj, u32 offset,393 dma_addr_t *dma_addr_out)394{395 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj);396 u32 accumulated_offset = 0;397 struct scatterlist *sgl;398 unsigned int sgt_idx;399 400 WARN_ON(!shmem_obj->sgt);401 for_each_sgtable_dma_sg(shmem_obj->sgt, sgl, sgt_idx) {402 u32 new_offset = accumulated_offset + sg_dma_len(sgl);403 404 if (offset >= accumulated_offset && offset < new_offset) {405 *dma_addr_out = sg_dma_address(sgl) +406 (offset - accumulated_offset);407 return 0;408 }409 410 accumulated_offset = new_offset;411 }412 413 return -EINVAL;414}415