473 lines · c
1/*2 * SPDX-License-Identifier: MIT3 *4 * Copyright © 2008 Intel Corporation5 */6 7#include <linux/string.h>8#include <linux/bitops.h>9 10#include "i915_drv.h"11#include "i915_gem.h"12#include "i915_gem_ioctls.h"13#include "i915_gem_mman.h"14#include "i915_gem_object.h"15#include "i915_gem_tiling.h"16#include "i915_reg.h"17 18/**19 * DOC: buffer object tiling20 *21 * i915_gem_set_tiling_ioctl() and i915_gem_get_tiling_ioctl() is the userspace22 * interface to declare fence register requirements.23 *24 * In principle GEM doesn't care at all about the internal data layout of an25 * object, and hence it also doesn't care about tiling or swizzling. There's two26 * exceptions:27 *28 * - For X and Y tiling the hardware provides detilers for CPU access, so called29 * fences. Since there's only a limited amount of them the kernel must manage30 * these, and therefore userspace must tell the kernel the object tiling if it31 * wants to use fences for detiling.32 * - On gen3 and gen4 platforms have a swizzling pattern for tiled objects which33 * depends upon the physical page frame number. When swapping such objects the34 * page frame number might change and the kernel must be able to fix this up35 * and hence now the tiling. Note that on a subset of platforms with36 * asymmetric memory channel population the swizzling pattern changes in an37 * unknown way, and for those the kernel simply forbids swapping completely.38 *39 * Since neither of this applies for new tiling layouts on modern platforms like40 * W, Ys and Yf tiling GEM only allows object tiling to be set to X or Y tiled.41 * Anything else can be handled in userspace entirely without the kernel's42 * invovlement.43 */44 45/**46 * i915_gem_fence_size - required global GTT size for a fence47 * @i915: i915 device48 * @size: object size49 * @tiling: tiling mode50 * @stride: tiling stride51 *52 * Return the required global GTT size for a fence (view of a tiled object),53 * taking into account potential fence register mapping.54 */55u32 i915_gem_fence_size(struct drm_i915_private *i915,56 u32 size, unsigned int tiling, unsigned int stride)57{58 u32 ggtt_size;59 60 GEM_BUG_ON(!size);61 62 if (tiling == I915_TILING_NONE)63 return size;64 65 GEM_BUG_ON(!stride);66 67 if (GRAPHICS_VER(i915) >= 4) {68 stride *= i915_gem_tile_height(tiling);69 GEM_BUG_ON(!IS_ALIGNED(stride, I965_FENCE_PAGE));70 return roundup(size, stride);71 }72 73 /* Previous chips need a power-of-two fence region when tiling */74 if (GRAPHICS_VER(i915) == 3)75 ggtt_size = 1024*1024;76 else77 ggtt_size = 512*1024;78 79 while (ggtt_size < size)80 ggtt_size <<= 1;81 82 return ggtt_size;83}84 85/**86 * i915_gem_fence_alignment - required global GTT alignment for a fence87 * @i915: i915 device88 * @size: object size89 * @tiling: tiling mode90 * @stride: tiling stride91 *92 * Return the required global GTT alignment for a fence (a view of a tiled93 * object), taking into account potential fence register mapping.94 */95u32 i915_gem_fence_alignment(struct drm_i915_private *i915, u32 size,96 unsigned int tiling, unsigned int stride)97{98 GEM_BUG_ON(!size);99 100 /*101 * Minimum alignment is 4k (GTT page size), but might be greater102 * if a fence register is needed for the object.103 */104 if (tiling == I915_TILING_NONE)105 return I915_GTT_MIN_ALIGNMENT;106 107 if (GRAPHICS_VER(i915) >= 4)108 return I965_FENCE_PAGE;109 110 /*111 * Previous chips need to be aligned to the size of the smallest112 * fence register that can contain the object.113 */114 return i915_gem_fence_size(i915, size, tiling, stride);115}116 117/* Check pitch constraints for all chips & tiling formats */118static bool119i915_tiling_ok(struct drm_i915_gem_object *obj,120 unsigned int tiling, unsigned int stride)121{122 struct drm_i915_private *i915 = to_i915(obj->base.dev);123 unsigned int tile_width;124 125 /* Linear is always fine */126 if (tiling == I915_TILING_NONE)127 return true;128 129 if (tiling > I915_TILING_LAST)130 return false;131 132 /* check maximum stride & object size */133 /* i965+ stores the end address of the gtt mapping in the fence134 * reg, so dont bother to check the size */135 if (GRAPHICS_VER(i915) >= 7) {136 if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL)137 return false;138 } else if (GRAPHICS_VER(i915) >= 4) {139 if (stride / 128 > I965_FENCE_MAX_PITCH_VAL)140 return false;141 } else {142 if (stride > 8192)143 return false;144 145 if (!is_power_of_2(stride))146 return false;147 }148 149 if (GRAPHICS_VER(i915) == 2 ||150 (tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(i915)))151 tile_width = 128;152 else153 tile_width = 512;154 155 if (!stride || !IS_ALIGNED(stride, tile_width))156 return false;157 158 return true;159}160 161static bool i915_vma_fence_prepare(struct i915_vma *vma,162 int tiling_mode, unsigned int stride)163{164 struct drm_i915_private *i915 = vma->vm->i915;165 u32 size, alignment;166 167 if (!i915_vma_is_map_and_fenceable(vma))168 return true;169 170 size = i915_gem_fence_size(i915, vma->size, tiling_mode, stride);171 if (i915_vma_size(vma) < size)172 return false;173 174 alignment = i915_gem_fence_alignment(i915, vma->size, tiling_mode, stride);175 if (!IS_ALIGNED(i915_ggtt_offset(vma), alignment))176 return false;177 178 return true;179}180 181/* Make the current GTT allocation valid for the change in tiling. */182static int183i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj,184 int tiling_mode, unsigned int stride)185{186 struct drm_i915_private *i915 = to_i915(obj->base.dev);187 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;188 struct i915_vma *vma, *vn;189 LIST_HEAD(unbind);190 int ret = 0;191 192 if (tiling_mode == I915_TILING_NONE)193 return 0;194 195 mutex_lock(&ggtt->vm.mutex);196 197 spin_lock(&obj->vma.lock);198 for_each_ggtt_vma(vma, obj) {199 GEM_BUG_ON(vma->vm != &ggtt->vm);200 201 if (i915_vma_fence_prepare(vma, tiling_mode, stride))202 continue;203 204 list_move(&vma->vm_link, &unbind);205 }206 spin_unlock(&obj->vma.lock);207 208 list_for_each_entry_safe(vma, vn, &unbind, vm_link) {209 ret = __i915_vma_unbind(vma);210 if (ret) {211 /* Restore the remaining vma on an error */212 list_splice(&unbind, &ggtt->vm.bound_list);213 break;214 }215 }216 217 mutex_unlock(&ggtt->vm.mutex);218 219 return ret;220}221 222bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)223{224 struct drm_i915_private *i915 = to_i915(obj->base.dev);225 226 return to_gt(i915)->ggtt->bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&227 i915_gem_object_is_tiled(obj);228}229 230int231i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,232 unsigned int tiling, unsigned int stride)233{234 struct drm_i915_private *i915 = to_i915(obj->base.dev);235 struct i915_vma *vma;236 int err;237 238 /* Make sure we don't cross-contaminate obj->tiling_and_stride */239 BUILD_BUG_ON(I915_TILING_LAST & STRIDE_MASK);240 241 GEM_BUG_ON(!i915_tiling_ok(obj, tiling, stride));242 GEM_BUG_ON(!stride ^ (tiling == I915_TILING_NONE));243 244 if ((tiling | stride) == obj->tiling_and_stride)245 return 0;246 247 if (i915_gem_object_is_framebuffer(obj))248 return -EBUSY;249 250 /* We need to rebind the object if its current allocation251 * no longer meets the alignment restrictions for its new252 * tiling mode. Otherwise we can just leave it alone, but253 * need to ensure that any fence register is updated before254 * the next fenced (either through the GTT or by the BLT unit255 * on older GPUs) access.256 *257 * After updating the tiling parameters, we then flag whether258 * we need to update an associated fence register. Note this259 * has to also include the unfenced register the GPU uses260 * whilst executing a fenced command for an untiled object.261 */262 263 i915_gem_object_lock(obj, NULL);264 if (i915_gem_object_is_framebuffer(obj)) {265 i915_gem_object_unlock(obj);266 return -EBUSY;267 }268 269 err = i915_gem_object_fence_prepare(obj, tiling, stride);270 if (err) {271 i915_gem_object_unlock(obj);272 return err;273 }274 275 /* If the memory has unknown (i.e. varying) swizzling, we pin the276 * pages to prevent them being swapped out and causing corruption277 * due to the change in swizzling.278 */279 if (i915_gem_object_has_pages(obj) &&280 obj->mm.madv == I915_MADV_WILLNEED &&281 i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) {282 if (tiling == I915_TILING_NONE) {283 GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));284 i915_gem_object_clear_tiling_quirk(obj);285 i915_gem_object_make_shrinkable(obj);286 }287 if (!i915_gem_object_is_tiled(obj)) {288 GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));289 i915_gem_object_make_unshrinkable(obj);290 i915_gem_object_set_tiling_quirk(obj);291 }292 }293 294 spin_lock(&obj->vma.lock);295 for_each_ggtt_vma(vma, obj) {296 vma->fence_size =297 i915_gem_fence_size(i915, vma->size, tiling, stride);298 vma->fence_alignment =299 i915_gem_fence_alignment(i915,300 vma->size, tiling, stride);301 302 if (vma->fence)303 vma->fence->dirty = true;304 }305 spin_unlock(&obj->vma.lock);306 307 obj->tiling_and_stride = tiling | stride;308 309 /* Try to preallocate memory required to save swizzling on put-pages */310 if (i915_gem_object_needs_bit17_swizzle(obj)) {311 if (!obj->bit_17) {312 obj->bit_17 = bitmap_zalloc(obj->base.size >> PAGE_SHIFT,313 GFP_KERNEL);314 }315 } else {316 bitmap_free(obj->bit_17);317 obj->bit_17 = NULL;318 }319 320 i915_gem_object_unlock(obj);321 322 /* Force the fence to be reacquired for GTT access */323 i915_gem_object_release_mmap_gtt(obj);324 325 return 0;326}327 328/**329 * i915_gem_set_tiling_ioctl - IOCTL handler to set tiling mode330 * @dev: DRM device331 * @data: data pointer for the ioctl332 * @file: DRM file for the ioctl call333 *334 * Sets the tiling mode of an object, returning the required swizzling of335 * bit 6 of addresses in the object.336 *337 * Called by the user via ioctl.338 *339 * Returns:340 * Zero on success, negative errno on failure.341 */342int343i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,344 struct drm_file *file)345{346 struct drm_i915_private *i915 = to_i915(dev);347 struct drm_i915_gem_set_tiling *args = data;348 struct drm_i915_gem_object *obj;349 int err;350 351 if (!to_gt(i915)->ggtt->num_fences)352 return -EOPNOTSUPP;353 354 obj = i915_gem_object_lookup(file, args->handle);355 if (!obj)356 return -ENOENT;357 358 /*359 * The tiling mode of proxy objects is handled by its generator, and360 * not allowed to be changed by userspace.361 */362 if (i915_gem_object_is_proxy(obj)) {363 err = -ENXIO;364 goto err;365 }366 367 if (!i915_tiling_ok(obj, args->tiling_mode, args->stride)) {368 err = -EINVAL;369 goto err;370 }371 372 if (args->tiling_mode == I915_TILING_NONE) {373 args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;374 args->stride = 0;375 } else {376 if (args->tiling_mode == I915_TILING_X)377 args->swizzle_mode = to_gt(i915)->ggtt->bit_6_swizzle_x;378 else379 args->swizzle_mode = to_gt(i915)->ggtt->bit_6_swizzle_y;380 381 /* Hide bit 17 swizzling from the user. This prevents old Mesa382 * from aborting the application on sw fallbacks to bit 17,383 * and we use the pread/pwrite bit17 paths to swizzle for it.384 * If there was a user that was relying on the swizzle385 * information for drm_intel_bo_map()ed reads/writes this would386 * break it, but we don't have any of those.387 */388 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)389 args->swizzle_mode = I915_BIT_6_SWIZZLE_9;390 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)391 args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;392 393 /* If we can't handle the swizzling, make it untiled. */394 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) {395 args->tiling_mode = I915_TILING_NONE;396 args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;397 args->stride = 0;398 }399 }400 401 err = i915_gem_object_set_tiling(obj, args->tiling_mode, args->stride);402 403 /* We have to maintain this existing ABI... */404 args->stride = i915_gem_object_get_stride(obj);405 args->tiling_mode = i915_gem_object_get_tiling(obj);406 407err:408 i915_gem_object_put(obj);409 return err;410}411 412/**413 * i915_gem_get_tiling_ioctl - IOCTL handler to get tiling mode414 * @dev: DRM device415 * @data: data pointer for the ioctl416 * @file: DRM file for the ioctl call417 *418 * Returns the current tiling mode and required bit 6 swizzling for the object.419 *420 * Called by the user via ioctl.421 *422 * Returns:423 * Zero on success, negative errno on failure.424 */425int426i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data,427 struct drm_file *file)428{429 struct drm_i915_gem_get_tiling *args = data;430 struct drm_i915_private *i915 = to_i915(dev);431 struct drm_i915_gem_object *obj;432 int err = -ENOENT;433 434 if (!to_gt(i915)->ggtt->num_fences)435 return -EOPNOTSUPP;436 437 rcu_read_lock();438 obj = i915_gem_object_lookup_rcu(file, args->handle);439 if (obj) {440 args->tiling_mode =441 READ_ONCE(obj->tiling_and_stride) & TILING_MASK;442 err = 0;443 }444 rcu_read_unlock();445 if (unlikely(err))446 return err;447 448 switch (args->tiling_mode) {449 case I915_TILING_X:450 args->swizzle_mode = to_gt(i915)->ggtt->bit_6_swizzle_x;451 break;452 case I915_TILING_Y:453 args->swizzle_mode = to_gt(i915)->ggtt->bit_6_swizzle_y;454 break;455 default:456 case I915_TILING_NONE:457 args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;458 break;459 }460 461 /* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */462 if (i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES)463 args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN;464 else465 args->phys_swizzle_mode = args->swizzle_mode;466 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)467 args->swizzle_mode = I915_BIT_6_SWIZZLE_9;468 if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)469 args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;470 471 return 0;472}473