1625 lines · c
1/*2 * Copyright © 2013 Intel Corporation3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice (including the next12 * paragraph) shall be included in all copies or substantial portions of the13 * Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS21 * IN THE SOFTWARE.22 *23 * Authors:24 * Brad Volkin <bradley.d.volkin@intel.com>25 *26 */27 28#include <linux/highmem.h>29 30#include <drm/drm_cache.h>31 32#include "gt/intel_engine.h"33#include "gt/intel_engine_regs.h"34#include "gt/intel_gpu_commands.h"35#include "gt/intel_gt_regs.h"36 37#include "i915_cmd_parser.h"38#include "i915_drv.h"39#include "i915_memcpy.h"40#include "i915_reg.h"41 42/**43 * DOC: batch buffer command parser44 *45 * Motivation:46 * Certain OpenGL features (e.g. transform feedback, performance monitoring)47 * require userspace code to submit batches containing commands such as48 * MI_LOAD_REGISTER_IMM to access various registers. Unfortunately, some49 * generations of the hardware will noop these commands in "unsecure" batches50 * (which includes all userspace batches submitted via i915) even though the51 * commands may be safe and represent the intended programming model of the52 * device.53 *54 * The software command parser is similar in operation to the command parsing55 * done in hardware for unsecure batches. However, the software parser allows56 * some operations that would be noop'd by hardware, if the parser determines57 * the operation is safe, and submits the batch as "secure" to prevent hardware58 * parsing.59 *60 * Threats:61 * At a high level, the hardware (and software) checks attempt to prevent62 * granting userspace undue privileges. There are three categories of privilege.63 *64 * First, commands which are explicitly defined as privileged or which should65 * only be used by the kernel driver. The parser rejects such commands66 *67 * Second, commands which access registers. To support correct/enhanced68 * userspace functionality, particularly certain OpenGL extensions, the parser69 * provides a whitelist of registers which userspace may safely access70 *71 * Third, commands which access privileged memory (i.e. GGTT, HWS page, etc).72 * The parser always rejects such commands.73 *74 * The majority of the problematic commands fall in the MI_* range, with only a75 * few specific commands on each engine (e.g. PIPE_CONTROL and MI_FLUSH_DW).76 *77 * Implementation:78 * Each engine maintains tables of commands and registers which the parser79 * uses in scanning batch buffers submitted to that engine.80 *81 * Since the set of commands that the parser must check for is significantly82 * smaller than the number of commands supported, the parser tables contain only83 * those commands required by the parser. This generally works because command84 * opcode ranges have standard command length encodings. So for commands that85 * the parser does not need to check, it can easily skip them. This is86 * implemented via a per-engine length decoding vfunc.87 *88 * Unfortunately, there are a number of commands that do not follow the standard89 * length encoding for their opcode range, primarily amongst the MI_* commands.90 * To handle this, the parser provides a way to define explicit "skip" entries91 * in the per-engine command tables.92 *93 * Other command table entries map fairly directly to high level categories94 * mentioned above: rejected, register whitelist. The parser implements a number95 * of checks, including the privileged memory checks, via a general bitmasking96 * mechanism.97 */98 99/*100 * A command that requires special handling by the command parser.101 */102struct drm_i915_cmd_descriptor {103 /*104 * Flags describing how the command parser processes the command.105 *106 * CMD_DESC_FIXED: The command has a fixed length if this is set,107 * a length mask if not set108 * CMD_DESC_SKIP: The command is allowed but does not follow the109 * standard length encoding for the opcode range in110 * which it falls111 * CMD_DESC_REJECT: The command is never allowed112 * CMD_DESC_REGISTER: The command should be checked against the113 * register whitelist for the appropriate ring114 */115 u32 flags;116#define CMD_DESC_FIXED (1<<0)117#define CMD_DESC_SKIP (1<<1)118#define CMD_DESC_REJECT (1<<2)119#define CMD_DESC_REGISTER (1<<3)120#define CMD_DESC_BITMASK (1<<4)121 122 /*123 * The command's unique identification bits and the bitmask to get them.124 * This isn't strictly the opcode field as defined in the spec and may125 * also include type, subtype, and/or subop fields.126 */127 struct {128 u32 value;129 u32 mask;130 } cmd;131 132 /*133 * The command's length. The command is either fixed length (i.e. does134 * not include a length field) or has a length field mask. The flag135 * CMD_DESC_FIXED indicates a fixed length. Otherwise, the command has136 * a length mask. All command entries in a command table must include137 * length information.138 */139 union {140 u32 fixed;141 u32 mask;142 } length;143 144 /*145 * Describes where to find a register address in the command to check146 * against the ring's register whitelist. Only valid if flags has the147 * CMD_DESC_REGISTER bit set.148 *149 * A non-zero step value implies that the command may access multiple150 * registers in sequence (e.g. LRI), in that case step gives the151 * distance in dwords between individual offset fields.152 */153 struct {154 u32 offset;155 u32 mask;156 u32 step;157 } reg;158 159#define MAX_CMD_DESC_BITMASKS 3160 /*161 * Describes command checks where a particular dword is masked and162 * compared against an expected value. If the command does not match163 * the expected value, the parser rejects it. Only valid if flags has164 * the CMD_DESC_BITMASK bit set. Only entries where mask is non-zero165 * are valid.166 *167 * If the check specifies a non-zero condition_mask then the parser168 * only performs the check when the bits specified by condition_mask169 * are non-zero.170 */171 struct {172 u32 offset;173 u32 mask;174 u32 expected;175 u32 condition_offset;176 u32 condition_mask;177 } bits[MAX_CMD_DESC_BITMASKS];178};179 180/*181 * A table of commands requiring special handling by the command parser.182 *183 * Each engine has an array of tables. Each table consists of an array of184 * command descriptors, which must be sorted with command opcodes in185 * ascending order.186 */187struct drm_i915_cmd_table {188 const struct drm_i915_cmd_descriptor *table;189 int count;190};191 192#define STD_MI_OPCODE_SHIFT (32 - 9)193#define STD_3D_OPCODE_SHIFT (32 - 16)194#define STD_2D_OPCODE_SHIFT (32 - 10)195#define STD_MFX_OPCODE_SHIFT (32 - 16)196#define MIN_OPCODE_SHIFT 16197 198#define CMD(op, opm, f, lm, fl, ...) \199 { \200 .flags = (fl) | ((f) ? CMD_DESC_FIXED : 0), \201 .cmd = { (op & ~0u << (opm)), ~0u << (opm) }, \202 .length = { (lm) }, \203 __VA_ARGS__ \204 }205 206/* Convenience macros to compress the tables */207#define SMI STD_MI_OPCODE_SHIFT208#define S3D STD_3D_OPCODE_SHIFT209#define S2D STD_2D_OPCODE_SHIFT210#define SMFX STD_MFX_OPCODE_SHIFT211#define F true212#define S CMD_DESC_SKIP213#define R CMD_DESC_REJECT214#define W CMD_DESC_REGISTER215#define B CMD_DESC_BITMASK216 217/* Command Mask Fixed Len Action218 ---------------------------------------------------------- */219static const struct drm_i915_cmd_descriptor gen7_common_cmds[] = {220 CMD( MI_NOOP, SMI, F, 1, S ),221 CMD( MI_USER_INTERRUPT, SMI, F, 1, R ),222 CMD( MI_WAIT_FOR_EVENT, SMI, F, 1, R ),223 CMD( MI_ARB_CHECK, SMI, F, 1, S ),224 CMD( MI_REPORT_HEAD, SMI, F, 1, S ),225 CMD( MI_SUSPEND_FLUSH, SMI, F, 1, S ),226 CMD( MI_SEMAPHORE_MBOX, SMI, !F, 0xFF, R ),227 CMD( MI_STORE_DWORD_INDEX, SMI, !F, 0xFF, R ),228 CMD( MI_LOAD_REGISTER_IMM(1), SMI, !F, 0xFF, W,229 .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 2 } ),230 CMD( MI_STORE_REGISTER_MEM, SMI, F, 3, W | B,231 .reg = { .offset = 1, .mask = 0x007FFFFC },232 .bits = {{233 .offset = 0,234 .mask = MI_GLOBAL_GTT,235 .expected = 0,236 }}, ),237 CMD( MI_LOAD_REGISTER_MEM, SMI, F, 3, W | B,238 .reg = { .offset = 1, .mask = 0x007FFFFC },239 .bits = {{240 .offset = 0,241 .mask = MI_GLOBAL_GTT,242 .expected = 0,243 }}, ),244 /*245 * MI_BATCH_BUFFER_START requires some special handling. It's not246 * really a 'skip' action but it doesn't seem like it's worth adding247 * a new action. See intel_engine_cmd_parser().248 */249 CMD( MI_BATCH_BUFFER_START, SMI, !F, 0xFF, S ),250};251 252static const struct drm_i915_cmd_descriptor gen7_render_cmds[] = {253 CMD( MI_FLUSH, SMI, F, 1, S ),254 CMD( MI_ARB_ON_OFF, SMI, F, 1, R ),255 CMD( MI_PREDICATE, SMI, F, 1, S ),256 CMD( MI_TOPOLOGY_FILTER, SMI, F, 1, S ),257 CMD( MI_SET_APPID, SMI, F, 1, S ),258 CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ),259 CMD( MI_SET_CONTEXT, SMI, !F, 0xFF, R ),260 CMD( MI_URB_CLEAR, SMI, !F, 0xFF, S ),261 CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3F, B,262 .bits = {{263 .offset = 0,264 .mask = MI_GLOBAL_GTT,265 .expected = 0,266 }}, ),267 CMD( MI_UPDATE_GTT, SMI, !F, 0xFF, R ),268 CMD( MI_CLFLUSH, SMI, !F, 0x3FF, B,269 .bits = {{270 .offset = 0,271 .mask = MI_GLOBAL_GTT,272 .expected = 0,273 }}, ),274 CMD( MI_REPORT_PERF_COUNT, SMI, !F, 0x3F, B,275 .bits = {{276 .offset = 1,277 .mask = MI_REPORT_PERF_COUNT_GGTT,278 .expected = 0,279 }}, ),280 CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B,281 .bits = {{282 .offset = 0,283 .mask = MI_GLOBAL_GTT,284 .expected = 0,285 }}, ),286 CMD( GFX_OP_3DSTATE_VF_STATISTICS, S3D, F, 1, S ),287 CMD( PIPELINE_SELECT, S3D, F, 1, S ),288 CMD( MEDIA_VFE_STATE, S3D, !F, 0xFFFF, B,289 .bits = {{290 .offset = 2,291 .mask = MEDIA_VFE_STATE_MMIO_ACCESS_MASK,292 .expected = 0,293 }}, ),294 CMD( GPGPU_OBJECT, S3D, !F, 0xFF, S ),295 CMD( GPGPU_WALKER, S3D, !F, 0xFF, S ),296 CMD( GFX_OP_3DSTATE_SO_DECL_LIST, S3D, !F, 0x1FF, S ),297 CMD( GFX_OP_PIPE_CONTROL(5), S3D, !F, 0xFF, B,298 .bits = {{299 .offset = 1,300 .mask = (PIPE_CONTROL_MMIO_WRITE | PIPE_CONTROL_NOTIFY),301 .expected = 0,302 },303 {304 .offset = 1,305 .mask = (PIPE_CONTROL_GLOBAL_GTT_IVB |306 PIPE_CONTROL_STORE_DATA_INDEX),307 .expected = 0,308 .condition_offset = 1,309 .condition_mask = PIPE_CONTROL_POST_SYNC_OP_MASK,310 }}, ),311};312 313static const struct drm_i915_cmd_descriptor hsw_render_cmds[] = {314 CMD( MI_SET_PREDICATE, SMI, F, 1, S ),315 CMD( MI_RS_CONTROL, SMI, F, 1, S ),316 CMD( MI_URB_ATOMIC_ALLOC, SMI, F, 1, S ),317 CMD( MI_SET_APPID, SMI, F, 1, S ),318 CMD( MI_RS_CONTEXT, SMI, F, 1, S ),319 CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, R ),320 CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ),321 CMD( MI_LOAD_REGISTER_REG, SMI, !F, 0xFF, W,322 .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 1 } ),323 CMD( MI_RS_STORE_DATA_IMM, SMI, !F, 0xFF, S ),324 CMD( MI_LOAD_URB_MEM, SMI, !F, 0xFF, S ),325 CMD( MI_STORE_URB_MEM, SMI, !F, 0xFF, S ),326 CMD( GFX_OP_3DSTATE_DX9_CONSTANTF_VS, S3D, !F, 0x7FF, S ),327 CMD( GFX_OP_3DSTATE_DX9_CONSTANTF_PS, S3D, !F, 0x7FF, S ),328 329 CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_VS, S3D, !F, 0x1FF, S ),330 CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_GS, S3D, !F, 0x1FF, S ),331 CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_HS, S3D, !F, 0x1FF, S ),332 CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_DS, S3D, !F, 0x1FF, S ),333 CMD( GFX_OP_3DSTATE_BINDING_TABLE_EDIT_PS, S3D, !F, 0x1FF, S ),334};335 336static const struct drm_i915_cmd_descriptor gen7_video_cmds[] = {337 CMD( MI_ARB_ON_OFF, SMI, F, 1, R ),338 CMD( MI_SET_APPID, SMI, F, 1, S ),339 CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B,340 .bits = {{341 .offset = 0,342 .mask = MI_GLOBAL_GTT,343 .expected = 0,344 }}, ),345 CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ),346 CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B,347 .bits = {{348 .offset = 0,349 .mask = MI_FLUSH_DW_NOTIFY,350 .expected = 0,351 },352 {353 .offset = 1,354 .mask = MI_FLUSH_DW_USE_GTT,355 .expected = 0,356 .condition_offset = 0,357 .condition_mask = MI_FLUSH_DW_OP_MASK,358 },359 {360 .offset = 0,361 .mask = MI_FLUSH_DW_STORE_INDEX,362 .expected = 0,363 .condition_offset = 0,364 .condition_mask = MI_FLUSH_DW_OP_MASK,365 }}, ),366 CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B,367 .bits = {{368 .offset = 0,369 .mask = MI_GLOBAL_GTT,370 .expected = 0,371 }}, ),372 /*373 * MFX_WAIT doesn't fit the way we handle length for most commands.374 * It has a length field but it uses a non-standard length bias.375 * It is always 1 dword though, so just treat it as fixed length.376 */377 CMD( MFX_WAIT, SMFX, F, 1, S ),378};379 380static const struct drm_i915_cmd_descriptor gen7_vecs_cmds[] = {381 CMD( MI_ARB_ON_OFF, SMI, F, 1, R ),382 CMD( MI_SET_APPID, SMI, F, 1, S ),383 CMD( MI_STORE_DWORD_IMM, SMI, !F, 0xFF, B,384 .bits = {{385 .offset = 0,386 .mask = MI_GLOBAL_GTT,387 .expected = 0,388 }}, ),389 CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ),390 CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B,391 .bits = {{392 .offset = 0,393 .mask = MI_FLUSH_DW_NOTIFY,394 .expected = 0,395 },396 {397 .offset = 1,398 .mask = MI_FLUSH_DW_USE_GTT,399 .expected = 0,400 .condition_offset = 0,401 .condition_mask = MI_FLUSH_DW_OP_MASK,402 },403 {404 .offset = 0,405 .mask = MI_FLUSH_DW_STORE_INDEX,406 .expected = 0,407 .condition_offset = 0,408 .condition_mask = MI_FLUSH_DW_OP_MASK,409 }}, ),410 CMD( MI_CONDITIONAL_BATCH_BUFFER_END, SMI, !F, 0xFF, B,411 .bits = {{412 .offset = 0,413 .mask = MI_GLOBAL_GTT,414 .expected = 0,415 }}, ),416};417 418static const struct drm_i915_cmd_descriptor gen7_blt_cmds[] = {419 CMD( MI_DISPLAY_FLIP, SMI, !F, 0xFF, R ),420 CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3FF, B,421 .bits = {{422 .offset = 0,423 .mask = MI_GLOBAL_GTT,424 .expected = 0,425 }}, ),426 CMD( MI_UPDATE_GTT, SMI, !F, 0x3F, R ),427 CMD( MI_FLUSH_DW, SMI, !F, 0x3F, B,428 .bits = {{429 .offset = 0,430 .mask = MI_FLUSH_DW_NOTIFY,431 .expected = 0,432 },433 {434 .offset = 1,435 .mask = MI_FLUSH_DW_USE_GTT,436 .expected = 0,437 .condition_offset = 0,438 .condition_mask = MI_FLUSH_DW_OP_MASK,439 },440 {441 .offset = 0,442 .mask = MI_FLUSH_DW_STORE_INDEX,443 .expected = 0,444 .condition_offset = 0,445 .condition_mask = MI_FLUSH_DW_OP_MASK,446 }}, ),447 CMD( COLOR_BLT, S2D, !F, 0x3F, S ),448 CMD( SRC_COPY_BLT, S2D, !F, 0x3F, S ),449};450 451static const struct drm_i915_cmd_descriptor hsw_blt_cmds[] = {452 CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, R ),453 CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, R ),454};455 456/*457 * For Gen9 we can still rely on the h/w to enforce cmd security, and only458 * need to re-enforce the register access checks. We therefore only need to459 * teach the cmdparser how to find the end of each command, and identify460 * register accesses. The table doesn't need to reject any commands, and so461 * the only commands listed here are:462 * 1) Those that touch registers463 * 2) Those that do not have the default 8-bit length464 *465 * Note that the default MI length mask chosen for this table is 0xFF, not466 * the 0x3F used on older devices. This is because the vast majority of MI467 * cmds on Gen9 use a standard 8-bit Length field.468 * All the Gen9 blitter instructions are standard 0xFF length mask, and469 * none allow access to non-general registers, so in fact no BLT cmds are470 * included in the table at all.471 *472 */473static const struct drm_i915_cmd_descriptor gen9_blt_cmds[] = {474 CMD( MI_NOOP, SMI, F, 1, S ),475 CMD( MI_USER_INTERRUPT, SMI, F, 1, S ),476 CMD( MI_WAIT_FOR_EVENT, SMI, F, 1, S ),477 CMD( MI_FLUSH, SMI, F, 1, S ),478 CMD( MI_ARB_CHECK, SMI, F, 1, S ),479 CMD( MI_REPORT_HEAD, SMI, F, 1, S ),480 CMD( MI_ARB_ON_OFF, SMI, F, 1, S ),481 CMD( MI_SUSPEND_FLUSH, SMI, F, 1, S ),482 CMD( MI_LOAD_SCAN_LINES_INCL, SMI, !F, 0x3F, S ),483 CMD( MI_LOAD_SCAN_LINES_EXCL, SMI, !F, 0x3F, S ),484 CMD( MI_STORE_DWORD_IMM, SMI, !F, 0x3FF, S ),485 CMD( MI_LOAD_REGISTER_IMM(1), SMI, !F, 0xFF, W,486 .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 2 } ),487 CMD( MI_UPDATE_GTT, SMI, !F, 0x3FF, S ),488 CMD( MI_STORE_REGISTER_MEM_GEN8, SMI, F, 4, W,489 .reg = { .offset = 1, .mask = 0x007FFFFC } ),490 CMD( MI_FLUSH_DW, SMI, !F, 0x3F, S ),491 CMD( MI_LOAD_REGISTER_MEM_GEN8, SMI, F, 4, W,492 .reg = { .offset = 1, .mask = 0x007FFFFC } ),493 CMD( MI_LOAD_REGISTER_REG, SMI, !F, 0xFF, W,494 .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 1 } ),495 496 /*497 * We allow BB_START but apply further checks. We just sanitize the498 * basic fields here.499 */500#define MI_BB_START_OPERAND_MASK GENMASK(SMI-1, 0)501#define MI_BB_START_OPERAND_EXPECT (MI_BATCH_PPGTT_HSW | 1)502 CMD( MI_BATCH_BUFFER_START_GEN8, SMI, !F, 0xFF, B,503 .bits = {{504 .offset = 0,505 .mask = MI_BB_START_OPERAND_MASK,506 .expected = MI_BB_START_OPERAND_EXPECT,507 }}, ),508};509 510static const struct drm_i915_cmd_descriptor noop_desc =511 CMD(MI_NOOP, SMI, F, 1, S);512 513#undef CMD514#undef SMI515#undef S3D516#undef S2D517#undef SMFX518#undef F519#undef S520#undef R521#undef W522#undef B523 524static const struct drm_i915_cmd_table gen7_render_cmd_table[] = {525 { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },526 { gen7_render_cmds, ARRAY_SIZE(gen7_render_cmds) },527};528 529static const struct drm_i915_cmd_table hsw_render_ring_cmd_table[] = {530 { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },531 { gen7_render_cmds, ARRAY_SIZE(gen7_render_cmds) },532 { hsw_render_cmds, ARRAY_SIZE(hsw_render_cmds) },533};534 535static const struct drm_i915_cmd_table gen7_video_cmd_table[] = {536 { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },537 { gen7_video_cmds, ARRAY_SIZE(gen7_video_cmds) },538};539 540static const struct drm_i915_cmd_table hsw_vebox_cmd_table[] = {541 { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },542 { gen7_vecs_cmds, ARRAY_SIZE(gen7_vecs_cmds) },543};544 545static const struct drm_i915_cmd_table gen7_blt_cmd_table[] = {546 { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },547 { gen7_blt_cmds, ARRAY_SIZE(gen7_blt_cmds) },548};549 550static const struct drm_i915_cmd_table hsw_blt_ring_cmd_table[] = {551 { gen7_common_cmds, ARRAY_SIZE(gen7_common_cmds) },552 { gen7_blt_cmds, ARRAY_SIZE(gen7_blt_cmds) },553 { hsw_blt_cmds, ARRAY_SIZE(hsw_blt_cmds) },554};555 556static const struct drm_i915_cmd_table gen9_blt_cmd_table[] = {557 { gen9_blt_cmds, ARRAY_SIZE(gen9_blt_cmds) },558};559 560 561/*562 * Register whitelists, sorted by increasing register offset.563 */564 565/*566 * An individual whitelist entry granting access to register addr. If567 * mask is non-zero the argument of immediate register writes will be568 * AND-ed with mask, and the command will be rejected if the result569 * doesn't match value.570 *571 * Registers with non-zero mask are only allowed to be written using572 * LRI.573 */574struct drm_i915_reg_descriptor {575 i915_reg_t addr;576 u32 mask;577 u32 value;578};579 580/* Convenience macro for adding 32-bit registers. */581#define REG32(_reg, ...) \582 { .addr = (_reg), __VA_ARGS__ }583 584#define REG32_IDX(_reg, idx) \585 { .addr = _reg(idx) }586 587/*588 * Convenience macro for adding 64-bit registers.589 *590 * Some registers that userspace accesses are 64 bits. The register591 * access commands only allow 32-bit accesses. Hence, we have to include592 * entries for both halves of the 64-bit registers.593 */594#define REG64(_reg) \595 { .addr = _reg }, \596 { .addr = _reg ## _UDW }597 598#define REG64_IDX(_reg, idx) \599 { .addr = _reg(idx) }, \600 { .addr = _reg ## _UDW(idx) }601 602#define REG64_BASE_IDX(_reg, base, idx) \603 { .addr = _reg(base, idx) }, \604 { .addr = _reg ## _UDW(base, idx) }605 606static const struct drm_i915_reg_descriptor gen7_render_regs[] = {607 REG64(GPGPU_THREADS_DISPATCHED),608 REG64(HS_INVOCATION_COUNT),609 REG64(DS_INVOCATION_COUNT),610 REG64(IA_VERTICES_COUNT),611 REG64(IA_PRIMITIVES_COUNT),612 REG64(VS_INVOCATION_COUNT),613 REG64(GS_INVOCATION_COUNT),614 REG64(GS_PRIMITIVES_COUNT),615 REG64(CL_INVOCATION_COUNT),616 REG64(CL_PRIMITIVES_COUNT),617 REG64(PS_INVOCATION_COUNT),618 REG64(PS_DEPTH_COUNT),619 REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE),620 REG64_IDX(MI_PREDICATE_SRC0, RENDER_RING_BASE),621 REG64_IDX(MI_PREDICATE_SRC1, RENDER_RING_BASE),622 REG32(GEN7_3DPRIM_END_OFFSET),623 REG32(GEN7_3DPRIM_START_VERTEX),624 REG32(GEN7_3DPRIM_VERTEX_COUNT),625 REG32(GEN7_3DPRIM_INSTANCE_COUNT),626 REG32(GEN7_3DPRIM_START_INSTANCE),627 REG32(GEN7_3DPRIM_BASE_VERTEX),628 REG32(GEN7_GPGPU_DISPATCHDIMX),629 REG32(GEN7_GPGPU_DISPATCHDIMY),630 REG32(GEN7_GPGPU_DISPATCHDIMZ),631 REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE),632 REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 0),633 REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 1),634 REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 2),635 REG64_IDX(GEN7_SO_NUM_PRIMS_WRITTEN, 3),636 REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 0),637 REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 1),638 REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 2),639 REG64_IDX(GEN7_SO_PRIM_STORAGE_NEEDED, 3),640 REG32(GEN7_SO_WRITE_OFFSET(0)),641 REG32(GEN7_SO_WRITE_OFFSET(1)),642 REG32(GEN7_SO_WRITE_OFFSET(2)),643 REG32(GEN7_SO_WRITE_OFFSET(3)),644 REG32(GEN7_L3SQCREG1),645 REG32(GEN7_L3CNTLREG2),646 REG32(GEN7_L3CNTLREG3),647 REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE),648};649 650static const struct drm_i915_reg_descriptor hsw_render_regs[] = {651 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 0),652 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 1),653 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 2),654 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 3),655 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 4),656 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 5),657 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 6),658 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 7),659 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 8),660 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 9),661 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 10),662 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 11),663 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 12),664 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 13),665 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 14),666 REG64_BASE_IDX(GEN8_RING_CS_GPR, RENDER_RING_BASE, 15),667 REG32(HSW_SCRATCH1,668 .mask = ~HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE,669 .value = 0),670 REG32(HSW_ROW_CHICKEN3,671 .mask = ~(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE << 16 |672 HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE),673 .value = 0),674};675 676static const struct drm_i915_reg_descriptor gen7_blt_regs[] = {677 REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE),678 REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE),679 REG32(BCS_SWCTRL),680 REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE),681};682 683static const struct drm_i915_reg_descriptor gen9_blt_regs[] = {684 REG64_IDX(RING_TIMESTAMP, RENDER_RING_BASE),685 REG64_IDX(RING_TIMESTAMP, BSD_RING_BASE),686 REG32(BCS_SWCTRL),687 REG64_IDX(RING_TIMESTAMP, BLT_RING_BASE),688 REG32_IDX(RING_CTX_TIMESTAMP, BLT_RING_BASE),689 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 0),690 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 1),691 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 2),692 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 3),693 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 4),694 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 5),695 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 6),696 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 7),697 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 8),698 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 9),699 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 10),700 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 11),701 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 12),702 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 13),703 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 14),704 REG64_BASE_IDX(GEN8_RING_CS_GPR, BLT_RING_BASE, 15),705};706 707#undef REG64708#undef REG32709 710struct drm_i915_reg_table {711 const struct drm_i915_reg_descriptor *regs;712 int num_regs;713};714 715static const struct drm_i915_reg_table ivb_render_reg_tables[] = {716 { gen7_render_regs, ARRAY_SIZE(gen7_render_regs) },717};718 719static const struct drm_i915_reg_table ivb_blt_reg_tables[] = {720 { gen7_blt_regs, ARRAY_SIZE(gen7_blt_regs) },721};722 723static const struct drm_i915_reg_table hsw_render_reg_tables[] = {724 { gen7_render_regs, ARRAY_SIZE(gen7_render_regs) },725 { hsw_render_regs, ARRAY_SIZE(hsw_render_regs) },726};727 728static const struct drm_i915_reg_table hsw_blt_reg_tables[] = {729 { gen7_blt_regs, ARRAY_SIZE(gen7_blt_regs) },730};731 732static const struct drm_i915_reg_table gen9_blt_reg_tables[] = {733 { gen9_blt_regs, ARRAY_SIZE(gen9_blt_regs) },734};735 736static u32 gen7_render_get_cmd_length_mask(u32 cmd_header)737{738 u32 client = cmd_header >> INSTR_CLIENT_SHIFT;739 u32 subclient =740 (cmd_header & INSTR_SUBCLIENT_MASK) >> INSTR_SUBCLIENT_SHIFT;741 742 if (client == INSTR_MI_CLIENT)743 return 0x3F;744 else if (client == INSTR_RC_CLIENT) {745 if (subclient == INSTR_MEDIA_SUBCLIENT)746 return 0xFFFF;747 else748 return 0xFF;749 }750 751 DRM_DEBUG("CMD: Abnormal rcs cmd length! 0x%08X\n", cmd_header);752 return 0;753}754 755static u32 gen7_bsd_get_cmd_length_mask(u32 cmd_header)756{757 u32 client = cmd_header >> INSTR_CLIENT_SHIFT;758 u32 subclient =759 (cmd_header & INSTR_SUBCLIENT_MASK) >> INSTR_SUBCLIENT_SHIFT;760 u32 op = (cmd_header & INSTR_26_TO_24_MASK) >> INSTR_26_TO_24_SHIFT;761 762 if (client == INSTR_MI_CLIENT)763 return 0x3F;764 else if (client == INSTR_RC_CLIENT) {765 if (subclient == INSTR_MEDIA_SUBCLIENT) {766 if (op == 6)767 return 0xFFFF;768 else769 return 0xFFF;770 } else771 return 0xFF;772 }773 774 DRM_DEBUG("CMD: Abnormal bsd cmd length! 0x%08X\n", cmd_header);775 return 0;776}777 778static u32 gen7_blt_get_cmd_length_mask(u32 cmd_header)779{780 u32 client = cmd_header >> INSTR_CLIENT_SHIFT;781 782 if (client == INSTR_MI_CLIENT)783 return 0x3F;784 else if (client == INSTR_BC_CLIENT)785 return 0xFF;786 787 DRM_DEBUG("CMD: Abnormal blt cmd length! 0x%08X\n", cmd_header);788 return 0;789}790 791static u32 gen9_blt_get_cmd_length_mask(u32 cmd_header)792{793 u32 client = cmd_header >> INSTR_CLIENT_SHIFT;794 795 if (client == INSTR_MI_CLIENT || client == INSTR_BC_CLIENT)796 return 0xFF;797 798 DRM_DEBUG("CMD: Abnormal blt cmd length! 0x%08X\n", cmd_header);799 return 0;800}801 802static bool validate_cmds_sorted(const struct intel_engine_cs *engine,803 const struct drm_i915_cmd_table *cmd_tables,804 int cmd_table_count)805{806 int i;807 bool ret = true;808 809 if (!cmd_tables || cmd_table_count == 0)810 return true;811 812 for (i = 0; i < cmd_table_count; i++) {813 const struct drm_i915_cmd_table *table = &cmd_tables[i];814 u32 previous = 0;815 int j;816 817 for (j = 0; j < table->count; j++) {818 const struct drm_i915_cmd_descriptor *desc =819 &table->table[j];820 u32 curr = desc->cmd.value & desc->cmd.mask;821 822 if (curr < previous) {823 drm_err(&engine->i915->drm,824 "CMD: %s [%d] command table not sorted: "825 "table=%d entry=%d cmd=0x%08X prev=0x%08X\n",826 engine->name, engine->id,827 i, j, curr, previous);828 ret = false;829 }830 831 previous = curr;832 }833 }834 835 return ret;836}837 838static bool check_sorted(const struct intel_engine_cs *engine,839 const struct drm_i915_reg_descriptor *reg_table,840 int reg_count)841{842 int i;843 u32 previous = 0;844 bool ret = true;845 846 for (i = 0; i < reg_count; i++) {847 u32 curr = i915_mmio_reg_offset(reg_table[i].addr);848 849 if (curr < previous) {850 drm_err(&engine->i915->drm,851 "CMD: %s [%d] register table not sorted: "852 "entry=%d reg=0x%08X prev=0x%08X\n",853 engine->name, engine->id,854 i, curr, previous);855 ret = false;856 }857 858 previous = curr;859 }860 861 return ret;862}863 864static bool validate_regs_sorted(struct intel_engine_cs *engine)865{866 int i;867 const struct drm_i915_reg_table *table;868 869 for (i = 0; i < engine->reg_table_count; i++) {870 table = &engine->reg_tables[i];871 if (!check_sorted(engine, table->regs, table->num_regs))872 return false;873 }874 875 return true;876}877 878struct cmd_node {879 const struct drm_i915_cmd_descriptor *desc;880 struct hlist_node node;881};882 883/*884 * Different command ranges have different numbers of bits for the opcode. For885 * example, MI commands use bits 31:23 while 3D commands use bits 31:16. The886 * problem is that, for example, MI commands use bits 22:16 for other fields887 * such as GGTT vs PPGTT bits. If we include those bits in the mask then when888 * we mask a command from a batch it could hash to the wrong bucket due to889 * non-opcode bits being set. But if we don't include those bits, some 3D890 * commands may hash to the same bucket due to not including opcode bits that891 * make the command unique. For now, we will risk hashing to the same bucket.892 */893static inline u32 cmd_header_key(u32 x)894{895 switch (x >> INSTR_CLIENT_SHIFT) {896 default:897 case INSTR_MI_CLIENT:898 return x >> STD_MI_OPCODE_SHIFT;899 case INSTR_RC_CLIENT:900 return x >> STD_3D_OPCODE_SHIFT;901 case INSTR_BC_CLIENT:902 return x >> STD_2D_OPCODE_SHIFT;903 }904}905 906static int init_hash_table(struct intel_engine_cs *engine,907 const struct drm_i915_cmd_table *cmd_tables,908 int cmd_table_count)909{910 int i, j;911 912 hash_init(engine->cmd_hash);913 914 for (i = 0; i < cmd_table_count; i++) {915 const struct drm_i915_cmd_table *table = &cmd_tables[i];916 917 for (j = 0; j < table->count; j++) {918 const struct drm_i915_cmd_descriptor *desc =919 &table->table[j];920 struct cmd_node *desc_node =921 kmalloc(sizeof(*desc_node), GFP_KERNEL);922 923 if (!desc_node)924 return -ENOMEM;925 926 desc_node->desc = desc;927 hash_add(engine->cmd_hash, &desc_node->node,928 cmd_header_key(desc->cmd.value));929 }930 }931 932 return 0;933}934 935static void fini_hash_table(struct intel_engine_cs *engine)936{937 struct hlist_node *tmp;938 struct cmd_node *desc_node;939 int i;940 941 hash_for_each_safe(engine->cmd_hash, i, tmp, desc_node, node) {942 hash_del(&desc_node->node);943 kfree(desc_node);944 }945}946 947/**948 * intel_engine_init_cmd_parser() - set cmd parser related fields for an engine949 * @engine: the engine to initialize950 *951 * Optionally initializes fields related to batch buffer command parsing in the952 * struct intel_engine_cs based on whether the platform requires software953 * command parsing.954 */955int intel_engine_init_cmd_parser(struct intel_engine_cs *engine)956{957 const struct drm_i915_cmd_table *cmd_tables;958 int cmd_table_count;959 int ret;960 961 if (GRAPHICS_VER(engine->i915) != 7 && !(GRAPHICS_VER(engine->i915) == 9 &&962 engine->class == COPY_ENGINE_CLASS))963 return 0;964 965 switch (engine->class) {966 case RENDER_CLASS:967 if (IS_HASWELL(engine->i915)) {968 cmd_tables = hsw_render_ring_cmd_table;969 cmd_table_count =970 ARRAY_SIZE(hsw_render_ring_cmd_table);971 } else {972 cmd_tables = gen7_render_cmd_table;973 cmd_table_count = ARRAY_SIZE(gen7_render_cmd_table);974 }975 976 if (IS_HASWELL(engine->i915)) {977 engine->reg_tables = hsw_render_reg_tables;978 engine->reg_table_count = ARRAY_SIZE(hsw_render_reg_tables);979 } else {980 engine->reg_tables = ivb_render_reg_tables;981 engine->reg_table_count = ARRAY_SIZE(ivb_render_reg_tables);982 }983 engine->get_cmd_length_mask = gen7_render_get_cmd_length_mask;984 break;985 case VIDEO_DECODE_CLASS:986 cmd_tables = gen7_video_cmd_table;987 cmd_table_count = ARRAY_SIZE(gen7_video_cmd_table);988 engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;989 break;990 case COPY_ENGINE_CLASS:991 engine->get_cmd_length_mask = gen7_blt_get_cmd_length_mask;992 if (GRAPHICS_VER(engine->i915) == 9) {993 cmd_tables = gen9_blt_cmd_table;994 cmd_table_count = ARRAY_SIZE(gen9_blt_cmd_table);995 engine->get_cmd_length_mask =996 gen9_blt_get_cmd_length_mask;997 998 /* BCS Engine unsafe without parser */999 engine->flags |= I915_ENGINE_REQUIRES_CMD_PARSER;1000 } else if (IS_HASWELL(engine->i915)) {1001 cmd_tables = hsw_blt_ring_cmd_table;1002 cmd_table_count = ARRAY_SIZE(hsw_blt_ring_cmd_table);1003 } else {1004 cmd_tables = gen7_blt_cmd_table;1005 cmd_table_count = ARRAY_SIZE(gen7_blt_cmd_table);1006 }1007 1008 if (GRAPHICS_VER(engine->i915) == 9) {1009 engine->reg_tables = gen9_blt_reg_tables;1010 engine->reg_table_count =1011 ARRAY_SIZE(gen9_blt_reg_tables);1012 } else if (IS_HASWELL(engine->i915)) {1013 engine->reg_tables = hsw_blt_reg_tables;1014 engine->reg_table_count = ARRAY_SIZE(hsw_blt_reg_tables);1015 } else {1016 engine->reg_tables = ivb_blt_reg_tables;1017 engine->reg_table_count = ARRAY_SIZE(ivb_blt_reg_tables);1018 }1019 break;1020 case VIDEO_ENHANCEMENT_CLASS:1021 cmd_tables = hsw_vebox_cmd_table;1022 cmd_table_count = ARRAY_SIZE(hsw_vebox_cmd_table);1023 /* VECS can use the same length_mask function as VCS */1024 engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;1025 break;1026 default:1027 MISSING_CASE(engine->class);1028 goto out;1029 }1030 1031 if (!validate_cmds_sorted(engine, cmd_tables, cmd_table_count)) {1032 drm_err(&engine->i915->drm,1033 "%s: command descriptions are not sorted\n",1034 engine->name);1035 goto out;1036 }1037 if (!validate_regs_sorted(engine)) {1038 drm_err(&engine->i915->drm,1039 "%s: registers are not sorted\n", engine->name);1040 goto out;1041 }1042 1043 ret = init_hash_table(engine, cmd_tables, cmd_table_count);1044 if (ret) {1045 drm_err(&engine->i915->drm,1046 "%s: initialised failed!\n", engine->name);1047 fini_hash_table(engine);1048 goto out;1049 }1050 1051 engine->flags |= I915_ENGINE_USING_CMD_PARSER;1052 1053out:1054 if (intel_engine_requires_cmd_parser(engine) &&1055 !intel_engine_using_cmd_parser(engine))1056 return -EINVAL;1057 1058 return 0;1059}1060 1061/**1062 * intel_engine_cleanup_cmd_parser() - clean up cmd parser related fields1063 * @engine: the engine to clean up1064 *1065 * Releases any resources related to command parsing that may have been1066 * initialized for the specified engine.1067 */1068void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine)1069{1070 if (!intel_engine_using_cmd_parser(engine))1071 return;1072 1073 fini_hash_table(engine);1074}1075 1076static const struct drm_i915_cmd_descriptor*1077find_cmd_in_table(struct intel_engine_cs *engine,1078 u32 cmd_header)1079{1080 struct cmd_node *desc_node;1081 1082 hash_for_each_possible(engine->cmd_hash, desc_node, node,1083 cmd_header_key(cmd_header)) {1084 const struct drm_i915_cmd_descriptor *desc = desc_node->desc;1085 if (((cmd_header ^ desc->cmd.value) & desc->cmd.mask) == 0)1086 return desc;1087 }1088 1089 return NULL;1090}1091 1092/*1093 * Returns a pointer to a descriptor for the command specified by cmd_header.1094 *1095 * The caller must supply space for a default descriptor via the default_desc1096 * parameter. If no descriptor for the specified command exists in the engine's1097 * command parser tables, this function fills in default_desc based on the1098 * engine's default length encoding and returns default_desc.1099 */1100static const struct drm_i915_cmd_descriptor*1101find_cmd(struct intel_engine_cs *engine,1102 u32 cmd_header,1103 const struct drm_i915_cmd_descriptor *desc,1104 struct drm_i915_cmd_descriptor *default_desc)1105{1106 u32 mask;1107 1108 if (((cmd_header ^ desc->cmd.value) & desc->cmd.mask) == 0)1109 return desc;1110 1111 desc = find_cmd_in_table(engine, cmd_header);1112 if (desc)1113 return desc;1114 1115 mask = engine->get_cmd_length_mask(cmd_header);1116 if (!mask)1117 return NULL;1118 1119 default_desc->cmd.value = cmd_header;1120 default_desc->cmd.mask = ~0u << MIN_OPCODE_SHIFT;1121 default_desc->length.mask = mask;1122 default_desc->flags = CMD_DESC_SKIP;1123 return default_desc;1124}1125 1126static const struct drm_i915_reg_descriptor *1127__find_reg(const struct drm_i915_reg_descriptor *table, int count, u32 addr)1128{1129 int start = 0, end = count;1130 while (start < end) {1131 int mid = start + (end - start) / 2;1132 int ret = addr - i915_mmio_reg_offset(table[mid].addr);1133 if (ret < 0)1134 end = mid;1135 else if (ret > 0)1136 start = mid + 1;1137 else1138 return &table[mid];1139 }1140 return NULL;1141}1142 1143static const struct drm_i915_reg_descriptor *1144find_reg(const struct intel_engine_cs *engine, u32 addr)1145{1146 const struct drm_i915_reg_table *table = engine->reg_tables;1147 const struct drm_i915_reg_descriptor *reg = NULL;1148 int count = engine->reg_table_count;1149 1150 for (; !reg && (count > 0); ++table, --count)1151 reg = __find_reg(table->regs, table->num_regs, addr);1152 1153 return reg;1154}1155 1156/* Returns a vmap'd pointer to dst_obj, which the caller must unmap */1157static u32 *copy_batch(struct drm_i915_gem_object *dst_obj,1158 struct drm_i915_gem_object *src_obj,1159 unsigned long offset, unsigned long length,1160 bool *needs_clflush_after)1161{1162 unsigned int src_needs_clflush;1163 unsigned int dst_needs_clflush;1164 void *dst, *src;1165 int ret;1166 1167 ret = i915_gem_object_prepare_write(dst_obj, &dst_needs_clflush);1168 if (ret)1169 return ERR_PTR(ret);1170 1171 dst = i915_gem_object_pin_map(dst_obj, I915_MAP_WB);1172 i915_gem_object_finish_access(dst_obj);1173 if (IS_ERR(dst))1174 return dst;1175 1176 ret = i915_gem_object_prepare_read(src_obj, &src_needs_clflush);1177 if (ret) {1178 i915_gem_object_unpin_map(dst_obj);1179 return ERR_PTR(ret);1180 }1181 1182 src = ERR_PTR(-ENODEV);1183 if (src_needs_clflush && i915_has_memcpy_from_wc()) {1184 src = i915_gem_object_pin_map(src_obj, I915_MAP_WC);1185 if (!IS_ERR(src)) {1186 i915_unaligned_memcpy_from_wc(dst,1187 src + offset,1188 length);1189 i915_gem_object_unpin_map(src_obj);1190 }1191 }1192 if (IS_ERR(src)) {1193 unsigned long x, n, remain;1194 void *ptr;1195 1196 /*1197 * We can avoid clflushing partial cachelines before the write1198 * if we only every write full cache-lines. Since we know that1199 * both the source and destination are in multiples of1200 * PAGE_SIZE, we can simply round up to the next cacheline.1201 * We don't care about copying too much here as we only1202 * validate up to the end of the batch.1203 */1204 remain = length;1205 if (dst_needs_clflush & CLFLUSH_BEFORE)1206 remain = round_up(remain,1207 boot_cpu_data.x86_clflush_size);1208 1209 ptr = dst;1210 x = offset_in_page(offset);1211 for (n = offset >> PAGE_SHIFT; remain; n++) {1212 int len = min(remain, PAGE_SIZE - x);1213 1214 src = kmap_local_page(i915_gem_object_get_page(src_obj, n));1215 if (src_needs_clflush)1216 drm_clflush_virt_range(src + x, len);1217 memcpy(ptr, src + x, len);1218 kunmap_local(src);1219 1220 ptr += len;1221 remain -= len;1222 x = 0;1223 }1224 }1225 1226 i915_gem_object_finish_access(src_obj);1227 1228 memset32(dst + length, 0, (dst_obj->base.size - length) / sizeof(u32));1229 1230 /* dst_obj is returned with vmap pinned */1231 *needs_clflush_after = dst_needs_clflush & CLFLUSH_AFTER;1232 1233 return dst;1234}1235 1236static inline bool cmd_desc_is(const struct drm_i915_cmd_descriptor * const desc,1237 const u32 cmd)1238{1239 return desc->cmd.value == (cmd & desc->cmd.mask);1240}1241 1242static bool check_cmd(const struct intel_engine_cs *engine,1243 const struct drm_i915_cmd_descriptor *desc,1244 const u32 *cmd, u32 length)1245{1246 if (desc->flags & CMD_DESC_SKIP)1247 return true;1248 1249 if (desc->flags & CMD_DESC_REJECT) {1250 DRM_DEBUG("CMD: Rejected command: 0x%08X\n", *cmd);1251 return false;1252 }1253 1254 if (desc->flags & CMD_DESC_REGISTER) {1255 /*1256 * Get the distance between individual register offset1257 * fields if the command can perform more than one1258 * access at a time.1259 */1260 const u32 step = desc->reg.step ? desc->reg.step : length;1261 u32 offset;1262 1263 for (offset = desc->reg.offset; offset < length;1264 offset += step) {1265 const u32 reg_addr = cmd[offset] & desc->reg.mask;1266 const struct drm_i915_reg_descriptor *reg =1267 find_reg(engine, reg_addr);1268 1269 if (!reg) {1270 DRM_DEBUG("CMD: Rejected register 0x%08X in command: 0x%08X (%s)\n",1271 reg_addr, *cmd, engine->name);1272 return false;1273 }1274 1275 /*1276 * Check the value written to the register against the1277 * allowed mask/value pair given in the whitelist entry.1278 */1279 if (reg->mask) {1280 if (cmd_desc_is(desc, MI_LOAD_REGISTER_MEM)) {1281 DRM_DEBUG("CMD: Rejected LRM to masked register 0x%08X\n",1282 reg_addr);1283 return false;1284 }1285 1286 if (cmd_desc_is(desc, MI_LOAD_REGISTER_REG)) {1287 DRM_DEBUG("CMD: Rejected LRR to masked register 0x%08X\n",1288 reg_addr);1289 return false;1290 }1291 1292 if (cmd_desc_is(desc, MI_LOAD_REGISTER_IMM(1)) &&1293 (offset + 2 > length ||1294 (cmd[offset + 1] & reg->mask) != reg->value)) {1295 DRM_DEBUG("CMD: Rejected LRI to masked register 0x%08X\n",1296 reg_addr);1297 return false;1298 }1299 }1300 }1301 }1302 1303 if (desc->flags & CMD_DESC_BITMASK) {1304 int i;1305 1306 for (i = 0; i < MAX_CMD_DESC_BITMASKS; i++) {1307 u32 dword;1308 1309 if (desc->bits[i].mask == 0)1310 break;1311 1312 if (desc->bits[i].condition_mask != 0) {1313 u32 offset =1314 desc->bits[i].condition_offset;1315 u32 condition = cmd[offset] &1316 desc->bits[i].condition_mask;1317 1318 if (condition == 0)1319 continue;1320 }1321 1322 if (desc->bits[i].offset >= length) {1323 DRM_DEBUG("CMD: Rejected command 0x%08X, too short to check bitmask (%s)\n",1324 *cmd, engine->name);1325 return false;1326 }1327 1328 dword = cmd[desc->bits[i].offset] &1329 desc->bits[i].mask;1330 1331 if (dword != desc->bits[i].expected) {1332 DRM_DEBUG("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (%s)\n",1333 *cmd,1334 desc->bits[i].mask,1335 desc->bits[i].expected,1336 dword, engine->name);1337 return false;1338 }1339 }1340 }1341 1342 return true;1343}1344 1345static int check_bbstart(u32 *cmd, u32 offset, u32 length,1346 u32 batch_length,1347 u64 batch_addr,1348 u64 shadow_addr,1349 const unsigned long *jump_whitelist)1350{1351 u64 jump_offset, jump_target;1352 u32 target_cmd_offset, target_cmd_index;1353 1354 /* For igt compatibility on older platforms */1355 if (!jump_whitelist) {1356 DRM_DEBUG("CMD: Rejecting BB_START for ggtt based submission\n");1357 return -EACCES;1358 }1359 1360 if (length != 3) {1361 DRM_DEBUG("CMD: Recursive BB_START with bad length(%u)\n",1362 length);1363 return -EINVAL;1364 }1365 1366 jump_target = *(u64 *)(cmd + 1);1367 jump_offset = jump_target - batch_addr;1368 1369 /*1370 * Any underflow of jump_target is guaranteed to be outside the range1371 * of a u32, so >= test catches both too large and too small1372 */1373 if (jump_offset >= batch_length) {1374 DRM_DEBUG("CMD: BB_START to 0x%llx jumps out of BB\n",1375 jump_target);1376 return -EINVAL;1377 }1378 1379 /*1380 * This cannot overflow a u32 because we already checked jump_offset1381 * is within the BB, and the batch_length is a u321382 */1383 target_cmd_offset = lower_32_bits(jump_offset);1384 target_cmd_index = target_cmd_offset / sizeof(u32);1385 1386 *(u64 *)(cmd + 1) = shadow_addr + target_cmd_offset;1387 1388 if (target_cmd_index == offset)1389 return 0;1390 1391 if (IS_ERR(jump_whitelist))1392 return PTR_ERR(jump_whitelist);1393 1394 if (!test_bit(target_cmd_index, jump_whitelist)) {1395 DRM_DEBUG("CMD: BB_START to 0x%llx not a previously executed cmd\n",1396 jump_target);1397 return -EINVAL;1398 }1399 1400 return 0;1401}1402 1403static unsigned long *alloc_whitelist(u32 batch_length)1404{1405 unsigned long *jmp;1406 1407 /*1408 * We expect batch_length to be less than 256KiB for known users,1409 * i.e. we need at most an 8KiB bitmap allocation which should be1410 * reasonably cheap due to kmalloc caches.1411 */1412 1413 /* Prefer to report transient allocation failure rather than hit oom */1414 jmp = bitmap_zalloc(DIV_ROUND_UP(batch_length, sizeof(u32)),1415 GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);1416 if (!jmp)1417 return ERR_PTR(-ENOMEM);1418 1419 return jmp;1420}1421 1422#define LENGTH_BIAS 21423 1424/**1425 * intel_engine_cmd_parser() - parse a batch buffer for privilege violations1426 * @engine: the engine on which the batch is to execute1427 * @batch: the batch buffer in question1428 * @batch_offset: byte offset in the batch at which execution starts1429 * @batch_length: length of the commands in batch_obj1430 * @shadow: validated copy of the batch buffer in question1431 * @trampoline: true if we need to trampoline into privileged execution1432 *1433 * Parses the specified batch buffer looking for privilege violations as1434 * described in the overview.1435 *1436 * Return: non-zero if the parser finds violations or otherwise fails; -EACCES1437 * if the batch appears legal but should use hardware parsing1438 */1439 1440int intel_engine_cmd_parser(struct intel_engine_cs *engine,1441 struct i915_vma *batch,1442 unsigned long batch_offset,1443 unsigned long batch_length,1444 struct i915_vma *shadow,1445 bool trampoline)1446{1447 u32 *cmd, *batch_end, offset = 0;1448 struct drm_i915_cmd_descriptor default_desc = noop_desc;1449 const struct drm_i915_cmd_descriptor *desc = &default_desc;1450 bool needs_clflush_after = false;1451 unsigned long *jump_whitelist;1452 u64 batch_addr, shadow_addr;1453 int ret = 0;1454 1455 GEM_BUG_ON(!IS_ALIGNED(batch_offset, sizeof(*cmd)));1456 GEM_BUG_ON(!IS_ALIGNED(batch_length, sizeof(*cmd)));1457 GEM_BUG_ON(range_overflows_t(u64, batch_offset, batch_length,1458 batch->size));1459 GEM_BUG_ON(!batch_length);1460 1461 cmd = copy_batch(shadow->obj, batch->obj,1462 batch_offset, batch_length,1463 &needs_clflush_after);1464 if (IS_ERR(cmd)) {1465 DRM_DEBUG("CMD: Failed to copy batch\n");1466 return PTR_ERR(cmd);1467 }1468 1469 jump_whitelist = NULL;1470 if (!trampoline)1471 /* Defer failure until attempted use */1472 jump_whitelist = alloc_whitelist(batch_length);1473 1474 shadow_addr = gen8_canonical_addr(i915_vma_offset(shadow));1475 batch_addr = gen8_canonical_addr(i915_vma_offset(batch) + batch_offset);1476 1477 /*1478 * We use the batch length as size because the shadow object is as1479 * large or larger and copy_batch() will write MI_NOPs to the extra1480 * space. Parsing should be faster in some cases this way.1481 */1482 batch_end = cmd + batch_length / sizeof(*batch_end);1483 do {1484 u32 length;1485 1486 if (*cmd == MI_BATCH_BUFFER_END)1487 break;1488 1489 desc = find_cmd(engine, *cmd, desc, &default_desc);1490 if (!desc) {1491 DRM_DEBUG("CMD: Unrecognized command: 0x%08X\n", *cmd);1492 ret = -EINVAL;1493 break;1494 }1495 1496 if (desc->flags & CMD_DESC_FIXED)1497 length = desc->length.fixed;1498 else1499 length = (*cmd & desc->length.mask) + LENGTH_BIAS;1500 1501 if ((batch_end - cmd) < length) {1502 DRM_DEBUG("CMD: Command length exceeds batch length: 0x%08X length=%u batchlen=%td\n",1503 *cmd,1504 length,1505 batch_end - cmd);1506 ret = -EINVAL;1507 break;1508 }1509 1510 if (!check_cmd(engine, desc, cmd, length)) {1511 ret = -EACCES;1512 break;1513 }1514 1515 if (cmd_desc_is(desc, MI_BATCH_BUFFER_START)) {1516 ret = check_bbstart(cmd, offset, length, batch_length,1517 batch_addr, shadow_addr,1518 jump_whitelist);1519 break;1520 }1521 1522 if (!IS_ERR_OR_NULL(jump_whitelist))1523 __set_bit(offset, jump_whitelist);1524 1525 cmd += length;1526 offset += length;1527 if (cmd >= batch_end) {1528 DRM_DEBUG("CMD: Got to the end of the buffer w/o a BBE cmd!\n");1529 ret = -EINVAL;1530 break;1531 }1532 } while (1);1533 1534 if (trampoline) {1535 /*1536 * With the trampoline, the shadow is executed twice.1537 *1538 * 1 - starting at offset 0, in privileged mode1539 * 2 - starting at offset batch_len, as non-privileged1540 *1541 * Only if the batch is valid and safe to execute, do we1542 * allow the first privileged execution to proceed. If not,1543 * we terminate the first batch and use the second batchbuffer1544 * entry to chain to the original unsafe non-privileged batch,1545 * leaving it to the HW to validate.1546 */1547 *batch_end = MI_BATCH_BUFFER_END;1548 1549 if (ret) {1550 /* Batch unsafe to execute with privileges, cancel! */1551 cmd = page_mask_bits(shadow->obj->mm.mapping);1552 *cmd = MI_BATCH_BUFFER_END;1553 1554 /* If batch is unsafe but valid, jump to the original */1555 if (ret == -EACCES) {1556 unsigned int flags;1557 1558 flags = MI_BATCH_NON_SECURE_I965;1559 if (IS_HASWELL(engine->i915))1560 flags = MI_BATCH_NON_SECURE_HSW;1561 1562 GEM_BUG_ON(!IS_GRAPHICS_VER(engine->i915, 6, 7));1563 __gen6_emit_bb_start(batch_end,1564 batch_addr,1565 flags);1566 1567 ret = 0; /* allow execution */1568 }1569 }1570 }1571 1572 i915_gem_object_flush_map(shadow->obj);1573 1574 if (!IS_ERR_OR_NULL(jump_whitelist))1575 kfree(jump_whitelist);1576 i915_gem_object_unpin_map(shadow->obj);1577 return ret;1578}1579 1580/**1581 * i915_cmd_parser_get_version() - get the cmd parser version number1582 * @dev_priv: i915 device private1583 *1584 * The cmd parser maintains a simple increasing integer version number suitable1585 * for passing to userspace clients to determine what operations are permitted.1586 *1587 * Return: the current version number of the cmd parser1588 */1589int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv)1590{1591 struct intel_engine_cs *engine;1592 bool active = false;1593 1594 /* If the command parser is not enabled, report 0 - unsupported */1595 for_each_uabi_engine(engine, dev_priv) {1596 if (intel_engine_using_cmd_parser(engine)) {1597 active = true;1598 break;1599 }1600 }1601 if (!active)1602 return 0;1603 1604 /*1605 * Command parser version history1606 *1607 * 1. Initial version. Checks batches and reports violations, but leaves1608 * hardware parsing enabled (so does not allow new use cases).1609 * 2. Allow access to the MI_PREDICATE_SRC0 and1610 * MI_PREDICATE_SRC1 registers.1611 * 3. Allow access to the GPGPU_THREADS_DISPATCHED register.1612 * 4. L3 atomic chicken bits of HSW_SCRATCH1 and HSW_ROW_CHICKEN3.1613 * 5. GPGPU dispatch compute indirect registers.1614 * 6. TIMESTAMP register and Haswell CS GPR registers1615 * 7. Allow MI_LOAD_REGISTER_REG between whitelisted registers.1616 * 8. Don't report cmd_check() failures as EINVAL errors to userspace;1617 * rely on the HW to NOOP disallowed commands as it would without1618 * the parser enabled.1619 * 9. Don't whitelist or handle oacontrol specially, as ownership1620 * for oacontrol state is moving to i915-perf.1621 * 10. Support for Gen9 BCS Parsing1622 */1623 return 10;1624}1625