487 lines · plain
1// RUN: mlir-opt %s --xevm-attach-target='module=xevm_* O=3 chip=pvc' -convert-vector-to-xegpu -split-input-file | FileCheck %s --check-prefix=LOAD-ND2// RUN: mlir-opt %s -convert-vector-to-xegpu -split-input-file | FileCheck %s --check-prefix=LOAD-GATHER3 4gpu.module @xevm_module {5gpu.func @load_1D_vector(%source: memref<8x16x32xf32>, %offset: index) -> vector<8xf32> {6 %c0 = arith.constant 0.0 : f327 %0 = vector.transfer_read %source[%offset, %offset, %offset], %c08 {in_bounds = [true]} : memref<8x16x32xf32>, vector<8xf32>9 gpu.return %0 : vector<8xf32>10}11 12// LOAD-ND-LABEL: @load_1D_vector(13// LOAD-ND-SAME: %[[SRC:.+]]: memref<8x16x32xf32>,14// LOAD-ND: %[[CST:.+]] = arith.constant dense<true> : vector<8xi1>15// LOAD-ND: %[[STEP:.+]] = vector.step : vector<8xindex>16// LOAD-ND-COUNT2: arith.muli {{.*}} : index17// LOAD-ND-COUNT2: arith.addi {{.*}} : index18// LOAD-ND: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8xindex>19// LOAD-ND: %[[IDX:.+]] = arith.addi %[[SPLAT]], %[[STEP]] : vector<8xindex>20// LOAD-ND: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<8x16x32xf32> -> index21// LOAD-ND: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i6422// LOAD-ND: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : i64, vector<8xindex>, vector<8xi1> -> vector<8xf32>23 24// LOAD-GATHER-LABEL: @load_1D_vector(25// LOAD-GATHER-SAME: %[[SRC:.+]]: memref<8x16x32xf32>,26// LOAD-GATHER: %[[CST:.+]] = arith.constant dense<true> : vector<8xi1>27// LOAD-GATHER: %[[STEP:.+]] = vector.step : vector<8xindex>28// LOAD-GATHER-COUNT2: arith.muli {{.*}} : index29// LOAD-GATHER-COUNT2: arith.addi {{.*}} : index30// LOAD-GATHER: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8xindex>31// LOAD-GATHER: %[[IDX:.+]] = arith.addi %[[SPLAT]], %[[STEP]] : vector<8xindex>32// LOAD-GATHER: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<8x16x32xf32> -> index33// LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i6434// LOAD-GATHER: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : i64, vector<8xindex>, vector<8xi1> -> vector<8xf32>35 36}37 38// -----39gpu.module @xevm_module {40gpu.func @load_2D_vector(%source: memref<8x16x32xf32>,41 %offset: index) -> vector<8x16xf32> {42 %c0 = arith.constant 0.0 : f3243 %0 = vector.transfer_read %source[%offset, %offset, %offset], %c044 {in_bounds = [true, true]} : memref<8x16x32xf32>, vector<8x16xf32>45 gpu.return %0 : vector<8x16xf32>46}47 48// LOAD-ND-LABEL: @load_2D_vector(49// LOAD-ND-SAME: %[[SRC:.+]]: memref<8x16x32xf32>,50// LOAD-ND-SAME: %[[OFFSET:.+]]: index51// LOAD-ND: %[[COLLAPSED:.+]] = memref.subview %[[SRC]][%[[OFFSET]], 0, 0]52// LOAD-ND: %[[DESC:.+]] = xegpu.create_nd_tdesc53// LOAD-ND-SAME: %[[COLLAPSED]]54// LOAD-ND-SAME: memref<16x32xf32, strided<[32, 1], offset: ?>> -> !xegpu.tensor_desc<8x16xf32,55// LOAD-ND-SAME: boundary_check = false56// LOAD-ND: %[[VEC:.+]] = xegpu.load_nd %[[DESC]][%[[OFFSET]], %[[OFFSET]]]{{.*}}-> vector<8x16xf32>57// LOAD-ND: return %[[VEC]]58 59// LOAD-GATHER-LABEL: @load_2D_vector(60// LOAD-GATHER-SAME: %[[SRC:.+]]: memref<8x16x32xf32>,61// LOAD-GATHER: %[[CST:.+]] = arith.constant dense<true> : vector<8x16xi1>62// LOAD-GATHER-COUNT2: vector.step63// LOAD-GATHER-COUNT2: vector.shape_cast64// LOAD-GATHER-COUNT2: vector.broadcast65// LOAD-GATHER-COUNT2: arith.muli {{.*}} : index66// LOAD-GATHER-COUNT2: arith.addi {{.*}} : index67// LOAD-GATHER: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8x16xindex>68// LOAD-GATHER: %[[IDX:.+]] = arith.addi %[[SPLAT]], {{.*}}: vector<8x16xindex>69// LOAD-GATHER: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<8x16x32xf32> -> index70// LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i6471// LOAD-GATHER: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : i64, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf32>72 73}74 75 76// -----77gpu.module @xevm_module {78gpu.func @load_zero_pad_out_of_bounds(%source: memref<32x64xf32>,79 %offset: index) -> vector<8x16xf32> {80 %c0 = arith.constant 0.0 : f3281 %0 = vector.transfer_read %source[%offset, %offset], %c082 {in_bounds = [false, true]} : memref<32x64xf32>, vector<8x16xf32>83 gpu.return %0 : vector<8x16xf32>84}85 86// LOAD-ND-LABEL: @load_zero_pad_out_of_bounds(87// LOAD-ND-SAME: %[[SRC:.+]]: memref<32x64xf32>,88// LOAD-ND-SAME: %[[OFFSET:.+]]: index89// LOAD-ND: %[[DESC:.+]] = xegpu.create_nd_tdesc %[[SRC]]90// LOAD-ND-SAME: memref<32x64xf32> -> !xegpu.tensor_desc<8x16xf32>91// LOAD-ND: %[[VEC:.+]] = xegpu.load_nd %[[DESC]][%[[OFFSET]], %[[OFFSET]]]{{.*}}-> vector<8x16xf32>92// LOAD-ND: return %[[VEC]]93 94// LOAD-GATHER-LABEL: @load_zero_pad_out_of_bounds(95// LOAD-GATHER: vector.transfer_read96 97}98 99 100// -----101gpu.module @xevm_module {102gpu.func @load_transposed(%source: memref<32x64xf32>,103 %i: index, %j: index) -> vector<8x16xf32> {104 %c0 = arith.constant 0.0 : f32105 %0 = vector.transfer_read %source[%i, %j], %c0106 {permutation_map = affine_map<(d0, d1) -> (d1, d0)>,107 in_bounds = [true, true]} : memref<32x64xf32>, vector<8x16xf32>108 gpu.return %0 : vector<8x16xf32>109}110 111// LOAD-ND-LABEL: @load_transposed(112// LOAD-ND-SAME: %[[SRC:.+]]: memref<32x64xf32>,113// LOAD-ND-SAME: %[[OFFSET1:.+]]: index, 114// LOAD-ND-SAME: %[[OFFSET2:.+]]: index 115// LOAD-ND: %[[DESC:.+]] = xegpu.create_nd_tdesc %[[SRC]]116// LOAD-ND-SAME: memref<32x64xf32> -> !xegpu.tensor_desc<16x8xf32117// LOAD-ND: %[[VEC:.+]] = xegpu.load_nd %[[DESC]][%[[OFFSET1]], %[[OFFSET2]]] <{transpose = array<i64: 1, 0>}>118// LOAD-ND-SAME: -> vector<8x16xf32>119// LOAD-ND: return %[[VEC]]120 121 122// LOAD-GATHER-LABEL: @load_transposed(123// LOAD-GATHER-SAME: %[[SRC:.+]]: memref<32x64xf32>,124// LOAD-GATHER: %[[CST:.+]] = arith.constant dense<true> : vector<8x16xi1>125// LOAD-GATHER-COUNT2: vector.step126// LOAD-GATHER-COUNT2: vector.shape_cast127// LOAD-GATHER-COUNT2: vector.broadcast128// LOAD-GATHER-COUNT2: arith.muli {{.*}} : index129// LOAD-GATHER-COUNT2: arith.addi {{.*}} : index130// LOAD-GATHER: %[[BCAST2:.+]] = vector.broadcast {{.*}} : index to vector<8x16xindex>131// LOAD-GATHER: %[[IDX:.+]] = arith.addi %[[BCAST2]], {{.*}}: vector<8x16xindex>132// LOAD-GATHER: %[[COLLAPSE:.*]] = memref.extract_aligned_pointer_as_index %arg0 : memref<32x64xf32> -> index133// LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64134// LOAD-GATHER: %[[LOAD:.*]] = xegpu.load %[[COLLAPSE_I]][%[[IDX]]], %[[CST]] : i64, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf32>135 136}137 138// -----139gpu.module @xevm_module {140gpu.func @load_dynamic_source(%source: memref<?x?x?xf32>,141 %i: index, %j: index, %k: index) -> vector<8x16xf32> {142 %c0 = arith.constant 0.0 : f32143 %0 = vector.transfer_read %source[%i, %j, %k], %c0144 {in_bounds = [true, true]} : memref<?x?x?xf32>, vector<8x16xf32>145 gpu.return %0 : vector<8x16xf32>146}147// LOAD-ND-LABEL: @load_dynamic_source(148// LOAD-ND-SAME: %[[SRC:.+]]: memref<?x?x?xf32>,149// LOAD-ND-SAME: %[[OFF0:.+]]: index, %[[OFF1:.+]]: index, %[[OFF2:.+]]: index150// LOAD-ND: %[[COLLAPSED:.+]] = memref.subview %[[SRC]][%[[OFF0]], 0, 0]151// LOAD-ND: {{.*}} %[[SIZES:.+]]:2, %[[STRIDES:.+]]:2 = memref.extract_strided_metadata %[[COLLAPSED]]152// LOAD-ND: %[[DESC:.+]] = xegpu.create_nd_tdesc %[[COLLAPSED]]153// LOAD-ND: %[[VEC:.+]] = xegpu.load_nd %[[DESC]][%[[OFF1]], %[[OFF2]]]{{.*}}-> vector<8x16xf32>154// LOAD-ND: return %[[VEC]]155 156 157// LOAD-GATHER-LABEL: @load_dynamic_source(158// LOAD-GATHER-SAME: %[[ARG0:.+]]: memref<?x?x?xf32>,159// LOAD-GATHER: %[[CST:.+]] = arith.constant dense<true> : vector<8x16xi1>160// LOAD-GATHER: memref.extract_strided_metadata %[[ARG0]]161// LOAD-GATHER-COUNT2: vector.step162// LOAD-GATHER-COUNT2: vector.shape_cast163// LOAD-GATHER-COUNT2: vector.broadcast164// LOAD-GATHER-COUNT2: arith.muli {{.*}} : index165// LOAD-GATHER-COUNT2: arith.addi {{.*}} : index166// LOAD-GATHER: %[[BROADIDX:.+]] = vector.broadcast {{.*}} : index to vector<8x16xindex>167// LOAD-GATHER: %[[FINALIDX:.+]] = arith.addi %[[BROADIDX]], {{.*}} : vector<8x16xindex>168// LOAD-GATHER: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[ARG0]] : memref<?x?x?xf32> -> index169// LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64170// LOAD-GATHER: %[[RES:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[FINALIDX]]{{\]}}, %[[CST]] : i64, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf32>171// LOAD-GATHER: gpu.return %[[RES]] : vector<8x16xf32>172}173 174// -----175gpu.module @xevm_module {176gpu.func @load_dynamic_source2(%source: memref<?x8x16xf32>,177 %i: index, %j: index, %k: index) -> vector<8x16xf32> {178 %c0 = arith.constant 0.0 : f32179 %0 = vector.transfer_read %source[%i, %j, %k], %c0180 {in_bounds = [true, true]} : memref<?x8x16xf32>, vector<8x16xf32>181 gpu.return %0 : vector<8x16xf32>182}183 184// LOAD-ND-LABEL: @load_dynamic_source2(185// LOAD-ND-SAME: %[[SRC:.+]]: memref<?x8x16xf32>,186// LOAD-ND-SAME: %[[OFF0:.+]]: index, %[[OFF1:.+]]: index, %[[OFF2:.+]]: index187// LOAD-ND: %[[COLLAPSED:.+]] = memref.subview %[[SRC]][%[[OFF0]], 0, 0]188// LOAD-ND: %[[DESC:.+]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x16xf32, strided<[16, 1], offset: ?>> -> !xegpu.tensor_desc<8x16xf32, #xegpu.block_tdesc_attr<boundary_check = false>>189// LOAD-ND: %[[VEC:.+]] = xegpu.load_nd %[[DESC]][%{{.*}}, %{{.*}}] : !xegpu.tensor_desc<8x16xf32, #xegpu.block_tdesc_attr<boundary_check = false>> -> vector<8x16xf32>190// LOAD-ND: return %[[VEC]] : vector<8x16xf32>191 192// LOAD-GATHER-LABEL: @load_dynamic_source2(193// LOAD-GATHER-DAG: %[[CST_0:.+]] = arith.constant dense<true> : vector<8x16xi1>194// LOAD-GATHER-COUNT2: vector.step195// LOAD-GATHER-COUNT2: vector.shape_cast196// LOAD-GATHER-COUNT2: vector.broadcast197// LOAD-GATHER-COUNT2: arith.muli {{.*}} : index198// LOAD-GATHER-COUNT2: arith.addi {{.*}} : index199// LOAD-GATHER-DAG: %[[BCASTIDX:.+]] = vector.broadcast {{.*}} : index to vector<8x16xindex>200// LOAD-GATHER-DAG: %[[OFFSETS:.+]] = arith.addi %[[BCASTIDX]], {{.*}} : vector<8x16xindex>201// LOAD-GATHER-DAG: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %arg0 : memref<?x8x16xf32> -> index202// LOAD-GATHER-DAG: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64203// LOAD-GATHER: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[OFFSETS]]{{\]}}, %[[CST_0]] : i64, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf32> 204 205}206 207// -----208gpu.module @xevm_module {209gpu.func @load_dynamic_source3(%source: memref<?x?x?x?x?xf32>,210 %i: index, %j: index, %k: index, %l: index, %m: index) -> vector<2x4x8x16xf32> {211 %c0 = arith.constant 0.0 : f32212 %0 = vector.transfer_read %source[%i, %j, %k, %l, %m], %c0213 {in_bounds = [true, true, true, true]} : memref<?x?x?x?x?xf32>, vector<2x4x8x16xf32>214 gpu.return %0 : vector<2x4x8x16xf32>215}216 217// LOAD-ND-LABEL: @load_dynamic_source3(218// LOAD-ND: vector.transfer_read219 220// LOAD-GATHER-LABEL: @load_dynamic_source3(221// LOAD-GATHER-SAME: %[[SRC:.+]]: memref<?x?x?x?x?xf32>222// LOAD-GATHER: %[[CST:.+]] = arith.constant dense<true> : vector<2x4x8x16xi1>223// LOAD-GATHER: memref.extract_strided_metadata %[[SRC]] : memref<?x?x?x?x?xf32> -> memref<f32>, index, index, index, index, index, index, index, index, index, index, index224// LOAD-GATHER-COUNT4: vector.step225// LOAD-GATHER-COUNT3: vector.broadcast226// LOAD-GATHER-COUNT4: vector.shape_cast227// LOAD-GATHER-COUNT4: vector.broadcast {{.*}} : vector<2x4x8x16xindex>228// LOAD-GATHER-COUNT3: arith.addi {{.*}} : vector<2x4x8x16xindex>229// LOAD-GATHER: %[[SPLAT:.+]] = vector.broadcast {{.*}} : index to vector<2x4x8x16xindex>230// LOAD-GATHER: %[[IDX:.+]] = arith.addi %[[SPLAT]], {{.*}} : vector<2x4x8x16xindex>231// LOAD-GATHER: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SRC]] : memref<?x?x?x?x?xf32> -> index232// LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64233// LOAD-GATHER: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : i64, vector<2x4x8x16xindex>, vector<2x4x8x16xi1> -> vector<2x4x8x16xf32>234// LOAD-GATHER: return %[[VEC]]235}236 237// -----238gpu.module @xevm_module {239gpu.func @load_high_dim_vector(%source: memref<16x32x64xf32>,240 %offset: index, %arg2: index) -> vector<8x16x32xf32> {241 %c0 = arith.constant 0.0 : f32242 %0 = vector.transfer_read %source[%offset, %arg2, %offset], %c0243 {in_bounds = [true, true, true]} : memref<16x32x64xf32>, vector<8x16x32xf32>244 gpu.return %0 : vector<8x16x32xf32>245}246 247// LOAD-ND-LABEL: @load_high_dim_vector(248// LOAD-ND: vector.transfer_read249 250// LOAD-GATHER-LABEL: @load_high_dim_vector(251// LOAD-GATHER: %[[CST:.+]] = arith.constant dense<true> : vector<8x16x32xi1>252// LOAD-GATHER: %[[CST_0:.+]] = arith.constant dense<64> : vector<16xindex>253// LOAD-GATHER: %[[CST_1:.+]] = arith.constant dense<2048> : vector<8xindex>254// LOAD-GATHER: %[[C2048:.+]] = arith.constant 2048 : index255// LOAD-GATHER: %[[C64:.+]] = arith.constant 64 : index256// LOAD-GATHER-COUNT3: vector.step257// LOAD-GATHER-COUNT3: vector.shape_cast258// LOAD-GATHER-COUNT3: vector.broadcast {{.*}} : vector<8x16x32xindex>259// LOAD-GATHER-COUNT2: arith.addi {{.*}} : vector<8x16x32xindex>260// LOAD-GATHER: %[[BCASTOFF:.+]] = vector.broadcast {{.*}} : index to vector<8x16x32xindex>261// LOAD-GATHER: %[[IDX:.+]] = arith.addi %[[BCASTOFF]], {{.*}} : vector<8x16x32xindex>262// LOAD-GATHER: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %arg0 : memref<16x32x64xf32> -> index263// LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64264// LOAD-GATHER: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]][%[[IDX]]], %[[CST]] : i64, vector<8x16x32xindex>, vector<8x16x32xi1> -> vector<8x16x32xf32>265 266}267 268// -----269gpu.module @xevm_module {270gpu.func @load_transpose_f16(%source: memref<32x64xf16>,271 %offset: index) -> vector<8x16xf16> {272 %c0 = arith.constant 0.0 : f16273 %0 = vector.transfer_read %source[%offset, %offset], %c0274 {permutation_map = affine_map<(d0, d1) -> (d1, d0)>,275 in_bounds = [true, true]} : memref<32x64xf16>, vector<8x16xf16>276 gpu.return %0 : vector<8x16xf16>277}278 279// LOAD-ND-LABEL: @load_transpose_f16(280// LOAD-ND: vector.transfer_read281 282// LOAD-GATHER-LABEL: @load_transpose_f16(283// LOAD-GATHER-SAME: %[[SRC:.+]]: memref<32x64xf16>,284// LOAD-GATHER: %[[CST:.+]] = arith.constant dense<true> : vector<8x16xi1>285// LOAD-GATHER-COUNT2: vector.step286// LOAD-GATHER-COUNT2: vector.shape_cast287// LOAD-GATHER-COUNT2: vector.broadcast288// LOAD-GATHER-COUNT2: arith.muli {{.*}} : index289// LOAD-GATHER-COUNT2: arith.addi {{.*}} : index290// LOAD-GATHER: %[[BCAST2:.+]] = vector.broadcast {{.*}} : index to vector<8x16xindex>291// LOAD-GATHER: %[[IDX:.+]] = arith.addi %[[BCAST2]], {{.*}}: vector<8x16xindex>292// LOAD-GATHER: %[[COLLAPSE:.*]] = memref.extract_aligned_pointer_as_index %arg0 : memref<32x64xf16> -> index293// LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64294// LOAD-GATHER: %[[LOAD:.*]] = xegpu.load %[[COLLAPSE_I]][%[[IDX]]], %[[CST]] : i64, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf16>295}296 297// -----298gpu.module @xevm_module {299gpu.func @no_load_out_of_bounds_non_zero_pad(%source: memref<32x64xf32>,300 %offset: index, %arg2: index, %pad: f32) -> (vector<8x16xf32>, vector<8x16xf32>) {301 %c1 = arith.constant 1.0 : f32302 %0 = vector.transfer_read %source[%offset, %arg2], %c1303 {in_bounds = [true, false]} : memref<32x64xf32>, vector<8x16xf32>304 %1 = vector.transfer_read %source[%arg2, %offset], %pad305 {in_bounds = [false, true]} : memref<32x64xf32>, vector<8x16xf32>306 gpu.return %0, %1 : vector<8x16xf32>, vector<8x16xf32>307}308 309// LOAD-ND-LABEL: @no_load_out_of_bounds_non_zero_pad(310// LOAD-ND-COUNT-2: vector.transfer_read311 312// LOAD-GATHER-LABEL: @no_load_out_of_bounds_non_zero_pad(313// LOAD-GATHER-COUNT-2: vector.transfer_read314}315 316// -----317gpu.module @xevm_module {318gpu.func @no_load_out_of_bounds_1D_vector(%source: memref<8x16x32xf32>,319 %offset: index) -> vector<8xf32> {320 %c0 = arith.constant 0.0 : f32321 %0 = vector.transfer_read %source[%offset, %offset, %offset], %c0322 {in_bounds = [false]} : memref<8x16x32xf32>, vector<8xf32>323 gpu.return %0 : vector<8xf32>324}325 326// LOAD-ND-LABEL: @no_load_out_of_bounds_1D_vector(327// LOAD-ND: vector.transfer_read328 329// LOAD-GATHER-LABEL: @no_load_out_of_bounds_1D_vector(330// LOAD-GATHER: vector.transfer_read331}332 333// -----334gpu.module @xevm_module {335gpu.func @no_load_masked(%source : memref<4xf32>,336 %offset : index) -> vector<4xf32> {337 %c0 = arith.constant 0.0 : f32338 %mask = arith.constant dense<[0, 1, 0, 1]> : vector<4xi1>339 %0 = vector.transfer_read %source[%offset], %c0, %mask340 {in_bounds = [true]} : memref<4xf32>, vector<4xf32>341 gpu.return %0 : vector<4xf32>342}343 344// LOAD-ND-LABEL: @no_load_masked(345// LOAD-ND: vector.transfer_read346 347// LOAD-GATHER-LABEL: @no_load_masked(348// LOAD-GATHER: vector.transfer_read349}350 351// -----352gpu.module @xevm_module {353gpu.func @no_load_tensor(%source: tensor<32x64xf32>,354 %offset: index, %arg2: index) -> vector<8x16xf32> {355 %c0 = arith.constant 0.0 : f32356 %0 = vector.transfer_read %source[%offset, %arg2], %c0357 {in_bounds = [true, true]} : tensor<32x64xf32>, vector<8x16xf32>358 gpu.return %0 : vector<8x16xf32>359}360 361// LOAD-ND-LABEL: @no_load_tensor(362// LOAD-ND: vector.transfer_read363 364// LOAD-GATHER-LABEL: @no_load_tensor(365// LOAD-GATHER: vector.transfer_read366}367 368 369// -----370gpu.module @xevm_module {371gpu.func @no_load_non_unit_inner_stride(372 %source: memref<32xf32, strided<[?], offset: ?>>,373 %offset: index) -> vector<8xf32> {374 %c0 = arith.constant 0.0 : f32375 %0 = vector.transfer_read %source[%offset], %c0 {in_bounds = [true]}376 : memref<32xf32, strided<[?], offset: ?>>, vector<8xf32>377 gpu.return %0 : vector<8xf32>378}379 380// LOAD-ND-LABEL: @no_load_non_unit_inner_stride(381// LOAD-ND: vector.transfer_read382 383// LOAD-GATHER-LABEL: @no_load_non_unit_inner_stride(384// LOAD-GATHER: vector.transfer_read385}386 387 388// -----389gpu.module @xevm_module {390gpu.func @no_load_unsupported_map(%source: memref<16x32x64xf32>,391 %offset: index) -> vector<8x16xf32> {392 %c0 = arith.constant 0.0 : f32393 %0 = vector.transfer_read %source[%offset, %offset, %offset], %c0394 {permutation_map = affine_map<(d0, d1, d2) -> (d0, d2)>,395 in_bounds = [true, true]} : memref<16x32x64xf32>, vector<8x16xf32>396 gpu.return %0 : vector<8x16xf32>397}398 399// LOAD-ND-LABEL: @no_load_unsupported_map(400// LOAD-ND: vector.transfer_read401 402// LOAD-GATHER-LABEL: @no_load_unsupported_map(403// LOAD-GATHER: vector.transfer_read404}405 406// -----407gpu.module @xevm_module {408gpu.func @load_from_subview_1D(%source: memref<4096x4096xf16>, %off1: index, %off2: index) -> vector<8xf16> {409 %c0 = arith.constant 0.0 : f16410 %subview = memref.subview %source[%off1, %off2] [256, 256] [1, 1] : memref<4096x4096xf16> to memref<256x256xf16, strided<[4096, 1], offset: ?>>411 %0 = vector.transfer_read %subview[%off2, %off2], %c0412 {in_bounds = [true]} : memref<256x256xf16, strided<[4096, 1], offset: ?>>, vector<8xf16>413 gpu.return %0 : vector<8xf16>414}415 416// LOAD-ND-LABEL: @load_from_subview_1D(417// LOAD-ND-SAME: %[[SRC:.+]]: memref<4096x4096xf16>,418// LOAD-ND-SAME: %[[OFF1:.+]]: index, %[[OFF2:.+]]: index419// LOAD-ND: %[[CST:.+]] = arith.constant dense<true> : vector<8xi1>420// LOAD-ND: %[[SUBVIEW:.+]] = memref.subview %[[SRC]][%[[OFF1]], %[[OFF2]]] [256, 256] [1, 1] : memref<4096x4096xf16> to memref<256x256xf16, strided<[4096, 1], offset: ?>> 421// LOAD-ND: %[[BB:.+]], %[[OFFSET:.+]],{{.*}},{{.*}} = memref.extract_strided_metadata %[[SUBVIEW]] : memref<256x256xf16, strided<[4096, 1], offset: ?>> -> memref<f16>, index, index, index, index, index422// LOAD-ND: %[[STEP:.+]] = vector.step : vector<8xindex>423// LOAD-ND: arith.muli {{.*}} : index424// LOAD-ND: arith.addi %[[OFFSET]]{{.*}} : index425// LOAD-ND: arith.addi {{.*}} : index426// LOAD-ND: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8xindex>427// LOAD-ND: %[[IDX:.+]] = arith.addi %[[SPLAT]], %[[STEP]] : vector<8xindex>428// LOAD-ND: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SUBVIEW]] : memref<256x256xf16, strided<[4096, 1], offset: ?>> -> index429// LOAD-ND: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64430// LOAD-ND: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : i64, vector<8xindex>, vector<8xi1> -> vector<8xf16>431 432// LOAD-GATHER-LABEL: @load_from_subview_1D(433// LOAD-GATHER-SAME: %[[SRC:.+]]: memref<4096x4096xf16>,434// LOAD-GATHER-SAME: %[[OFF1:.+]]: index, %[[OFF2:.+]]: index435// LOAD-GATHER: %[[CST:.+]] = arith.constant dense<true> : vector<8xi1>436// LOAD-GATHER: %[[SUBVIEW:.+]] = memref.subview %[[SRC]][%[[OFF1]], %[[OFF2]]] [256, 256] [1, 1] : memref<4096x4096xf16> to memref<256x256xf16, strided<[4096, 1], offset: ?>> 437// LOAD-GATHER: %[[BB:.+]], %[[OFFSET:.+]],{{.*}},{{.*}} = memref.extract_strided_metadata %[[SUBVIEW]] : memref<256x256xf16, strided<[4096, 1], offset: ?>> -> memref<f16>, index, index, index, index, index438// LOAD-GATHER: %[[STEP:.+]] = vector.step : vector<8xindex>439// LOAD-GATHER: arith.muli {{.*}} : index440// LOAD-GATHER: arith.addi %[[OFFSET]]{{.*}} : index441// LOAD-GATHER: arith.addi {{.*}} : index442// LOAD-GATHER: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8xindex>443// LOAD-GATHER: %[[IDX:.+]] = arith.addi %[[SPLAT]], %[[STEP]] : vector<8xindex>444// LOAD-GATHER: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SUBVIEW]] : memref<256x256xf16, strided<[4096, 1], offset: ?>> -> index445// LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64446// LOAD-GATHER: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : i64, vector<8xindex>, vector<8xi1> -> vector<8xf16>447}448 449// -----450gpu.module @xevm_module {451gpu.func @load_from_subview_2D(%source: memref<4096x4096xf16>, %off1: index, %off2: index) -> vector<8x16xf16> {452 %c0 = arith.constant 0.0 : f16453 %subview = memref.subview %source[%off1, %off2] [256, 256] [1, 1] : memref<4096x4096xf16> to memref<256x256xf16, strided<[4096, 1], offset: ?>>454 %0 = vector.transfer_read %subview[%off2, %off2], %c0455 {in_bounds = [true, true]} : memref<256x256xf16, strided<[4096, 1], offset: ?>>, vector<8x16xf16>456 gpu.return %0 : vector<8x16xf16>457}458 459// LOAD-ND-LABEL: @load_from_subview_2D(460// LOAD-ND-SAME: %[[SRC:.+]]: memref<4096x4096xf16>,461// LOAD-ND-SAME: %[[OFF1:.+]]: index, %[[OFF2:.+]]: index462// LOAD-ND: %[[SUBVIEW:.+]] = memref.subview %[[SRC]][%[[OFF1]], %[[OFF2]]] [256, 256] [1, 1] : memref<4096x4096xf16> to memref<256x256xf16, strided<[4096, 1], offset: ?>>463// LOAD-ND: %[[DESC:.+]] = xegpu.create_nd_tdesc464// LOAD-ND-SAME: %[[SUBVIEW]]465// LOAD-ND-SAME: memref<256x256xf16, strided<[4096, 1], offset: ?>> -> !xegpu.tensor_desc<8x16xf16,466// LOAD-ND-SAME: boundary_check = false467// LOAD-ND: %[[VEC:.+]] = xegpu.load_nd %[[DESC]][%[[OFF2]], %[[OFF2]]]{{.*}}-> vector<8x16xf16>468// LOAD-ND: return %[[VEC]]469 470// LOAD-GATHER-LABEL: @load_from_subview_2D(471// LOAD-GATHER-SAME: %[[SRC:.+]]: memref<4096x4096xf16>,472// LOAD-GATHER-SAME: %[[OFF1:.+]]: index, %[[OFF2:.+]]: index473// LOAD-GATHER: %[[CST:.+]] = arith.constant dense<true> : vector<8x16xi1>474// LOAD-GATHER: %[[SUBVIEW:.+]] = memref.subview %[[SRC]][%[[OFF1]], %[[OFF2]]] [256, 256] [1, 1] : memref<4096x4096xf16> to memref<256x256xf16, strided<[4096, 1], offset: ?>>475// LOAD-GATHER: %[[BB:.+]], %[[OFFSET:.+]],{{.*}},{{.*}} = memref.extract_strided_metadata %[[SUBVIEW]] : memref<256x256xf16, strided<[4096, 1], offset: ?>> -> memref<f16>, index, index, index, index, index476// LOAD-GATHER-COUNT2: vector.step477// LOAD-GATHER-COUNT2: vector.shape_cast478// LOAD-GATHER-COUNT2: vector.broadcast479// LOAD-GATHER-COUNT2: arith.muli {{.*}} : index480// LOAD-GATHER-COUNT2: arith.addi {{.*}} : index481// LOAD-GATHER: %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8x16xindex>482// LOAD-GATHER: %[[IDX:.+]] = arith.addi %[[SPLAT]], {{.*}} : vector<8x16xindex>483// LOAD-GATHER: %[[COLLAPSE:.+]] = memref.extract_aligned_pointer_as_index %[[SUBVIEW]] : memref<256x256xf16, strided<[4096, 1], offset: ?>> -> index484// LOAD-GATHER: %[[COLLAPSE_I:.+]] = arith.index_cast %[[COLLAPSE]] : index to i64485// LOAD-GATHER: %[[VEC:.+]] = xegpu.load %[[COLLAPSE_I]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : i64, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf16>486}487