494 lines · plain
1// RUN: mlir-opt %s --transform-interpreter --split-input-file | FileCheck %s2 3// TODO: Review the usage of `in_bounds` and remove where not affecting the4// generated output.5 6/// CHECK: #[[$MAP:.*]] = affine_map<(d0, d1, d2, d3) -> (d1, 0, d3)>7 8///----------------------------------------------------------------------------------------9/// [Pattern: TransferWritePermutationLowering]10///11/// IN: vector.transfer_write (_transposed_ minor identity permutation map)12/// OUT: vector.transpose + vector.transfer_write (minor identity permutation map)13///----------------------------------------------------------------------------------------14 15// CHECK-LABEL: func.func @xfer_write_minor_identity_transposed16// CHECK-SAME: %[[VEC:.*]]: vector<4x8xi16>,17// CHECK-SAME: %[[MEM:.*]]: memref<2x2x8x4xi16>18// CHECK: %[[TR:.*]] = vector.transpose %[[VEC]], [1, 0] : vector<4x8xi16> to vector<8x4xi16>19// CHECK: vector.transfer_write20// CHECK-NOT: permutation_map21// CHECK-SAME: %[[TR]], %[[MEM]]{{.*}} {in_bounds = [true, true]} : vector<8x4xi16>, memref<2x2x8x4xi16>22func.func @xfer_write_minor_identity_transposed(23 %vec: vector<4x8xi16>,24 %mem: memref<2x2x8x4xi16>,25 %idx: index) {26 27 vector.transfer_write %vec, %mem[%idx, %idx, %idx, %idx] {28 in_bounds = [true, true],29 permutation_map = affine_map<(d0, d1, d2, d3) -> (d3, d2)>30 } : vector<4x8xi16>, memref<2x2x8x4xi16>31 32 return33}34 35// Even with out-of-bounds accesses, it is safe to apply this pattern36 37// CHECK-LABEL: func.func @xfer_write_minor_identity_transposed_out_of_bounds38// CHECK-SAME: %[[VEC:.*]]: vector<4x8xi16>,39// CHECK-SAME: %[[MEM:.*]]: memref<2x2x?x?xi16>,40// CHECK-SAME: %[[IDX:.*]]: index) {41// CHECK: %[[TR:.*]] = vector.transpose %[[VEC]], [1, 0] : vector<4x8xi16> to vector<8x4xi16>42// Expect the in_bounds attribute to be preserved. Since we don't print it when43// all flags are "false", it should not appear in the output.44// CHECK-NOT: in_bounds45// CHECK: vector.transfer_write46// CHECK-NOT: permutation_map47// CHECK-SAME: %[[TR]], %[[MEM]][%[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]]] : vector<8x4xi16>, memref<2x2x?x?xi16>48func.func @xfer_write_minor_identity_transposed_out_of_bounds(49 %vec: vector<4x8xi16>,50 %mem: memref<2x2x?x?xi16>,51 %idx: index) {52 53 vector.transfer_write %vec, %mem[%idx, %idx, %idx, %idx] {54 in_bounds = [false, false],55 permutation_map = affine_map<(d0, d1, d2, d3) -> (d3, d2)>56 } : vector<4x8xi16>, memref<2x2x?x?xi16>57 58 return59}60 61// CHECK-LABEL: func.func @xfer_write_minor_identity_transposed_with_mask_scalable62// CHECK-SAME: %[[VEC:.*]]: vector<4x[8]xi16>,63// CHECK-SAME: %[[MEM:.*]]: memref<2x2x?x4xi16>,64// CHECK-SAME: %[[MASK:.*]]: vector<[8]x4xi1>65// CHECK: %[[TR:.*]] = vector.transpose %[[VEC]], [1, 0] : vector<4x[8]xi16> to vector<[8]x4xi16>66// CHECK: vector.transfer_write67// CHECK-NOT: permutation_map68// CHECK-SAME: %[[TR]], %[[MEM]]{{.*}}, %[[MASK]] {in_bounds = [true, true]} : vector<[8]x4xi16>, memref<2x2x?x4xi16>69func.func @xfer_write_minor_identity_transposed_with_mask_scalable(70 %vec: vector<4x[8]xi16>,71 %mem: memref<2x2x?x4xi16>,72 %mask: vector<[8]x4xi1>,73 %idx: index) {74 75 %c0 = arith.constant 0 : index76 vector.transfer_write %vec, %mem[%idx, %idx, %idx, %idx], %mask {77 in_bounds = [true, true],78 permutation_map = affine_map<(d0, d1, d2, d3) -> (d3, d2)>79 } : vector<4x[8]xi16>, memref<2x2x?x4xi16>80 81 return82}83 84// Masked version is not supported85 86// CHECK-LABEL: func.func @xfer_write_minor_identity_transposed_map_masked87// CHECK-NOT: vector.transpose88func.func @xfer_write_minor_identity_transposed_map_masked(89 %vec: vector<4x8xi16>,90 %mem: memref<2x2x8x4xi16>,91 %mask: vector<8x4xi1>,92 %idx: index) {93 94 %c0 = arith.constant 0 : index95 vector.mask %mask {96 vector.transfer_write %vec, %mem[%idx, %idx, %idx, %idx] {97 in_bounds = [true, true],98 permutation_map = affine_map<(d0, d1, d2, d3) -> (d3, d2)>99 } : vector<4x8xi16>, memref<2x2x8x4xi16>100 } : vector<8x4xi1>101 102 return103}104 105///----------------------------------------------------------------------------------------106/// [Patterns: TransferWriteNonPermutationLowering + TransferWritePermutationLowering]107///108/// IN: vector.transfer_write109/// (neither a minor identity nor transposed minor identity map)110/// OUT 1: vector.broadcast + vector.transfer_write111/// (transposed minor identity)112/// OUT 2: vector.transfer_write -> vector.broadcast + vector.transpose113/// + vector.transfer_write (minor identity)114///----------------------------------------------------------------------------------------115 116// CHECK-LABEL: func.func @xfer_write_non_minor_identity(117// CHECK-SAME: %[[MEM:.*]]: memref<?x?xf32>,118// CHECK-SAME: %[[VEC:.*]]: vector<7xf32>,119// CHECK-SAME: %[[IDX_1:.*]]: index, %[[IDX_2:.*]]: index) {120// CHECK: %[[BC:.*]] = vector.broadcast %[[VEC]] : vector<7xf32> to vector<1x7xf32>121// CHECK: %[[TR:.*]] = vector.transpose %[[BC]], [1, 0] : vector<1x7xf32> to vector<7x1xf32>122// CHECK: vector.transfer_write %[[TR]], %[[MEM]]{{\[}}%[[IDX_1]], %[[IDX_2]]] {in_bounds = [false, true]} : vector<7x1xf32>, memref<?x?xf32>123func.func @xfer_write_non_minor_identity(124 %mem : memref<?x?xf32>,125 %vec : vector<7xf32>,126 %idx_1 : index,127 %idx_2 : index) {128 129 vector.transfer_write %vec, %mem[%idx_1, %idx_2] {130 permutation_map = affine_map<(d0, d1) -> (d0)>131 } : vector<7xf32>, memref<?x?xf32>132 133 return134}135 136// Even with out-of-bounds accesses, it is safe to apply this pattern137 138// CHECK-LABEL: func.func @xfer_write_non_minor_identity_with_mask_out_of_bounds(139// CHECK-SAME: %[[MEM:.*]]: memref<?x?xf32>,140// CHECK-SAME: %[[VEC:.*]]: vector<7xf32>,141// CHECK-SAME: %[[IDX_1:.*]]: index, %[[IDX_2:.*]]: index,142// CHECK-SAME: %[[MASK:.*]]: vector<7xi1>) {143// CHECK: %[[BC_VEC:.*]] = vector.broadcast %[[VEC]] : vector<7xf32> to vector<1x7xf32>144// CHECK: %[[BC_MASK:.*]] = vector.broadcast %[[MASK]] : vector<7xi1> to vector<1x7xi1>145// CHECK: %[[TR_MASK:.*]] = vector.transpose %[[BC_MASK]], [1, 0] : vector<1x7xi1> to vector<7x1xi1>146// CHECK: %[[TR_VEC:.*]] = vector.transpose %[[BC_VEC]], [1, 0] : vector<1x7xf32> to vector<7x1xf32>147// CHECK: vector.transfer_write %[[TR_VEC]], %[[MEM]]{{\[}}%[[IDX_1]], %[[IDX_2]]], %[[TR_MASK]] {in_bounds = [false, true]} : vector<7x1xf32>, memref<?x?xf32>148func.func @xfer_write_non_minor_identity_with_mask_out_of_bounds(149 %mem : memref<?x?xf32>,150 %vec : vector<7xf32>,151 %idx_1 : index,152 %idx_2 : index,153 %mask : vector<7xi1>) {154 155 vector.transfer_write %vec, %mem[%idx_1, %idx_2], %mask {156 permutation_map = affine_map<(d0, d1) -> (d0)>,157 in_bounds = [false]158 } : vector<7xf32>, memref<?x?xf32>159 160 return161}162 163// CHECK-LABEL: func.func @xfer_write_non_minor_identity_with_mask_scalable(164// CHECK-SAME: %[[VEC:.*]]: vector<4x[8]xi16>,165// CHECK-SAME: %[[MEM:.*]]: memref<1x4x?x1xi16>,166// CHECK-SAME: %[[MASK:.*]]: vector<4x[8]xi1>167// CHECK: %[[BC_1:.*]] = vector.broadcast %[[VEC]] : vector<4x[8]xi16> to vector<1x4x[8]xi16>168// CHECK: %[[BC_2:.*]] = vector.broadcast %[[MASK]] : vector<4x[8]xi1> to vector<1x4x[8]xi1>169// CHECK: %[[TRANSPOSE_1:.*]] = vector.transpose %[[BC_2]], [1, 2, 0] : vector<1x4x[8]xi1> to vector<4x[8]x1xi1>170// CHECK: %[[TRANSPOSE_2:.*]] = vector.transpose %[[BC_1]], [1, 2, 0] : vector<1x4x[8]xi16> to vector<4x[8]x1xi16>171// CHECK: vector.transfer_write %[[TRANSPOSE_2]], %[[MEM]]{{.*}}, %[[TRANSPOSE_1]] {in_bounds = [true, true, true]} : vector<4x[8]x1xi16>, memref<1x4x?x1xi16>172func.func @xfer_write_non_minor_identity_with_mask_scalable(173 %vec: vector<4x[8]xi16>,174 %mem: memref<1x4x?x1xi16>,175 %mask: vector<4x[8]xi1>,176 %idx: index){177 178 vector.transfer_write %vec, %mem[%idx, %idx, %idx, %idx], %mask {179 in_bounds = [true, true],180 permutation_map = affine_map<(d0, d1, d2, d3) -> (d1, d2)>181 } : vector<4x[8]xi16>, memref<1x4x?x1xi16>182 183 return184}185 186// Masked version is not supported187 188// CHECK-LABEL: func @xfer_write_non_minor_identity_masked189// CHECK-SAME: %[[DEST:.*]]: tensor<?x?xf32>,190// CHECK-SAME: %[[VEC:.*]]: vector<16xf32>,191// CHECK-SAME: %[[IDX:.*]]: index,192// CHECK-SAME: %[[MASK:.*]]: vector<16xi1>193// CHECK-NOT: vector.transpose194// CHECK: vector.mask %[[MASK]] { vector.transfer_write %[[VEC]], %[[DEST]]{{.*}} vector<16xf32>, tensor<?x?xf32> } : vector<16xi1> -> tensor<?x?xf32>195func.func @xfer_write_non_minor_identity_masked(196 %dest: tensor<?x?xf32>,197 %vec: vector<16xf32>,198 %idx: index,199 %mask: vector<16xi1>) -> tensor<?x?xf32> {200 201 %res = vector.mask %mask {202 vector.transfer_write %vec, %dest[%idx, %idx] {203 permutation_map = affine_map<(d0, d1) -> (d0)>204 } : vector<16xf32>, tensor<?x?xf32>205 } : vector<16xi1> -> tensor<?x?xf32>206 207 return %res : tensor<?x?xf32>208}209 210// CHECK-LABEL: func.func @xfer_write_non_minor_identity_masked_scalable211// CHECK-SAME: %[[VEC:.*]]: vector<4x[8]xi16>,212// CHECK-SAME: %[[DEST:.*]]: tensor<?x?x?x?xf32>,213// CHECK-SAME: %[[MASK:.*]]: vector<4x[8]xi1>214// CHECK-SAME: -> tensor<?x?x?x?xf32> {215// CHECK-NOT: vector.transpose216// CHECK: vector.mask %[[MASK]] { vector.transfer_write %[[VEC]], %[[DEST]]{{.*}} : vector<4x[8]xi16>, tensor<?x?x?x?xf32> } : vector<4x[8]xi1> -> tensor<?x?x?x?xf32>217func.func @xfer_write_non_minor_identity_masked_scalable(218 %vec: vector<4x[8]xi16>,219 %dest: tensor<?x?x?x?xf32>,220 %mask: vector<4x[8]xi1>,221 %idx: index) -> tensor<?x?x?x?xf32> {222 223 %c0 = arith.constant 0 : index224 %res = vector.mask %mask {225 vector.transfer_write %vec, %dest[%idx, %idx, %idx, %idx] {226 in_bounds = [true, true],227 permutation_map = affine_map<(d0, d1, d2, d3) -> (d1, d2)>228 } : vector<4x[8]xi16>, tensor<?x?x?x?xf32>229 } : vector<4x[8]xi1> -> tensor<?x?x?x?xf32>230 231 return %res : tensor<?x?x?x?xf32>232}233 234// Masked version is not supported235 236// CHECK-LABEL: func @xfer_write_non_minor_identity_masked_2237// CHECK-SAME: %[[DEST:.*]]: tensor<?x?x?x?xf32>238// CHECK-SAME: %[[VEC:.*]]: vector<14x8x16xf32>239// CHECK-SAME: %[[MASK:.*]]: vector<14x8x16xi1>240// CHECK-SAME: %[[DIM:.*]]: index241// CHECK-NOT: vector.broadcast242// CHECK: vector.mask %[[MASK]] { vector.transfer_write %[[VEC]], %[[DEST]]{{.*}} : vector<14x8x16xf32>, tensor<?x?x?x?xf32> } : vector<14x8x16xi1> -> tensor<?x?x?x?xf32>243func.func @xfer_write_non_minor_identity_masked_2(244 %dest : tensor<?x?x?x?xf32>,245 %vec : vector<14x8x16xf32>,246 %mask: vector<14x8x16xi1>,247 %idx: index) -> tensor<?x?x?x?xf32> {248 249 %res = vector.mask %mask {250 vector.transfer_write %vec, %dest[%idx, %idx, %idx, %idx] {251 in_bounds = [false, false, true],252 permutation_map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>253 } : vector<14x8x16xf32>, tensor<?x?x?x?xf32>254 } : vector<14x8x16xi1> -> tensor<?x?x?x?xf32>255 256 return %res : tensor<?x?x?x?xf32>257}258 259///----------------------------------------------------------------------------------------260/// [Pattern: TransferOpReduceRank (for leading 0 dim) + 261/// TransferReadPermutationLowering (for transposed minor identity)]262///263/// IN: vector.transfer_read264/// (_transposed_ minor identity permutation map, with 0 or more broadcast dims)265/// OUT: vector.transfer_read + vector.broadcast + vector.transpose266/// (minor identity permutation map with 0 or more leading broadcast dims)267///----------------------------------------------------------------------------------------268/// TODO: Inner broadcast dim - see also the block at the bottom of this file269 270// CHECK-LABEL: func.func @xfer_read_minor_identity_transposed_with_mask271// CHECK-SAME: %[[MEM:.*]]: memref<?x?xf32>,272// CHECK-SAME: %[[MASK:.*]]: vector<2x4xi1>273// CHECK-SAME: %[[IDX:.*]]: index274// CHECK: %[[PASS_THROUGH:.*]] = arith.constant 0.000000e+00 : f32275// CHECK: %[[T_READ:.*]] = vector.transfer_read %[[MEM]]{{\[}}%[[IDX]], %[[IDX]]], %[[PASS_THROUGH]], %[[MASK]] {in_bounds = [true, true]} : memref<?x?xf32>, vector<2x4xf32>276// CHECK: %[[BCAST:.*]] = vector.broadcast %[[T_READ]] : vector<2x4xf32> to vector<8x2x4xf32>277// CHECK: %[[TRANSPOSE:.*]] = vector.transpose %[[BCAST]], [0, 2, 1] : vector<8x2x4xf32> to vector<8x4x2xf32>278// CHECK: return %[[TRANSPOSE]] : vector<8x4x2xf32>279func.func @xfer_read_minor_identity_transposed_with_mask(280 %mem: memref<?x?xf32>,281 %mask: vector<2x4xi1>,282 %idx: index) -> (vector<8x4x2xf32>) {283 284 %pad = arith.constant 0.000000e+00 : f32285 286 %res = vector.transfer_read %mem[%idx, %idx], %pad, %mask {287 in_bounds = [true, true, true],288 permutation_map = affine_map<(d0, d1) -> (0, d1, d0)>289 } : memref<?x?xf32>, vector<8x4x2xf32>290 291 return %res : vector<8x4x2xf32>292}293 294// CHECK-LABEL: func.func @xfer_read_minor_identity_transposed_with_mask_scalable(295// CHECK-SAME: %[[MEM:.*]]: memref<?x?xf32>,296// CHECK-SAME: %[[MASK:.*]]: vector<2x[4]xi1>297// CHECK-SAME: %[[IDX:.*]]: index298// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32299// CHECK: %[[T_READ:.*]] = vector.transfer_read %[[MEM]]{{\[}}%[[IDX]], %[[IDX]]], %[[PAD]], %[[MASK]] {in_bounds = [true, true]} : memref<?x?xf32>, vector<2x[4]xf32>300// CHECK: %[[BCAST:.*]] = vector.broadcast %[[T_READ]] : vector<2x[4]xf32> to vector<8x2x[4]xf32>301// CHECK: %[[TRANSPOSE:.*]] = vector.transpose %[[BCAST]], [0, 2, 1] : vector<8x2x[4]xf32> to vector<8x[4]x2xf32>302// CHECK: return %[[TRANSPOSE]] : vector<8x[4]x2xf32>303func.func @xfer_read_minor_identity_transposed_with_mask_scalable(304 %mem: memref<?x?xf32>,305 %mask: vector<2x[4]xi1>,306 %idx: index) -> (vector<8x[4]x2xf32>) {307 308 %pad = arith.constant 0.000000e+00 : f32309 310 %res = vector.transfer_read %mem[%idx, %idx], %pad, %mask {311 in_bounds = [true, true, true],312 permutation_map = affine_map<(d0, d1) -> (0, d1, d0)>313 } : memref<?x?xf32>, vector<8x[4]x2xf32>314 315 return %res : vector<8x[4]x2xf32>316}317 318// Masked version is not supported319 320// CHECK-LABEL: func @xfer_read_minor_identity_transposed_masked(321// CHECK-SAME: %[[DEST:.*]]: tensor<?x?xf32>,322// CHECK-SAME: %[[MASK:.*]]: vector<2x4xi1>323// CHECK-SAME: %[[IDX:.*]]: index324// CHECK-NOT: vector.transpose325// CHECK: vector.mask %[[MASK]] { vector.transfer_read %[[DEST]]{{.*}}: tensor<?x?xf32>, vector<8x4x2xf32> } : vector<2x4xi1> -> vector<8x4x2xf32>326func.func @xfer_read_minor_identity_transposed_masked(327 %dest: tensor<?x?xf32>,328 %mask: vector<2x4xi1>,329 %idx: index) -> (vector<8x4x2xf32>) {330 331 %pad = arith.constant 0.000000e+00 : f32332 333 %res = vector.mask %mask {334 vector.transfer_read %dest[%idx, %idx], %pad {335 in_bounds = [true, true, true],336 permutation_map = affine_map<(d0, d1) -> (0, d1, d0)>337 } : tensor<?x?xf32>, vector<8x4x2xf32>338 } : vector<2x4xi1> -> vector<8x4x2xf32>339 340 return %res : vector<8x4x2xf32>341}342 343// CHECK-LABEL: func.func @xfer_read_minor_identity_transposed_masked_scalable(344// CHECK-SAME: %[[DEST:.*]]: tensor<?x?xf32>,345// CHECK-SAME: %[[MASK:.*]]: vector<2x[4]xi1>346// CHECK-NOT: vector.transpose347// CHECK: %[[T_READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[DEST]]{{.*}} : tensor<?x?xf32>, vector<8x[4]x2xf32> } : vector<2x[4]xi1> -> vector<8x[4]x2xf32>348func.func @xfer_read_minor_identity_transposed_masked_scalable(349 %dest: tensor<?x?xf32>,350 %mask: vector<2x[4]xi1>,351 %idx: index) -> vector<8x[4]x2xf32> {352 353 %pad = arith.constant 0.000000e+00 : f32354 355 %res = vector.mask %mask {356 vector.transfer_read %dest[%idx, %idx], %pad {357 in_bounds = [true, true, true],358 permutation_map = affine_map<(d0, d1) -> (0, d1, d0)>359 } : tensor<?x?xf32>, vector<8x[4]x2xf32>360 } :vector<2x[4]xi1> -> vector<8x[4]x2xf32>361 362 return %res : vector<8x[4]x2xf32>363}364 365///----------------------------------------------------------------------------------------366/// [Pattern: TransferOpReduceRank]367///368/// IN: vector.transfer_read (minor identity map + broadcast)369/// OUT: vector.transfer_read + vector.broadcast370///----------------------------------------------------------------------------------------371 372// CHECK-LABEL: func.func @xfer_read_minor_identitiy_bcast_dims373// CHECK-SAME: %[[MEM:.*]]: memref<?x?x?x?xf32>, %[[IDX:.*]]: index) -> vector<8x4x2x3xf32> {374// CHECK: %[[T_READ:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]]]{{.*}} permutation_map = #[[$MAP]]} : memref<?x?x?x?xf32>, vector<4x2x3xf32>375// CHECK: %[[BC:.*]] = vector.broadcast %[[T_READ]] : vector<4x2x3xf32> to vector<8x4x2x3xf32>376// CHECK: return %[[BC]] : vector<8x4x2x3xf32>377func.func @xfer_read_minor_identitiy_bcast_dims(378 %mem: memref<?x?x?x?xf32>,379 %idx: index) -> vector<8x4x2x3xf32> {380 381 %pad = arith.constant 0.000000e+00 : f32382 383 %res = vector.transfer_read %mem[%idx, %idx, %idx, %idx], %pad {384 in_bounds = [true, true, true, true],385 permutation_map = affine_map<(d0, d1, d2, d3) -> (0, d1, 0, d3)>386 } : memref<?x?x?x?xf32>, vector<8x4x2x3xf32>387 388 return %res : vector<8x4x2x3xf32>389}390 391// CHECK-LABEL: func.func @xfer_read_minor_identitiy_bcast_dims_scalable392// CHECK-SAME: %[[MEM:.*]]: memref<?x?x?x?xf32>, %[[IDX:.*]]: index) -> vector<8x[4]x2x3xf32> {393// CHECK: %[[T_READ:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]]]{{.*}} permutation_map = #[[$MAP]]} : memref<?x?x?x?xf32>, vector<[4]x2x3xf32>394// CHECK: %[[BC:.*]] = vector.broadcast %[[T_READ]] : vector<[4]x2x3xf32> to vector<8x[4]x2x3xf32>395// CHECK: return %[[BC]] : vector<8x[4]x2x3xf32>396func.func @xfer_read_minor_identitiy_bcast_dims_scalable(397 %mem: memref<?x?x?x?xf32>,398 %idx: index) -> vector<8x[4]x2x3xf32> {399 400 %pad = arith.constant 0.000000e+00 : f32401 402 %res = vector.transfer_read %mem[%idx, %idx, %idx, %idx], %pad {403 in_bounds = [true, true, true, true],404 permutation_map = affine_map<(d0, d1, d2, d3) -> (0, d1, 0, d3)>405 } : memref<?x?x?x?xf32>, vector<8x[4]x2x3xf32>406 407 return %res : vector<8x[4]x2x3xf32>408}409 410// CHECK-LABEL: func.func @xfer_read_minor_identitiy_bcast_dims_with_mask411// CHECK-SAME: %[[MEM:.*]]: memref<?x?x?x?xf32>412// CHECK-SAME: %[[MASK:.*]]: vector<4x3xi1>413// CHECK-SAME: %[[IDX:.*]]: index) -> vector<8x4x2x3xf32>414// CHECK: %[[PASS_THROUGH:.*]] = arith.constant 0.000000e+00 : f32415// CHECK: %[[T_READ:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]]], %[[PASS_THROUGH]], %[[MASK]]{{.*}} permutation_map = #[[$MAP]]} : memref<?x?x?x?xf32>, vector<4x2x3xf32>416// CHECK: %[[BC:.*]] = vector.broadcast %[[T_READ]] : vector<4x2x3xf32> to vector<8x4x2x3xf32>417// CHECK: return %[[BC]] : vector<8x4x2x3xf32>418func.func @xfer_read_minor_identitiy_bcast_dims_with_mask(419 %mem: memref<?x?x?x?xf32>,420 %mask: vector<4x3xi1>,421 %idx: index) -> vector<8x4x2x3xf32> {422 423 %pad = arith.constant 0.000000e+00 : f32424 425 %res = vector.transfer_read %mem[%idx, %idx, %idx, %idx], %pad, %mask {426 in_bounds = [true, true, true, true],427 permutation_map = affine_map<(d0, d1, d2, d3) -> (0, d1, 0, d3)>428 } : memref<?x?x?x?xf32>, vector<8x4x2x3xf32>429 430 return %res : vector<8x4x2x3xf32>431}432 433// CHECK-LABEL: func.func @xfer_read_minor_identitiy_bcast_dims_with_mask_scalable434// CHECK-SAME: %[[MEM:.*]]: memref<?x?x?x?xf32>435// CHECK-SAME: %[[MASK:.*]]: vector<[4]x3xi1>436// CHECK-SAME: %[[IDX:.*]]: index) -> vector<8x[4]x2x3xf32>437// CHECK: %[[PASS_THROUGH:.*]] = arith.constant 0.000000e+00 : f32438// CHECK: %[[T_READ:.*]] = vector.transfer_read %[[MEM]][%[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]]], %[[PASS_THROUGH]], %[[MASK]]{{.*}} permutation_map = #[[$MAP]]} : memref<?x?x?x?xf32>, vector<[4]x2x3xf32>439// CHECK: %[[BC:.*]] = vector.broadcast %[[T_READ]] : vector<[4]x2x3xf32> to vector<8x[4]x2x3xf32>440// CHECK: return %[[BC]] : vector<8x[4]x2x3xf32>441func.func @xfer_read_minor_identitiy_bcast_dims_with_mask_scalable(442 %mem: memref<?x?x?x?xf32>,443 %mask: vector<[4]x3xi1>,444 %idx: index) -> vector<8x[4]x2x3xf32> {445 446 %pad = arith.constant 0.000000e+00 : f32447 448 %res = vector.transfer_read %mem[%idx, %idx, %idx, %idx], %pad, %mask {449 in_bounds = [true, true, true, true],450 permutation_map = affine_map<(d0, d1, d2, d3) -> (0, d1, 0, d3)>451 } : memref<?x?x?x?xf32>, vector<8x[4]x2x3xf32>452 453 return %res : vector<8x[4]x2x3xf32>454}455 456// Masked version is not supported457 458// CHECK-LABEL: func.func @xfer_read_minor_identitiy_bcast_dims_masked459// CHECK-SAME: %[[MEM:.*]]: memref<?x?x?x?xf32>,460// CHECK-SAME: %[[MASK:.*]]: vector<4x3xi1>461// CHECK-SAME: %[[IDX:.*]]: index) -> vector<8x4x2x3xf32> {462// CHECK-NOT: vector.broadcast463// CHECK: vector.mask %[[MASK]] { vector.transfer_read %[[MEM]]{{.*}} : memref<?x?x?x?xf32>, vector<8x4x2x3xf32> } : vector<4x3xi1> -> vector<8x4x2x3xf32>464func.func @xfer_read_minor_identitiy_bcast_dims_masked(465 %mem: memref<?x?x?x?xf32>,466 %mask: vector<4x3xi1>,467 %idx: index) -> vector<8x4x2x3xf32> {468 469 %pad = arith.constant 0.000000e+00 : f32470 471 %res = vector.mask %mask {472 vector.transfer_read %mem[%idx, %idx, %idx, %idx], %pad {473 in_bounds = [true, true, true, true],474 permutation_map = affine_map<(d0, d1, d2, d3) -> (0, d1, 0, d3)>475 } : memref<?x?x?x?xf32>, vector<8x4x2x3xf32>476 } : vector<4x3xi1> -> vector<8x4x2x3xf32>477 478 return %res : vector<8x4x2x3xf32>479}480 481///----------------------------------------------------------------------------------------482// TD sequence483///----------------------------------------------------------------------------------------484module attributes {transform.with_named_sequence} {485 transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {486 %f = transform.structured.match ops{["func.func"]} in %module_op487 : (!transform.any_op) -> !transform.any_op488 transform.apply_patterns to %f {489 transform.apply_patterns.vector.transfer_permutation_patterns490 } : !transform.any_op491 transform.yield492 }493}494