610 lines · plain
1// RUN: mlir-opt %s -test-vector-transferop-opt | FileCheck %s2 3// CHECK-LABEL: func @forward_dead_store4// CHECK-NOT: vector.transfer_write5// CHECK-NOT: vector.transfer_read6// CHECK: scf.for7// CHECK: }8// CHECK: vector.transfer_write9// CHECK: return10func.func @forward_dead_store(%arg0: i1, %arg1 : memref<4x4xf32>,11 %v0 : vector<1x4xf32>, %v1 : vector<1x4xf32>, %i : index) {12 %c1 = arith.constant 1 : index13 %c4 = arith.constant 4 : index14 %c0 = arith.constant 0 : index15 %cf0 = arith.constant 0.0 : f3216 vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} :17 vector<1x4xf32>, memref<4x4xf32>18 %0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {in_bounds = [true, true]} :19 memref<4x4xf32>, vector<1x4xf32>20 %x = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%acc = %0)21 -> (vector<1x4xf32>) {22 %1 = arith.addf %acc, %acc : vector<1x4xf32>23 scf.yield %1 : vector<1x4xf32>24 }25 vector.transfer_write %x, %arg1[%c1, %c0] {in_bounds = [true, true]} :26 vector<1x4xf32>, memref<4x4xf32>27 return28}29 30// CHECK-LABEL: func @forward_nested31// CHECK: vector.transfer_write32// CHECK: vector.transfer_write33// CHECK: scf.if34// CHECK-NOT: vector.transfer_read35// CHECK: }36// CHECK: vector.transfer_write37// CHECK: return38func.func @forward_nested(%arg0: i1, %arg1 : memref<4x4xf32>, %v0 : vector<1x4xf32>,39 %v1 : vector<1x4xf32>, %i : index) {40 %c0 = arith.constant 0 : index41 %c1 = arith.constant 1 : index42 %cf0 = arith.constant 0.0 : f3243 vector.transfer_write %v1, %arg1[%i, %c0] {in_bounds = [true, true]} :44 vector<1x4xf32>, memref<4x4xf32>45 vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} :46 vector<1x4xf32>, memref<4x4xf32>47 %x = scf.if %arg0 -> (vector<1x4xf32>) {48 %0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {in_bounds = [true, true]} :49 memref<4x4xf32>, vector<1x4xf32>50 scf.yield %0 : vector<1x4xf32>51 } else {52 scf.yield %v1 : vector<1x4xf32>53 }54 vector.transfer_write %x, %arg1[%c0, %c0] {in_bounds = [true, true]} :55 vector<1x4xf32>, memref<4x4xf32>56 return57}58 59// Negative test, the transfer_write in the scf.if region block the store to60// load forwarding because we don't recursively look into the region to realize61// that the transfer_write cannot reach the transfer_read.62// CHECK-LABEL: func @forward_nested_negative63// CHECK: vector.transfer_write64// CHECK: scf.if65// CHECK: vector.transfer_read66// CHECK: } else {67// CHECK: vector.transfer_write68// CHECK: }69// CHECK: vector.transfer_write70// CHECK: return71func.func @forward_nested_negative(%arg0: i1, %arg1 : memref<4x4xf32>,72 %v0 : vector<1x4xf32>, %v1 : vector<1x4xf32>, %i : index) {73 %c0 = arith.constant 0 : index74 %c1 = arith.constant 1 : index75 %cf0 = arith.constant 0.0 : f3276 vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} :77 vector<1x4xf32>, memref<4x4xf32>78 %x = scf.if %arg0 -> (vector<1x4xf32>) {79 %0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {in_bounds = [true, true]} :80 memref<4x4xf32>, vector<1x4xf32>81 scf.yield %0 : vector<1x4xf32>82 } else {83 vector.transfer_write %v1, %arg1[%i, %c0] {in_bounds = [true, true]} :84 vector<1x4xf32>, memref<4x4xf32>85 scf.yield %v1 : vector<1x4xf32>86 }87 vector.transfer_write %x, %arg1[%c0, %i] {in_bounds = [true, true]} :88 vector<1x4xf32>, memref<4x4xf32>89 return90}91 92// CHECK-LABEL: func @dead_store_region93// CHECK: vector.transfer_write94// CHECK: scf.if95// CHECK: } else {96// CHECK: vector.transfer_read97// CHECK: }98// CHECK: scf.if99// CHECK-NOT: vector.transfer_write100// CHECK: }101// CHECK: vector.transfer_write102// CHECK-NOT: vector.transfer_write103// CHECK: vector.transfer_read104// CHECK: return105func.func @dead_store_region(%arg0: i1, %arg1 : memref<4x4xf32>,106 %v0 : vector<1x4xf32>, %v1 : vector<1x4xf32>, %i : index)107 -> (vector<1x4xf32>) {108 %c0 = arith.constant 0 : index109 %c1 = arith.constant 1 : index110 %cf0 = arith.constant 0.0 : f32111 vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} :112 vector<1x4xf32>, memref<4x4xf32>113 %x = scf.if %arg0 -> (vector<1x4xf32>) {114 scf.yield %v1 : vector<1x4xf32>115 } else {116 %0 = vector.transfer_read %arg1[%i, %c0], %cf0 {in_bounds = [true, true]} :117 memref<4x4xf32>, vector<1x4xf32>118 scf.yield %0 : vector<1x4xf32>119 }120 scf.if %arg0 {121 vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} :122 vector<1x4xf32>, memref<4x4xf32>123 }124 vector.transfer_write %x, %arg1[%c1, %c0] {in_bounds = [true, true]} :125 vector<1x4xf32>, memref<4x4xf32>126 vector.transfer_write %x, %arg1[%c1, %c0] {in_bounds = [true, true]} :127 vector<1x4xf32>, memref<4x4xf32>128 %1 = vector.transfer_read %arg1[%i, %c0], %cf0 {in_bounds = [true, true]} :129 memref<4x4xf32>, vector<1x4xf32>130 return %1 : vector<1x4xf32>131}132 133// CHECK-LABEL: func @dead_store_negative134// CHECK: scf.if135// CHECK: vector.transfer_write136// CHECK: vector.transfer_read137// CHECK: } else {138// CHECK: }139// CHECK: vector.transfer_write140// CHECK: return141func.func @dead_store_negative(%arg0: i1, %arg1 : memref<4x4xf32>,142 %v0 :vector<1x4xf32>, %v1 : vector<1x4xf32>, %i : index) {143 %c0 = arith.constant 0 : index144 %c1 = arith.constant 1 : index145 %cf0 = arith.constant 0.0 : f32146 %x = scf.if %arg0 -> (vector<1x4xf32>) {147 vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} :148 vector<1x4xf32>, memref<4x4xf32>149 %0 = vector.transfer_read %arg1[%i, %c0], %cf0 {in_bounds = [true, true]} :150 memref<4x4xf32>, vector<1x4xf32>151 scf.yield %0 : vector<1x4xf32>152 } else {153 scf.yield %v1 : vector<1x4xf32>154 }155 vector.transfer_write %x, %arg1[%c1, %c0] {in_bounds = [true, true]} :156 vector<1x4xf32>, memref<4x4xf32>157 return158}159 160// CHECK-LABEL: func @dead_store_nested_region161// CHECK: scf.if162// CHECK: vector.transfer_read163// CHECK: scf.if164// CHECK-NOT: vector.transfer_write165// CHECK: }166// CHECK: vector.transfer_write167// CHECK: }168// CHECK: return169func.func @dead_store_nested_region(%arg0: i1, %arg1: i1, %arg2 : memref<4x4xf32>,170 %v0 : vector<1x4xf32>, %v1 : vector<1x4xf32>, %i : index) {171 %c0 = arith.constant 0 : index172 %c1 = arith.constant 1 : index173 %cf0 = arith.constant 0.0 : f32174 scf.if %arg0 {175 %0 = vector.transfer_read %arg2[%i, %c0], %cf0 {in_bounds = [true, true]} :176 memref<4x4xf32>, vector<1x4xf32>177 scf.if %arg1 {178 vector.transfer_write %v1, %arg2[%c1, %c0] {in_bounds = [true, true]} :179 vector<1x4xf32>, memref<4x4xf32>180 }181 vector.transfer_write %v0, %arg2[%c1, %c0] {in_bounds = [true, true]} :182 vector<1x4xf32>, memref<4x4xf32>183 }184 return185}186 187// CHECK-LABEL: func @forward_dead_store_negative188// CHECK: vector.transfer_write189// CHECK: vector.transfer_write190// CHECK: vector.transfer_write191// CHECK: vector.transfer_write192// CHECK: vector.transfer_read193// CHECK: vector.transfer_write194// CHECK: return195func.func @forward_dead_store_negative(%arg0: i1, %arg1 : memref<4x4xf32>,196 %v0 : vector<1x4xf32>, %v1 : vector<1x1xf32>, %v2 : vector<1x4xf32>, %i : index) -> vector<1x4xf32> {197 %alias = memref.subview %arg1[0, 0] [2, 2] [1, 1] :198 memref<4x4xf32> to memref<2x2xf32, strided<[4, 1]>>199 %c1 = arith.constant 1 : index200 %c4 = arith.constant 4 : index201 %c0 = arith.constant 0 : index202 %cf0 = arith.constant 0.0 : f32203 vector.transfer_write %v0, %arg1[%c1, %c0] {in_bounds = [true, true]} :204 vector<1x4xf32>, memref<4x4xf32>205 // blocking write.206 vector.transfer_write %v1, %alias[%c0, %c0] {in_bounds = [true, true]} :207 vector<1x1xf32>, memref<2x2xf32, strided<[4, 1]>>208 vector.transfer_write %v2, %arg1[%c1, %c0] {in_bounds = [true, true]} :209 vector<1x4xf32>, memref<4x4xf32>210 // blocking write.211 vector.transfer_write %v1, %alias[%c1, %c0] {in_bounds = [true, true]} :212 vector<1x1xf32>, memref<2x2xf32, strided<[4, 1]>>213 %0 = vector.transfer_read %arg1[%c1, %c0], %cf0 {in_bounds = [true, true]} :214 memref<4x4xf32>, vector<1x4xf32>215 vector.transfer_write %v2, %arg1[%c1, %c0] {in_bounds = [true, true]} :216 vector<1x4xf32>, memref<4x4xf32>217 return %0 : vector<1x4xf32>218}219 220 221// Regression test - the following _potential forwarding_ of %1 to the final222// `vector.transfer_write` would not be safe:223// %1 = vector.transfer_read %subview224// vector.transfer_write %1, %alloca225// vector.transfer_write %vec, %collapse_shape226// %2 = vector.transfer_read %alloca227// vector.transfer_write %1, %subview228// Indeed, %alloca and %collapse_shape alias and hence %2 != %1. Instead, the229// final `vector.transfer_write` should be preserved as:230// vector.transfer_write %2, %subview231 232// CHECK-LABEL: func.func @collapse_shape_and_read_from_source233// CHECK: scf.for {{.*}} {234// CHECK: vector.transfer_read235// CHECK: vector.transfer_write236// CHECK: vector.transfer_write237// CHECK: vector.transfer_read238// CHECK: vector.transfer_write239 240func.func @collapse_shape_and_read_from_source(%in_0: memref<1x20x1xi32>, %vec: vector<4xi32>) {241 %c0_i32 = arith.constant 0 : i32242 %c0 = arith.constant 0 : index243 %c4 = arith.constant 4 : index244 %c20 = arith.constant 20 : index245 246 %alloca = memref.alloca() {alignment = 64 : i64} : memref<1x4x1xi32>247 %collapse_shape = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x4x1xi32> into memref<4xi32>248 scf.for %arg0 = %c0 to %c20 step %c4 {249 %subview = memref.subview %in_0[0, %arg0, 0] [1, 4, 1] [1, 1, 1] : memref<1x20x1xi32> to memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>250 %1 = vector.transfer_read %subview[%c0, %c0, %c0], %c0_i32 {in_bounds = [true, true, true]} : memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>, vector<1x4x1xi32>251 // $alloca and $collapse_shape alias252 vector.transfer_write %1, %alloca[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x4x1xi32>, memref<1x4x1xi32>253 vector.transfer_write %vec, %collapse_shape[%c0] {in_bounds = [true]} : vector<4xi32>, memref<4xi32>254 %2 = vector.transfer_read %alloca[%c0, %c0, %c0], %c0_i32 {in_bounds = [true, true, true]} : memref<1x4x1xi32>, vector<1x4x1xi32>255 vector.transfer_write %2, %subview[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x4x1xi32>, memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>256 }257 return258}259 260// The same regression test for expand_shape.261 262// CHECK-LABEL: func.func @expand_shape_and_read_from_source263// CHECK: scf.for {{.*}} {264// CHECK: vector.transfer_read265// CHECK: vector.transfer_write266// CHECK: vector.transfer_write267// CHECK: vector.transfer_read268// CHECK: vector.transfer_write269 270func.func @expand_shape_and_read_from_source(%in_0: memref<20xi32>, %vec: vector<1x4x1xi32>) {271 %c0_i32 = arith.constant 0 : i32272 %c0 = arith.constant 0 : index273 %c4 = arith.constant 4 : index274 %c20 = arith.constant 20 : index275 276 %alloca = memref.alloca() {alignment = 64 : i64} : memref<4xi32>277 %expand_shape = memref.expand_shape %alloca [[0, 1, 2]] output_shape [1, 4, 1] : memref<4xi32> into memref<1x4x1xi32>278 scf.for %arg0 = %c0 to %c20 step %c4 {279 %subview = memref.subview %in_0[%arg0] [4] [1] : memref<20xi32> to memref<4xi32, strided<[1], offset: ?>>280 %1 = vector.transfer_read %subview[%c0], %c0_i32 {in_bounds = [true]} : memref<4xi32, strided<[1], offset: ?>>, vector<4xi32>281 // $alloca and $expand_shape alias282 vector.transfer_write %1, %alloca[%c0] {in_bounds = [true]} : vector<4xi32>, memref<4xi32>283 vector.transfer_write %vec, %expand_shape[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x4x1xi32>, memref<1x4x1xi32>284 %2 = vector.transfer_read %alloca[%c0], %c0_i32 {in_bounds = [true]} : memref<4xi32>, vector<4xi32>285 vector.transfer_write %2, %subview[%c0] {in_bounds = [true]} : vector<4xi32>, memref<4xi32, strided<[1], offset: ?>>286 }287 return288}289 290// The same regression test, but the initial write is to the collapsed memref,291// and the subsequent unforwardable read is from the collapse shape.292 293// CHECK-LABEL: func.func @collapse_shape_and_read_from_collapse294// CHECK: scf.for {{.*}} {295// CHECK: vector.transfer_read296// CHECK: vector.transfer_write297// CHECK: vector.transfer_write298// CHECK: vector.transfer_read299// CHECK: vector.transfer_write300 301func.func @collapse_shape_and_read_from_collapse(%in_0: memref<20xi32>, %vec: vector<1x4x1xi32>) {302 %c0_i32 = arith.constant 0 : i32303 %c0 = arith.constant 0 : index304 %c4 = arith.constant 4 : index305 %c20 = arith.constant 20 : index306 307 %alloca = memref.alloca() {alignment = 64 : i64} : memref<1x4x1xi32>308 %collapse_shape = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x4x1xi32> into memref<4xi32>309 scf.for %arg0 = %c0 to %c20 step %c4 {310 %subview = memref.subview %in_0[%arg0] [4] [1] : memref<20xi32> to memref<4xi32, strided<[1], offset: ?>>311 %1 = vector.transfer_read %subview[%c0], %c0_i32 {in_bounds = [true]} : memref<4xi32, strided<[1], offset: ?>>, vector<4xi32>312 vector.transfer_write %1, %collapse_shape[%c0] {in_bounds = [true]} : vector<4xi32>, memref<4xi32>313 // $alloca and $collapse_shape alias314 vector.transfer_write %vec, %alloca[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x4x1xi32>, memref<1x4x1xi32>315 %2 = vector.transfer_read %collapse_shape[%c0], %c0_i32 {in_bounds = [true]} : memref<4xi32>, vector<4xi32>316 vector.transfer_write %2, %subview[%c0] {in_bounds = [true]} : vector<4xi32>, memref<4xi32, strided<[1], offset: ?>>317 }318 return319}320 321// The same test except writing to the expanded source first (same as the322// previous collapse test but for expand).323 324// CHECK-LABEL: func.func @expand_shape_and_read_from_expand325// CHECK: scf.for {{.*}} {326// CHECK: vector.transfer_read327// CHECK: vector.transfer_write328// CHECK: vector.transfer_write329// CHECK: vector.transfer_read330// CHECK: vector.transfer_write331 332func.func @expand_shape_and_read_from_expand(%in_0: memref<1x20x1xi32>, %vec: vector<4xi32>) {333 %c0_i32 = arith.constant 0 : i32334 %c0 = arith.constant 0 : index335 %c4 = arith.constant 4 : index336 %c20 = arith.constant 20 : index337 338 %alloca = memref.alloca() {alignment = 64 : i64} : memref<4xi32>339 %expand_shape = memref.expand_shape %alloca [[0, 1, 2]] output_shape [1, 4, 1] : memref<4xi32> into memref<1x4x1xi32>340 scf.for %arg0 = %c0 to %c20 step %c4 {341 %subview = memref.subview %in_0[0, %arg0, 0] [1, 4, 1] [1, 1, 1] : memref<1x20x1xi32> to memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>342 %1 = vector.transfer_read %subview[%c0, %c0, %c0], %c0_i32 {in_bounds = [true, true, true]} : memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>, vector<1x4x1xi32>343 vector.transfer_write %1, %expand_shape[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x4x1xi32>, memref<1x4x1xi32>344 // $alloca and $expand_shape alias345 vector.transfer_write %vec, %alloca[%c0] {in_bounds = [true]} : vector<4xi32>, memref<4xi32>346 %2 = vector.transfer_read %expand_shape[%c0, %c0, %c0], %c0_i32 {in_bounds = [true, true, true]} : memref<1x4x1xi32>, vector<1x4x1xi32>347 vector.transfer_write %2, %subview[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x4x1xi32>, memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>348 }349 return350}351 352// CHECK-LABEL: func @forward_dead_store_dynamic_same_index353// CHECK-NOT: vector.transfer_write354// CHECK-NOT: vector.transfer_read355// CHECK: scf.for356// CHECK: }357// CHECK: vector.transfer_write358// CHECK: return359func.func @forward_dead_store_dynamic_same_index(360 %buffer : memref<?x?xf32>, %v0 : vector<4xf32>, %v1 : vector<4xf32>, %i : index) {361 %c1 = arith.constant 1 : index362 %c4 = arith.constant 4 : index363 %c0 = arith.constant 0 : index364 %cf0 = arith.constant 0.0 : f32365 vector.transfer_write %v0, %buffer[%i, %i] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>366 // The following transfer op reads/writes to the same address so that we can forward.367 %0 = vector.transfer_read %buffer[%i, %i], %cf0 {in_bounds = [true]} : memref<?x?xf32>, vector<4xf32>368 %x = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%acc = %0) -> (vector<4xf32>) {369 %1 = arith.addf %acc, %acc : vector<4xf32>370 scf.yield %1 : vector<4xf32>371 }372 vector.transfer_write %x, %buffer[%i, %i] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>373 return374}375 376// CHECK-LABEL: func @dont_forward_dead_store_dynamic_overlap377// CHECK-COUNT-2: vector.transfer_write378// CHECK: vector.transfer_read379// CHECK: scf.for380// CHECK: }381// CHECK: vector.transfer_write382// CHECK: return383func.func @dont_forward_dead_store_dynamic_overlap(384 %buffer : memref<?x?xf32>, %v0 : vector<4xf32>, %v1 : vector<4xf32>, %i0 : index) {385 %c1 = arith.constant 1 : index386 %c4 = arith.constant 4 : index387 %c0 = arith.constant 0 : index388 %cf0 = arith.constant 0.0 : f32389 %i1 = affine.apply affine_map<(d0) -> (d0 + 3)>(%i0)390 vector.transfer_write %v0, %buffer[%i0, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>391 // The following transfer op writes to an overlapping range so we cannot forward.392 vector.transfer_write %v0, %buffer[%i0, %i1] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>393 %0 = vector.transfer_read %buffer[%i0, %i0], %cf0 {in_bounds = [true]} : memref<?x?xf32>, vector<4xf32>394 %x = scf.for %iv = %c0 to %c4 step %c1 iter_args(%acc = %0) -> (vector<4xf32>) {395 %1 = arith.addf %acc, %acc : vector<4xf32>396 scf.yield %1 : vector<4xf32>397 }398 vector.transfer_write %x, %buffer[%i0, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>399 return400}401 402// CHECK-LABEL: func @forward_dead_store_dynamic_non_overlap_leading_dim403// CHECK: vector.transfer_write404// CHECK-NOT: vector.transfer_write405// CHECK-NOT: vector.transfer_read406// CHECK: scf.for407// CHECK: }408// CHECK: vector.transfer_write409// CHECK: return410func.func @forward_dead_store_dynamic_non_overlap_leading_dim(411 %buffer : memref<?x?xf32>, %v0 : vector<4xf32>, %v1 : vector<4xf32>, %i0 : index) {412 %c1 = arith.constant 1 : index413 %c4 = arith.constant 4 : index414 %c0 = arith.constant 0 : index415 %cf0 = arith.constant 0.0 : f32416 %i1 = affine.apply affine_map<(d0) -> (d0 + 1)>(%i0)417 vector.transfer_write %v0, %buffer[%i0, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>418 // The following transfer op writes to an non-overlapping range so we can forward.419 vector.transfer_write %v0, %buffer[%i1, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>420 %0 = vector.transfer_read %buffer[%i0, %i0], %cf0 {in_bounds = [true]} : memref<?x?xf32>, vector<4xf32>421 %x = scf.for %iv = %c0 to %c4 step %c1 iter_args(%acc = %0) -> (vector<4xf32>) {422 %1 = arith.addf %acc, %acc : vector<4xf32>423 scf.yield %1 : vector<4xf32>424 }425 vector.transfer_write %x, %buffer[%i0, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>426 return427}428 429// CHECK-LABEL: func @forward_dead_store_dynamic_non_overlap_trailing_dim430// CHECK: vector.transfer_write431// CHECK-NOT: vector.transfer_write432// CHECK-NOT: vector.transfer_read433// CHECK: scf.for434// CHECK: }435// CHECK: vector.transfer_write436// CHECK: return437func.func @forward_dead_store_dynamic_non_overlap_trailing_dim(438 %buffer : memref<?x?xf32>, %v0 : vector<4xf32>, %v1 : vector<4xf32>, %i0 : index) {439 %c1 = arith.constant 1 : index440 %c4 = arith.constant 4 : index441 %c0 = arith.constant 0 : index442 %cf0 = arith.constant 0.0 : f32443 %i1 = affine.apply affine_map<(d0) -> (d0 + 4)>(%i0)444 vector.transfer_write %v0, %buffer[%i0, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>445 // The following transfer op writes to an non-overlapping range so we can forward.446 vector.transfer_write %v0, %buffer[%i0, %i1] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>447 %0 = vector.transfer_read %buffer[%i0, %i0], %cf0 {in_bounds = [true]} : memref<?x?xf32>, vector<4xf32>448 %x = scf.for %iv = %c0 to %c4 step %c1 iter_args(%acc = %0) -> (vector<4xf32>) {449 %1 = arith.addf %acc, %acc : vector<4xf32>450 scf.yield %1 : vector<4xf32>451 }452 vector.transfer_write %x, %buffer[%i0, %i0] {in_bounds = [true]} : vector<4xf32>, memref<?x?xf32>453 return454}455 456// CHECK-LABEL: func @forward_dead_constant_splat_store_with_masking457// CHECK: %[[SPLAT:.*]] = arith.constant dense<0.000000e+00> : vector<[8]x[8]xf32>458// CHECK-NOT: vector.transfer_write459// CHECK-NOT: vector.transfer_read460// CHECK: scf.for461// CHECK-SAME: iter_args(%{{.*}} = %[[SPLAT]])462// CHECK: }463// CHECK: vector.transfer_write464// CHECK: return465func.func @forward_dead_constant_splat_store_with_masking(%buffer : memref<?x?xf32>, %mask: vector<[8]x[8]xi1>) {466 %zero_splat = arith.constant dense<0.0> : vector<[8]x[8]xf32>467 %read_padding = arith.constant 0.0 : f32468 %c1 = arith.constant 1 : index469 %c0 = arith.constant 0 : index470 %c512 = arith.constant 512 : index471 vector.transfer_write %zero_splat, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>472 %0 = vector.transfer_read %buffer[%c0, %c0], %read_padding, %mask {in_bounds = [true, true]} : memref<?x?xf32>, vector<[8]x[8]xf32>473 %x = scf.for %arg2 = %c0 to %c512 step %c1 iter_args(%acc = %0) -> (vector<[8]x[8]xf32>) {474 %1 = arith.addf %acc, %acc : vector<[8]x[8]xf32>475 scf.yield %1 : vector<[8]x[8]xf32>476 }477 vector.transfer_write %x, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>478 return479}480 481// Here the read can be eliminated but not the write (due to mismatched masks).482// CHECK-LABEL: func @forward_dead_constant_splat_store_with_masking_unmasked_write483// CHECK: %[[SPLAT:.*]] = arith.constant dense<0.000000e+00> : vector<[8]x[8]xf32>484// CHECK: vector.transfer_write %[[SPLAT]]485// CHECK: scf.for486// CHECK-SAME: iter_args(%{{.*}} = %[[SPLAT]])487// CHECK: }488// CHECK: vector.transfer_write489// CHECK: return490func.func @forward_dead_constant_splat_store_with_masking_unmasked_write(%buffer : memref<?x?xf32>, %mask: vector<[8]x[8]xi1>) {491 %zero_splat = arith.constant dense<0.0> : vector<[8]x[8]xf32>492 %read_padding = arith.constant 0.0 : f32493 %c1 = arith.constant 1 : index494 %c0 = arith.constant 0 : index495 %c512 = arith.constant 512 : index496 vector.transfer_write %zero_splat, %buffer[%c0, %c0] {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>497 %0 = vector.transfer_read %buffer[%c0, %c0], %read_padding, %mask {in_bounds = [true, true]} : memref<?x?xf32>, vector<[8]x[8]xf32>498 %x = scf.for %arg2 = %c0 to %c512 step %c1 iter_args(%acc = %0) -> (vector<[8]x[8]xf32>) {499 %1 = arith.addf %acc, %acc : vector<[8]x[8]xf32>500 scf.yield %1 : vector<[8]x[8]xf32>501 }502 vector.transfer_write %x, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>503 return504}505 506// Negative test, the padding does not match the constant splat, so we can't507// forward the store.508// CHECK-LABEL: func @forward_dead_constant_splat_store_with_masking_negative_0509// CHECK: vector.transfer_write510// CHECK: vector.transfer_read511// CHECK: scf.for512// CHECK: }513// CHECK: vector.transfer_write514// CHECK: return515func.func @forward_dead_constant_splat_store_with_masking_negative_0(%buffer : memref<?x?xf32>, %mask: vector<[8]x[8]xi1>) {516 %zero_splat = arith.constant dense<0.0> : vector<[8]x[8]xf32>517 %read_padding = arith.constant 1.0 : f32518 %c1 = arith.constant 1 : index519 %c0 = arith.constant 0 : index520 %c512 = arith.constant 512 : index521 vector.transfer_write %zero_splat, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>522 %0 = vector.transfer_read %buffer[%c0, %c0], %read_padding, %mask {in_bounds = [true, true]} : memref<?x?xf32>, vector<[8]x[8]xf32>523 %x = scf.for %arg2 = %c0 to %c512 step %c1 iter_args(%acc = %0) -> (vector<[8]x[8]xf32>) {524 %1 = arith.addf %acc, %acc : vector<[8]x[8]xf32>525 scf.yield %1 : vector<[8]x[8]xf32>526 }527 vector.transfer_write %x, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>528 return529}530 531// Negative test, the masks don't match between the read and write, so we can't532// foward the store.533// CHECK-LABEL: func @forward_dead_constant_splat_store_with_masking_negative_1534// CHECK: vector.transfer_write535// CHECK: vector.transfer_read536// CHECK: scf.for537// CHECK: }538// CHECK: vector.transfer_write539// CHECK: return540func.func @forward_dead_constant_splat_store_with_masking_negative_1(%buffer : memref<?x?xf32>, %mask_a: vector<[8]x[8]xi1>, %mask_b: vector<[8]x[8]xi1>) {541 %zero_splat = arith.constant dense<0.0> : vector<[8]x[8]xf32>542 %read_padding = arith.constant 0.0 : f32543 %c1 = arith.constant 1 : index544 %c0 = arith.constant 0 : index545 %c512 = arith.constant 512 : index546 vector.transfer_write %zero_splat, %buffer[%c0, %c0], %mask_a {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>547 %0 = vector.transfer_read %buffer[%c0, %c0], %read_padding, %mask_b {in_bounds = [true, true]} : memref<?x?xf32>, vector<[8]x[8]xf32>548 %x = scf.for %arg2 = %c0 to %c512 step %c1 iter_args(%acc = %0) -> (vector<[8]x[8]xf32>) {549 %1 = arith.addf %acc, %acc : vector<[8]x[8]xf32>550 scf.yield %1 : vector<[8]x[8]xf32>551 }552 vector.transfer_write %x, %buffer[%c0, %c0], %mask_a {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>553 return554}555 556// Negative test, here the write is masked but the read is unmasked. We can't557// forward the store (as the write could be of less elements then the read).558// CHECK-LABEL: func @forward_dead_constant_splat_store_with_masking_negative_3559// CHECK: vector.transfer_write560// CHECK: vector.transfer_read561// CHECK: scf.for562// CHECK: }563// CHECK: vector.transfer_write564// CHECK: return565func.func @forward_dead_constant_splat_store_with_masking_negative_3(%buffer : memref<?x?xf32>, %mask: vector<[8]x[8]xi1>) {566 %zero_splat = arith.constant dense<0.0> : vector<[8]x[8]xf32>567 %read_padding = arith.constant 0.0 : f32568 %c1 = arith.constant 1 : index569 %c0 = arith.constant 0 : index570 %c512 = arith.constant 512 : index571 vector.transfer_write %zero_splat, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>572 %0 = vector.transfer_read %buffer[%c0, %c0], %read_padding {in_bounds = [true, true]} : memref<?x?xf32>, vector<[8]x[8]xf32>573 %x = scf.for %arg2 = %c0 to %c512 step %c1 iter_args(%acc = %0) -> (vector<[8]x[8]xf32>) {574 %1 = arith.addf %acc, %acc : vector<[8]x[8]xf32>575 scf.yield %1 : vector<[8]x[8]xf32>576 }577 vector.transfer_write %x, %buffer[%c0, %c0], %mask {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>578 return579}580 581// Here each read/write is to a different subview, but they all point to exact582// same bit of memory (just through casts and subviews with unit strides and583// zero offsets).584// CHECK-LABEL: func @forward_and_eliminate_stores_through_trivial_aliases585// CHECK-NOT: vector.transfer_write586// CHECK-NOT: vector.transfer_read587// CHECK: scf.for588// CHECK: }589// CHECK: vector.transfer_write590// CHECK: return591func.func @forward_and_eliminate_stores_through_trivial_aliases(592 %buffer : memref<?x?xf32>, %vec: vector<[8]x[8]xf32>, %size: index, %a_size: index, %another_size: index593) {594 %c0 = arith.constant 0 : index595 %c1 = arith.constant 1 : index596 %c32 = arith.constant 32 : index597 %cst = arith.constant 0.0 : f32598 vector.transfer_write %vec, %buffer[%c0, %c0] {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32>599 %direct_subview = memref.subview %buffer[0, 0] [%a_size, %a_size] [1, 1] : memref<?x?xf32> to memref<?x?xf32, strided<[?, 1]>>600 %cast = memref.cast %direct_subview : memref<?x?xf32, strided<[?, 1]>> to memref<?x?xf32>601 %subview_of_cast = memref.subview %cast[0, 0] [%another_size, %another_size] [1, 1] : memref<?x?xf32> to memref<?x?xf32, strided<[?, 1]>>602 %21 = vector.transfer_read %direct_subview[%c0, %c0], %cst {in_bounds = [true, true]} : memref<?x?xf32, strided<[?, 1]>>, vector<[8]x[8]xf32>603 %23 = scf.for %arg2 = %c0 to %c32 step %c1 iter_args(%arg3 = %21) -> (vector<[8]x[8]xf32>) {604 %24 = arith.addf %arg3, %arg3 : vector<[8]x[8]xf32>605 scf.yield %24 : vector<[8]x[8]xf32>606 }607 vector.transfer_write %23, %subview_of_cast[%c0, %c0] {in_bounds = [true, true]} : vector<[8]x[8]xf32>, memref<?x?xf32, strided<[?, 1]>>608 return609}610