1000 lines · plain
1// RUN: mlir-opt -allow-unregistered-dialect %s -affine-scalrep | FileCheck %s2 3// CHECK-DAG: [[$MAP0:#map[0-9]*]] = affine_map<(d0, d1) -> (d1 + 1)>4// CHECK-DAG: [[$MAP1:#map[0-9]*]] = affine_map<(d0, d1) -> (d0)>5// CHECK-DAG: [[$MAP2:#map[0-9]*]] = affine_map<(d0, d1) -> (d1)>6// CHECK-DAG: [[$MAP3:#map[0-9]*]] = affine_map<(d0, d1) -> (d0 - 1)>7// CHECK-DAG: [[$MAP4:#map[0-9]*]] = affine_map<(d0) -> (d0 + 1)>8// CHECK-DAG: [[$IDENT:#map[0-9]*]] = affine_map<(d0) -> (d0)>9 10// CHECK-LABEL: func @simple_store_load() {11func.func @simple_store_load() {12 %cf7 = arith.constant 7.0 : f3213 %m = memref.alloc() : memref<10xf32>14 affine.for %i0 = 0 to 10 {15 affine.store %cf7, %m[%i0] : memref<10xf32>16 %v0 = affine.load %m[%i0] : memref<10xf32>17 %v1 = arith.addf %v0, %v0 : f3218 }19 memref.dealloc %m : memref<10xf32>20 return21// CHECK: %[[C7:.*]] = arith.constant 7.000000e+00 : f3222// CHECK-NEXT: affine.for %{{.*}} = 0 to 10 {23// CHECK-NEXT: arith.addf %[[C7]], %[[C7]] : f3224// CHECK-NEXT: }25// CHECK-NEXT: return26}27 28// CHECK-LABEL: func @multi_store_load() {29func.func @multi_store_load() {30 %cf7 = arith.constant 7.0 : f3231 %cf8 = arith.constant 8.0 : f3232 %cf9 = arith.constant 9.0 : f3233 %m = gpu.alloc() : memref<10xf32>34 affine.for %i0 = 0 to 10 {35 affine.store %cf7, %m[%i0] : memref<10xf32>36 %v0 = affine.load %m[%i0] : memref<10xf32>37 %v1 = arith.addf %v0, %v0 : f3238 affine.store %cf8, %m[%i0] : memref<10xf32>39 affine.store %cf9, %m[%i0] : memref<10xf32>40 %v2 = affine.load %m[%i0] : memref<10xf32>41 %v3 = affine.load %m[%i0] : memref<10xf32>42 %v4 = arith.mulf %v2, %v3 : f3243 }44 gpu.dealloc %m : memref<10xf32>45 return46// CHECK-NEXT: %[[C7:.*]] = arith.constant 7.000000e+00 : f3247// CHECK-NEXT: arith.constant 8.000000e+00 : f3248// CHECK-NEXT: %[[C9:.*]] = arith.constant 9.000000e+00 : f3249// CHECK-NEXT: affine.for %{{.*}} = 0 to 10 {50// CHECK-NEXT: arith.addf %[[C7]], %[[C7]] : f3251// CHECK-NEXT: arith.mulf %[[C9]], %[[C9]] : f3252// CHECK-NEXT: }53// CHECK-NEXT: return54}55 56// The store-load forwarding can see through affine apply's since it relies on57// dependence information.58// CHECK-LABEL: func @store_load_affine_apply59func.func @store_load_affine_apply() -> memref<10x10xf32> {60 %cf7 = arith.constant 7.0 : f3261 %m = memref.alloc() : memref<10x10xf32>62 affine.for %i0 = 0 to 10 {63 affine.for %i1 = 0 to 10 {64 %t0 = affine.apply affine_map<(d0, d1) -> (d1 + 1)>(%i0, %i1)65 %t1 = affine.apply affine_map<(d0, d1) -> (d0)>(%i0, %i1)66 %idx0 = affine.apply affine_map<(d0, d1) -> (d1)> (%t0, %t1)67 %idx1 = affine.apply affine_map<(d0, d1) -> (d0 - 1)> (%t0, %t1)68 affine.store %cf7, %m[%idx0, %idx1] : memref<10x10xf32>69 // CHECK-NOT: affine.load %{{[0-9]+}}70 %v0 = affine.load %m[%i0, %i1] : memref<10x10xf32>71 %v1 = arith.addf %v0, %v0 : f3272 }73 }74 // The memref and its stores won't be erased due to this memref return.75 return %m : memref<10x10xf32>76// CHECK: %{{.*}} = arith.constant 7.000000e+00 : f3277// CHECK-NEXT: %{{.*}} = memref.alloc() : memref<10x10xf32>78// CHECK-NEXT: affine.for %{{.*}} = 0 to 10 {79// CHECK-NEXT: affine.for %{{.*}} = 0 to 10 {80// CHECK-NEXT: %{{.*}} = affine.apply [[$MAP0]](%{{.*}}, %{{.*}})81// CHECK-NEXT: %{{.*}} = affine.apply [[$MAP1]](%{{.*}}, %{{.*}})82// CHECK-NEXT: %{{.*}} = affine.apply [[$MAP2]](%{{.*}}, %{{.*}})83// CHECK-NEXT: %{{.*}} = affine.apply [[$MAP3]](%{{.*}}, %{{.*}})84// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[%{{.*}}, %{{.*}}] : memref<10x10xf32>85// CHECK-NEXT: %{{.*}} = arith.addf %{{.*}}, %{{.*}} : f3286// CHECK-NEXT: }87// CHECK-NEXT: }88// CHECK-NEXT: return %{{.*}} : memref<10x10xf32>89}90 91// CHECK-LABEL: func @store_load_nested92func.func @store_load_nested(%N : index) {93 %cf7 = arith.constant 7.0 : f3294 %m = memref.alloc() : memref<10xf32>95 affine.for %i0 = 0 to 10 {96 affine.store %cf7, %m[%i0] : memref<10xf32>97 affine.for %i1 = 0 to %N {98 %v0 = affine.load %m[%i0] : memref<10xf32>99 %v1 = arith.addf %v0, %v0 : f32100 }101 }102 return103// CHECK: %{{.*}} = arith.constant 7.000000e+00 : f32104// CHECK-NEXT: affine.for %{{.*}} = 0 to 10 {105// CHECK-NEXT: affine.for %{{.*}} = 0 to %{{.*}} {106// CHECK-NEXT: %{{.*}} = arith.addf %{{.*}}, %{{.*}} : f32107// CHECK-NEXT: }108// CHECK-NEXT: }109// CHECK-NEXT: return110}111 112// No forwarding happens here since either of the two stores could be the last113// writer; store/load forwarding will however be possible here once loop live114// out SSA scalars are available.115// CHECK-LABEL: func @multi_store_load_nested_no_fwd116func.func @multi_store_load_nested_no_fwd(%N : index) {117 %cf7 = arith.constant 7.0 : f32118 %cf8 = arith.constant 8.0 : f32119 %m = memref.alloc() : memref<10xf32>120 affine.for %i0 = 0 to 10 {121 affine.store %cf7, %m[%i0] : memref<10xf32>122 affine.for %i1 = 0 to %N {123 affine.store %cf8, %m[%i1] : memref<10xf32>124 }125 affine.for %i2 = 0 to %N {126 // CHECK: %{{[0-9]+}} = affine.load %{{.*}}[%{{.*}}] : memref<10xf32>127 %v0 = affine.load %m[%i0] : memref<10xf32>128 %v1 = arith.addf %v0, %v0 : f32129 }130 }131 return132}133 134// No forwarding happens here since both stores have a value going into135// the load.136// CHECK-LABEL: func @store_load_store_nested_no_fwd137func.func @store_load_store_nested_no_fwd(%N : index) {138 %cf7 = arith.constant 7.0 : f32139 %cf9 = arith.constant 9.0 : f32140 %m = memref.alloc() : memref<10xf32>141 affine.for %i0 = 0 to 10 {142 affine.store %cf7, %m[%i0] : memref<10xf32>143 affine.for %i1 = 0 to %N {144 // CHECK: %{{[0-9]+}} = affine.load %{{.*}}[%{{.*}}] : memref<10xf32>145 %v0 = affine.load %m[%i0] : memref<10xf32>146 %v1 = arith.addf %v0, %v0 : f32147 affine.store %cf9, %m[%i0] : memref<10xf32>148 }149 }150 return151}152 153// Forwarding happens here since the last store postdominates all other stores154// and other forwarding criteria are satisfied.155// CHECK-LABEL: func @multi_store_load_nested_fwd156func.func @multi_store_load_nested_fwd(%N : index) {157 %cf7 = arith.constant 7.0 : f32158 %cf8 = arith.constant 8.0 : f32159 %cf9 = arith.constant 9.0 : f32160 %cf10 = arith.constant 10.0 : f32161 %m = memref.alloc() : memref<10xf32>162 affine.for %i0 = 0 to 10 {163 affine.store %cf7, %m[%i0] : memref<10xf32>164 affine.for %i1 = 0 to %N {165 affine.store %cf8, %m[%i1] : memref<10xf32>166 }167 affine.for %i2 = 0 to %N {168 affine.store %cf9, %m[%i2] : memref<10xf32>169 }170 affine.store %cf10, %m[%i0] : memref<10xf32>171 affine.for %i3 = 0 to %N {172 // CHECK-NOT: %{{[0-9]+}} = affine.load173 %v0 = affine.load %m[%i0] : memref<10xf32>174 %v1 = arith.addf %v0, %v0 : f32175 }176 }177 return178}179 180// There is no unique load location for the store to forward to.181// CHECK-LABEL: func @store_load_no_fwd182func.func @store_load_no_fwd() {183 %cf7 = arith.constant 7.0 : f32184 %m = memref.alloc() : memref<10xf32>185 affine.for %i0 = 0 to 10 {186 affine.store %cf7, %m[%i0] : memref<10xf32>187 affine.for %i1 = 0 to 10 {188 affine.for %i2 = 0 to 10 {189 // CHECK: affine.load190 %v0 = affine.load %m[%i2] : memref<10xf32>191 %v1 = arith.addf %v0, %v0 : f32192 }193 }194 }195 return196}197 198// Forwarding happens here as there is a one-to-one store-load correspondence.199// CHECK-LABEL: func @store_load_fwd200func.func @store_load_fwd() {201 %cf7 = arith.constant 7.0 : f32202 %c0 = arith.constant 0 : index203 %m = memref.alloc() : memref<10xf32>204 affine.store %cf7, %m[%c0] : memref<10xf32>205 affine.for %i0 = 0 to 10 {206 affine.for %i1 = 0 to 10 {207 affine.for %i2 = 0 to 10 {208 // CHECK-NOT: affine.load %{{[0-9]}}+209 %v0 = affine.load %m[%c0] : memref<10xf32>210 %v1 = arith.addf %v0, %v0 : f32211 }212 }213 }214 return215}216 217// Although there is a dependence from the second store to the load, it is218// satisfied by the outer surrounding loop, and does not prevent the first219// store to be forwarded to the load.220func.func @store_load_store_nested_fwd(%N : index) -> f32 {221 %cf7 = arith.constant 7.0 : f32222 %cf9 = arith.constant 9.0 : f32223 %c0 = arith.constant 0 : index224 %c1 = arith.constant 1 : index225 %m = memref.alloc() : memref<10xf32>226 affine.for %i0 = 0 to 10 {227 affine.store %cf7, %m[%i0] : memref<10xf32>228 affine.for %i1 = 0 to %N {229 %v0 = affine.load %m[%i0] : memref<10xf32>230 %v1 = arith.addf %v0, %v0 : f32231 %idx = affine.apply affine_map<(d0) -> (d0 + 1)> (%i0)232 affine.store %cf9, %m[%idx] : memref<10xf32>233 }234 }235 // Due to this load, the memref isn't optimized away.236 %v3 = affine.load %m[%c1] : memref<10xf32>237 return %v3 : f32238// CHECK: %{{.*}} = memref.alloc() : memref<10xf32>239// CHECK-NEXT: affine.for %{{.*}} = 0 to 10 {240// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>241// CHECK-NEXT: affine.for %{{.*}} = 0 to %{{.*}} {242// CHECK-NEXT: %{{.*}} = arith.addf %{{.*}}, %{{.*}} : f32243// CHECK-NEXT: %{{.*}} = affine.apply [[$MAP4]](%{{.*}})244// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>245// CHECK-NEXT: }246// CHECK-NEXT: }247// CHECK-NEXT: %{{.*}} = affine.load %{{.*}}[%{{.*}}] : memref<10xf32>248// CHECK-NEXT: return %{{.*}} : f32249}250 251// CHECK-LABEL: func @should_not_fwd252func.func @should_not_fwd(%A: memref<100xf32>, %M : index, %N : index) -> f32 {253 %cf = arith.constant 0.0 : f32254 affine.store %cf, %A[%M] : memref<100xf32>255 // CHECK: affine.load %{{.*}}[%{{.*}}]256 %v = affine.load %A[%N] : memref<100xf32>257 return %v : f32258}259 260// Can store forward to A[%j, %i], but no forwarding to load on %A[%i, %j]261// CHECK-LABEL: func @refs_not_known_to_be_equal262func.func @refs_not_known_to_be_equal(%A : memref<100 x 100 x f32>, %M : index) {263 %N = affine.apply affine_map<(d0) -> (d0 + 1)> (%M)264 %cf1 = arith.constant 1.0 : f32265 affine.for %i = 0 to 100 {266 // CHECK: affine.for %[[I:.*]] =267 affine.for %j = 0 to 100 {268 // CHECK: affine.for %[[J:.*]] =269 // CHECK: affine.load %{{.*}}[%[[I]], %[[J]]]270 %u = affine.load %A[%i, %j] : memref<100x100xf32>271 // CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[%[[J]], %[[I]]]272 affine.store %cf1, %A[%j, %i] : memref<100x100xf32>273 // CHECK-NEXT: affine.load %{{.*}}[%[[I]], %[[J]]]274 %v = affine.load %A[%i, %j] : memref<100x100xf32>275 // This load should disappear.276 %w = affine.load %A[%j, %i] : memref<100x100xf32>277 // CHECK-NEXT: "foo"278 "foo" (%u, %v, %w) : (f32, f32, f32) -> ()279 }280 }281 return282}283 284// CHECK-LABEL: func @elim_load_after_store285func.func @elim_load_after_store(%arg0: memref<100xf32>, %arg1: memref<100xf32>) {286 %alloc = memref.alloc() : memref<1xf32>287 %alloc_0 = memref.alloc() : memref<1xf32>288 // CHECK: affine.for289 affine.for %arg2 = 0 to 100 {290 // CHECK: affine.load291 %0 = affine.load %arg0[%arg2] : memref<100xf32>292 %1 = affine.load %arg0[%arg2] : memref<100xf32>293 // CHECK: arith.addf294 %2 = arith.addf %0, %1 : f32295 affine.store %2, %alloc_0[0] : memref<1xf32>296 %3 = affine.load %arg0[%arg2] : memref<100xf32>297 %4 = affine.load %alloc_0[0] : memref<1xf32>298 // CHECK-NEXT: arith.addf299 %5 = arith.addf %3, %4 : f32300 affine.store %5, %alloc[0] : memref<1xf32>301 %6 = affine.load %arg0[%arg2] : memref<100xf32>302 %7 = affine.load %alloc[0] : memref<1xf32>303 %8 = arith.addf %6, %7 : f32304 affine.store %8, %arg1[%arg2] : memref<100xf32>305 }306 return307}308 309// The test checks for value forwarding from vector stores to vector loads.310// The value loaded from %in can directly be stored to %out by eliminating311// store and load from %tmp.312func.func @vector_forwarding(%in : memref<512xf32>, %out : memref<512xf32>) {313 %tmp = memref.alloc() : memref<512xf32>314 affine.for %i = 0 to 16 {315 %ld0 = affine.vector_load %in[32*%i] : memref<512xf32>, vector<32xf32>316 affine.vector_store %ld0, %tmp[32*%i] : memref<512xf32>, vector<32xf32>317 %ld1 = affine.vector_load %tmp[32*%i] : memref<512xf32>, vector<32xf32>318 affine.vector_store %ld1, %out[32*%i] : memref<512xf32>, vector<32xf32>319 }320 return321}322 323// CHECK-LABEL: func @vector_forwarding324// CHECK: affine.for %{{.*}} = 0 to 16 {325// CHECK-NEXT: %[[LDVAL:.*]] = affine.vector_load326// CHECK-NEXT: affine.vector_store %[[LDVAL]],{{.*}}327// CHECK-NEXT: }328 329func.func @vector_no_forwarding(%in : memref<512xf32>, %out : memref<512xf32>) {330 %tmp = memref.alloc() : memref<512xf32>331 affine.for %i = 0 to 16 {332 %ld0 = affine.vector_load %in[32*%i] : memref<512xf32>, vector<32xf32>333 affine.vector_store %ld0, %tmp[32*%i] : memref<512xf32>, vector<32xf32>334 %ld1 = affine.vector_load %tmp[32*%i] : memref<512xf32>, vector<16xf32>335 affine.vector_store %ld1, %out[32*%i] : memref<512xf32>, vector<16xf32>336 }337 return338}339 340// CHECK-LABEL: func @vector_no_forwarding341// CHECK: affine.for %{{.*}} = 0 to 16 {342// CHECK-NEXT: %[[LDVAL:.*]] = affine.vector_load343// CHECK-NEXT: affine.vector_store %[[LDVAL]],{{.*}}344// CHECK-NEXT: %[[LDVAL1:.*]] = affine.vector_load345// CHECK-NEXT: affine.vector_store %[[LDVAL1]],{{.*}}346// CHECK-NEXT: }347 348// CHECK-LABEL: func @simple_three_loads349func.func @simple_three_loads(%in : memref<10xf32>) {350 affine.for %i0 = 0 to 10 {351 // CHECK: affine.load352 %v0 = affine.load %in[%i0] : memref<10xf32>353 // CHECK-NOT: affine.load354 %v1 = affine.load %in[%i0] : memref<10xf32>355 %v2 = arith.addf %v0, %v1 : f32356 %v3 = affine.load %in[%i0] : memref<10xf32>357 %v4 = arith.addf %v2, %v3 : f32358 }359 return360}361 362// CHECK-LABEL: func @nested_loads_const_index363func.func @nested_loads_const_index(%in : memref<10xf32>) {364 %c0 = arith.constant 0 : index365 // CHECK: affine.load366 %v0 = affine.load %in[%c0] : memref<10xf32>367 affine.for %i0 = 0 to 10 {368 affine.for %i1 = 0 to 20 {369 affine.for %i2 = 0 to 30 {370 // CHECK-NOT: affine.load371 %v1 = affine.load %in[%c0] : memref<10xf32>372 %v2 = arith.addf %v0, %v1 : f32373 }374 }375 }376 return377}378 379// CHECK-LABEL: func @nested_loads380func.func @nested_loads(%N : index, %in : memref<10xf32>) {381 affine.for %i0 = 0 to 10 {382 // CHECK: affine.load383 %v0 = affine.load %in[%i0] : memref<10xf32>384 affine.for %i1 = 0 to %N {385 // CHECK-NOT: affine.load386 %v1 = affine.load %in[%i0] : memref<10xf32>387 %v2 = arith.addf %v0, %v1 : f32388 }389 }390 return391}392 393// CHECK-LABEL: func @nested_loads_different_memref_accesses_no_cse394func.func @nested_loads_different_memref_accesses_no_cse(%in : memref<10xf32>) {395 affine.for %i0 = 0 to 10 {396 // CHECK: affine.load397 %v0 = affine.load %in[%i0] : memref<10xf32>398 affine.for %i1 = 0 to 20 {399 // CHECK: affine.load400 %v1 = affine.load %in[%i1] : memref<10xf32>401 %v2 = arith.addf %v0, %v1 : f32402 }403 }404 return405}406 407// CHECK-LABEL: func @load_load_store408func.func @load_load_store(%m : memref<10xf32>) {409 affine.for %i0 = 0 to 10 {410 // CHECK: affine.load411 %v0 = affine.load %m[%i0] : memref<10xf32>412 // CHECK-NOT: affine.load413 %v1 = affine.load %m[%i0] : memref<10xf32>414 %v2 = arith.addf %v0, %v1 : f32415 affine.store %v2, %m[%i0] : memref<10xf32>416 }417 return418}419 420// CHECK-LABEL: func @load_load_store_2_loops_no_cse421func.func @load_load_store_2_loops_no_cse(%N : index, %m : memref<10xf32>) {422 affine.for %i0 = 0 to 10 {423 // CHECK: affine.load424 %v0 = affine.load %m[%i0] : memref<10xf32>425 affine.for %i1 = 0 to %N {426 // CHECK: affine.load427 %v1 = affine.load %m[%i0] : memref<10xf32>428 %v2 = arith.addf %v0, %v1 : f32429 affine.store %v2, %m[%i0] : memref<10xf32>430 }431 }432 return433}434 435// CHECK-LABEL: func @load_load_store_3_loops_no_cse436func.func @load_load_store_3_loops_no_cse(%m : memref<10xf32>) {437%cf1 = arith.constant 1.0 : f32438 affine.for %i0 = 0 to 10 {439 // CHECK: affine.load440 %v0 = affine.load %m[%i0] : memref<10xf32>441 affine.for %i1 = 0 to 20 {442 affine.for %i2 = 0 to 30 {443 // CHECK: affine.load444 %v1 = affine.load %m[%i0] : memref<10xf32>445 %v2 = arith.addf %v0, %v1 : f32446 }447 affine.store %cf1, %m[%i0] : memref<10xf32>448 }449 }450 return451}452 453// CHECK-LABEL: func @load_load_store_3_loops454func.func @load_load_store_3_loops(%m : memref<10xf32>) {455%cf1 = arith.constant 1.0 : f32456 affine.for %i0 = 0 to 10 {457 affine.for %i1 = 0 to 20 {458 // CHECK: affine.load459 %v0 = affine.load %m[%i0] : memref<10xf32>460 affine.for %i2 = 0 to 30 {461 // CHECK-NOT: affine.load462 %v1 = affine.load %m[%i0] : memref<10xf32>463 %v2 = arith.addf %v0, %v1 : f32464 }465 }466 affine.store %cf1, %m[%i0] : memref<10xf32>467 }468 return469}470 471// CHECK-LABEL: func @loads_in_sibling_loops_const_index_no_cse472func.func @loads_in_sibling_loops_const_index_no_cse(%m : memref<10xf32>) {473 %c0 = arith.constant 0 : index474 affine.for %i0 = 0 to 10 {475 // CHECK: affine.load476 %v0 = affine.load %m[%c0] : memref<10xf32>477 }478 affine.for %i1 = 0 to 10 {479 // CHECK: affine.load480 %v0 = affine.load %m[%c0] : memref<10xf32>481 %v1 = arith.addf %v0, %v0 : f32482 }483 return484}485 486// CHECK-LABEL: func @load_load_affine_apply487func.func @load_load_affine_apply(%in : memref<10x10xf32>) {488 affine.for %i0 = 0 to 10 {489 affine.for %i1 = 0 to 10 {490 %t0 = affine.apply affine_map<(d0, d1) -> (d1 + 1)>(%i0, %i1)491 %t1 = affine.apply affine_map<(d0, d1) -> (d0)>(%i0, %i1)492 %idx0 = affine.apply affine_map<(d0, d1) -> (d1)> (%t0, %t1)493 %idx1 = affine.apply affine_map<(d0, d1) -> (d0 - 1)> (%t0, %t1)494 // CHECK: affine.load495 %v0 = affine.load %in[%idx0, %idx1] : memref<10x10xf32>496 // CHECK-NOT: affine.load497 %v1 = affine.load %in[%i0, %i1] : memref<10x10xf32>498 %v2 = arith.addf %v0, %v1 : f32499 }500 }501 return502}503 504// CHECK-LABEL: func @vector_loads505func.func @vector_loads(%in : memref<512xf32>, %out : memref<512xf32>) {506 affine.for %i = 0 to 16 {507 // CHECK: affine.vector_load508 %ld0 = affine.vector_load %in[32*%i] : memref<512xf32>, vector<32xf32>509 // CHECK-NOT: affine.vector_load510 %ld1 = affine.vector_load %in[32*%i] : memref<512xf32>, vector<32xf32>511 %add = arith.addf %ld0, %ld1 : vector<32xf32>512 affine.vector_store %ld1, %out[32*%i] : memref<512xf32>, vector<32xf32>513 }514 return515}516 517// CHECK-LABEL: func @vector_loads_no_cse518func.func @vector_loads_no_cse(%in : memref<512xf32>, %out : memref<512xf32>) {519 affine.for %i = 0 to 16 {520 // CHECK: affine.vector_load521 %ld0 = affine.vector_load %in[32*%i] : memref<512xf32>, vector<32xf32>522 // CHECK: affine.vector_load523 %ld1 = affine.vector_load %in[32*%i] : memref<512xf32>, vector<16xf32>524 affine.vector_store %ld1, %out[32*%i] : memref<512xf32>, vector<16xf32>525 }526 return527}528 529// CHECK-LABEL: func @vector_load_store_load_no_cse530func.func @vector_load_store_load_no_cse(%in : memref<512xf32>, %out : memref<512xf32>) {531 affine.for %i = 0 to 16 {532 // CHECK: affine.vector_load533 %ld0 = affine.vector_load %in[32*%i] : memref<512xf32>, vector<32xf32>534 affine.vector_store %ld0, %in[16*%i] : memref<512xf32>, vector<32xf32>535 // CHECK: affine.vector_load536 %ld1 = affine.vector_load %in[32*%i] : memref<512xf32>, vector<32xf32>537 %add = arith.addf %ld0, %ld1 : vector<32xf32>538 affine.vector_store %ld1, %out[32*%i] : memref<512xf32>, vector<32xf32>539 }540 return541}542 543// CHECK-LABEL: func @reduction_multi_store544func.func @reduction_multi_store() -> memref<1xf32> {545 %A = memref.alloc() : memref<1xf32>546 %cf0 = arith.constant 0.0 : f32547 %cf5 = arith.constant 5.0 : f32548 549 affine.store %cf0, %A[0] : memref<1xf32>550 affine.for %i = 0 to 100 step 2 {551 %l = affine.load %A[0] : memref<1xf32>552 %s = arith.addf %l, %cf5 : f32553 // Store to load forwarding from this store should happen.554 affine.store %s, %A[0] : memref<1xf32>555 %m = affine.load %A[0] : memref<1xf32>556 "test.foo"(%m) : (f32) -> ()557 }558 559// CHECK: affine.for560// CHECK: affine.load561// CHECK: affine.store %[[S:.*]],562// CHECK-NEXT: "test.foo"(%[[S]])563 564 return %A : memref<1xf32>565}566 567// CHECK-LABEL: func @vector_load_affine_apply_store_load568func.func @vector_load_affine_apply_store_load(%in : memref<512xf32>, %out : memref<512xf32>) {569 %cf1 = arith.constant 1: index570 affine.for %i = 0 to 15 {571 // CHECK: affine.vector_load572 %ld0 = affine.vector_load %in[32*%i] : memref<512xf32>, vector<32xf32>573 %idx = affine.apply affine_map<(d0) -> (d0 + 1)> (%i)574 affine.vector_store %ld0, %in[32*%idx] : memref<512xf32>, vector<32xf32>575 // CHECK-NOT: affine.vector_load576 %ld1 = affine.vector_load %in[32*%i] : memref<512xf32>, vector<32xf32>577 %add = arith.addf %ld0, %ld1 : vector<32xf32>578 affine.vector_store %ld1, %out[32*%i] : memref<512xf32>, vector<32xf32>579 }580 return581}582 583// CHECK-LABEL: func @external_no_forward_load584 585func.func @external_no_forward_load(%in : memref<512xf32>, %out : memref<512xf32>) {586 affine.for %i = 0 to 16 {587 %ld0 = affine.load %in[32*%i] : memref<512xf32>588 affine.store %ld0, %out[32*%i] : memref<512xf32>589 "memop"(%in, %out) : (memref<512xf32>, memref<512xf32>) -> ()590 %ld1 = affine.load %in[32*%i] : memref<512xf32>591 affine.store %ld1, %out[32*%i] : memref<512xf32>592 }593 return594}595// CHECK: affine.load596// CHECK: affine.store597// CHECK: affine.load598// CHECK: affine.store599 600// CHECK-LABEL: func @external_no_forward_store601 602func.func @external_no_forward_store(%in : memref<512xf32>, %out : memref<512xf32>) {603 %cf1 = arith.constant 1.0 : f32604 affine.for %i = 0 to 16 {605 affine.store %cf1, %in[32*%i] : memref<512xf32>606 "memop"(%in, %out) : (memref<512xf32>, memref<512xf32>) -> ()607 %ld1 = affine.load %in[32*%i] : memref<512xf32>608 affine.store %ld1, %out[32*%i] : memref<512xf32>609 }610 return611}612// CHECK: affine.store613// CHECK: affine.load614// CHECK: affine.store615 616// CHECK-LABEL: func @no_forward_cast617 618func.func @no_forward_cast(%in : memref<512xf32>, %out : memref<512xf32>) {619 %cf1 = arith.constant 1.0 : f32620 %cf2 = arith.constant 2.0 : f32621 %m2 = memref.cast %in : memref<512xf32> to memref<?xf32>622 affine.for %i = 0 to 16 {623 affine.store %cf1, %in[32*%i] : memref<512xf32>624 affine.store %cf2, %m2[32*%i] : memref<?xf32>625 %ld1 = affine.load %in[32*%i] : memref<512xf32>626 affine.store %ld1, %out[32*%i] : memref<512xf32>627 }628 return629}630// CHECK: affine.store631// CHECK-NEXT: affine.store632// CHECK-NEXT: affine.load633// CHECK-NEXT: affine.store634 635// Although there is a dependence from the second store to the load, it is636// satisfied by the outer surrounding loop, and does not prevent the first637// store to be forwarded to the load.638 639// CHECK-LABEL: func @overlap_no_fwd640func.func @overlap_no_fwd(%N : index) -> f32 {641 %cf7 = arith.constant 7.0 : f32642 %cf9 = arith.constant 9.0 : f32643 %c0 = arith.constant 0 : index644 %c1 = arith.constant 1 : index645 %m = memref.alloc() : memref<10xf32>646 affine.for %i0 = 0 to 5 {647 affine.store %cf7, %m[2 * %i0] : memref<10xf32>648 affine.for %i1 = 0 to %N {649 %v0 = affine.load %m[2 * %i0] : memref<10xf32>650 %v1 = arith.addf %v0, %v0 : f32651 affine.store %cf9, %m[%i0 + 1] : memref<10xf32>652 }653 }654 // Due to this load, the memref isn't optimized away.655 %v3 = affine.load %m[%c1] : memref<10xf32>656 return %v3 : f32657 658// CHECK: affine.for %{{.*}} = 0 to 5 {659// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>660// CHECK-NEXT: affine.for %{{.*}} = 0 to %{{.*}} {661// CHECK-NEXT: %{{.*}} = affine.load662// CHECK-NEXT: %{{.*}} = arith.addf %{{.*}}, %{{.*}} : f32663// CHECK-NEXT: affine.store %{{.*}}, %{{.*}}[%{{.*}}] : memref<10xf32>664// CHECK-NEXT: }665// CHECK-NEXT: }666// CHECK-NEXT: %{{.*}} = affine.load %{{.*}}[%{{.*}}] : memref<10xf32>667// CHECK-NEXT: return %{{.*}} : f32668}669 670// CHECK-LABEL: func @redundant_store_elim671 672func.func @redundant_store_elim(%out : memref<512xf32>) {673 %cf1 = arith.constant 1.0 : f32674 %cf2 = arith.constant 2.0 : f32675 affine.for %i = 0 to 16 {676 affine.store %cf1, %out[32*%i] : memref<512xf32>677 affine.store %cf2, %out[32*%i] : memref<512xf32>678 }679 return680}681 682// CHECK: affine.for683// CHECK-NEXT: affine.store684// CHECK-NEXT: }685 686// CHECK-LABEL: func @redundant_store_elim_nonintervening687 688func.func @redundant_store_elim_nonintervening(%in : memref<512xf32>) {689 %cf1 = arith.constant 1.0 : f32690 %out = memref.alloc() : memref<512xf32>691 affine.for %i = 0 to 16 {692 affine.store %cf1, %out[32*%i] : memref<512xf32>693 %0 = affine.load %in[32*%i] : memref<512xf32>694 affine.store %0, %out[32*%i] : memref<512xf32>695 }696 return697}698 699// CHECK: affine.for700// CHECK-NEXT: affine.load701// CHECK-NEXT: affine.store702// CHECK-NEXT: }703 704// CHECK-LABEL: func @redundant_store_elim_fail705 706func.func @redundant_store_elim_fail(%out : memref<512xf32>) {707 %cf1 = arith.constant 1.0 : f32708 %cf2 = arith.constant 2.0 : f32709 affine.for %i = 0 to 16 {710 affine.store %cf1, %out[32*%i] : memref<512xf32>711 "test.use"(%out) : (memref<512xf32>) -> ()712 affine.store %cf2, %out[32*%i] : memref<512xf32>713 }714 return715}716// CHECK: affine.for717// CHECK-NEXT: affine.store718// CHECK-NEXT: "test.use"719// CHECK-NEXT: affine.store720// CHECK-NEXT: }721 722// CHECK-LABEL: @with_inner_ops723func.func @with_inner_ops(%arg0: memref<?xf64>, %arg1: memref<?xf64>, %arg2: i1) {724 %cst = arith.constant 0.000000e+00 : f64725 %cst_0 = arith.constant 3.140000e+00 : f64726 %cst_1 = arith.constant 1.000000e+00 : f64727 affine.for %arg3 = 0 to 28 {728 affine.store %cst, %arg1[%arg3] : memref<?xf64>729 affine.store %cst_0, %arg1[%arg3] : memref<?xf64>730 %0 = scf.if %arg2 -> (f64) {731 scf.yield %cst_1 : f64732 } else {733 %1 = affine.load %arg1[%arg3] : memref<?xf64>734 scf.yield %1 : f64735 }736 affine.store %0, %arg0[%arg3] : memref<?xf64>737 }738 return739}740 741// Semantics of non-affine region ops would be unknown.742 743// CHECK: } else {744// CHECK-NEXT: %[[Y:.*]] = affine.load745// CHECK-NEXT: scf.yield %[[Y]] : f64746 747// Check if scalar replacement works correctly when affine memory ops are in the748// body of an scf.for.749 750// CHECK-LABEL: func @affine_store_load_in_scope751func.func @affine_store_load_in_scope(%memref: memref<1x4094x510x1xf32>, %memref_2: memref<4x4x1x64xf32>, %memref_0: memref<1x2046x254x1x64xf32>) {752 %c0 = arith.constant 0 : index753 %c1 = arith.constant 1 : index754 %c2 = arith.constant 2 : index755 %c64 = arith.constant 64 : index756 %c768 = arith.constant 768 : index757 scf.for %i = %c0 to %c768 step %c1 {758 %9 = arith.remsi %i, %c64 : index759 %10 = arith.divsi %i, %c64 : index760 %11 = arith.remsi %10, %c2 : index761 %12 = arith.divsi %10, %c2 : index762 test.affine_scope {763 %14 = arith.muli %12, %c2 : index764 %15 = arith.addi %c2, %14 : index765 %16 = arith.addi %15, %c0 : index766 %18 = arith.muli %11, %c2 : index767 %19 = arith.addi %c2, %18 : index768 %20 = affine.load %memref[0, symbol(%16), symbol(%19), 0] : memref<1x4094x510x1xf32>769 %21 = affine.load %memref_2[0, 0, 0, symbol(%9)] : memref<4x4x1x64xf32>770 %24 = affine.load %memref_0[0, symbol(%12), symbol(%11), 0, symbol(%9)] : memref<1x2046x254x1x64xf32>771 %25 = arith.mulf %20, %21 : f32772 %26 = arith.addf %24, %25 : f32773 // CHECK: %[[A:.*]] = arith.addf774 affine.store %26, %memref_0[0, symbol(%12), symbol(%11), 0, symbol(%9)] : memref<1x2046x254x1x64xf32>775 %27 = arith.addi %19, %c1 : index776 %28 = affine.load %memref[0, symbol(%16), symbol(%27), 0] : memref<1x4094x510x1xf32>777 %29 = affine.load %memref_2[0, 1, 0, symbol(%9)] : memref<4x4x1x64xf32>778 %30 = affine.load %memref_0[0, symbol(%12), symbol(%11), 0, symbol(%9)] : memref<1x2046x254x1x64xf32>779 %31 = arith.mulf %28, %29 : f32780 %32 = arith.addf %30, %31 : f32781 // The addf above will get the forwarded value from the store on782 // %memref_0 above which is being loaded into %30..783 // CHECK: arith.addf %[[A]],784 "terminate"() : () -> ()785 }786 }787 return788}789 790// No scalrep will be performed here but we ensure dependence correctly fails.791 792// CHECK-LABEL: func @affine_load_store_in_different_scopes793func.func @affine_load_store_in_different_scopes() -> memref<1xf32> {794 %A = memref.alloc() : memref<1xf32>795 %cf0 = arith.constant 0.0 : f32796 %cf5 = arith.constant 5.0 : f32797 798 affine.store %cf0, %A[0] : memref<1xf32>799 test.affine_scope {800 affine.store %cf5, %A[0] : memref<1xf32>801 "test.terminate"() : () -> ()802 }803 %v = affine.load %A[0] : memref<1xf32>804 // CHECK: affine.store805 // CHECK-NEXT: test.affine_scope806 // CHECK: affine.store807 // CHECK: affine.load808 return %A : memref<1xf32>809}810 811// No forwarding should again happen here.812 813// CHECK-LABEL: func.func @no_forwarding_across_scopes814func.func @no_forwarding_across_scopes() -> memref<1xf32> {815 %A = memref.alloc() : memref<1xf32>816 %cf0 = arith.constant 0.0 : f32817 %cf5 = arith.constant 5.0 : f32818 %c0 = arith.constant 0 : index819 %c100 = arith.constant 100 : index820 %c1 = arith.constant 1 : index821 822 // Store shouldn't be forwarded to the load.823 affine.store %cf0, %A[0] : memref<1xf32>824 // CHECK: test.affine_scope825 // CHECK-NEXT: affine.load826 test.affine_scope {827 %l = affine.load %A[0] : memref<1xf32>828 %s = arith.addf %l, %cf5 : f32829 affine.store %s, %A[0] : memref<1xf32>830 "terminator"() : () -> ()831 }832 return %A : memref<1xf32>833}834 835// CHECK-LABEL: func @parallel_store_load() {836func.func @parallel_store_load() {837 %cf7 = arith.constant 7.0 : f32838 %m = memref.alloc() : memref<10xf32>839 affine.parallel (%i0) = (0) to (10) {840 affine.store %cf7, %m[%i0] : memref<10xf32>841 %v0 = affine.load %m[%i0] : memref<10xf32>842 %v1 = arith.addf %v0, %v0 : f32843 }844 memref.dealloc %m : memref<10xf32>845 return846// CHECK: %[[C7:.*]] = arith.constant 7.000000e+00 : f32847// CHECK-NEXT: affine.parallel (%{{.*}}) = (0) to (10) {848// CHECK-NEXT: arith.addf %[[C7]], %[[C7]] : f32849// CHECK-NEXT: }850// CHECK-NEXT: return851}852 853func.func @non_constant_parallel_store_load(%N : index) {854 %cf7 = arith.constant 7.0 : f32855 %m = memref.alloc() : memref<10xf32>856 affine.parallel (%i0) = (0) to (%N) {857 affine.store %cf7, %m[%i0] : memref<10xf32>858 %v0 = affine.load %m[%i0] : memref<10xf32>859 %v1 = arith.addf %v0, %v0 : f32860 }861 memref.dealloc %m : memref<10xf32>862 return863}864// CHECK: func.func @non_constant_parallel_store_load(%[[ARG0:.*]]: index) {865// CHECK-NEXT: %[[C7:.*]] = arith.constant 7.000000e+00 : f32866// CHECK-NEXT: affine.parallel (%{{.*}}) = (0) to (%[[ARG0]]) {867// CHECK-NEXT: arith.addf %[[C7]], %[[C7]] : f32868// CHECK-NEXT: }869// CHECK-NEXT: return870 871// CHECK-LABEL: func @parallel_surrounding_for() {872func.func @parallel_surrounding_for() {873 %cf7 = arith.constant 7.0 : f32874 %m = memref.alloc() : memref<10x10xf32>875 affine.parallel (%i0) = (0) to (10) {876 affine.for %i1 = 0 to 10 {877 affine.store %cf7, %m[%i0,%i1] : memref<10x10xf32>878 %v0 = affine.load %m[%i0,%i1] : memref<10x10xf32>879 %v1 = arith.addf %v0, %v0 : f32880 }881 }882 memref.dealloc %m : memref<10x10xf32>883 return884// CHECK: %[[C7:.*]] = arith.constant 7.000000e+00 : f32885// CHECK-NEXT: affine.parallel (%{{.*}}) = (0) to (10) {886// CHECK-NEXT: affine.for %{{.*}} = 0 to 10 {887// CHECK-NEXT: arith.addf %[[C7]], %[[C7]] : f32888// CHECK-NEXT: }889// CHECK-NEXT: }890// CHECK-NEXT: return891}892 893// CHECK-LABEL: func.func @dead_affine_region_op894func.func @dead_affine_region_op() {895 %c1 = arith.constant 1 : index896 %alloc = memref.alloc() : memref<15xi1>897 %true = arith.constant true898 affine.store %true, %alloc[%c1] : memref<15xi1>899 // Dead store.900 affine.store %true, %alloc[%c1] : memref<15xi1>901 // This affine.if is dead.902 affine.if affine_set<(d0, d1, d2, d3) : ((d0 + 1) mod 8 >= 0, d0 * -8 >= 0)>(%c1, %c1, %c1, %c1){903 // No forwarding will happen.904 affine.load %alloc[%c1] : memref<15xi1>905 }906 // CHECK-NEXT: arith.constant907 // CHECK-NEXT: memref.alloc908 // CHECK-NEXT: arith.constant909 // CHECK-NEXT: affine.store910 // CHECK-NEXT: affine.if911 // CHECK-NEXT: affine.load912 return913}914 915// We perform no scalar replacement here since we don't depend on dominance916// info, which would be needed in such cases when ops fall in different blocks917// of a CFG region.918 919// CHECK-LABEL: func @cross_block920func.func @cross_block() {921 %c10 = arith.constant 10 : index922 %alloc_83 = memref.alloc() : memref<1x13xf32>923 %alloc_99 = memref.alloc() : memref<13xi1>924 %true_110 = arith.constant true925 affine.store %true_110, %alloc_99[%c10] : memref<13xi1>926 %true = arith.constant true927 affine.store %true, %alloc_99[%c10] : memref<13xi1>928 cf.br ^bb1(%alloc_83 : memref<1x13xf32>)929^bb1(%35: memref<1x13xf32>):930 // CHECK: affine.load931 %69 = affine.load %alloc_99[%c10] : memref<13xi1>932 return933}934 935#map1 = affine_map<(d0) -> (d0)>936 937// CHECK-LABEL: func @consecutive_store938func.func @consecutive_store() {939 // CHECK: %[[CST:.*]] = arith.constant940 %tmp = arith.constant 1.1 : f16941 // CHECK: %[[ALLOC:.*]] = memref.alloc942 %alloc_66 = memref.alloc() : memref<f16, 1>943 affine.for %arg2 = 4 to 6 {944 affine.for %arg3 = #map1(%arg2) to #map1(%arg2) step 4 {945 // CHECK: affine.store %[[CST]], %[[ALLOC]][]946 affine.store %tmp, %alloc_66[] : memref<f16, 1>947 // CHECK-NOT: affine.store %[[CST]], %[[ALLOC]][]948 affine.store %tmp, %alloc_66[] : memref<f16, 1>949 %270 = affine.load %alloc_66[] : memref<f16, 1>950 }951 }952 return953}954 955// CHECK-LABEL: func @scf_for_if956func.func @scf_for_if(%arg0: memref<?xi32>, %arg1: i32) -> i32 attributes {llvm.linkage = #llvm.linkage<external>} {957 %c1 = arith.constant 1 : index958 %c0 = arith.constant 0 : index959 %c0_i32 = arith.constant 0 : i32960 %c5_i32 = arith.constant 5 : i32961 %c10_i32 = arith.constant 10 : i32962 %0 = memref.alloca() : memref<1xi32>963 %1 = llvm.mlir.undef : i32964 affine.store %1, %0[0] : memref<1xi32>965 affine.store %c0_i32, %0[0] : memref<1xi32>966 %2 = arith.index_cast %arg1 : i32 to index967 scf.for %arg2 = %c0 to %2 step %c1 {968 %4 = memref.load %arg0[%arg2] : memref<?xi32>969 %5 = arith.muli %4, %c5_i32 : i32970 %6 = arith.cmpi sgt, %5, %c10_i32 : i32971 // CHECK: scf.if972 scf.if %6 {973 // No forwarding should happen here since we have an scf.for around and we974 // can't analyze the flow of values.975 // CHECK: affine.load976 %7 = affine.load %0[0] : memref<1xi32>977 %8 = arith.addi %5, %7 : i32978 // CHECK: affine.store979 affine.store %8, %0[0] : memref<1xi32>980 }981 }982 // CHECK: affine.load983 %3 = affine.load %0[0] : memref<1xi32>984 return %3 : i32985}986 987// CHECK-LABEL: func @zero_d_memrefs988func.func @zero_d_memrefs() {989 // CHECK: %[[C0:.*]] = arith.constant 0990 %c0_i32 = arith.constant 0 : i32991 %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<i32>992 affine.store %c0_i32, %alloc_0[] : memref<i32>993 affine.for %arg0 = 0 to 9 {994 %2 = affine.load %alloc_0[] : memref<i32>995 arith.addi %2, %2 : i32996 // CHECK: arith.addi %[[C0]], %[[C0]]997 }998 return999}1000