374 lines · plain
1// RUN: mlir-opt %s -scf-for-loop-peeling -canonicalize -split-input-file -verify-diagnostics | FileCheck %s2// RUN: mlir-opt %s -scf-for-loop-peeling=skip-partial=false -canonicalize -split-input-file -verify-diagnostics | FileCheck %s -check-prefix=CHECK-NO-SKIP3 4// CHECK-DAG: #[[MAP0:.*]] = affine_map<()[s0, s1, s2] -> (s1 - (-s0 + s1) mod s2)>5// CHECK-DAG: #[[MAP1:.*]] = affine_map<(d0)[s0] -> (-d0 + s0)>6// CHECK: func @fully_dynamic_bounds(7// CHECK-SAME: %[[LB:.*]]: index, %[[UB:.*]]: index, %[[STEP:.*]]: index8// CHECK: %[[C0_I32:.*]] = arith.constant 0 : i329// CHECK: %[[NEW_UB:.*]] = affine.apply #[[MAP0]]()[%[[LB]], %[[UB]], %[[STEP]]]10// CHECK: %[[LOOP:.*]] = scf.for %[[IV:.*]] = %[[LB]] to %[[NEW_UB]]11// CHECK-SAME: step %[[STEP]] iter_args(%[[ACC:.*]] = %[[C0_I32]]) -> (i32) {12// CHECK: %[[CAST:.*]] = arith.index_cast %[[STEP]] : index to i3213// CHECK: %[[ADD:.*]] = arith.addi %[[ACC]], %[[CAST]] : i3214// CHECK: scf.yield %[[ADD]]15// CHECK: }16// CHECK: %[[RESULT:.*]] = scf.for %[[IV2:.*]] = %[[NEW_UB]] to %[[UB]]17// CHECK-SAME: step %[[STEP]] iter_args(%[[ACC2:.*]] = %[[LOOP]]) -> (i32) {18// CHECK: %[[REM:.*]] = affine.apply #[[MAP1]](%[[IV2]])[%[[UB]]]19// CHECK: %[[CAST2:.*]] = arith.index_cast %[[REM]]20// CHECK: %[[ADD2:.*]] = arith.addi %[[ACC2]], %[[CAST2]]21// CHECK: scf.yield %[[ADD2]]22// CHECK: }23// CHECK: return %[[RESULT]]24#map = affine_map<(d0, d1)[s0] -> (s0, d0 - d1)>25func.func @fully_dynamic_bounds(%lb : index, %ub: index, %step: index) -> i32 {26 %c0 = arith.constant 0 : i3227 %r = scf.for %iv = %lb to %ub step %step iter_args(%arg = %c0) -> i32 {28 %s = affine.min #map(%ub, %iv)[%step]29 %casted = arith.index_cast %s : index to i3230 %0 = arith.addi %arg, %casted : i3231 scf.yield %0 : i3232 }33 return %r : i3234}35 36// -----37 38// CHECK-LABEL: func.func @static_two_iterations_ub_used_in_loop(39// CHECK-SAME: %[[IFM1:.*]]: memref<1xi32>) -> i32 {40// CHECK: %[[C7_I32:.*]] = arith.constant 7 : i3241// CHECK: %[[C7_IDX:.*]] = arith.constant 7 : index42// CHECK: %[[LOAD1:.*]] = memref.load %[[IFM1]]{{\[}}%[[C7_IDX]]] : memref<1xi32>43// CHECK: %[[ADDI1:.*]] = arith.addi %[[LOAD1]], %[[C7_I32]] : i3244// CHECK: %[[LOAD2:.*]] = memref.load %[[IFM1]]{{\[}}%[[C7_IDX]]] : memref<1xi32>45// CHECK: %[[ADDI2:.*]] = arith.addi %[[ADDI1]], %[[LOAD2]] : i3246// CHECK: return %[[ADDI2]] : i3247// CHECK: }48 49#map = affine_map<(d0, d1)[s0] -> (s0, d0 - d1)>50func.func @static_two_iterations_ub_used_in_loop(%arg0: memref<1xi32>) -> i32 {51 %c0_i32 = arith.constant 0 : i3252 %lb = arith.constant 0 : index53 %step = arith.constant 4 : index54 %ub = arith.constant 7 : index55 %r = scf.for %iv = %lb to %ub step %step iter_args(%arg = %c0_i32) -> i32 {56 %s = affine.min #map(%ub, %iv)[%step]57 %casted = arith.index_cast %s : index to i3258 %0 = arith.addi %arg, %casted : i3259 %1 = memref.load %arg0[%ub] : memref<1xi32>60 %result = arith.addi %0, %1 : i3261 scf.yield %result : i3262 }63 return %r : i3264}65// -----66 67// CHECK: func @fully_static_bounds(68// CHECK-DAG: %[[C0_I32:.*]] = arith.constant 0 : i3269// CHECK-DAG: %[[C1_I32:.*]] = arith.constant 1 : i3270// CHECK-DAG: %[[C4_I32:.*]] = arith.constant 4 : i3271// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index72// CHECK-DAG: %[[C4:.*]] = arith.constant 4 : index73// CHECK-DAG: %[[C16:.*]] = arith.constant 16 : index74// CHECK: %[[LOOP:.*]] = scf.for %[[IV:.*]] = %[[C0]] to %[[C16]]75// CHECK-SAME: step %[[C4]] iter_args(%[[ACC:.*]] = %[[C0_I32]]) -> (i32) {76// CHECK: %[[ADD:.*]] = arith.addi %[[ACC]], %[[C4_I32]] : i3277// CHECK: scf.yield %[[ADD]]78// CHECK: }79// CHECK: %[[RESULT:.*]] = arith.addi %[[LOOP]], %[[C1_I32]] : i3280// CHECK: return %[[RESULT]]81#map = affine_map<(d0, d1)[s0] -> (s0, d0 - d1)>82func.func @fully_static_bounds() -> i32 {83 %c0_i32 = arith.constant 0 : i3284 %lb = arith.constant 0 : index85 %step = arith.constant 4 : index86 %ub = arith.constant 17 : index87 %r = scf.for %iv = %lb to %ub step %step88 iter_args(%arg = %c0_i32) -> i32 {89 %s = affine.min #map(%ub, %iv)[%step]90 %casted = arith.index_cast %s : index to i3291 %0 = arith.addi %arg, %casted : i3292 scf.yield %0 : i3293 }94 return %r : i3295}96 97// -----98 99// CHECK: func @fully_static_bounds_integers(100// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : i32101// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : i32102// CHECK-DAG: %[[C4:.*]] = arith.constant 4 : i32103// CHECK-DAG: %[[C16:.*]] = arith.constant 16 : i32104// CHECK: %[[LOOP:.*]] = scf.for %[[IV:.*]] = %[[C0]] to %[[C16]]105// CHECK-SAME: step %[[C4]] iter_args(%[[ACC:.*]] = %[[C0]]) -> (i32)106// CHECK: %[[MAP:.*]] = affine.min107// CHECK: %[[MAP_CAST:.*]] = arith.index_cast %[[MAP]]108// CHECK: %[[ADD:.*]] = arith.addi %[[ACC]], %[[MAP_CAST]] : i32109// CHECK: scf.yield %[[ADD]]110// CHECK: }111// CHECK: %[[RESULT:.*]] = arith.addi %[[LOOP]], %[[C1]] : i32112// CHECK: return %[[RESULT]]113#map = affine_map<(d0, d1)[s0] -> (s0, d0 - d1)>114func.func @fully_static_bounds_integers() -> i32 {115 %c0_i32 = arith.constant 0 : i32116 %lb = arith.constant 0 : i32117 %step = arith.constant 4 : i32118 %ub = arith.constant 17 : i32119 %r = scf.for %iv = %lb to %ub step %step120 iter_args(%arg = %c0_i32) -> i32 : i32 {121 %ub_index = arith.index_cast %ub : i32 to index122 %iv_index = arith.index_cast %iv : i32 to index123 %step_index = arith.index_cast %step : i32 to index124 %s = affine.min #map(%ub_index, %iv_index)[%step_index]125 %casted = arith.index_cast %s : index to i32126 %0 = arith.addi %arg, %casted : i32127 scf.yield %0 : i32128 }129 return %r : i32130}131 132// -----133 134// CHECK-DAG: #[[MAP0:.*]] = affine_map<()[s0] -> ((s0 floordiv 4) * 4)>135// CHECK-DAG: #[[MAP1:.*]] = affine_map<(d0)[s0] -> (-d0 + s0)>136// CHECK: func @dynamic_upper_bound(137// CHECK-SAME: %[[UB:.*]]: index138// CHECK-DAG: %[[C0_I32:.*]] = arith.constant 0 : i32139// CHECK-DAG: %[[C4_I32:.*]] = arith.constant 4 : i32140// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index141// CHECK-DAG: %[[C4:.*]] = arith.constant 4 : index142// CHECK: %[[NEW_UB:.*]] = affine.apply #[[MAP0]]()[%[[UB]]]143// CHECK: %[[LOOP:.*]] = scf.for %[[IV:.*]] = %[[C0]] to %[[NEW_UB]]144// CHECK-SAME: step %[[C4]] iter_args(%[[ACC:.*]] = %[[C0_I32]]) -> (i32) {145// CHECK: %[[ADD:.*]] = arith.addi %[[ACC]], %[[C4_I32]] : i32146// CHECK: scf.yield %[[ADD]]147// CHECK: }148// CHECK: %[[RESULT:.*]] = scf.for %[[IV2:.*]] = %[[NEW_UB]] to %[[UB]]149// CHECK-SAME: step %[[C4]] iter_args(%[[ACC2:.*]] = %[[LOOP]]) -> (i32) {150// CHECK: %[[REM:.*]] = affine.apply #[[MAP1]](%[[IV2]])[%[[UB]]]151// CHECK: %[[CAST2:.*]] = arith.index_cast %[[REM]]152// CHECK: %[[ADD2:.*]] = arith.addi %[[ACC2]], %[[CAST2]]153// CHECK: scf.yield %[[ADD2]]154// CHECK: }155// CHECK: return %[[RESULT]]156#map = affine_map<(d0, d1)[s0] -> (s0, d0 - d1)>157func.func @dynamic_upper_bound(%ub : index) -> i32 {158 %c0_i32 = arith.constant 0 : i32159 %lb = arith.constant 0 : index160 %step = arith.constant 4 : index161 %r = scf.for %iv = %lb to %ub step %step162 iter_args(%arg = %c0_i32) -> i32 {163 %s = affine.min #map(%ub, %iv)[%step]164 %casted = arith.index_cast %s : index to i32165 %0 = arith.addi %arg, %casted : i32166 scf.yield %0 : i32167 }168 return %r : i32169}170 171// -----172 173// CHECK-DAG: #[[MAP0:.*]] = affine_map<()[s0] -> ((s0 floordiv 4) * 4)>174// CHECK-DAG: #[[MAP1:.*]] = affine_map<(d0)[s0] -> (-d0 + s0)>175// CHECK: func @no_loop_results(176// CHECK-SAME: %[[UB:.*]]: index, %[[MEMREF:.*]]: memref<i32>177// CHECK-DAG: %[[C4_I32:.*]] = arith.constant 4 : i32178// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index179// CHECK-DAG: %[[C4:.*]] = arith.constant 4 : index180// CHECK: %[[NEW_UB:.*]] = affine.apply #[[MAP0]]()[%[[UB]]]181// CHECK: scf.for %[[IV:.*]] = %[[C0]] to %[[NEW_UB]] step %[[C4]] {182// CHECK: %[[LOAD:.*]] = memref.load %[[MEMREF]][]183// CHECK: %[[ADD:.*]] = arith.addi %[[LOAD]], %[[C4_I32]] : i32184// CHECK: memref.store %[[ADD]], %[[MEMREF]]185// CHECK: }186// CHECK: scf.for %[[IV2:.*]] = %[[NEW_UB]] to %[[UB]] step %[[C4]] {187// CHECK: %[[REM:.*]] = affine.apply #[[MAP1]](%[[IV2]])[%[[UB]]]188// CHECK: %[[LOAD2:.*]] = memref.load %[[MEMREF]][]189// CHECK: %[[CAST2:.*]] = arith.index_cast %[[REM]]190// CHECK: %[[ADD2:.*]] = arith.addi %[[LOAD2]], %[[CAST2]]191// CHECK: memref.store %[[ADD2]], %[[MEMREF]]192// CHECK: }193// CHECK: return194#map = affine_map<(d0, d1)[s0] -> (s0, d0 - d1)>195func.func @no_loop_results(%ub : index, %d : memref<i32>) {196 %c0_i32 = arith.constant 0 : i32197 %lb = arith.constant 0 : index198 %step = arith.constant 4 : index199 scf.for %iv = %lb to %ub step %step {200 %s = affine.min #map(%ub, %iv)[%step]201 %r = memref.load %d[] : memref<i32>202 %casted = arith.index_cast %s : index to i32203 %0 = arith.addi %r, %casted : i32204 memref.store %0, %d[] : memref<i32>205 }206 return207}208 209// -----210 211// Test rewriting of affine.min/max ops. Make sure that more general cases than212// the ones above are successfully rewritten. Also make sure that the pattern213// does not rewrite ops that should not be rewritten.214 215// CHECK-DAG: #[[MAP1:.*]] = affine_map<()[s0] -> (s0 + 1)>216// CHECK-DAG: #[[MAP2:.*]] = affine_map<(d0)[s0, s1] -> (-d0 + s1 - 1, s0)>217// CHECK-DAG: #[[MAP3:.*]] = affine_map<(d0)[s0, s1, s2] -> (-d0 + s1, s2, s0)>218// CHECK-DAG: #[[MAP4:.*]] = affine_map<()[s0] -> (-s0)>219// CHECK-DAG: #[[MAP5:.*]] = affine_map<(d0)[s0] -> (-d0 + s0)>220// CHECK-DAG: #[[MAP6:.*]] = affine_map<(d0)[s0] -> (-d0 + s0 + 1)>221// CHECK-DAG: #[[MAP7:.*]] = affine_map<(d0)[s0] -> (-d0 + s0 - 1)>222// CHECK-DAG: #[[MAP8:.*]] = affine_map<(d0)[s0] -> (d0 - s0)>223// CHECK: func @test_affine_op_rewrite(224// CHECK-SAME: %[[LB:.*]]: index, %[[UB:.*]]: index, %[[STEP:.*]]: index,225// CHECK-SAME: %[[MEMREF:.*]]: memref<?xindex>, %[[SOME_VAL:.*]]: index226// CHECK: scf.for %[[IV:.*]] = %[[LB]] to %{{.*}} step %[[STEP]] {227// (affine.min folded away)228// CHECK: memref.store %[[STEP]]229// (affine.min folded away)230// CHECK: memref.store %[[STEP]]231// CHECK: %[[RES2:.*]] = affine.apply #[[MAP1]]()[%[[STEP]]]232// CHECK: memref.store %[[RES2]]233// CHECK: %[[RES3:.*]] = affine.min #[[MAP2]](%[[IV]])[%[[STEP]], %[[UB]]]234// CHECK: memref.store %[[RES3]]235// CHECK: %[[RES4:.*]] = affine.min #[[MAP3]](%[[IV]])[%[[STEP]], %[[UB]], %[[SOME_VAL]]]236// CHECK: memref.store %[[RES4]]237// CHECK: %[[RES5:.*]] = affine.apply #[[MAP4]]()[%[[STEP]]]238// CHECK: memref.store %[[RES5]]239// CHECK: }240// CHECK: scf.for %[[IV2:.*]] = {{.*}} to %[[UB]] step %[[STEP]] {241// CHECK: %[[RES_IF_0:.*]] = affine.apply #[[MAP5]](%[[IV2]])[%[[UB]]]242// CHECK: memref.store %[[RES_IF_0]]243// CHECK: %[[RES_IF_1:.*]] = affine.apply #[[MAP6]](%[[IV2]])[%[[UB]]]244// CHECK: memref.store %[[RES_IF_1]]245// CHECK: %[[RES_IF_2:.*]] = affine.apply #[[MAP6]](%[[IV2]])[%[[UB]]]246// CHECK: memref.store %[[RES_IF_2]]247// CHECK: %[[RES_IF_3:.*]] = affine.apply #[[MAP7]](%[[IV2]])[%[[UB]]]248// CHECK: memref.store %[[RES_IF_3]]249// CHECK: %[[RES_IF_4:.*]] = affine.min #[[MAP3]](%[[IV2]])[%[[STEP]], %[[UB]], %[[SOME_VAL]]]250// CHECK: memref.store %[[RES_IF_4]]251// CHECK: %[[RES_IF_5:.*]] = affine.apply #[[MAP8]](%[[IV2]])[%[[UB]]]252// CHECK: memref.store %[[RES_IF_5]]253#map0 = affine_map<(d0, d1)[s0] -> (s0, d0 - d1)>254#map1 = affine_map<(d0, d1)[s0] -> (d0 - d1 + 1, s0)>255#map2 = affine_map<(d0, d1)[s0] -> (s0 + 1, d0 - d1 + 1)>256#map3 = affine_map<(d0, d1)[s0] -> (s0, d0 - d1 - 1)>257#map4 = affine_map<(d0, d1, d2)[s0] -> (s0, d0 - d1, d2)>258#map5 = affine_map<(d0, d1)[s0] -> (-s0, -d0 + d1)>259func.func @test_affine_op_rewrite(%lb : index, %ub: index,260 %step: index, %d : memref<?xindex>,261 %some_val: index) {262 %c0 = arith.constant 0 : index263 %c1 = arith.constant 1 : index264 %c2 = arith.constant 2 : index265 %c3 = arith.constant 3 : index266 %c4 = arith.constant 4 : index267 %c5 = arith.constant 5 : index268 scf.for %iv = %lb to %ub step %step {269 // Most common case: Rewrite min(%ub - %iv, %step) to %step.270 %m0 = affine.min #map0(%ub, %iv)[%step]271 memref.store %m0, %d[%c0] : memref<?xindex>272 273 // Increase %ub - %iv a little bit, pattern should still apply.274 %m1 = affine.min #map1(%ub, %iv)[%step]275 memref.store %m1, %d[%c1] : memref<?xindex>276 277 // Rewrite min(%ub - %iv + 1, %step + 1) to %step + 1.278 %m2 = affine.min #map2(%ub, %iv)[%step]279 memref.store %m2, %d[%c2] : memref<?xindex>280 281 // min(%ub - %iv - 1, %step) cannot be simplified because %ub - %iv - 1282 // can be smaller than %step. (Can be simplified in if-statement.)283 %m3 = affine.min #map3(%ub, %iv)[%step]284 memref.store %m3, %d[%c3] : memref<?xindex>285 286 // min(%ub - %iv, %step, %some_val) cannot be simplified because the range287 // of %some_val is unknown.288 %m4 = affine.min #map4(%ub, %iv, %some_val)[%step]289 memref.store %m4, %d[%c4] : memref<?xindex>290 291 // Rewrite max(-%ub + %iv, -%step) to -%ub + %iv (and -%step in the scf.if).292 %m5 = affine.max #map5(%ub, %iv)[%step]293 memref.store %m5, %d[%c5] : memref<?xindex>294 }295 return296}297 298// -----299 300// CHECK: func @nested_loops301// CHECK: scf.for {{.*}} {302// CHECK: scf.for {{.*}} {303// CHECK: }304// CHECK: scf.for {{.*}} {305// CHECK: }306// CHECK: }307// CHECK: scf.for {{.*}} {308// CHECK: scf.for {{.*}} {309// CHECK: }310// CHECK-NOT: scf.for311// CHECK: }312 313// CHECK-NO-SKIP: func @nested_loops314// CHECK-NO-SKIP: scf.for {{.*}} {315// CHECK-NO-SKIP: scf.for {{.*}} {316// CHECK-NO-SKIP: }317// CHECK-NO-SKIP: scf.for {{.*}} {318// CHECK-NO-SKIP: }319// CHECK-NO-SKIP: }320// CHECK-NO-SKIP: scf.for {{.*}} {321// CHECK-NO-SKIP: scf.for {{.*}} {322// CHECK-NO-SKIP: }323// CHECK-NO-SKIP: scf.for {{.*}} {324// CHECK-NO-SKIP: }325// CHECK-NO-SKIP: }326#map = affine_map<(d0, d1)[s0] -> (s0, d0 - d1)>327func.func @nested_loops(%lb0: index, %lb1 : index, %ub0: index, %ub1: index,328 %step: index) -> i32 {329 %c0 = arith.constant 0 : i32330 %r0 = scf.for %iv0 = %lb0 to %ub0 step %step iter_args(%arg0 = %c0) -> i32 {331 %r1 = scf.for %iv1 = %lb1 to %ub1 step %step iter_args(%arg1 = %arg0) -> i32 {332 %s = affine.min #map(%ub1, %iv1)[%step]333 %casted = arith.index_cast %s : index to i32334 %0 = arith.addi %arg1, %casted : i32335 scf.yield %0 : i32336 }337 %1 = arith.addi %arg0, %r1 : i32338 scf.yield %1 : i32339 }340 return %r0 : i32341}342 343// -----344 345// CHECK-LABEL: func @regression346func.func @regression(%arg0: memref<i64>, %arg1: index) {347 %c0 = arith.constant 0 : index348 %0 = affine.apply affine_map<()[s0] -> (s0 * s0)>()[%arg1]349 scf.for %arg2 = %c0 to %0 step %arg1 {350 %1 = affine.min affine_map<(d0)[s0] -> (s0, -d0 + s0 * s0)>(%arg2)[%arg1]351 %2 = arith.index_cast %0 : index to i64352 memref.store %2, %arg0[] : memref<i64>353 }354 return355}356 357// -----358 359// Regression test: Make sure that we do not crash.360// The step is 0, the loop will be eliminated.361// CHECK-LABEL: func @zero_step(362// CHECK-NOT: scf.for363func.func @zero_step(%arg0: memref<i64>) {364 %c0 = arith.constant 0 : index365 %c1 = arith.constant 1 : index366 %foldto0 = arith.subi %c1, %c1 : index367 scf.for %arg2 = %c0 to %c1 step %foldto0 {368 %2 = arith.index_cast %arg2 : index to i64369 memref.store %2, %arg0[] : memref<i64>370 }371 return372}373 374