1585 lines · plain
1// RUN: mlir-opt %s -split-input-file -loop-invariant-code-motion | FileCheck %s2 3func.func @nested_loops_both_having_invariant_code() {4 %m = memref.alloc() : memref<10xf32>5 %cf7 = arith.constant 7.0 : f326 %cf8 = arith.constant 8.0 : f327 8 affine.for %arg0 = 0 to 10 {9 %v0 = arith.addf %cf7, %cf8 : f3210 affine.for %arg1 = 0 to 10 {11 %v1 = arith.addf %v0, %cf8 : f3212 affine.store %v0, %m[%arg0] : memref<10xf32>13 }14 }15 16 // CHECK: memref.alloc() : memref<10xf32>17 // CHECK-NEXT: %[[CST0:.*]] = arith.constant 7.000000e+00 : f3218 // CHECK-NEXT: %[[CST1:.*]] = arith.constant 8.000000e+00 : f3219 // CHECK-NEXT: %[[ADD0:.*]] = arith.addf %[[CST0]], %[[CST1]] : f3220 // CHECK-NEXT: arith.addf %[[ADD0]], %[[CST1]] : f3221 // CHECK-NEXT: affine.for22 // CHECK-NEXT: affine.for23 // CHECK-NEXT: affine.store24 25 return26}27 28// -----29 30func.func @nested_loops_code_invariant_to_both() {31 %m = memref.alloc() : memref<10xf32>32 %cf7 = arith.constant 7.0 : f3233 %cf8 = arith.constant 8.0 : f3234 35 affine.for %arg0 = 0 to 10 {36 affine.for %arg1 = 0 to 10 {37 %v0 = arith.addf %cf7, %cf8 : f3238 }39 }40 41 // CHECK: memref.alloc() : memref<10xf32>42 // CHECK-NEXT: arith.constant 7.000000e+00 : f3243 // CHECK-NEXT: arith.constant 8.000000e+00 : f3244 // CHECK-NEXT: arith.addf45 46 return47}48 49// -----50 51func.func @single_loop_nothing_invariant() {52 %m1 = memref.alloc() : memref<10xf32>53 %m2 = memref.alloc() : memref<10xf32>54 affine.for %arg0 = 0 to 10 {55 %v0 = affine.load %m1[%arg0] : memref<10xf32>56 %v1 = affine.load %m2[%arg0] : memref<10xf32>57 %v2 = arith.addf %v0, %v1 : f3258 affine.store %v2, %m1[%arg0] : memref<10xf32>59 }60 61 // CHECK: memref.alloc() : memref<10xf32>62 // CHECK-NEXT: memref.alloc() : memref<10xf32>63 // CHECK-NEXT: affine.for64 // CHECK-NEXT: affine.load65 // CHECK-NEXT: affine.load66 // CHECK-NEXT: arith.addf67 // CHECK-NEXT: affine.store68 69 return70}71 72// -----73 74func.func @invariant_code_inside_affine_if() {75 %m = memref.alloc() : memref<10xf32>76 %cf8 = arith.constant 8.0 : f3277 78 affine.for %arg0 = 0 to 10 {79 %t0 = affine.apply affine_map<(d1) -> (d1 + 1)>(%arg0)80 affine.if affine_set<(d0, d1) : (d1 - d0 >= 0)> (%arg0, %t0) {81 %cf9 = arith.addf %cf8, %cf8 : f3282 affine.store %cf9, %m[%arg0] : memref<10xf32>83 84 }85 }86 87 // CHECK: memref.alloc() : memref<10xf32>88 // CHECK-NEXT: arith.constant 8.000000e+00 : f3289 // CHECK-NEXT: affine.for90 // CHECK-NEXT: affine.apply91 // CHECK-NEXT: affine.if92 // CHECK-NEXT: arith.addf93 // CHECK-NEXT: affine.store94 // CHECK-NEXT: }95 96 97 return98}99 100// -----101 102func.func @invariant_affine_if() {103 %m = memref.alloc() : memref<10xf32>104 %cf8 = arith.constant 8.0 : f32105 affine.for %arg0 = 0 to 10 {106 affine.for %arg1 = 0 to 20 {107 affine.if affine_set<(d0, d1) : (d1 - d0 >= 0)> (%arg0, %arg0) {108 %cf9 = arith.addf %cf8, %cf8 : f32109 }110 }111 }112 113 // CHECK: memref.alloc() : memref<10xf32>114 // CHECK-NEXT: %[[CST:.*]] = arith.constant 8.000000e+00 : f32115 // CHECK-NEXT: affine.for %[[ARG:.*]] = 0 to 20 {116 // CHECK-NEXT: }117 // CHECK-NEXT: affine.for %[[ARG:.*]] = 0 to 10 {118 // CHECK-NEXT: affine.if #set(%[[ARG]], %[[ARG]]) {119 // CHECK-NEXT: arith.addf %[[CST]], %[[CST]] : f32120 // CHECK-NEXT: }121 122 return123}124 125// -----126 127func.func @hoist_invariant_affine_if_success(%lb: index, %ub: index, %step: index) -> i32 {128 %cst_0 = arith.constant 0 : i32129 %cst_42 = arith.constant 42 : i32130 %sum_result = affine.for %i = %lb to %ub iter_args(%acc = %cst_0) -> i32 {131 %conditional_add = affine.if affine_set<() : ()> () -> (i32) {132 %add = arith.addi %cst_42, %cst_42 : i32133 affine.yield %add : i32134 } else {135 %poison = ub.poison : i32136 affine.yield %poison : i32137 }138 %sum = arith.addi %acc, %conditional_add : i32139 affine.yield %sum : i32140 }141 142 // CHECK-LABEL: hoist_invariant_affine_if_success143 // CHECK-NEXT: arith.constant 0 : i32144 // CHECK-NEXT: %[[CST:.*]] = arith.constant 42 : i32145 // CHECK-NEXT: %[[IF:.*]] = affine.if146 // CHECK-NEXT: arith.addi %[[CST]], %[[CST]] : i32147 // CHECK: affine.for148 // CHECK-NOT: affine.if149 // CHECK-NEXT: arith.addi %{{.*}}, %[[IF]]150 151 return %sum_result : i32152}153 154// -----155 156func.func @hoist_variant_affine_if_failure(%lb: index, %ub: index, %step: index) -> i32 {157 %cst_0 = arith.constant 0 : i32158 %cst_42 = arith.constant 42 : i32159 %ind_7 = arith.constant 7 : index160 %sum_result = affine.for %i = %lb to %ub iter_args(%acc = %cst_0) -> i32 {161 %conditional_add = affine.if affine_set<(d0, d1) : (d1 - d0 >= 0)> (%i, %ind_7) -> (i32) {162 %add = arith.addi %cst_42, %cst_42 : i32163 affine.yield %add : i32164 } else {165 %poison = ub.poison : i32166 affine.yield %poison : i32167 }168 %sum = arith.addi %acc, %conditional_add : i32169 affine.yield %sum : i32170 }171 172 // CHECK-LABEL: hoist_variant_affine_if_failure173 // CHECK-NEXT: arith.constant 0 : i32174 // CHECK-NEXT: %[[CST:.*]] = arith.constant 42 : i32175 // CHECK-NEXT: arith.constant 7 : index176 // CHECK-NEXT: affine.for177 // CHECK-NEXT: %[[IF:.*]] = affine.if178 // CHECK: arith.addi %{{.*}}, %[[IF]]179 180 return %sum_result : i32181}182 183// -----184 185func.func @hoist_affine_for_with_unknown_trip_count(%lb: index, %ub: index) {186 affine.for %arg0 = 0 to 10 {187 affine.for %arg1 = %lb to %ub {188 }189 }190 191 // CHECK: @hoist_affine_for_with_unknown_trip_count(%[[ARG0:.*]]: index, %[[ARG1:.*]]: index) {192 // CHECK-NEXT: affine.for %[[ARG2:.*]] = %[[ARG0]] to %[[ARG1]] {193 // CHECK-NEXT: }194 // CHECK-NEXT: affine.for %[[ARG3:.*]] = 0 to 10 {195 // CHECK-NEXT: }196 197 return198}199 200// -----201 202func.func @hoist_affine_for_with_unknown_trip_count_non_unit_step(%lb: index, %ub: index) {203 affine.for %arg0 = 0 to 10 {204 affine.for %arg1 = %lb to %ub step 2 {205 }206 }207 208 // CHECK: @hoist_affine_for_with_unknown_trip_count_non_unit_step(%[[ARG0:.*]]: index, %[[ARG1:.*]]: index) {209 // CHECK-NEXT: affine.for %[[ARG2:.*]] = 0 to 10 {210 // CHECK-NEXT: affine.for %[[ARG3:.*]] = %[[ARG0]] to %[[ARG1]] step 2 {211 // CHECK-NEXT: }212 // CHECK-NEXT: }213 214 return215}216 217// -----218 219func.func @hoist_scf_for_with_unknown_trip_count_unit_step(%lb: index, %ub: index) {220 %c1 = arith.constant 1 : index221 scf.for %arg0 = %lb to %ub step %c1 {222 scf.for %arg1 = %lb to %ub step %c1 {223 }224 }225 226 // CHECK: @hoist_scf_for_with_unknown_trip_count_unit_step(%[[ARG0:.*]]: index, %[[ARG1:.*]]: index) {227 // CHECK: scf.for %[[ARG2:.*]] = %[[ARG0]] to %[[ARG1]]228 // CHECK-NEXT: }229 // CHECK-NEXT: scf.for %[[ARG3:.*]] = %[[ARG0]] to %[[ARG1]]230 // CHECK-NEXT: }231 232 return233}234 235// -----236 237func.func @hoist_scf_for_with_unknown_trip_count_non_unit_constant_step(%lb: index, %ub: index) {238 %c1 = arith.constant 1 : index239 %c2 = arith.constant 2 : index240 scf.for %arg0 = %lb to %ub step %c1 {241 scf.for %arg1 = %lb to %ub step %c2 {242 }243 }244 245 // CHECK: @hoist_scf_for_with_unknown_trip_count_non_unit_constant_step(%[[ARG0:.*]]: index, %[[ARG1:.*]]: index) {246 // CHECK: scf.for %[[ARG2:.*]] = %[[ARG0]] to %[[ARG1]]247 // CHECK-NEXT: scf.for %[[ARG3:.*]] = %[[ARG0]] to %[[ARG1]]248 // CHECK-NEXT: }249 // CHECK-NEXT: }250 251 return252}253 254// -----255 256func.func @hoist_scf_for_with_unknown_trip_count_unknown_step(%lb: index, %ub: index, %step: index) {257 %c1 = arith.constant 1 : index258 scf.for %arg0 = %lb to %ub step %c1 {259 scf.for %arg1 = %lb to %ub step %step {260 }261 }262 263 // CHECK: @hoist_scf_for_with_unknown_trip_count_unknown_step(%[[ARG0:.*]]: index, %[[ARG1:.*]]: index, %[[STEP:.*]]: index) {264 // CHECK: scf.for %[[ARG2:.*]] = %[[ARG0]] to %[[ARG1]]265 // CHECK-NEXT: scf.for %[[ARG3:.*]] = %[[ARG0]] to %[[ARG1]] step %[[STEP]]266 // CHECK-NEXT: }267 // CHECK-NEXT: }268 269 return270}271 272// -----273 274func.func @invariant_affine_if2() {275 %m = memref.alloc() : memref<10xf32>276 %cf8 = arith.constant 8.0 : f32277 affine.for %arg0 = 0 to 10 {278 affine.for %arg1 = 0 to 10 {279 affine.if affine_set<(d0, d1) : (d1 - d0 >= 0)> (%arg0, %arg0) {280 %cf9 = arith.addf %cf8, %cf8 : f32281 affine.store %cf9, %m[%arg1] : memref<10xf32>282 }283 }284 }285 286 // CHECK: memref.alloc287 // CHECK-NEXT: arith.constant288 // CHECK-NEXT: affine.for289 // CHECK-NEXT: affine.for290 // CHECK-NEXT: affine.if291 // CHECK-NEXT: arith.addf292 // CHECK-NEXT: affine.store293 // CHECK-NEXT: }294 // CHECK-NEXT: }295 296 return297}298 299// -----300 301func.func @invariant_affine_nested_if() {302 %m = memref.alloc() : memref<10xf32>303 %cf8 = arith.constant 8.0 : f32304 affine.for %arg0 = 0 to 10 {305 affine.for %arg1 = 0 to 10 {306 affine.if affine_set<(d0, d1) : (d1 - d0 >= 0)> (%arg0, %arg0) {307 %cf9 = arith.addf %cf8, %cf8 : f32308 affine.if affine_set<(d0, d1) : (d1 - d0 >= 0)> (%arg0, %arg0) {309 %cf10 = arith.addf %cf9, %cf9 : f32310 }311 }312 }313 }314 315 // CHECK: memref.alloc316 // CHECK-NEXT: arith.constant317 // CHECK-NEXT: affine.for318 // CHECK-NEXT: }319 // CHECK-NEXT: affine.for320 // CHECK-NEXT: affine.if321 // CHECK-NEXT: arith.addf322 // CHECK-NEXT: affine.if323 // CHECK-NEXT: arith.addf324 // CHECK-NEXT: }325 // CHECK-NEXT: }326 327 328 return329}330 331// -----332 333func.func @invariant_affine_nested_if_else() {334 %m = memref.alloc() : memref<10xf32>335 %cf8 = arith.constant 8.0 : f32336 affine.for %arg0 = 0 to 10 {337 affine.for %arg1 = 0 to 10 {338 affine.if affine_set<(d0, d1) : (d1 - d0 >= 0)> (%arg0, %arg0) {339 %cf9 = arith.addf %cf8, %cf8 : f32340 affine.store %cf9, %m[%arg0] : memref<10xf32>341 affine.if affine_set<(d0, d1) : (d1 - d0 >= 0)> (%arg0, %arg0) {342 %cf10 = arith.addf %cf9, %cf9 : f32343 } else {344 affine.store %cf9, %m[%arg1] : memref<10xf32>345 }346 }347 }348 }349 350 // CHECK: memref.alloc351 // CHECK-NEXT: arith.constant352 // CHECK-NEXT: affine.for353 // CHECK-NEXT: affine.for354 // CHECK-NEXT: affine.if355 // CHECK-NEXT: arith.addf356 // CHECK-NEXT: affine.store357 // CHECK-NEXT: affine.if358 // CHECK-NEXT: arith.addf359 // CHECK-NEXT: } else {360 // CHECK-NEXT: affine.store361 // CHECK-NEXT: }362 // CHECK-NEXT: }363 // CHECK-NEXT: }364 365 366 return367}368 369// -----370 371func.func @invariant_loop_dialect() {372 %ci0 = arith.constant 0 : index373 %ci10 = arith.constant 10 : index374 %ci1 = arith.constant 1 : index375 %m = memref.alloc() : memref<10xf32>376 %cf7 = arith.constant 7.0 : f32377 %cf8 = arith.constant 8.0 : f32378 scf.for %arg0 = %ci0 to %ci10 step %ci1 {379 scf.for %arg1 = %ci0 to %ci10 step %ci1 {380 %v0 = arith.addf %cf7, %cf8 : f32381 }382 }383 384 // CHECK: memref.alloc() : memref<10xf32>385 // CHECK-NEXT: arith.constant 7.000000e+00 : f32386 // CHECK-NEXT: arith.constant 8.000000e+00 : f32387 // CHECK-NEXT: arith.addf388 389 return390}391 392// -----393 394func.func @variant_loop_dialect() {395 %ci0 = arith.constant 0 : index396 %ci10 = arith.constant 10 : index397 %ci1 = arith.constant 1 : index398 %m = memref.alloc() : memref<10xf32>399 scf.for %arg0 = %ci0 to %ci10 step %ci1 {400 scf.for %arg1 = %ci0 to %ci10 step %ci1 {401 %v0 = arith.addi %arg0, %arg1 : index402 }403 }404 405 // CHECK: memref.alloc() : memref<10xf32>406 // CHECK-NEXT: scf.for407 // CHECK-NEXT: scf.for408 // CHECK-NEXT: arith.addi409 410 return411}412 413// -----414 415func.func @parallel_loop_with_invariant() {416 %c0 = arith.constant 0 : index417 %c10 = arith.constant 10 : index418 %c1 = arith.constant 1 : index419 %c7 = arith.constant 7 : i32420 %c8 = arith.constant 8 : i32421 scf.parallel (%arg0, %arg1) = (%c0, %c0) to (%c10, %c10) step (%c1, %c1) {422 %v0 = arith.addi %c7, %c8 : i32423 %v3 = arith.addi %arg0, %arg1 : index424 }425 426 // CHECK-LABEL: func @parallel_loop_with_invariant427 // CHECK: arith.constant 0 : index428 // CHECK-NEXT: arith.constant 10 : index429 // CHECK-NEXT: arith.constant 1 : index430 // CHECK-NEXT: arith.constant 7 : i32431 // CHECK-NEXT: arith.constant 8 : i32432 // CHECK-NEXT: arith.addi433 // CHECK-NEXT: scf.parallel (%[[A:.*]],{{.*}}) =434 // CHECK-NEXT: arith.addi %[[A]]435 // CHECK-NEXT: reduce436 // CHECK-NEXT: }437 // CHECK-NEXT: return438 439 return440}441 442// -----443 444func.func @hoist_invariant_scf_if_success(%lb: index, %ub: index, %step: index) -> i32 {445 %cst_0 = arith.constant 0 : i32446 %cst_42 = arith.constant 42 : i32447 %true = arith.constant true448 %sum_result = scf.for %i = %lb to %ub step %step iter_args(%acc = %cst_0) -> i32 {449 %conditional_add = scf.if %true -> (i32) {450 %add = arith.addi %cst_42, %cst_42 : i32451 scf.yield %add : i32452 } else {453 %poison = ub.poison : i32454 scf.yield %poison : i32455 }456 %sum = arith.addi %acc, %conditional_add : i32457 scf.yield %sum : i32458 }459 460 // CHECK-LABEL: hoist_invariant_scf_if_success461 // CHECK-NEXT: arith.constant 0 : i32462 // CHECK-NEXT: %[[CST:.*]] = arith.constant 42 : i32463 // CHECK-NEXT: %[[TRUE:.*]] = arith.constant true464 // CHECK-NEXT: %[[IF:.*]] = scf.if %[[TRUE]]465 // CHECK-NEXT: arith.addi %[[CST]], %[[CST]] : i32466 // CHECK: scf.for467 // CHECK-NOT: scf.if468 // CHECK-NEXT: arith.addi %{{.*}}, %[[IF]]469 470 return %sum_result : i32471}472 473// -----474 475func.func @hoist_variant_scf_if_failure(%lb: index, %ub: index, %step: index) -> i32 {476 %cst_0 = arith.constant 0 : i32477 %cst_42 = arith.constant 42 : i32478 %ind_7 = arith.constant 7 : index479 %sum_result = scf.for %i = %lb to %ub step %step iter_args(%acc = %cst_0) -> i32 {480 %cond = arith.cmpi ult, %i, %ind_7 : index481 %conditional_add = scf.if %cond -> (i32) {482 %add = arith.addi %cst_42, %cst_42 : i32483 scf.yield %add : i32484 } else {485 %poison = ub.poison : i32486 scf.yield %poison : i32487 }488 %sum = arith.addi %acc, %conditional_add : i32489 scf.yield %sum : i32490 }491 492 // CHECK-LABEL: hoist_variant_scf_if_failure493 // CHECK-NEXT: arith.constant 0 : i32494 // CHECK-NEXT: %[[CST_42:.*]] = arith.constant 42 : i32495 // CHECK-NEXT: %[[CST_7:.*]] = arith.constant 7 : index496 // CHECK-NEXT: scf.for %[[IV:.*]] = %{{.*}} to %{{.*}}497 // CHECK-NEXT: %[[CMP:.*]] = arith.cmpi ult, %[[IV]], %[[CST_7]]498 // CHECK-NEXT: %[[IF:.*]] = scf.if %[[CMP]]499 // CHECK-NEXT: arith.addi %[[CST_42]], %[[CST_42]] : i32500 // CHECK: arith.addi %{{.*}}, %[[IF]]501 502 return %sum_result : i32503}504 505// -----506 507func.func private @make_val() -> (index)508 509// CHECK-LABEL: func @nested_uses_inside510func.func @nested_uses_inside(%lb: index, %ub: index, %step: index) {511 %true = arith.constant true512 513 // Check that ops that contain nested uses to values not defiend outside514 // remain in the loop.515 // CHECK-NEXT: arith.constant516 // CHECK-NEXT: scf.for517 // CHECK-NEXT: call @518 // CHECK-NEXT: call @519 // CHECK-NEXT: scf.if520 // CHECK-NEXT: scf.yield521 // CHECK-NEXT: else522 // CHECK-NEXT: scf.yield523 scf.for %i = %lb to %ub step %step {524 %val = func.call @make_val() : () -> (index)525 %val2 = func.call @make_val() : () -> (index)526 %r = scf.if %true -> (index) {527 scf.yield %val: index528 } else {529 scf.yield %val2: index530 }531 }532 return533}534 535// -----536 537// Test that two ops that feed into each other are moved without violating538// dominance in non-graph regions.539// CHECK-LABEL: func @invariant_subgraph540// CHECK-SAME: %{{.*}}: index, %{{.*}}: index, %{{.*}}: index, %[[ARG:.*]]: i32541func.func @invariant_subgraph(%lb: index, %ub: index, %step: index, %arg: i32) {542 // CHECK: %[[V0:.*]] = arith.addi %[[ARG]], %[[ARG]]543 // CHECK-NEXT: %[[V1:.*]] = arith.addi %[[ARG]], %[[V0]]544 // CHECK-NEXT: scf.for545 scf.for %i = %lb to %ub step %step {546 // CHECK-NEXT: "test.sink"(%[[V1]])547 %v0 = arith.addi %arg, %arg : i32548 %v1 = arith.addi %arg, %v0 : i32549 "test.sink"(%v1) : (i32) -> ()550 }551 return552}553 554// -----555 556// Test invariant nested loop is hoisted.557// CHECK-LABEL: func @test_invariant_nested_loop558func.func @test_invariant_nested_loop() {559 // CHECK: %[[C:.*]] = arith.constant560 %0 = arith.constant 5 : i32561 // CHECK: %[[V0:.*]] = arith.addi %[[C]], %[[C]]562 // CHECK-NEXT: %[[V1:.*]] = arith.addi %[[V0]], %[[C]]563 // CHECK-NEXT: test.graph_loop564 // CHECK-NEXT: ^bb0(%[[ARG0:.*]]: i32)565 // CHECK-NEXT: %[[V2:.*]] = arith.subi %[[ARG0]], %[[ARG0]]566 // CHECK-NEXT: test.region_yield %[[V2]]567 // CHECK: test.graph_loop568 // CHECK-NEXT: test.region_yield %[[V1]]569 test.graph_loop {570 %1 = arith.addi %0, %0 : i32571 %2 = arith.addi %1, %0 : i32572 test.graph_loop {573 ^bb0(%arg0: i32):574 %3 = arith.subi %arg0, %arg0 : i32575 test.region_yield %3 : i32576 } : () -> ()577 test.region_yield %2 : i32578 } : () -> ()579 return580}581 582 583// -----584 585// Test ops in a graph region are hoisted.586// CHECK-LABEL: func @test_invariants_in_graph_region587func.func @test_invariants_in_graph_region() {588 // CHECK: test.single_no_terminator_op589 test.single_no_terminator_op : {590 // CHECK-NEXT: %[[C:.*]] = arith.constant591 // CHECK-NEXT: %[[V1:.*]] = arith.addi %[[C]], %[[C]]592 // CHECK-NEXT: %[[V0:.*]] = arith.addi %[[C]], %[[V1]]593 test.graph_loop {594 %v0 = arith.addi %c0, %v1 : i32595 %v1 = arith.addi %c0, %c0 : i32596 %c0 = arith.constant 5 : i32597 test.region_yield %v0 : i32598 } : () -> ()599 }600 return601}602 603// -----604 605// Test ops in a graph region are hoisted in topological order into non-graph606// regions and that dominance is preserved.607// CHECK-LABEL: func @test_invariant_backedge608func.func @test_invariant_backedge() {609 // CHECK-NEXT: %[[C:.*]] = arith.constant610 // CHECK-NEXT: %[[V1:.*]] = arith.addi %[[C]], %[[C]]611 // CHECK-NEXT: %[[V0:.*]] = arith.addi %[[C]], %[[V1]]612 // CHECK-NEXT: test.graph_loop613 test.graph_loop {614 // CHECK-NEXT: test.region_yield %[[V0]]615 %v0 = arith.addi %c0, %v1 : i32616 %v1 = arith.addi %c0, %c0 : i32617 %c0 = arith.constant 5 : i32618 test.region_yield %v0 : i32619 } : () -> ()620 return621}622 623// -----624 625// Test that cycles aren't hoisted from graph regions to non-graph regions.626// CHECK-LABEL: func @test_invariant_cycle_not_hoisted627func.func @test_invariant_cycle_not_hoisted() {628 // CHECK: test.graph_loop629 test.graph_loop {630 // CHECK-NEXT: %[[A:.*]] = "test.a"(%[[B:.*]]) :631 // CHECK-NEXT: %[[B]] = "test.b"(%[[A]]) :632 // CHECK-NEXT: test.region_yield %[[A]]633 %a = "test.a"(%b) : (i32) -> i32634 %b = "test.b"(%a) : (i32) -> i32635 test.region_yield %a : i32636 } : () -> ()637 return638}639 640// -----641 642// CHECK-LABEL: test_always_speculatable_op643func.func @test_always_speculatable_op(%lb: index, %ub: index, %step: index) {644 // CHECK: test.always_speculatable_op645 // CHECK-NEXT: scf.for646 scf.for %i = %lb to %ub step %step {647 %val = "test.always_speculatable_op"() : () -> i32648 }649 650 return651}652 653// CHECK-LABEL: test_never_speculatable_op654func.func @test_never_speculatable_op(%lb: index, %ub: index, %step: index) {655 // CHECK: scf.for656 // CHECK-NEXT: test.never_speculatable_op657 scf.for %i = %lb to %ub step %step {658 %val = "test.never_speculatable_op"() : () -> i32659 }660 661 return662}663 664// CHECK-LABEL: test_conditionally_speculatable_op_success665func.func @test_conditionally_speculatable_op_success(%lb: index, %ub: index, %step: index) {666 // CHECK: test.conditionally_speculatable_op667 // CHECK-NEXT: scf.for668 scf.for %i = %lb to %ub step %step {669 %const_val = arith.constant 5 : i32670 %val = "test.conditionally_speculatable_op"(%const_val) : (i32) -> i32671 }672 673 return674}675 676// CHECK-LABEL: test_conditionally_speculatable_op_failure677func.func @test_conditionally_speculatable_op_failure(%lb: index, %ub: index, %step: index, %arg: i32) {678 // CHECK: scf.for679 // CHECK-NEXT: test.conditionally_speculatable_op680 %const_5 = arith.constant 5 : i32681 %non_const = arith.addi %arg, %const_5 : i32682 scf.for %i = %lb to %ub step %step {683 %val = "test.conditionally_speculatable_op"(%non_const) : (i32) -> i32684 }685 686 return687}688 689// CHECK-LABEL: test_recursively_speculatable_op_success690func.func @test_recursively_speculatable_op_success(%lb: index, %ub: index, %step: index, %arg: i32) {691 // CHECK: test.recursively_speculatable_op692 // CHECK: scf.for693 scf.for %i = %lb to %ub step %step {694 %val = "test.recursively_speculatable_op"()({695 %result = arith.addi %arg, %arg : i32696 test.region_yield %result : i32697 }) : () -> i32698 }699 700 return701}702 703// CHECK-LABEL: test_recursively_speculatable_op_failure704func.func @test_recursively_speculatable_op_failure(%lb: index, %ub: index, %step: index, %arg: i32) {705 // CHECK: scf.for706 // CHECK-NEXT: test.recursively_speculatable_op707 scf.for %i = %lb to %ub step %step {708 %val = "test.recursively_speculatable_op"()({709 %result = "test.never_speculatable_op"() : () -> i32710 test.region_yield %result : i32711 }) : () -> i32712 }713 714 return715}716 717// -----718 719func.func @speculate_tensor_dim_unknown_rank_unknown_dim(720// CHECK-LABEL: @speculate_tensor_dim_unknown_rank_unknown_dim721 %t: tensor<*xf32>, %dim_idx: index, %lb: index, %ub: index, %step: index) {722 // CHECK: scf.for723 // CHECK-NEXT: tensor.dim724 scf.for %i = %lb to %ub step %step {725 %val = tensor.dim %t, %dim_idx : tensor<*xf32>726 }727 728 return729}730 731func.func @speculate_tensor_dim_known_rank_unknown_dim(732// CHECK-LABEL: @speculate_tensor_dim_known_rank_unknown_dim733 %t: tensor<?x?x?x?xf32>, %dim_idx: index, %lb: index, %ub: index, %step: index) {734 // CHECK: scf.for735 // CHECK-NEXT: tensor.dim736 scf.for %i = %lb to %ub step %step {737 %val = tensor.dim %t, %dim_idx : tensor<?x?x?x?xf32>738 }739 740 return741}742 743func.func @speculate_tensor_dim_unknown_rank_known_dim(744// CHECK-LABEL: @speculate_tensor_dim_unknown_rank_known_dim745 %t: tensor<*xf32>, %dim_idx: index, %lb: index, %ub: index, %step: index) {746 %c0 = arith.constant 0 : index747 // CHECK: scf.for748 // CHECK-NEXT: tensor.dim749 scf.for %i = %lb to %ub step %step {750 %val = tensor.dim %t, %c0 : tensor<*xf32>751 }752 753 return754}755 756func.func @speculate_tensor_dim_known_rank_known_dim_inbounds(757// CHECK-LABEL: @speculate_tensor_dim_known_rank_known_dim_inbounds758 %t: tensor<?x?x?x?xf32>, %dim_idx: index, %lb: index, %ub: index, %step: index) {759 %c1 = arith.constant 1 : index760 // CHECK: tensor.dim761 // CHECK-NEXT: scf.for762 scf.for %i = %lb to %ub step %step {763 %val = tensor.dim %t, %c1 : tensor<?x?x?x?xf32>764 }765 766 return767}768 769// -----770 771func.func @speculate_memref_dim_unknown_rank_unknown_dim(772// CHECK-LABEL: @speculate_memref_dim_unknown_rank_unknown_dim773 %t: memref<*xf32>, %dim_idx: index, %lb: index, %ub: index, %step: index) {774 // CHECK: scf.for775 // CHECK-NEXT: memref.dim776 scf.for %i = %lb to %ub step %step {777 %val = memref.dim %t, %dim_idx : memref<*xf32>778 }779 780 return781}782 783func.func @speculate_memref_dim_known_rank_unknown_dim(784// CHECK-LABEL: @speculate_memref_dim_known_rank_unknown_dim785 %t: memref<?x?x?x?xf32>, %dim_idx: index, %lb: index, %ub: index, %step: index) {786 // CHECK: scf.for787 // CHECK-NEXT: memref.dim788 scf.for %i = %lb to %ub step %step {789 %val = memref.dim %t, %dim_idx : memref<?x?x?x?xf32>790 }791 792 return793}794 795func.func @speculate_memref_dim_unknown_rank_known_dim(796// CHECK-LABEL: @speculate_memref_dim_unknown_rank_known_dim797 %t: memref<*xf32>, %dim_idx: index, %lb: index, %ub: index, %step: index) {798 %c0 = arith.constant 0 : index799 // CHECK: scf.for800 // CHECK-NEXT: memref.dim801 scf.for %i = %lb to %ub step %step {802 %val = memref.dim %t, %c0 : memref<*xf32>803 }804 805 return806}807 808func.func @speculate_memref_dim_known_rank_known_dim_inbounds(809// CHECK-LABEL: @speculate_memref_dim_known_rank_known_dim_inbounds810 %t: memref<?x?x?x?xf32>, %dim_idx: index, %lb: index, %ub: index, %step: index) {811 %c1 = arith.constant 1 : index812 // CHECK: memref.dim813 // CHECK-NEXT: scf.for814 scf.for %i = %lb to %ub step %step {815 %val = memref.dim %t, %c1 : memref<?x?x?x?xf32>816 }817 818 return819}820 821// -----822 823// CHECK-LABEL: @speculate_memref_dim_known_rank_known_dim_inbounds824func.func @speculate_memref_dim_known_rank_known_dim_inbounds() {825 %c0 = arith.constant 0 : index826 %c1 = arith.constant 1 : index827 %c22 = arith.constant 22 : index828 %alloc = memref.alloc(%c22) : memref<?xi1>829 scf.for %arg4 = %c0 to %c22 step %c1 {830 %dim = memref.dim %alloc, %c0 : memref<?xi1>831 }832 return833}834// CHECK: memref.dim835// CHECK-NEXT: scf.for836 837// -----838 839// CHECK-LABEL: @speculate_tensor_dim_known_rank_known_dim_inbounds840func.func @speculate_tensor_dim_known_rank_known_dim_inbounds() {841 %c0 = arith.constant 0 : index842 %c1 = arith.constant 1 : index843 %c22 = arith.constant 22 : index844 %t = tensor.empty(%c22, %c22) : tensor<?x?xi1>845 scf.for %arg4 = %c0 to %c22 step %c1 {846 %dim = tensor.dim %t, %c1 : tensor<?x?xi1>847 }848 return849}850// CHECK: tensor.dim851// CHECK-NEXT: scf.for852 853// -----854 855// CHECK-LABEL: @no_speculate_memref_dim_known_rank_known_dim_out_of_bounds856func.func @no_speculate_memref_dim_known_rank_known_dim_out_of_bounds() {857 %c0 = arith.constant 0 : index858 %c1 = arith.constant 1 : index859 %c22 = arith.constant 22 : index860 %alloc = memref.alloc(%c22) : memref<?xi1>861 scf.for %arg4 = %c0 to %c22 step %c1 {862 %dim = memref.dim %alloc, %c1 : memref<?xi1>863 }864 return865}866// CHECK: scf.for867// CHECK-NEXT: memref.dim868 869// -----870 871func.func @no_speculate_divui(872// CHECK-LABEL: @no_speculate_divui(873 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {874 scf.for %i = %lb to %ub step %step {875// CHECK: scf.for876// CHECK: arith.divui877 %val = arith.divui %num, %denom : i32878 }879 880 return881}882 883func.func @no_speculate_udiv(884// CHECK-LABEL: @no_speculate_udiv(885 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {886 scf.for %i = %lb to %ub step %step {887// CHECK: scf.for888// CHECK: llvm.udiv889 %val = llvm.udiv %num, %denom : i32890 }891 892 return893}894 895func.func @no_speculate_divsi(896// CHECK-LABEL: @no_speculate_divsi(897 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {898 scf.for %i = %lb to %ub step %step {899// CHECK: scf.for900// CHECK: arith.divsi901 %val = arith.divsi %num, %denom : i32902 }903 904 return905}906 907func.func @no_speculate_sdiv(908// CHECK-LABEL: @no_speculate_sdiv(909 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {910 scf.for %i = %lb to %ub step %step {911// CHECK: scf.for912// CHECK: llvm.sdiv913 %val = llvm.sdiv %num, %denom : i32914 }915 916 return917}918 919func.func @no_speculate_ceildivui(920// CHECK-LABEL: @no_speculate_ceildivui(921 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {922 scf.for %i = %lb to %ub step %step {923// CHECK: scf.for924// CHECK: arith.ceildivui925 %val = arith.ceildivui %num, %denom : i32926 }927 928 return929}930 931func.func @no_speculate_ceildivsi(932// CHECK-LABEL: @no_speculate_ceildivsi(933 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {934 scf.for %i = %lb to %ub step %step {935// CHECK: scf.for936// CHECK: arith.ceildivsi937 %val = arith.ceildivsi %num, %denom : i32938 }939 940 return941}942 943func.func @no_speculate_divui_const(%num: i32, %lb: index, %ub: index, %step: index) {944// CHECK-LABEL: @no_speculate_divui_const(945 %c0 = arith.constant 0 : i32946 scf.for %i = %lb to %ub step %step {947// CHECK: scf.for948// CHECK: arith.divui949 %val = arith.divui %num, %c0 : i32950 }951 952 return953}954 955func.func @no_speculate_udiv_const(%num: i32, %lb: index, %ub: index, %step: index) {956// CHECK-LABEL: @no_speculate_udiv_const(957 %c0 = arith.constant 0 : i32958 scf.for %i = %lb to %ub step %step {959// CHECK: scf.for960// CHECK: llvm.udiv961 %val = llvm.udiv %num, %c0 : i32962 }963 964 return965}966 967func.func @speculate_divui_const(968// CHECK-LABEL: @speculate_divui_const(969 %num: i32, %lb: index, %ub: index, %step: index) {970 %c5 = arith.constant 5 : i32971// CHECK: arith.divui972// CHECK: scf.for973 scf.for %i = %lb to %ub step %step {974 %val = arith.divui %num, %c5 : i32975 }976 977 return978}979 980func.func @speculate_udiv_const(981// CHECK-LABEL: @speculate_udiv_const(982 %num: i32, %lb: index, %ub: index, %step: index) {983 %c5 = llvm.mlir.constant(5 : i32) : i32984// CHECK: llvm.udiv985// CHECK: scf.for986 scf.for %i = %lb to %ub step %step {987 %val = llvm.udiv %num, %c5 : i32988 }989 990 return991}992 993func.func @no_speculate_ceildivui_const(%num: i32, %lb: index, %ub: index, %step: index) {994// CHECK-LABEL: @no_speculate_ceildivui_const(995 %c0 = arith.constant 0 : i32996 scf.for %i = %lb to %ub step %step {997// CHECK: scf.for998// CHECK: arith.ceildivui999 %val = arith.ceildivui %num, %c0 : i321000 }1001 1002 return1003}1004 1005func.func @speculate_ceildivui_const(1006// CHECK-LABEL: @speculate_ceildivui_const(1007 %num: i32, %lb: index, %ub: index, %step: index) {1008 %c5 = arith.constant 5 : i321009// CHECK: arith.ceildivui1010// CHECK: scf.for1011 scf.for %i = %lb to %ub step %step {1012 %val = arith.ceildivui %num, %c5 : i321013 }1014 1015 return1016}1017 1018func.func @no_speculate_divsi_const0(1019// CHECK-LABEL: @no_speculate_divsi_const0(1020 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {1021 %c0 = arith.constant 0 : i321022 scf.for %i = %lb to %ub step %step {1023// CHECK: scf.for1024// CHECK: arith.divsi1025 %val = arith.divsi %num, %c0 : i321026 }1027 1028 return1029}1030 1031func.func @no_speculate_sdiv_const0(1032// CHECK-LABEL: @no_speculate_sdiv_const0(1033 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {1034 %c0 = arith.constant 0 : i321035 scf.for %i = %lb to %ub step %step {1036// CHECK: scf.for1037// CHECK: llvm.sdiv1038 %val = llvm.sdiv %num, %c0 : i321039 }1040 1041 return1042}1043 1044func.func @no_speculate_divsi_const_minus1(1045// CHECK-LABEL: @no_speculate_divsi_const_minus1(1046 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {1047 %cm1 = arith.constant -1 : i321048 scf.for %i = %lb to %ub step %step {1049// CHECK: scf.for1050// CHECK: arith.divsi1051 %val = arith.divsi %num, %cm1 : i321052 }1053 1054 return1055}1056 1057func.func @no_speculate_sdiv_const_minus1(1058// CHECK-LABEL: @no_speculate_sdiv_const_minus1(1059 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {1060 %cm1 = arith.constant -1 : i321061 scf.for %i = %lb to %ub step %step {1062// CHECK: scf.for1063// CHECK: llvm.sdiv1064 %val = llvm.sdiv %num, %cm1 : i321065 }1066 1067 return1068}1069 1070func.func @speculate_divsi_const(1071// CHECK-LABEL: @speculate_divsi_const(1072 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {1073 %c5 = arith.constant 5 : i321074 scf.for %i = %lb to %ub step %step {1075// CHECK: arith.divsi1076// CHECK: scf.for1077 %val = arith.divsi %num, %c5 : i321078 }1079 1080 return1081}1082 1083func.func @speculate_sdiv_const(1084// CHECK-LABEL: @speculate_sdiv_const(1085 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {1086 %c5 = arith.constant 5 : i321087 scf.for %i = %lb to %ub step %step {1088// CHECK: llvm.sdiv1089// CHECK: scf.for1090 %val = llvm.sdiv %num, %c5 : i321091 }1092 1093 return1094}1095 1096func.func @no_speculate_ceildivsi_const0(1097// CHECK-LABEL: @no_speculate_ceildivsi_const0(1098 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {1099 %c0 = arith.constant 0 : i321100 scf.for %i = %lb to %ub step %step {1101// CHECK: scf.for1102// CHECK: arith.ceildivsi1103 %val = arith.ceildivsi %num, %c0 : i321104 }1105 1106 return1107}1108 1109func.func @no_speculate_ceildivsi_const_minus1(1110// CHECK-LABEL: @no_speculate_ceildivsi_const_minus1(1111 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {1112 %cm1 = arith.constant -1 : i321113 scf.for %i = %lb to %ub step %step {1114// CHECK: scf.for1115// CHECK: arith.ceildivsi1116 %val = arith.ceildivsi %num, %cm1 : i321117 }1118 1119 return1120}1121 1122func.func @speculate_ceildivsi_const(1123// CHECK-LABEL: @speculate_ceildivsi_const(1124 %num: i32, %denom: i32, %lb: index, %ub: index, %step: index) {1125 %c5 = arith.constant 5 : i321126 scf.for %i = %lb to %ub step %step {1127// CHECK: arith.ceildivsi1128// CHECK: scf.for1129 %val = arith.ceildivsi %num, %c5 : i321130 }1131 1132 return1133}1134 1135func.func @no_speculate_divui_range(1136// CHECK-LABEL: @no_speculate_divui_range(1137 %num: i8, %lb: index, %ub: index, %step: index) {1138 %denom = test.with_bounds {smax = 127 : i8, smin = -128 : i8, umax = 255 : i8, umin = 0 : i8} : i81139 scf.for %i = %lb to %ub step %step {1140// CHECK: scf.for1141// CHECK: arith.divui1142 %val = arith.divui %num, %denom : i81143 }1144 1145 return1146}1147 1148func.func @no_speculate_udiv_range(1149// CHECK-LABEL: @no_speculate_udiv_range(1150 %num: i8, %lb: index, %ub: index, %step: index) {1151 %denom = test.with_bounds {smax = 127 : i8, smin = -128 : i8, umax = 255 : i8, umin = 0 : i8} : i81152 scf.for %i = %lb to %ub step %step {1153// CHECK: scf.for1154// CHECK: llvm.udiv1155 %val = llvm.udiv %num, %denom : i81156 }1157 1158 return1159}1160 1161func.func @no_speculate_divsi_range(1162// CHECK-LABEL: @no_speculate_divsi_range(1163 %num: i8, %lb: index, %ub: index, %step: index) {1164 %denom0 = test.with_bounds {smax = -1: i8, smin = -128 : i8, umax = 255 : i8, umin = 0 : i8} : i81165 %denom1 = test.with_bounds {smax = 127 : i8, smin = 0 : i8, umax = 255 : i8, umin = 0 : i8} : i81166 scf.for %i = %lb to %ub step %step {1167// CHECK: scf.for1168// CHECK-COUNT-2: arith.divsi1169 %val0 = arith.divsi %num, %denom0 : i81170 %val1 = arith.divsi %num, %denom1 : i81171 }1172 1173 return1174}1175 1176func.func @no_speculate_sdiv_range(1177// CHECK-LABEL: @no_speculate_sdiv_range(1178 %num: i8, %lb: index, %ub: index, %step: index) {1179 %denom0 = test.with_bounds {smax = -1: i8, smin = -128 : i8, umax = 255 : i8, umin = 0 : i8} : i81180 %denom1 = test.with_bounds {smax = 127 : i8, smin = 0 : i8, umax = 255 : i8, umin = 0 : i8} : i81181 scf.for %i = %lb to %ub step %step {1182// CHECK: scf.for1183// CHECK-COUNT-2: llvm.sdiv1184 %val0 = llvm.sdiv %num, %denom0 : i81185 %val1 = llvm.sdiv %num, %denom1 : i81186 }1187 1188 return1189}1190 1191func.func @no_speculate_ceildivui_range(1192// CHECK-LABEL: @no_speculate_ceildivui_range(1193 %num: i8, %lb: index, %ub: index, %step: index) {1194 %denom = test.with_bounds {smax = 127 : i8, smin = -128 : i8, umax = 255 : i8, umin = 0 : i8} : i81195 scf.for %i = %lb to %ub step %step {1196// CHECK: scf.for1197// CHECK: arith.ceildivui1198 %val = arith.ceildivui %num, %denom : i81199 }1200 1201 return1202}1203 1204func.func @no_speculate_ceildivsi_range(1205// CHECK-LABEL: @no_speculate_ceildivsi_range(1206 %num: i8, %lb: index, %ub: index, %step: index) {1207 %denom0 = test.with_bounds {smax = -1 : i8, smin = -128 : i8, umax = 255 : i8, umin = 0 : i8} : i81208 %denom1 = test.with_bounds {smax = 127 : i8, smin = 0 : i8, umax = 255 : i8, umin = 0 : i8} : i81209 scf.for %i = %lb to %ub step %step {1210// CHECK: scf.for1211// CHECK-COUNT-2: arith.ceildivsi1212 %val0 = arith.ceildivsi %num, %denom0 : i81213 %val1 = arith.ceildivsi %num, %denom1 : i81214 }1215 1216 return1217}1218 1219func.func @speculate_divui_range(1220// CHECK-LABEL: @speculate_divui_range(1221 %num: i8, %lb: index, %ub: index, %step: index) {1222 %denom = test.with_bounds {smax = 127 : i8, smin = -128 : i8, umax = 255 : i8, umin = 1 : i8} : i81223 scf.for %i = %lb to %ub step %step {1224// CHECK: arith.divui1225// CHECK: scf.for1226 %val = arith.divui %num, %denom : i81227 }1228 1229 return1230}1231 1232func.func @speculate_udiv_range(1233// CHECK-LABEL: @speculate_udiv_range(1234 %num: i8, %lb: index, %ub: index, %step: index) {1235 %denom = test.with_bounds {smax = 127 : i8, smin = -128 : i8, umax = 255 : i8, umin = 1 : i8} : i81236 scf.for %i = %lb to %ub step %step {1237// CHECK: llvm.udiv1238// CHECK: scf.for1239 %val = llvm.udiv %num, %denom : i81240 }1241 1242 return1243}1244 1245func.func @speculate_divsi_range(1246// CHECK-LABEL: @speculate_divsi_range(1247 %num: i8, %lb: index, %ub: index, %step: index) {1248 %denom0 = test.with_bounds {smax = 127 : i8, smin = 1 : i8, umax = 255 : i8, umin = 0 : i8} : i81249 %denom1 = test.with_bounds {smax = -2 : i8, smin = -128 : i8, umax = 255 : i8, umin = 0 : i8} : i81250 scf.for %i = %lb to %ub step %step {1251// CHECK-COUNT-2: arith.divsi1252// CHECK: scf.for1253 %val0 = arith.divsi %num, %denom0 : i81254 %val1 = arith.divsi %num, %denom1 : i81255 1256 }1257 1258 return1259}1260 1261func.func @speculate_sdiv_range(1262// CHECK-LABEL: @speculate_sdiv_range(1263 %num: i8, %lb: index, %ub: index, %step: index) {1264 %denom0 = test.with_bounds {smax = 127 : i8, smin = 1 : i8, umax = 255 : i8, umin = 0 : i8} : i81265 %denom1 = test.with_bounds {smax = -2 : i8, smin = -128 : i8, umax = 255 : i8, umin = 0 : i8} : i81266 scf.for %i = %lb to %ub step %step {1267// CHECK-COUNT-2: llvm.sdiv1268// CHECK: scf.for1269 %val0 = llvm.sdiv %num, %denom0 : i81270 %val1 = llvm.sdiv %num, %denom1 : i81271 1272 }1273 1274 return1275}1276 1277func.func @speculate_ceildivui_range(1278// CHECK-LABEL: @speculate_ceildivui_range(1279 %num: i8, %lb: index, %ub: index, %step: index) {1280 %denom = test.with_bounds {smax = 127 : i8, smin = -128 : i8, umax = 255 : i8, umin = 1 : i8} : i81281 scf.for %i = %lb to %ub step %step {1282// CHECK: arith.ceildivui1283// CHECK: scf.for1284 %val = arith.ceildivui %num, %denom : i81285 }1286 1287 return1288}1289 1290func.func @speculate_ceildivsi_range(1291// CHECK-LABEL: @speculate_ceildivsi_range(1292 %num: i8, %lb: index, %ub: index, %step: index) {1293 %denom0 = test.with_bounds {smax = 127 : i8, smin = 1 : i8, umax = 255 : i8, umin = 0 : i8} : i81294 %denom1 = test.with_bounds {smax = -2 : i8, smin = -128 : i8, umax = 255 : i8, umin = 0 : i8} : i81295 scf.for %i = %lb to %ub step %step {1296// CHECK-COUNT-2: arith.ceildivsi1297// CHECK: scf.for1298 %val0 = arith.ceildivsi %num, %denom0 : i81299 %val1 = arith.ceildivsi %num, %denom1 : i81300 1301 }1302 1303 return1304}1305 1306// -----1307 1308func.func @speculate_static_pack_and_unpack(%source: tensor<128x256xf32>,1309 %dest: tensor<4x16x32x16xf32>, %lb: index, %ub: index, %step: index) {1310 1311 // CHECK: linalg.pack1312 // CHECK-NEXT: scf.for1313 scf.for %i = %lb to %ub step %step {1314 %packed = linalg.pack %source1315 inner_dims_pos = [0, 1]1316 inner_tiles = [32, 16] into %dest : tensor<128x256xf32> -> tensor<4x16x32x16xf32>1317 }1318 1319 // CHECK: linalg.unpack1320 // CHECK-NEXT: scf.for1321 scf.for %i = %lb to %ub step %step {1322 %unpacked = linalg.unpack %dest1323 inner_dims_pos = [0, 1]1324 inner_tiles = [32, 16] into %source : tensor<4x16x32x16xf32> -> tensor<128x256xf32>1325 }1326 return1327}1328 1329// -----1330 1331func.func @speculate_dynamic_pack_and_unpack(%source: tensor<?x?xf32>,1332 %dest: tensor<?x?x?x?xf32>, %lb: index, %ub: index, %step: index,1333 %tile_m: index, %tile_n: index, %pad: f32) {1334 1335 // CHECK: scf.for1336 // CHECK-NEXT: linalg.pack1337 scf.for %i = %lb to %ub step %step {1338 %packed = linalg.pack %source1339 inner_dims_pos = [0, 1]1340 inner_tiles = [%tile_n, %tile_m] into %dest : tensor<?x?xf32> -> tensor<?x?x?x?xf32>1341 }1342 1343 // CHECK: scf.for1344 // CHECK-NEXT: linalg.unpack1345 scf.for %i = %lb to %ub step %step {1346 %unpacked = linalg.unpack %dest1347 inner_dims_pos = [0, 1]1348 inner_tiles = [%tile_n, %tile_m] into %source : tensor<?x?x?x?xf32> -> tensor<?x?xf32>1349 }1350 1351 // CHECK: linalg.pack1352 // CHECK-NEXT: scf.for1353 scf.for %i = %lb to %ub step %step {1354 %packed = linalg.pack %source padding_value(%pad : f32)1355 inner_dims_pos = [0, 1]1356 inner_tiles = [%tile_n, %tile_m] into %dest : tensor<?x?xf32> -> tensor<?x?x?x?xf32>1357 }1358 return1359}1360 1361// -----1362 1363// CHECK-LABEL: func @hoist_from_scf_while(1364// CHECK-SAME: %[[arg0:.*]]: i32, %{{.*}}: i32)1365// CHECK-DAG: arith.constant 1 : i321366// CHECK-DAG: %[[c2:.*]] = arith.constant 2 : i321367// CHECK-DAG: %[[c10:.*]] = arith.constant 10 : i321368// CHECK-DAG: %[[added:.*]] = arith.addi %[[arg0]], %[[c2]]1369// CHECK: scf.while1370// CHECK: %[[cmpi:.*]] = arith.cmpi slt, %{{.*}}, %[[added]]1371// CHECK: scf.condition(%[[cmpi]])1372func.func @hoist_from_scf_while(%arg0: i32, %arg1: i32) -> i32 {1373 %0 = scf.while (%arg2 = %arg1) : (i32) -> (i32) {1374 %c2 = arith.constant 2 : i321375 %c10 = arith.constant 10 : i321376 %added = arith.addi %arg0, %c2 : i321377 %1 = arith.cmpi slt, %arg2, %added : i321378 scf.condition(%1) %arg2 : i321379 } do {1380 ^bb0(%arg2: i32):1381 %c1 = arith.constant 1 : i321382 %added2 = arith.addi %c1, %arg2 : i321383 scf.yield %added2 : i321384 }1385 return %0 : i321386}1387 1388// -----1389 1390#trait = {1391 indexing_maps = [1392 affine_map<(m, n, k) -> (m, k)>,1393 affine_map<(m, n, k) -> (k, n)>,1394 affine_map<(m, n, k) -> (m, n)>1395 ],1396 iterator_types = ["parallel", "parallel", "reduction"] 1397}1398 1399// CHECK-LABEL: func @hoist_linalg_ops1400// CHECK: linalg.generic1401// CHECK: scf.for1402// CHECK-NOT: linalg.generic1403// CHECK: tensor.insert_slice1404// CHECK: scf.yield1405func.func @hoist_linalg_ops(%a : tensor<128x128xf32>, 1406 %b : tensor<128x128xf32>, 1407 %c: tensor<128x128xf32>,1408 %lb : index,1409 %ub : index,1410 %step : index,1411 %output : tensor<?x128xf32>) -> tensor<?x128xf32> {1412 %final = 1413 scf.for %i = %lb to %ub step %step iter_args(%acc = %output) 1414 -> tensor<?x128xf32> {1415 %compute = linalg.generic #trait1416 ins(%a, %b : tensor<128x128xf32>, tensor<128x128xf32>) 1417 outs(%c : tensor<128x128xf32>) {1418 ^bb0(%in : f32, %in2 : f32, %in3 : f32):1419 %mul = arith.mulf %in, %in2 : f321420 %add = arith.addf %mul, %in3 : f321421 linalg.yield %in3 : f321422 } -> tensor<128x128xf32>1423 1424 %newacc = tensor.insert_slice %compute into 1425 %output[%i, 0][128, 128][1, 1] 1426 : tensor<128x128xf32> into tensor<?x128xf32>1427 scf.yield %newacc : tensor<?x128xf32>1428 }1429 1430 func.return %final : tensor<?x128xf32>1431}1432 1433// -----1434 1435#trait = {1436 indexing_maps = [1437 affine_map<(m, n, k) -> (m, k)>,1438 affine_map<(m, n, k) -> (k, n)>,1439 affine_map<(m, n, k) -> (m, n)>1440 ],1441 iterator_types = ["parallel", "parallel", "reduction"] 1442}1443 1444// CHECK-LABEL: func @hoist_linalg_ops_div_by_zero1445// CHECK-NOT: linalg.generic1446// CHECK: scf.for1447// CHECK: linalg.generic1448// CHECK: tensor.insert_slice1449// CHECK: scf.yield1450func.func @hoist_linalg_ops_div_by_zero(%a : tensor<128x128xi32>, 1451 %b : tensor<128x128xi32>, 1452 %c: tensor<128x128xi32>,1453 %lb : index,1454 %ub : index,1455 %step : index,1456 %output : tensor<?x128xi32>) -> tensor<?x128xi32> {1457 %cst0 = arith.constant 0 : i321458 %final = 1459 scf.for %i = %lb to %ub step %step iter_args(%acc = %output) 1460 -> tensor<?x128xi32> {1461 %compute = linalg.generic #trait1462 ins(%a, %b : tensor<128x128xi32>, tensor<128x128xi32>) 1463 outs(%c : tensor<128x128xi32>) {1464 ^bb0(%in : i32, %in2 : i32, %in3 : i32):1465 %div = arith.divui %in, %in2 : i321466 %add = arith.addi %div, %in3 : i321467 linalg.yield %in3 : i321468 } -> tensor<128x128xi32>1469 1470 %newacc = tensor.insert_slice %compute into 1471 %output[%i, 0][128, 128][1, 1] 1472 : tensor<128x128xi32> into tensor<?x128xi32>1473 scf.yield %newacc : tensor<?x128xi32>1474 }1475 1476 func.return %final : tensor<?x128xi32>1477}1478 1479// -----1480 1481// CHECK-LABEL: func @hoist_vector_transfer_ops1482// CHECK: vector.transfer_read1483// CHECK: scf.for1484// CHECK-NOT: vector.transfer_read1485// CHECK: arith.addf1486// CHECK: scf.yield1487func.func @hoist_vector_transfer_ops(1488 %a : tensor<128x128xf32>, 1489 %lb : index,1490 %ub : index,1491 %step : index,1492 %ida : index,1493 %idb : index) -> vector<4x4xf32> {1494 %cst_0 = arith.constant 0.0 : f321495 %cst = arith.constant dense<0.0> : vector<4x4xf32>1496 %final = 1497 scf.for %i = %lb to %ub step %step iter_args(%acc = %cst) -> vector<4x4xf32> {1498 %read = vector.transfer_read %a[%ida, %idb], %cst_0 : tensor<128x128xf32>, vector<4x4xf32>1499 %out = arith.addf %read, %acc : vector<4x4xf32>1500 scf.yield %out : vector<4x4xf32>1501 }1502 func.return %final : vector<4x4xf32>1503}1504 1505// -----1506 1507// CHECK-LABEL: func @hoist_vector_transfer_ops1508// CHECK: vector.transfer_write1509// CHECK: vector.transfer_read1510// CHECK: scf.for1511// CHECK-NOT: vector.transfer_write1512// CHECK-NOT: vector.transfer_read1513// CHECK: arith.addf1514// CHECK: scf.yield1515func.func @hoist_vector_transfer_ops(1516 %lb : index,1517 %ub : index,1518 %step : index,1519 %ida : index,1520 %idb : index) -> vector<4x4xf32> {1521 %c0 = arith.constant 0 : index1522 %cst_0 = arith.constant 0.0 : f321523 %cst = arith.constant dense<0.0> : vector<4x4xf32>1524 %empty = tensor.empty() : tensor<4x4xf32>1525 %final = 1526 scf.for %i = %lb to %ub step %step iter_args(%acc = %cst) -> vector<4x4xf32> {1527 %a = vector.transfer_write %cst, %empty[%c0, %c0] : vector<4x4xf32>, tensor<4x4xf32>1528 %read = vector.transfer_read %a[%c0, %c0], %cst_0 : tensor<4x4xf32>, vector<4x4xf32>1529 %out = arith.addf %read, %acc : vector<4x4xf32>1530 scf.yield %out : vector<4x4xf32>1531 }1532 func.return %final : vector<4x4xf32>1533}1534 1535// -----1536 1537// CHECK-LABEL: func @do_not_hoist_vector_transfer_ops_loop_dep1538// CHECK-NOT: vector.transfer_read1539// CHECK: scf.for1540// CHECK: vector.transfer_read1541// CHECK: arith.addf1542// CHECK: scf.yield1543func.func @do_not_hoist_vector_transfer_ops_loop_dep(1544 %a : tensor<128x128xf32>, 1545 %lb : index,1546 %ub : index,1547 %step : index,1548 %ida : index) -> vector<4x4xf32> {1549 %cst_0 = arith.constant 0.0 : f321550 %cst = arith.constant dense<0.0> : vector<4x4xf32>1551 %final = 1552 scf.for %i = %lb to %ub step %step iter_args(%acc = %cst) -> vector<4x4xf32> {1553 %read = vector.transfer_read %a[%ida, %i], %cst_0 : tensor<128x128xf32>, vector<4x4xf32>1554 %out = arith.addf %read, %acc : vector<4x4xf32>1555 scf.yield %out : vector<4x4xf32>1556 }1557 func.return %final : vector<4x4xf32>1558}1559 1560// -----1561 1562// CHECK-LABEL: func @do_not_hoist_vector_transfer_ops_memref1563// CHECK-NOT: vector.transfer_read1564// CHECK: scf.for1565// CHECK: vector.transfer_read1566// CHECK: arith.addf1567// CHECK: scf.yield1568func.func @do_not_hoist_vector_transfer_ops_memref(1569 %a : memref<128x128xf32>, 1570 %lb : index,1571 %ub : index,1572 %step : index,1573 %ida : index,1574 %idb : index) -> vector<4x4xf32> {1575 %cst_0 = arith.constant 0.0 : f321576 %cst = arith.constant dense<0.0> : vector<4x4xf32>1577 %final = 1578 scf.for %i = %lb to %ub step %step iter_args(%acc = %cst) -> vector<4x4xf32> {1579 %read = vector.transfer_read %a[%ida, %idb], %cst_0 : memref<128x128xf32>, vector<4x4xf32>1580 %out = arith.addf %read, %acc : vector<4x4xf32>1581 scf.yield %out : vector<4x4xf32>1582 }1583 func.return %final : vector<4x4xf32>1584}1585