1040 lines · plain
1// RUN: mlir-opt %s -test-scf-pipelining -split-input-file -verify-diagnostics | FileCheck %s2// RUN: mlir-opt %s -test-scf-pipelining=annotate -split-input-file | FileCheck %s --check-prefix ANNOTATE3// RUN: mlir-opt %s -test-scf-pipelining=no-epilogue-peeling -split-input-file | FileCheck %s --check-prefix NOEPILOGUE4 5// CHECK-LABEL: simple_pipeline(6// CHECK-SAME: %[[A:.*]]: memref<?xf32>, %[[R:.*]]: memref<?xf32>) {7// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index8// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index9// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index10// Prologue:11// CHECK: %[[L0:.*]] = memref.load %[[A]][%[[C0]]] : memref<?xf32>12// Kernel:13// CHECK-NEXT: %[[L1:.*]] = scf.for %[[IV:.*]] = %[[C0]] to %[[C3]]14// CHECK-SAME: step %[[C1]] iter_args(%[[LARG:.*]] = %[[L0]]) -> (f32) {15// CHECK-NEXT: %[[ADD0:.*]] = arith.addf %[[LARG]], %{{.*}} : f3216// CHECK-NEXT: memref.store %[[ADD0]], %[[R]][%[[IV]]] : memref<?xf32>17// CHECK-NEXT: %[[IV1:.*]] = arith.addi %[[IV]], %[[C1]] : index18// CHECK-NEXT: %[[LR:.*]] = memref.load %[[A]][%[[IV1]]] : memref<?xf32>19// CHECK-NEXT: scf.yield %[[LR]] : f3220// CHECK-NEXT: }21// Epilogue:22// CHECK-NEXT: %[[ADD1:.*]] = arith.addf %[[L1]], %{{.*}} : f3223// CHECK-NEXT: memref.store %[[ADD1]], %[[R]][%[[C3]]] : memref<?xf32>24func.func @simple_pipeline(%A: memref<?xf32>, %result: memref<?xf32>) {25 %c0 = arith.constant 0 : index26 %c1 = arith.constant 1 : index27 %c4 = arith.constant 4 : index28 %cf = arith.constant 1.0 : f3229 scf.for %i0 = %c0 to %c4 step %c1 {30 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>31 %A1_elem = arith.addf %A_elem, %cf { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 0 } : f3232 memref.store %A1_elem, %result[%i0] { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : memref<?xf32>33 } { __test_pipelining_loop__ }34 return35}36 37// -----38 39// A static loop does not satisfy `numIteration >= maxStage`40 41// CHECK-LABEL: func.func @iteration_lt_stage(42// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index43// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index44// CHECK-DAG: %[[C_NEG1:.*]] = arith.constant -1 : index45// CHECK-DAG: %[[TRUE:.*]] = arith.constant true46// CHECK-DAG: %[[FALSE:.*]] = arith.constant false47// Prologue:48// CHECK: scf.if %[[TRUE]]49// CHECK: scf.if %[[TRUE]]50// CHECK: scf.if %[[FALSE]]51// Kernel:52// CHECK: scf.for %[[IV:.*]] = %[[C0:.*]] to %[[C_NEG1:.*]] step %[[C1:.*]]53// Epilogue:54// CHECK: scf.if %[[TRUE]]55// CHECK: scf.if %[[TRUE]]56// CHECK: scf.if %[[TRUE]]57// CHECK: scf.if %[[TRUE]]58// CHECK: scf.if %[[FALSE]]59// CHECK: scf.if %[[FALSE]]60func.func @iteration_lt_stage(%A: memref<?xf32>, %result: memref<?xf32>) {61 %c0 = arith.constant 0 : index62 %c1 = arith.constant 1 : index63 %c2 = arith.constant 2 : index64 %cf = arith.constant 1.0 : f3265 scf.for %i0 = %c0 to %c2 step %c1 {66 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>67 %A1_elem = arith.addf %A_elem, %cf { __test_pipelining_stage__ = 3, __test_pipelining_op_order__ = 0 } : f3268 memref.store %A1_elem, %result[%i0] { __test_pipelining_stage__ = 3, __test_pipelining_op_order__ = 1 } : memref<?xf32>69 } { __test_pipelining_loop__ }70 return71}72 73 74// -----75 76// CHECK-LABEL: simple_pipeline_region(77// CHECK-SAME: %[[A:.*]]: memref<?xf32>, %[[R:.*]]: memref<?xf32>) {78// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index79// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index80// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index81// Prologue:82// CHECK: %[[L0:.*]] = scf.execute_region83// CHECK-NEXT: memref.load %[[A]][%[[C0]]] : memref<?xf32>84// Kernel:85// CHECK: %[[L1:.*]] = scf.for %[[IV:.*]] = %[[C0]] to %[[C3]]86// CHECK-SAME: step %[[C1]] iter_args(%[[LARG:.*]] = %[[L0]]) -> (f32) {87// CHECK-NEXT: %[[ADD0:.*]] = scf.execute_region88// CHECK-NEXT: arith.addf %[[LARG]], %{{.*}} : f3289// CHECK: memref.store %[[ADD0]], %[[R]][%[[IV]]] : memref<?xf32>90// CHECK-NEXT: %[[IV1:.*]] = arith.addi %[[IV]], %[[C1]] : index91// CHECK-NEXT: %[[LR:.*]] = scf.execute_region92// CHECK-NEXT: memref.load %[[A]][%[[IV1]]] : memref<?xf32>93// CHECK: scf.yield %[[LR]] : f3294// CHECK-NEXT: }95// Epilogue:96// CHECK-NEXT: %[[ADD1:.*]] = scf.execute_region97// CHECK-NEXT: arith.addf %[[L1]], %{{.*}} : f3298// CHECK: memref.store %[[ADD1]], %[[R]][%[[C3]]] : memref<?xf32>99func.func @simple_pipeline_region(%A: memref<?xf32>, %result: memref<?xf32>) {100 %c0 = arith.constant 0 : index101 %c1 = arith.constant 1 : index102 %c4 = arith.constant 4 : index103 %cf = arith.constant 1.0 : f32104 scf.for %i0 = %c0 to %c4 step %c1 {105 106 %A_elem = scf.execute_region -> f32 {107 %A_elem1 = memref.load %A[%i0] : memref<?xf32>108 scf.yield %A_elem1 : f32109 } { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 }110 111 %A1_elem = scf.execute_region -> f32 {112 %A1_elem1 = arith.addf %A_elem, %cf : f32113 scf.yield %A1_elem1 : f32114 } { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 0 }115 116 memref.store %A1_elem, %result[%i0] { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : memref<?xf32>117 } { __test_pipelining_loop__ }118 return119}120 121// -----122 123// CHECK-LABEL: simple_pipeline_step(124// CHECK-SAME: %[[A:.*]]: memref<?xf32>, %[[R:.*]]: memref<?xf32>) {125// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index126// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index127// CHECK-DAG: %[[C5:.*]] = arith.constant 5 : index128// CHECK-DAG: %[[C6:.*]] = arith.constant 6 : index129// CHECK-DAG: %[[C9:.*]] = arith.constant 9 : index130// Prologue:131// CHECK: %[[L0:.*]] = memref.load %[[A]][%[[C0]]] : memref<?xf32>132// CHECK: %[[L1:.*]] = memref.load %[[A]][%[[C3]]] : memref<?xf32>133// Kernel:134// CHECK-NEXT: %[[L2:.*]]:2 = scf.for %[[IV:.*]] = %[[C0]] to %[[C5]]135// CHECK-SAME: step %[[C3]] iter_args(%[[LARG0:.*]] = %[[L0]], %[[LARG1:.*]] = %[[L1]]) -> (f32, f32) {136// CHECK-NEXT: %[[ADD0:.*]] = arith.addf %[[LARG0]], %{{.*}} : f32137// CHECK-NEXT: memref.store %[[ADD0]], %[[R]][%[[IV]]] : memref<?xf32>138// CHECK-NEXT: %[[IV1:.*]] = arith.addi %[[IV]], %[[C6]] : index139// CHECK-NEXT: %[[LR:.*]] = memref.load %[[A]][%[[IV1]]] : memref<?xf32>140// CHECK-NEXT: scf.yield %[[LARG1]], %[[LR]] : f32, f32141// CHECK-NEXT: }142// Epilogue:143// CHECK-NEXT: %[[ADD1:.*]] = arith.addf %[[L2]]#0, %{{.*}} : f32144// CHECK-NEXT: memref.store %[[ADD1]], %[[R]][%[[C6]]] : memref<?xf32>145// CHECK-NEXT: %[[ADD2:.*]] = arith.addf %[[L2]]#1, %{{.*}} : f32146// CHECK-NEXT: memref.store %[[ADD2]], %[[R]][%[[C9]]] : memref<?xf32>147func.func @simple_pipeline_step(%A: memref<?xf32>, %result: memref<?xf32>) {148 %c0 = arith.constant 0 : index149 %c3 = arith.constant 3 : index150 %c11 = arith.constant 11 : index151 %cf = arith.constant 1.0 : f32152 scf.for %i0 = %c0 to %c11 step %c3 {153 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>154 %A1_elem = arith.addf %A_elem, %cf { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 0 } : f32155 memref.store %A1_elem, %result[%i0] { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 1 } : memref<?xf32>156 } { __test_pipelining_loop__ }157 return158}159 160// -----161 162// CHECK-LABEL: three_stage(163// CHECK-SAME: %[[A:.*]]: memref<?xf32>, %[[R:.*]]: memref<?xf32>) {164// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index165// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index166// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index167// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index168// Prologue:169// CHECK: %[[L0:.*]] = memref.load %[[A]][%[[C0]]] : memref<?xf32>170// CHECK-NEXT: %[[ADD0:.*]] = arith.addf %[[L0]], %{{.*}} : f32171// CHECK-NEXT: %[[L1:.*]] = memref.load %[[A]][%[[C1]]] : memref<?xf32>172// Kernel:173// CHECK-NEXT: %[[LR:.*]]:2 = scf.for %[[IV:.*]] = %[[C0]] to %[[C2]]174// CHECK-SAME: step %[[C1]] iter_args(%[[ADDARG:.*]] = %[[ADD0]],175// CHECK-SAME: %[[LARG:.*]] = %[[L1]]) -> (f32, f32) {176// CHECK-NEXT: memref.store %[[ADDARG]], %[[R]][%[[IV]]] : memref<?xf32>177// CHECK-NEXT: %[[ADD1:.*]] = arith.addf %[[LARG]], %{{.*}} : f32178// CHECK-NEXT: %[[IV2:.*]] = arith.addi %[[IV]], %[[C2]] : index179// CHECK-NEXT: %[[L3:.*]] = memref.load %[[A]][%[[IV2]]] : memref<?xf32>180// CHECK-NEXT: scf.yield %[[ADD1]], %[[L3]] : f32, f32181// CHECK-NEXT: }182// Epilogue:183// CHECK-NEXT: memref.store %[[LR]]#0, %[[R]][%[[C2]]] : memref<?xf32>184// CHECK-NEXT: %[[ADD2:.*]] = arith.addf %[[LR]]#1, %{{.*}} : f32185// CHECK-NEXT: memref.store %[[ADD2]], %[[R]][%[[C3]]] : memref<?xf32>186 187// Prologue:188// ANNOTATE: memref.load {{.*}} {__test_pipelining_iteration = 0 : i32, __test_pipelining_part = "prologue"}189// ANNOTATE: memref.load {{.*}} {__test_pipelining_iteration = 1 : i32, __test_pipelining_part = "prologue"}190// Kernel:191// ANNOTATE: scf.for192// ANNOTATE: memref.store {{.*}} {__test_pipelining_iteration = 0 : i32, __test_pipelining_part = "kernel"}193// ANNOTATE: arith.addf {{.*}} {__test_pipelining_iteration = 0 : i32, __test_pipelining_part = "kernel"}194// ANNOTATE: memref.load {{.*}} {__test_pipelining_iteration = 0 : i32, __test_pipelining_part = "kernel"}195// ANNOTATE: scf.yield196// ANNOTATE: }197// Epilogue:198// ANNOTATE: memref.store {{.*}} {__test_pipelining_iteration = 0 : i32, __test_pipelining_part = "epilogue"}199// ANNOTATE: arith.addf {{.*}} {__test_pipelining_iteration = 0 : i32, __test_pipelining_part = "epilogue"}200// ANNOTATE: memref.store {{.*}} {__test_pipelining_iteration = 1 : i32, __test_pipelining_part = "epilogue"}201 202// NOEPILOGUE-LABEL: three_stage(203// NOEPILOGUE-SAME: %[[A:.*]]: memref<?xf32>, %[[R:.*]]: memref<?xf32>) {204// NOEPILOGUE-DAG: %[[C0:.*]] = arith.constant 0 : index205// NOEPILOGUE-DAG: %[[C1:.*]] = arith.constant 1 : index206// NOEPILOGUE-DAG: %[[C2:.*]] = arith.constant 2 : index207// NOEPILOGUE-DAG: %[[C3:.*]] = arith.constant 3 : index208// NOEPILOGUE-DAG: %[[C4:.*]] = arith.constant 4 : index209// NOEPILOGUE-DAG: %[[CF:.*]] = arith.constant 0.000000e+00 : f32210// Prologue:211// NOEPILOGUE: %[[L0:.*]] = memref.load %[[A]][%[[C0]]] : memref<?xf32>212// NOEPILOGUE-NEXT: %[[ADD0:.*]] = arith.addf %[[L0]], %{{.*}} : f32213// NOEPILOGUE-NEXT: %[[L1:.*]] = memref.load %[[A]][%[[C1]]] : memref<?xf32>214// Kernel:215// NOEPILOGUE-NEXT: %[[LR:.*]]:2 = scf.for %[[IV:.*]] = %[[C0]] to %[[C4]]216// NOEPILOGUE-SAME: step %[[C1]] iter_args(%[[ADDARG:.*]] = %[[ADD0]],217// NOEPILOGUE-SAME: %[[LARG:.*]] = %[[L1]]) -> (f32, f32) {218// NOEPILOGUE-DAG: %[[S0:.*]] = arith.cmpi slt, %[[IV]], %[[C2]] : index219// NOEPILOGUE-DAG: %[[S1:.*]] = arith.cmpi slt, %[[IV]], %[[C3]] : index220// NOEPILOGUE-NEXT: memref.store %[[ADDARG]], %[[R]][%[[IV]]] : memref<?xf32>221// NOEPILOGUE-NEXT: %[[ADD1:.*]] = scf.if %[[S1]] -> (f32) {222// NOEPILOGUE-NEXT: %[[PADD:.*]] = arith.addf %[[LARG]], %{{.*}} : f32223// NOEPILOGUE-NEXT: scf.yield %[[PADD]] : f32224// NOEPILOGUE-NEXT: } else {225// NOEPILOGUE-NEXT: scf.yield %[[CF]] : f32226// NOEPILOGUE-NEXT: }227// NOEPILOGUE-NEXT: %[[IV2:.*]] = arith.addi %[[IV]], %[[C2]] : index228// NOEPILOGUE-NEXT: %[[L3:.*]] = scf.if %[[S0]] -> (f32) {229// NOEPILOGUE-NEXT: %[[PL:.*]] = memref.load %[[A]][%[[IV2]]] : memref<?xf32>230// NOEPILOGUE-NEXT: scf.yield %[[PL]] : f32231// NOEPILOGUE-NEXT: } else {232// NOEPILOGUE-NEXT: scf.yield %[[CF]] : f32233// NOEPILOGUE-NEXT: }234// NOEPILOGUE-NEXT: scf.yield %[[ADD1]], %[[L3]] : f32, f32235// NOEPILOGUE-NEXT: }236// No epilogue should be generated.237// NOEPILOGUE-NOT: memref.store238// NOEPILOGUE: return239 240func.func @three_stage(%A: memref<?xf32>, %result: memref<?xf32>) {241 %c0 = arith.constant 0 : index242 %c1 = arith.constant 1 : index243 %c4 = arith.constant 4 : index244 %cf = arith.constant 1.0 : f32245 scf.for %i0 = %c0 to %c4 step %c1 {246 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>247 %A1_elem = arith.addf %A_elem, %cf { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : f32248 memref.store %A1_elem, %result[%i0] { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 0 } : memref<?xf32>249 } { __test_pipelining_loop__ }250 return251}252 253// -----254// CHECK-LABEL: long_liverange(255// CHECK-SAME: %[[A:.*]]: memref<?xf32>, %[[R:.*]]: memref<?xf32>) {256// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index257// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index258// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index259// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index260// CHECK-DAG: %[[C4:.*]] = arith.constant 4 : index261// CHECK-DAG: %[[C6:.*]] = arith.constant 6 : index262// CHECK-DAG: %[[C7:.*]] = arith.constant 7 : index263// CHECK-DAG: %[[C8:.*]] = arith.constant 8 : index264// CHECK-DAG: %[[C9:.*]] = arith.constant 9 : index265// Prologue:266// CHECK: %[[L0:.*]] = memref.load %[[A]][%[[C0]]] : memref<?xf32>267// CHECK-NEXT: %[[L1:.*]] = memref.load %[[A]][%[[C1]]] : memref<?xf32>268// CHECK-NEXT: %[[L2:.*]] = memref.load %[[A]][%[[C2]]] : memref<?xf32>269// CHECK-NEXT: %[[L3:.*]] = memref.load %[[A]][%[[C3]]] : memref<?xf32>270// Kernel:271// CHECK-NEXT: %[[LR:.*]]:4 = scf.for %[[IV:.*]] = %[[C0]] to %[[C6]]272// CHECK-SAME: step %[[C1]] iter_args(%[[LA0:.*]] = %[[L0]],273// CHECK-SAME: %[[LA1:.*]] = %[[L1]], %[[LA2:.*]] = %[[L2]],274// CHECK-SAME: %[[LA3:.*]] = %[[L3]]) -> (f32, f32, f32, f32) {275// CHECK-NEXT: %[[ADD0:.*]] = arith.addf %[[LA0]], %{{.*}} : f32276// CHECK-NEXT: memref.store %[[ADD0]], %[[R]][%[[IV]]] : memref<?xf32>277// CHECK-NEXT: %[[IV4:.*]] = arith.addi %[[IV]], %[[C4]] : index278// CHECK-NEXT: %[[L4:.*]] = memref.load %[[A]][%[[IV4]]] : memref<?xf32>279// CHECK-NEXT: scf.yield %[[LA1]], %[[LA2]], %[[LA3]], %[[L4]] : f32, f32, f32, f32280// CHECK-NEXT: }281// Epilogue:282// CHECK-NEXT: %[[ADD1:.*]] = arith.addf %[[LR]]#0, %{{.*}} : f32283// CHECK-NEXT: memref.store %[[ADD1]], %[[R]][%[[C6]]] : memref<?xf32>284// CHECK-NEXT: %[[ADD2:.*]] = arith.addf %[[LR]]#1, %{{.*}} : f32285// CHECK-NEXT: memref.store %[[ADD2]], %[[R]][%[[C7]]] : memref<?xf32>286// CHECK-NEXT: %[[ADD3:.*]] = arith.addf %[[LR]]#2, %{{.*}} : f32287// CHECK-NEXT: memref.store %[[ADD3]], %[[R]][%[[C8]]] : memref<?xf32>288// CHECK-NEXT: %[[ADD4:.*]] = arith.addf %[[LR]]#3, %{{.*}} : f32289// CHECK-NEXT: memref.store %[[ADD4]], %[[R]][%[[C9]]] : memref<?xf32>290func.func @long_liverange(%A: memref<?xf32>, %result: memref<?xf32>) {291 %c0 = arith.constant 0 : index292 %c1 = arith.constant 1 : index293 %c10 = arith.constant 10 : index294 %cf = arith.constant 1.0 : f32295 scf.for %i0 = %c0 to %c10 step %c1 {296 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>297 %A1_elem = arith.addf %A_elem, %cf { __test_pipelining_stage__ = 4, __test_pipelining_op_order__ = 0 } : f32298 memref.store %A1_elem, %result[%i0] { __test_pipelining_stage__ = 4, __test_pipelining_op_order__ = 1 } : memref<?xf32>299 } { __test_pipelining_loop__ }300 return301}302 303// -----304 305// CHECK-LABEL: multiple_uses(306// CHECK-SAME: %[[A:.*]]: memref<?xf32>, %[[R:.*]]: memref<?xf32>) {307// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index308// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index309// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index310// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index311// CHECK-DAG: %[[C7:.*]] = arith.constant 7 : index312// CHECK-DAG: %[[C8:.*]] = arith.constant 8 : index313// CHECK-DAG: %[[C9:.*]] = arith.constant 9 : index314// Prologue:315// CHECK: %[[L0:.*]] = memref.load %[[A]][%[[C0]]] : memref<?xf32>316// CHECK-NEXT: %[[ADD0:.*]] = arith.addf %[[L0]], %{{.*}} : f32317// CHECK-NEXT: %[[L1:.*]] = memref.load %[[A]][%[[C1]]] : memref<?xf32>318// CHECK-NEXT: %[[ADD1:.*]] = arith.addf %[[L1]], %{{.*}} : f32319// CHECK-NEXT: %[[MUL0:.*]] = arith.mulf %[[ADD0]], %[[L0]] : f32320// CHECK-NEXT: %[[L2:.*]] = memref.load %[[A]][%[[C2]]] : memref<?xf32>321// Kernel:322// CHECK-NEXT: %[[LR:.*]]:4 = scf.for %[[IV:.*]] = %[[C0]] to %[[C7]]323// CHECK-SAME: step %[[C1]] iter_args(%[[LA1:.*]] = %[[L1]],324// CHECK-SAME: %[[LA2:.*]] = %[[L2]], %[[ADDARG1:.*]] = %[[ADD1]],325// CHECK-SAME: %[[MULARG0:.*]] = %[[MUL0]]) -> (f32, f32, f32, f32) {326// CHECK-NEXT: %[[ADD2:.*]] = arith.addf %[[LA2]], %{{.*}} : f32327// CHECK-NEXT: %[[MUL1:.*]] = arith.mulf %[[ADDARG1]], %[[LA1]] : f32328// CHECK-NEXT: memref.store %[[MULARG0]], %[[R]][%[[IV]]] : memref<?xf32>329// CHECK-NEXT: %[[IV3:.*]] = arith.addi %[[IV]], %[[C3]] : index330// CHECK-NEXT: %[[L3:.*]] = memref.load %[[A]][%[[IV3]]] : memref<?xf32>331// CHECK-NEXT: scf.yield %[[LA2]], %[[L3]], %[[ADD2]], %[[MUL1]] : f32, f32, f32, f32332// CHECK-NEXT: }333// Epilogue:334// CHECK-NEXT: %[[ADD3:.*]] = arith.addf %[[LR]]#1, %{{.*}} : f32335// CHECK-NEXT: %[[MUL2:.*]] = arith.mulf %[[LR]]#2, %[[LR]]#0 : f32336// CHECK-NEXT: memref.store %[[LR]]#3, %[[R]][%[[C7]]] : memref<?xf32>337// CHECK-NEXT: %[[MUL3:.*]] = arith.mulf %[[ADD3]], %[[LR]]#1 : f32338// CHECK-NEXT: memref.store %[[MUL2]], %[[R]][%[[C8]]] : memref<?xf32>339// CHECK-NEXT: memref.store %[[MUL3]], %[[R]][%[[C9]]] : memref<?xf32>340func.func @multiple_uses(%A: memref<?xf32>, %result: memref<?xf32>) {341 %c0 = arith.constant 0 : index342 %c1 = arith.constant 1 : index343 %c10 = arith.constant 10 : index344 %cf = arith.constant 1.0 : f32345 scf.for %i0 = %c0 to %c10 step %c1 {346 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 3 } : memref<?xf32>347 %A1_elem = arith.addf %A_elem, %cf { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 0 } : f32348 %A2_elem = arith.mulf %A1_elem, %A_elem { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 1 } : f32349 memref.store %A2_elem, %result[%i0] { __test_pipelining_stage__ = 3, __test_pipelining_op_order__ = 2 } : memref<?xf32>350 } { __test_pipelining_loop__ }351 return352}353 354// -----355 356// CHECK-LABEL: region_multiple_uses(357// CHECK-SAME: %[[A:.*]]: memref<?xf32>, %[[R:.*]]: memref<?xf32>) {358// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index359// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index360// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index361// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index362// CHECK-DAG: %[[C7:.*]] = arith.constant 7 : index363// CHECK-DAG: %[[C8:.*]] = arith.constant 8 : index364// CHECK-DAG: %[[C9:.*]] = arith.constant 9 : index365// Prologue:366// CHECK: %[[L0:.*]] = memref.load %[[A]][%[[C0]]] : memref<?xf32>367// CHECK-NEXT: %[[ADD0:.*]] = arith.addf %[[L0]], %{{.*}} : f32368// CHECK-NEXT: %[[L1:.*]] = memref.load %[[A]][%[[C1]]] : memref<?xf32>369// CHECK-NEXT: %[[ADD1:.*]] = arith.addf %[[L1]], %{{.*}} : f32370// CHECK-NEXT: %[[MUL0:.*]] = scf.execute_region371// arith.mulf %[[ADD0]], %[[L0]] : f32372// CHECK: %[[L2:.*]] = memref.load %[[A]][%[[C2]]] : memref<?xf32>373// Kernel:374// CHECK-NEXT: %[[LR:.*]]:4 = scf.for %[[IV:.*]] = %[[C0]] to %[[C7]]375// CHECK-SAME: step %[[C1]] iter_args(%[[LA1:.*]] = %[[L1]],376// CHECK-SAME: %[[LA2:.*]] = %[[L2]], %[[ADDARG1:.*]] = %[[ADD1]],377// CHECK-SAME: %[[MULARG0:.*]] = %[[MUL0]]) -> (f32, f32, f32, f32) {378// CHECK-NEXT: %[[ADD2:.*]] = arith.addf %[[LA2]], %{{.*}} : f32379// CHECK-NEXT: %[[MUL1:.*]] = scf.execute_region380// arith.mulf %[[ADDARG1]], %[[LA1]] : f32381// CHECK: memref.store %[[MULARG0]], %[[R]][%[[IV]]] : memref<?xf32>382// CHECK-NEXT: %[[IV3:.*]] = arith.addi %[[IV]], %[[C3]] : index383// CHECK-NEXT: %[[L3:.*]] = memref.load %[[A]][%[[IV3]]] : memref<?xf32>384// CHECK-NEXT: scf.yield %[[LA2]], %[[L3]], %[[ADD2]], %[[MUL1]] : f32, f32, f32, f32385// CHECK-NEXT: }386// Epilogue:387// CHECK-NEXT: %[[ADD3:.*]] = arith.addf %[[LR]]#1, %{{.*}} : f32388// CHECK-NEXT: %[[MUL2:.*]] = scf.execute_region389// arith.mulf %[[LR]]#2, %[[LR]]#0 : f32390// CHECK: memref.store %[[LR]]#3, %[[R]][%[[C7]]] : memref<?xf32>391// CHECK-NEXT: %[[MUL3:.*]] = scf.execute_region392/// %[[ADD3]], %[[LR]]#1 : f32393// CHECK: memref.store %[[MUL2]], %[[R]][%[[C8]]] : memref<?xf32>394// CHECK-NEXT: memref.store %[[MUL3]], %[[R]][%[[C9]]] : memref<?xf32>395 396func.func @region_multiple_uses(%A: memref<?xf32>, %result: memref<?xf32>) {397 %c0 = arith.constant 0 : index398 %c1 = arith.constant 1 : index399 %c10 = arith.constant 10 : index400 %cf = arith.constant 1.0 : f32401 scf.for %i0 = %c0 to %c10 step %c1 {402 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 3 } : memref<?xf32>403 %A1_elem = arith.addf %A_elem, %cf { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 0 } : f32404 %A2_elem = scf.execute_region -> f32 {405 %A2_elem1 = arith.mulf %A1_elem, %A_elem : f32406 scf.yield %A2_elem1 : f32407 } { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 1 }408 memref.store %A2_elem, %result[%i0] { __test_pipelining_stage__ = 3, __test_pipelining_op_order__ = 2 } : memref<?xf32>409 } { __test_pipelining_loop__ }410 return411}412 413// -----414 415// CHECK-LABEL: loop_carried(416// CHECK-SAME: %[[A:.*]]: memref<?xf32>, %[[R:.*]]: memref<?xf32>) {417// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index418// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index419// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index420// CHECK-DAG: %[[CSTF:.*]] = arith.constant 1.000000e+00 : f32421// Prologue:422// CHECK: %[[L0:.*]] = memref.load %[[A]][%[[C0]]] : memref<?xf32>423// Kernel:424// CHECK-NEXT: %[[LR:.*]]:2 = scf.for %[[IV:.*]] = %[[C0]] to %[[C3]]425// CHECK-SAME: step %[[C1]] iter_args(%[[C:.*]] = %[[CSTF]],426// CHECK-SAME: %[[LARG:.*]] = %[[L0]]) -> (f32, f32) {427// CHECK-NEXT: %[[ADD0:.*]] = arith.addf %[[LARG]], %[[C]] : f32428// CHECK-NEXT: %[[IV1:.*]] = arith.addi %[[IV]], %[[C1]] : index429// CHECK-NEXT: %[[L1:.*]] = memref.load %[[A]][%[[IV1]]] : memref<?xf32>430// CHECK-NEXT: scf.yield %[[ADD0]], %[[L1]] : f32, f32431// CHECK-NEXT: }432// Epilogue:433// CHECK-NEXT: %[[ADD1:.*]] = arith.addf %[[LR]]#1, %[[LR]]#0 : f32434// CHECK-NEXT: memref.store %[[ADD1]], %[[R]][%[[C0]]] : memref<?xf32>435func.func @loop_carried(%A: memref<?xf32>, %result: memref<?xf32>) {436 %c0 = arith.constant 0 : index437 %c1 = arith.constant 1 : index438 %c4 = arith.constant 4 : index439 %cf = arith.constant 1.0 : f32440 %r = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%arg0 = %cf) -> (f32) {441 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 1 } : memref<?xf32>442 %A1_elem = arith.addf %A_elem, %arg0 { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 0 } : f32443 scf.yield %A1_elem : f32444 } { __test_pipelining_loop__ }445 memref.store %r, %result[%c0] : memref<?xf32>446 return447}448 449// -----450 451// CHECK-LABEL: backedge_different_stage452// CHECK-SAME: (%[[A:.*]]: memref<?xf32>) -> f32 {453// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index454// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index455// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index456// CHECK-DAG: %[[CSTF:.*]] = arith.constant 2.000000e+00 : f32457// Prologue:458// CHECK: %[[L0:.*]] = memref.load %[[A]][%[[C0]]] : memref<?xf32>459// CHECK-NEXT: %[[ADD0:.*]] = arith.addf %[[L0]], %[[CSTF]] : f32460// CHECK-NEXT: %[[L1:.*]] = memref.load %[[A]][%[[C1]]] : memref<?xf32>461// Kernel:462// CHECK-NEXT: %[[R:.*]]:3 = scf.for %[[IV:.*]] = %[[C0]] to %[[C2]]463// CHECK-SAME: step %[[C1]] iter_args(%[[C:.*]] = %[[CSTF]],464// CHECK-SAME: %[[ADDARG:.*]] = %[[ADD0]], %[[LARG:.*]] = %[[L1]]) -> (f32, f32, f32) {465// CHECK-NEXT: %[[MUL0:.*]] = arith.mulf %[[ADDARG]], %[[CSTF]] : f32466// CHECK-NEXT: %[[ADD1:.*]] = arith.addf %[[LARG]], %[[MUL0]] : f32467// CHECK-NEXT: %[[IV2:.*]] = arith.addi %[[IV]], %[[C2]] : index468// CHECK-NEXT: %[[L2:.*]] = memref.load %[[A]][%[[IV2]]] : memref<?xf32>469// CHECK-NEXT: scf.yield %[[MUL0]], %[[ADD1]], %[[L2]] : f32, f32, f32470// CHECK-NEXT: }471// Epilogue:472// CHECK-NEXT: %[[MUL1:.*]] = arith.mulf %[[R]]#1, %[[CSTF]] : f32473// CHECK-NEXT: %[[ADD2:.*]] = arith.addf %[[R]]#2, %[[MUL1]] : f32474// CHECK-NEXT: %[[MUL2:.*]] = arith.mulf %[[ADD2]], %[[CSTF]] : f32475// CHECK-NEXT: return %[[MUL2]] : f32476func.func @backedge_different_stage(%A: memref<?xf32>) -> f32 {477 %c0 = arith.constant 0 : index478 %c1 = arith.constant 1 : index479 %c4 = arith.constant 4 : index480 %cf = arith.constant 2.0 : f32481 %r = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%arg0 = %cf) -> (f32) {482 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>483 %A1_elem = arith.addf %A_elem, %arg0 { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : f32484 %A2_elem = arith.mulf %cf, %A1_elem { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 0 } : f32485 scf.yield %A2_elem : f32486 } { __test_pipelining_loop__ }487 return %r : f32488}489 490// -----491 492// CHECK-LABEL: region_backedge_different_stage493// CHECK-SAME: (%[[A:.*]]: memref<?xf32>) -> f32 {494// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index495// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index496// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index497// CHECK-DAG: %[[CSTF:.*]] = arith.constant 2.000000e+00 : f32498// Prologue:499// CHECK: %[[L0:.*]] = scf.execute_region500// CHECK-NEXT: memref.load %[[A]][%[[C0]]] : memref<?xf32>501// CHECK: %[[ADD0:.*]] = scf.execute_region502// CHECK-NEXT: arith.addf %[[L0]], %[[CSTF]] : f32503// CHECK: %[[L1:.*]] = scf.execute_region504// CHECK-NEXT: memref.load %[[A]][%[[C1]]] : memref<?xf32>505// Kernel:506// CHECK: %[[R:.*]]:3 = scf.for %[[IV:.*]] = %[[C0]] to %[[C2]]507// CHECK-SAME: step %[[C1]] iter_args(%[[C:.*]] = %[[CSTF]],508// CHECK-SAME: %[[ADDARG:.*]] = %[[ADD0]], %[[LARG:.*]] = %[[L1]]) -> (f32, f32, f32) {509// CHECK: %[[MUL0:.*]] = arith.mulf %[[ADDARG]], %[[CSTF]] : f32510// CHECK: %[[ADD1:.*]] = scf.execute_region511// CHECK-NEXT: arith.addf %[[LARG]], %[[MUL0]] : f32512// CHECK: %[[IV2:.*]] = arith.addi %[[IV]], %[[C2]] : index513// CHECK: %[[L2:.*]] = scf.execute_region514// CHECK-NEXT: memref.load %[[A]][%[[IV2]]] : memref<?xf32>515// CHECK: scf.yield %[[MUL0]], %[[ADD1]], %[[L2]] : f32, f32, f32516// CHECK-NEXT: }517// Epilogue:518// CHECK: %[[MUL1:.*]] = arith.mulf %[[R]]#1, %[[CSTF]] : f32519// CHECK: %[[ADD2:.*]] = scf.execute_region520// CHECK-NEXT: arith.addf %[[R]]#2, %[[MUL1]] : f32521// CHECK: %[[MUL2:.*]] = arith.mulf %[[ADD2]], %[[CSTF]] : f32522// CHECK: return %[[MUL2]] : f32523 524func.func @region_backedge_different_stage(%A: memref<?xf32>) -> f32 {525 %c0 = arith.constant 0 : index526 %c1 = arith.constant 1 : index527 %c4 = arith.constant 4 : index528 %cf = arith.constant 2.0 : f32529 %r = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%arg0 = %cf) -> (f32) {530 %A_elem = scf.execute_region -> f32 {531 %A_elem1 = memref.load %A[%i0] : memref<?xf32>532 scf.yield %A_elem1 : f32533 } { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 }534 %A1_elem = scf.execute_region -> f32 {535 %inner = arith.addf %A_elem, %arg0 : f32536 scf.yield %inner : f32537 } { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 }538 %A2_elem = arith.mulf %cf, %A1_elem { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 0 } : f32539 scf.yield %A2_elem : f32540 } { __test_pipelining_loop__ }541 return %r : f32542}543 544 545// -----546 547// CHECK-LABEL: backedge_same_stage548// CHECK-SAME: (%[[A:.*]]: memref<?xf32>) -> f32 {549// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index550// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index551// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index552// CHECK-DAG: %[[CSTF:.*]] = arith.constant 2.000000e+00 : f32553// Prologue:554// CHECK: %[[L0:.*]] = memref.load %[[A]][%[[C0]]] : memref<?xf32>555// Kernel:556// CHECK-NEXT: %[[R:.*]]:2 = scf.for %[[IV:.*]] = %[[C0]] to %[[C3]]557// CHECK-SAME: step %[[C1]] iter_args(%[[C:.*]] = %[[CSTF]],558// CHECK-SAME: %[[LARG:.*]] = %[[L0]]) -> (f32, f32) {559// CHECK-NEXT: %[[ADD0:.*]] = arith.addf %[[LARG]], %[[C]] : f32560// CHECK-NEXT: %[[MUL0:.*]] = arith.mulf %[[ADD0]], %[[CSTF]] : f32561// CHECK-NEXT: %[[IV1:.*]] = arith.addi %[[IV]], %[[C1]] : index562// CHECK-NEXT: %[[L2:.*]] = memref.load %[[A]][%[[IV1]]] : memref<?xf32>563// CHECK-NEXT: scf.yield %[[MUL0]], %[[L2]] : f32, f32564// CHECK-NEXT: }565// Epilogue:566// CHECK-NEXT: %[[ADD1:.*]] = arith.addf %[[R]]#1, %[[R]]#0 : f32567// CHECK-NEXT: %[[MUL1:.*]] = arith.mulf %[[ADD1]], %[[CSTF]] : f32568// CHECK-NEXT: return %[[MUL1]] : f32569func.func @backedge_same_stage(%A: memref<?xf32>) -> f32 {570 %c0 = arith.constant 0 : index571 %c1 = arith.constant 1 : index572 %c4 = arith.constant 4 : index573 %cf = arith.constant 2.0 : f32574 %r = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%arg0 = %cf) -> (f32) {575 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>576 %A1_elem = arith.addf %A_elem, %arg0 { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 0 } : f32577 %A2_elem = arith.mulf %cf, %A1_elem { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : f32578 scf.yield %A2_elem : f32579 } { __test_pipelining_loop__ }580 return %r : f32581}582 583// -----584 585// CHECK: @pipeline_op_with_region(%[[ARG0:.+]]: memref<?xf32>, %[[ARG1:.+]]: memref<?xf32>, %[[ARG2:.+]]: memref<?xf32>, %[[CF:.*]]: f32) {586// CHECK-DAG: %[[C0:.+]] = arith.constant 0 :587// CHECK-DAG: %[[C3:.+]] = arith.constant 3 :588// CHECK-DAG: %[[C1:.+]] = arith.constant 1 :589// CHECK: %[[APRO:.+]] = memref.alloc() :590// CHECK: %[[BPRO:.+]] = memref.alloc() :591// CHECK: %[[ASV0:.+]] = memref.subview %[[ARG0]][%[[C0]]] [8] [1] :592// CHECK: %[[BSV0:.+]] = memref.subview %[[ARG1]][%[[C0]]] [8] [1] :593 594// Prologue:595// CHECK: %[[PAV0:.+]] = memref.subview %[[APRO]][%[[C0]], 0] [1, 8] [1, 1] :596// CHECK: %[[PBV0:.+]] = memref.subview %[[BPRO]][%[[C0]], 0] [1, 8] [1, 1] :597// CHECK: memref.copy %[[ASV0]], %[[PAV0]] :598// CHECK: memref.copy %[[BSV0]], %[[PBV0]] :599 600// Kernel:601// CHECK: %[[R:.+]]:2 = scf.for %[[IV:.+]] = %[[C0]] to %[[C3]] step %[[C1]]602// CHECK-SAME: iter_args(%[[IA:.+]] = %[[PAV0]], %[[IB:.+]] = %[[PBV0:.+]])603// CHECK: %[[CV:.+]] = memref.subview %[[ARG2]]604// CHECK: linalg.generic605// CHECK-SAME: ins(%[[IA]], %[[IB]], %{{.*}} : {{.*}}) outs(%[[CV]] :606// CHECK: %[[NEXT:.+]] = arith.addi %[[IV]], %[[C1]]607// CHECK: %[[ASV:.+]] = memref.subview %[[ARG0]][%[[NEXT]]] [8] [1] :608// CHECK: %[[NEXT:.+]] = arith.addi %[[IV]], %[[C1]] :609// CHECK: %[[BSV:.+]] = memref.subview %[[ARG1]][%[[NEXT]]] [8] [1] :610// CHECK: %[[NEXT:.+]] = arith.addi %[[IV]], %[[C1]] :611// CHECK: %[[BUFIDX:.+]] = affine.apply612// CHECK: %[[APROSV:.+]] = memref.subview %[[APRO]][%[[BUFIDX]], 0] [1, 8] [1, 1] :613// CHECK: %[[BPROSV:.+]] = memref.subview %[[BPRO]][%[[BUFIDX]], 0] [1, 8] [1, 1] :614// CHECK: memref.copy %[[ASV]], %[[APROSV]] :615// CHECK: memref.copy %[[BSV]], %[[BPROSV]] :616// CHECK: scf.yield %[[APROSV]], %[[BPROSV]] :617// CHECK: }618// CHECK: %[[CV:.+]] = memref.subview %[[ARG2]][%[[C3]]] [8] [1] :619// CHECK: linalg.generic620// CHECK-SAME: ins(%[[R]]#0, %[[R]]#1, %{{.*}} : {{.*}}) outs(%[[CV]] :621 622 623#map = affine_map<(d0)[s0]->(d0 + s0)>624#map1 = affine_map<(d0)->(d0)>625#map2 = affine_map<(d0)->()>626#linalg_attrs = {627 indexing_maps = [628 #map1,629 #map1,630 #map2,631 #map1632 ],633 iterator_types = ["parallel"],634 __test_pipelining_stage__ = 1,635 __test_pipelining_op_order__ = 2636}637func.func @pipeline_op_with_region(%A: memref<?xf32>, %B: memref<?xf32>, %result: memref<?xf32>, %cf: f32) {638 %c0 = arith.constant 0 : index639 %c1 = arith.constant 1 : index640 %c4 = arith.constant 4 : index641 %a_buf = memref.alloc() : memref<2x8xf32>642 %b_buf = memref.alloc() : memref<2x8xf32>643 scf.for %i0 = %c0 to %c4 step %c1 {644 %A_view = memref.subview %A[%i0][8][1] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 3 } : memref<?xf32> to memref<8xf32, #map>645 %B_view = memref.subview %B[%i0][8][1] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 4 } : memref<?xf32> to memref<8xf32, #map>646 %buf_idx = affine.apply affine_map<(d0)->(d0 mod 2)> (%i0)[] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 5 }647 %a_buf_view = memref.subview %a_buf[%buf_idx,0][1,8][1,1] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 6 } : memref<2x8xf32> to memref<8xf32, #map>648 %b_buf_view = memref.subview %b_buf[%buf_idx,0][1,8][1,1] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 7 } : memref<2x8xf32> to memref<8xf32, #map>649 memref.copy %A_view , %a_buf_view {__test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 8} : memref<8xf32, #map> to memref<8xf32, #map>650 memref.copy %B_view , %b_buf_view {__test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 9} : memref<8xf32, #map> to memref<8xf32, #map>651 %C_view = memref.subview %result[%i0][8][1] { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 0 } : memref<?xf32> to memref<8xf32, #map>652 %scalar = arith.addf %cf, %cf {__test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1} : f32653 linalg.generic #linalg_attrs ins(%a_buf_view, %b_buf_view, %scalar : memref<8xf32, #map>, memref<8xf32, #map>, f32)654 outs(%C_view: memref<8xf32, #map>) {655 ^bb0(%a: f32, %b: f32, %s: f32, %c: f32):656 %add = arith.addf %a, %b : f32657 %accum = arith.addf %add, %c : f32658 %accum1 = arith.addf %scalar, %accum : f32659 %accum2 = arith.addf %s, %accum1 : f32660 linalg.yield %accum2 : f32661 }662 scf.yield663 } { __test_pipelining_loop__ }664 return665}666 667// -----668 669// CHECK-LABEL: @backedge_mix_order670// CHECK-SAME: (%[[A:.*]]: memref<?xf32>) -> f32 {671// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index672// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index673// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index674// CHECK-DAG: %[[CSTF:.*]] = arith.constant 2.000000e+00 : f32675// Prologue:676// CHECK: %[[L0:.*]] = memref.load %[[A]][%[[C0]]] : memref<?xf32>677// CHECK-NEXT: %[[L1:.*]] = memref.load %[[A]][%[[C1]]] : memref<?xf32>678// Kernel:679// CHECK-NEXT: %[[R:.*]]:3 = scf.for %[[IV:.*]] = %[[C0]] to %[[C3]]680// CHECK-SAME: step %[[C1]] iter_args(%[[C:.*]] = %[[CSTF]],681// CHECK-SAME: %[[ARG1:.*]] = %[[L0]], %[[ARG2:.*]] = %[[L1]]) -> (f32, f32, f32) {682// CHECK-NEXT: %[[IV2:.*]] = arith.addi %[[IV]], %[[C1]] : index683// CHECK-NEXT: %[[L2:.*]] = memref.load %[[A]][%[[IV2]]] : memref<?xf32>684// CHECK-NEXT: %[[MUL0:.*]] = arith.mulf %[[C]], %[[ARG1]] : f32685// CHECK-NEXT: %[[IV3:.*]] = arith.addi %[[IV]], %[[C1]] : index686// CHECK-NEXT: %[[IV4:.*]] = arith.addi %[[IV3]], %[[C1]] : index687// CHECK-NEXT: %[[L3:.*]] = memref.load %[[A]][%[[IV4]]] : memref<?xf32>688// CHECK-NEXT: %[[MUL1:.*]] = arith.mulf %[[ARG2]], %[[MUL0]] : f32689// CHECK-NEXT: scf.yield %[[MUL1]], %[[L2]], %[[L3]] : f32, f32, f32690// CHECK-NEXT: }691// Epilogue:692// CHECK-NEXT: %[[MUL1:.*]] = arith.mulf %[[R]]#0, %[[R]]#1 : f32693// CHECK-NEXT: %[[MUL2:.*]] = arith.mulf %[[R]]#2, %[[MUL1]] : f32694// CHECK-NEXT: return %[[MUL2]] : f32695func.func @backedge_mix_order(%A: memref<?xf32>) -> f32 {696 %c0 = arith.constant 0 : index697 %c1 = arith.constant 1 : index698 %c4 = arith.constant 4 : index699 %cf = arith.constant 2.0 : f32700 %r = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%arg0 = %cf) -> (f32) {701 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 0 } : memref<?xf32>702 %A2_elem = arith.mulf %arg0, %A_elem { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : f32703 %i1 = arith.addi %i0, %c1 { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : index704 %A1_elem = memref.load %A[%i1] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 3 } : memref<?xf32>705 %A3_elem = arith.mulf %A1_elem, %A2_elem { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 4 } : f32706 scf.yield %A3_elem : f32707 } { __test_pipelining_loop__ }708 return %r : f32709}710 711// -----712 713// CHECK-LABEL: @distance_1_use714// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index715// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index716// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index717// Prologue:718// CHECK: %[[L0:.+]] = memref.load %{{.*}}[%[[C0]]] : memref<?xf32>719// CHECK: %[[L1:.+]] = memref.load %{{.*}}[%[[C1]]] : memref<?xf32>720// CHECK: %[[R:.+]]:5 = scf.for {{.*}} iter_args(%[[IDX0:.+]] = %[[C2]], %[[L2:.+]] = %[[L0]], %[[L3:.+]] = %[[L1]]721// CHECK: %[[L4:.+]] = memref.load %{{.*}}[%[[IDX0]]] : memref<?xf32>722// CHECK: %[[IDX1:.+]] = arith.addi %[[IDX0]], %[[C1]] : index723// CHECK: memref.store %[[L2]]724// CHECK: scf.yield %[[IDX1]], %[[L3]], %[[L4]]725func.func @distance_1_use(%A: memref<?xf32>, %result: memref<?xf32>) {726 %c0 = arith.constant 0 : index727 %c1 = arith.constant 1 : index728 %c4 = arith.constant 4 : index729 %cf = arith.constant 1.0 : f32730 %r = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%idx = %c0) -> (index) {731 %A_elem = memref.load %A[%idx] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 0 } : memref<?xf32>732 %idx1 = arith.addi %idx, %c1 { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 1 } : index733 memref.store %A_elem, %result[%idx] { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 2 } : memref<?xf32>734 scf.yield %idx1 : index735 } { __test_pipelining_loop__ }736 return737}738 739// -----740 741// NOEPILOGUE-LABEL: stage_0_value_escape(742func.func @stage_0_value_escape(%A: memref<?xf32>, %result: memref<?xf32>, %ub: index) {743 %c0 = arith.constant 0 : index744 %c1 = arith.constant 1 : index745 %cf = arith.constant 1.0 : f32746// NOEPILOGUE: %[[UB:[^,]+]]: index)747// NOEPILOGUE-DAG: %[[C0:.+]] = arith.constant 0 : index748// NOEPILOGUE-DAG: %[[C1:.+]] = arith.constant 1 : index749// NOEPILOGUE-DAG: %[[CF:.+]] = arith.constant 1.000000e+00750// NOEPILOGUE: %[[CND0:.+]] = arith.cmpi sgt, %[[UB]], %[[C0]]751// NOEPILOGUE: scf.if752// NOEPILOGUE: %[[IF:.+]] = scf.if %[[CND0]]753// NOEPILOGUE: %[[A:.+]] = arith.addf754// NOEPILOGUE: scf.yield %[[A]]755// NOEPILOGUE: %[[S0:.+]] = arith.select %[[CND0]], %[[IF]], %[[CF]]756// NOEPILOGUE: scf.for %[[IV:.+]] = {{.*}} iter_args(%[[ARG:.+]] = %[[S0]],757// NOEPILOGUE: %[[UB_1:.+]] = arith.subi %[[UB]], %[[C1]] : index758// NOEPILOGUE: %[[CND1:.+]] = arith.cmpi slt, %[[IV]], %[[UB_1]] : index759// NOEPILOGUE: %[[S1:.+]] = arith.select %[[CND1]], %{{.+}}, %[[ARG]] : f32760// NOEPILOGUE: scf.yield %[[S1]]761 %r = scf.for %i0 = %c0 to %ub step %c1 iter_args(%arg0 = %cf) -> (f32) {762 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 1 } : memref<?xf32>763 %A1_elem = arith.addf %A_elem, %arg0 { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 0 } : f32764 memref.store %A1_elem, %result[%c0] { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 2 } : memref<?xf32>765 scf.yield %A1_elem : f32766 } { __test_pipelining_loop__ }767 memref.store %r, %result[%c1] : memref<?xf32>768 return769}770 771// -----772 773// NOEPILOGUE-LABEL: dynamic_loop(774// NOEPILOGUE-SAME: %[[A:.*]]: memref<?xf32>, %[[R:.*]]: memref<?xf32>, %[[LB:.+]]: index, %[[UB:.+]]: index, %[[STEP:.+]]: index) {775// NOEPILOGUE-DAG: %[[C2:.+]] = arith.constant 2 : index776// NOEPILOGUE-DAG: %[[CSTF:.+]] = arith.constant 1.000000e+00 : f32777// Prologue:778// NOEPILOGUE: %[[P_I0:.+]] = arith.cmpi slt, %[[LB]], %[[UB]] : index779// NOEPILOGUE: %[[L0:.+]] = scf.if %[[P_I0]] -> (f32) {780// NOEPILOGUE-NEXT: memref.load %[[A]][%[[LB]]] : memref<?xf32>781// NOEPILOGUE: %[[IV1:.+]] = arith.addi %[[LB]], %[[STEP]] : index782// NOEPILOGUE: %[[P_I1:.+]] = arith.cmpi slt, %[[IV1]], %[[UB]] : index783// NOEPILOGUE: %[[IV1_2:.+]] = arith.addi %[[LB]], %[[STEP]] : index784// NOEPILOGUE: %[[V0:.+]] = scf.if %[[P_I0]] -> (f32) {785// NOEPILOGUE-NEXT: arith.addf %[[L0]], %[[CSTF]] : f32786// NOEPILOGUE: %[[L1:.+]] = scf.if %[[P_I1]] -> (f32) {787// NOEPILOGUE-NEXT: memref.load %[[A]][%[[IV1_2]]] : memref<?xf32>788// NOEPILOGUE: scf.for %[[IV2:.+]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[V1:.+]] = %[[V0]], %[[L2:.+]] = %[[L1]]) -> (f32, f32) {789// NOEPILOGUE-DAG: %[[S2:.+]] = arith.muli %[[STEP]], %[[C2]] : index790// NOEPILOGUE-DAG: %[[IT2:.+]] = arith.subi %[[UB]], %[[S2]] : index791// NOEPILOGUE-DAG: %[[P_I2:.+]] = arith.cmpi slt, %[[IV2]], %[[IT2]] : index792// NOEPILOGUE-DAG: %[[IT3:.+]] = arith.subi %[[UB]], %[[STEP]] : index793// NOEPILOGUE-DAG: %[[P_I3:.+]] = arith.cmpi slt, %[[IV2]], %[[IT3]] : index794// NOEPILOGUE: memref.store %[[V1]], %[[R]][%[[IV2]]] : memref<?xf32>795// NOEPILOGUE: %[[V2:.+]] = scf.if %[[P_I3]] -> (f32) {796// NOEPILOGUE: arith.addf %[[L2]], %[[CSTF]] : f32797// NOEPILOGUE: %[[IT4:.+]] = arith.muli %[[STEP]], %[[C2]] : index798// NOEPILOGUE: %[[IV3:.+]] = arith.addi %[[IV2]], %[[IT4]] : index799// NOEPILOGUE: %[[L3:.+]] = scf.if %[[P_I2]] -> (f32) {800// NOEPILOGUE: memref.load %[[A]][%[[IV3]]] : memref<?xf32>801// NOEPILOGUE: scf.yield %[[V2]], %[[L3]] : f32, f32802 803// Check for predicated epilogue for dynamic loop.804// CHECK-LABEL: dynamic_loop(805// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index806// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index807// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index808// CHECK-DAG: %[[CM1:.*]] = arith.constant -1 : index809// CHECK: %[[UBM:.*]] = arith.subi %[[UB:.*]], %{{.*}}810// CHECK: %{{.*}}:2 = scf.for %[[ARG5:.*]] = %[[LB:.*]] to %[[UBM]] step %[[STEP:.*]] iter_args(%[[ARG6:.*]] = %{{.*}}, %[[ARG7:.*]] = %{{.*}})811// CHECK: memref.store %[[ARG6]], %{{.*}}[%[[ARG5]]]812// CHECK: %[[ADDF_24:.*]] = arith.addf %[[ARG7]], %{{.*}}813// CHECK: %[[MULI_25:.*]] = arith.muli %{{.*}}, %{{.*}}814// CHECK: %[[ADDI_26:.*]] = arith.addi %[[ARG5]], %[[MULI_25]]815// CHECK: %[[LOAD_27:.*]] = memref.load %{{.*}}[%[[ADDI_26]]]816// CHECK: scf.yield %[[ADDF_24]], %[[LOAD_27]]817// CHECK: }818// CHECK: %[[CMPI_10:.*]] = arith.cmpi slt, %[[STEP]], %[[C0]]819// CHECK: %[[SELECT_11:.*]] = arith.select %[[CMPI_10]], %[[C1]], %[[CM1]]820// CHECK: %[[SUBI_12:.*]] = arith.subi %[[UB]], %[[LB]]821// CHECK: %[[ADDI_13:.*]] = arith.addi %[[SUBI_12]], %[[STEP]]822// CHECK: %[[ADDI_14:.*]] = arith.addi %[[ADDI_13]], %[[SELECT_11]]823// CHECK: %[[DIVSI_15:.*]] = arith.divsi %[[ADDI_14]], %[[STEP]]824// CHECK: %[[SUBI_17:.*]] = arith.subi %[[DIVSI_15]], %[[C2]]825// CHECK: %[[MAXSI_18:.*]] = arith.maxsi %[[SUBI_17]], %[[C0]]826// CHECK: %[[MULI_19:.*]] = arith.muli %[[STEP]], %[[MAXSI_18]]827// CHECK: %[[ADDI_20:.*]] = arith.addi %[[LB]], %[[MULI_19]]828// CHECK: %[[ADDI_21:.*]] = arith.addi %[[MAXSI_18]], %[[C1]]829// CHECK: %[[CMPI_22:.*]] = arith.cmpi sge, %[[DIVSI_15]], %[[C1]]830// CHECK: %[[MULI_23:.*]] = arith.muli %[[STEP]], %[[ADDI_21]]831// CHECK: %[[ADDI_24:.*]] = arith.addi %[[LB]], %[[MULI_23]]832// CHECK: %[[CMPI_25:.*]] = arith.cmpi sge, %[[DIVSI_15]], %[[C2]]833// CHECK: scf.if %[[CMPI_22]] {834// CHECK: memref.store %{{.*}}#0, %{{.*}}[%[[ADDI_20]]]835// CHECK: } else {836// CHECK: }837// CHECK: %[[IF_26:.*]] = scf.if %[[CMPI_25]]838// CHECK: %[[ADDF_27:.*]] = arith.addf %{{.*}}#1, %{{.*}}839// CHECK: scf.yield %[[ADDF_27]]840// CHECK: } else {841// CHECK: scf.yield %{{.*}}842// CHECK: }843// CHECK: scf.if %[[CMPI_25]] {844// CHECK: memref.store %[[IF_26]], %{{.*}}[%[[ADDI_24]]]845// CHECK: } else {846// CHECK: }847// CHECK: return848func.func @dynamic_loop(%A: memref<?xf32>, %result: memref<?xf32>, %lb: index, %ub: index, %step: index) {849 %cf = arith.constant 1.0 : f32850 scf.for %i0 = %lb to %ub step %step {851 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>852 %A1_elem = arith.addf %A_elem, %cf { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : f32853 memref.store %A1_elem, %result[%i0] { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 0 } : memref<?xf32>854 } { __test_pipelining_loop__ }855 return856}857 858// -----859 860// NOEPILOGUE-LABEL: func.func @dynamic_loop_result861// NOEPILOGUE: %{{.*}}:2 = scf.for %[[ARG5:.*]] = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%[[ARG6:.*]] = %{{.*}}, %[[ARG7:.*]] = %{{.*}})862// NOEPILOGUE: %[[SUBI_3:.*]] = arith.subi %{{.*}}, %{{.*}}863// NOEPILOGUE: %[[CMPI_4:.*]] = arith.cmpi slt, %[[ARG5]], %[[SUBI_3]]864// NOEPILOGUE: %[[ADDF_5:.*]] = arith.addf %[[ARG7]], %[[ARG6]]865// NOEPILOGUE: %[[MULF_6:.*]] = arith.mulf %[[ADDF_5]], %{{.*}}866// NOEPILOGUE: %[[ADDI_7:.*]] = arith.addi %[[ARG5]], %{{.*}}867// NOEPILOGUE: %[[IF_8:.*]] = scf.if %[[CMPI_4]]868// NOEPILOGUE: %[[LOAD_9:.*]] = memref.load %{{.*}}[%[[ADDI_7]]]869// NOEPILOGUE: scf.yield %[[LOAD_9]]870// NOEPILOGUE: } else {871// NOEPILOGUE: scf.yield %{{.*}}872// NOEPILOGUE: }873// NOEPILOGUE: scf.yield %[[MULF_6]], %[[IF_8]]874// NOEPILOGUE: }875// NOEPILOGUE: memref.store %{{.*}}#0, %{{.*}}[%{{.*}}]876 877// Check for predicated epilogue for dynamic loop.878// CHECK-LABEL: func.func @dynamic_loop_result879// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index880// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index881// CHECK-DAG: %[[CM1:.*]] = arith.constant -1 : index882// CHECK-DAG: %[[CF0:.*]] = arith.constant 0.000000e+00883// CHECK: %[[UBM:.*]] = arith.subi %[[UB:.*]], %{{.*}}884// CHECK: %{{.*}}:2 = scf.for %[[ARG5:.*]] = %[[LB:.*]] to %[[UBM]] step %[[STEP:.*]] iter_args(%[[ARG6:.*]] = %{{.*}}, %[[ARG7:.*]] = %{{.*}})885// CHECK: %[[ADDF_13:.*]] = arith.addf %[[ARG7]], %[[ARG6]]886// CHECK: %[[MULF_14:.*]] = arith.mulf %[[ADDF_13]], %{{.*}}887// CHECK: %[[ADDI_15:.*]] = arith.addi %[[ARG5]], %{{.*}}888// CHECK: %[[LOAD_16:.*]] = memref.load %{{.*}}[%[[ADDI_15]]]889// CHECK: scf.yield %[[MULF_14]], %[[LOAD_16]]890// CHECK: }891// CHECK: %[[CMPI_4:.*]] = arith.cmpi slt, %[[STEP]], %[[C0]]892// CHECK: %[[SELECT_5:.*]] = arith.select %[[CMPI_4]], %[[C1]], %[[CM1]]893// CHECK: %[[SUBI_6:.*]] = arith.subi %[[UB]], %[[LB]]894// CHECK: %[[ADDI_7:.*]] = arith.addi %[[SUBI_6]], %[[STEP]]895// CHECK: %[[ADDI_8:.*]] = arith.addi %[[ADDI_7]], %[[SELECT_5]]896// CHECK: %[[DIVSI_9:.*]] = arith.divsi %[[ADDI_8]], %[[STEP]]897// CHECK: %[[CMPI_10:.*]] = arith.cmpi sge, %[[DIVSI_9]], %[[C1]]898// CHECK: %[[IF_11:.*]] = scf.if %[[CMPI_10]]899// CHECK: %[[ADDF_14:.*]] = arith.addf %{{.*}}#1, %{{.*}}#0900// CHECK: scf.yield %[[ADDF_14]]901// CHECK: } else {902// CHECK: scf.yield %[[CF0]]903// CHECK: }904// CHECK: %[[IF_12:.*]] = scf.if %[[CMPI_10]]905// CHECK: %[[MULF_14:.*]] = arith.mulf %[[IF_11]], %{{.*}}906// CHECK: scf.yield %[[MULF_14]]907// CHECK: } else {908// CHECK: scf.yield %[[CF0]]909// CHECK: }910// CHECK: %[[SELECT_13:.*]] = arith.select %[[CMPI_10]], %[[IF_12]], %{{.*}}#0911// CHECK: memref.store %[[SELECT_13]], %{{.*}}[%[[C0]]]912func.func @dynamic_loop_result(%A: memref<?xf32>, %result: memref<?xf32>, %lb: index, %ub: index, %step: index) {913 %cf0 = arith.constant 1.0 : f32914 %cf1 = arith.constant 33.0 : f32915 %cst = arith.constant 0 : index916 %res:1 = scf.for %i0 = %lb to %ub step %step iter_args (%arg0 = %cf0) -> (f32) {917 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>918 %A1_elem = arith.addf %A_elem, %arg0 { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 0 } : f32919 %A2_elem = arith.mulf %A1_elem, %cf1 { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : f32920 scf.yield %A2_elem : f32921 } { __test_pipelining_loop__ }922 memref.store %res#0, %result[%cst] : memref<?xf32>923 return924}925 926// -----927 928// CHECK-LABEL: yield_constant_loop(929// CHECK-SAME: %[[A:.*]]: memref<?xf32>) -> f32 {930// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index931// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index932// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index933// CHECK-DAG: %[[CST0:.*]] = arith.constant 0.000000e+00 : f32934// CHECK-DAG: %[[CST2:.*]] = arith.constant 2.000000e+00 : f32935// Prologue:936// CHECK: %[[L0:.*]] = memref.load %[[A]][%[[C0]]] : memref<?xf32>937// Kernel:938// CHECK-NEXT: %[[L1:.*]]:2 = scf.for %[[IV:.*]] = %[[C0]] to %[[C3]]939// CHECK-SAME: step %[[C1]] iter_args(%[[ARG0:.*]] = %[[CST2]], %[[ARG1:.*]] = %[[L0]]) -> (f32, f32) {940// CHECK-NEXT: %[[ADD0:.*]] = arith.addf %[[ARG1]], %[[ARG0]] : f32941// CHECK-NEXT: %[[MUL0:.*]] = arith.mulf %[[ADD0]], %[[CST0]] : f32942// CHECK-NEXT: memref.store %[[MUL0]], %[[A]][%[[IV]]] : memref<?xf32>943// CHECK-NEXT: %[[IV1:.*]] = arith.addi %[[IV]], %[[C1]] : index944// CHECK-NEXT: %[[L2:.*]] = memref.load %[[A]][%[[IV1]]] : memref<?xf32>945// CHECK-NEXT: scf.yield %[[CST0]], %[[L2]] : f32946// CHECK-NEXT: }947// Epilogue:948// CHECK-NEXT: %[[ADD1:.*]] = arith.addf %[[L1]]#1, %[[CST0]] : f32949// CHECK-NEXT: %[[MUL1:.*]] = arith.mulf %[[ADD1]], %[[CST0]] : f32950// CHECK-NEXT: memref.store %[[MUL1]], %[[A]][%[[C3]]] : memref<?xf32>951// CHECK-NEXT: return %[[L1]]#0 : f32952 953func.func @yield_constant_loop(%A: memref<?xf32>) -> f32 {954 %c0 = arith.constant 0 : index955 %c1 = arith.constant 1 : index956 %c4 = arith.constant 4 : index957 %cf0 = arith.constant 0.0 : f32958 %cf2 = arith.constant 2.0 : f32959 %r = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%arg0 = %cf2) -> f32 {960 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 3 } : memref<?xf32>961 %A1_elem = arith.addf %A_elem, %arg0 { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 0 } : f32962 %A2_elem = arith.mulf %cf0, %A1_elem { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : f32963 memref.store %A2_elem, %A[%i0] { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 2 } : memref<?xf32>964 scf.yield %cf0: f32965 } { __test_pipelining_loop__ }966 return %r : f32967}968 969// -----970 971func.func @invalid_schedule(%A: memref<?xf32>, %result: memref<?xf32>) {972 %c0 = arith.constant 0 : index973 %c1 = arith.constant 1 : index974 %c4 = arith.constant 4 : index975 %cf = arith.constant 1.0 : f32976 scf.for %i0 = %c0 to %c4 step %c1 {977 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>978 %A1_elem = arith.addf %A_elem, %cf { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 0 } : f32979 // expected-error@+1 {{operation scheduled before its operands}}980 memref.store %A1_elem, %result[%i0] { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : memref<?xf32>981 } { __test_pipelining_loop__ }982 return983}984 985// -----986 987func.func @invalid_schedule2(%A: memref<?xf32>, %result: memref<?xf32>) {988 %c0 = arith.constant 0 : index989 %c1 = arith.constant 1 : index990 %c4 = arith.constant 4 : index991 %cf = arith.constant 1.0 : f32992 %r = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%idx = %c0) -> (index) {993 // expected-error@+1 {{operation scheduled before its operands}}994 %A_elem = memref.load %A[%idx] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 0 } : memref<?xf32>995 %idx1 = arith.addi %idx, %c1 { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : index996 memref.store %A_elem, %result[%idx] { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 2 } : memref<?xf32>997 scf.yield %idx1 : index998 } { __test_pipelining_loop__ }999 return1000}1001 1002// -----1003 1004func.func @invalid_schedule3(%A: memref<?xf32>, %result: memref<?xf32>, %ext: index) {1005 %c0 = arith.constant 0 : index1006 %c1 = arith.constant 1 : index1007 %c4 = arith.constant 4 : index1008 %r = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%idx = %c0) -> (index) {1009 %cnd = arith.cmpi slt, %ext, %c4 { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 0 } : index1010 // expected-error@+1 {{operation scheduled before its operands}}1011 %idx1 = scf.if %cnd -> (index) {1012 %idxinc = arith.addi %idx, %c1 : index1013 scf.yield %idxinc : index1014 } else {1015 scf.yield %idx : index1016 } { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 1 }1017 %A_elem = memref.load %A[%idx1] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>1018 %idx2 = arith.addi %idx1, %c1 { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 3 } : index1019 memref.store %A_elem, %result[%idx1] { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 4 } : memref<?xf32>1020 scf.yield %idx2 : index1021 } { __test_pipelining_loop__ }1022 return1023}1024 1025// -----1026 1027// Ensure this case not crash when step is zero.1028 1029// CHECK-LABEL: @invalid_loop_step1030func.func @invalid_loop_step(%A: memref<?xf32>, %result: memref<?xf32>) {1031 %c0 = arith.constant 0 : index1032 %cf = arith.constant 1.0 : f321033 scf.for %i0 = %c0 to %c0 step %c0 {1034 %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>1035 %A1_elem = arith.addf %A_elem, %cf { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 0 } : f321036 memref.store %A1_elem, %result[%i0] { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : memref<?xf32>1037 } { __test_pipelining_loop__ }1038 return1039}1040