1174 lines · plain
1// RUN: mlir-opt -transform-interpreter -canonicalize --split-input-file --allow-unregistered-dialect %s | FileCheck %s2 3///----------------------------------------------------------------------------------------4/// Tests for vector.transfer_read + vector.transfer_write pairs5///6/// * Nested inside a single loop7// * Indices are constant8///----------------------------------------------------------------------------------------9 10// The most basic example - hoisting is safe.11 12// CHECK-LABEL: func.func @hoist_basic_vector_xfer_pair(13// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,14// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,15// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,16// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index) {17func.func @hoist_basic_vector_xfer_pair(18 %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {19 %c0 = arith.constant 0 : index20 %pad = arith.constant 0.0 : f3221 22// CHECK: %[[C0:.*]] = arith.constant 0 : index23// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f3224// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>25// CHECK: %[[SCF:.*]] = scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[INIT:.*]] = %[[READ]]) -> (vector<1xf32>) {26// CHECK: %[[VAL_6:.*]] = "val_use"(%[[INIT]]) : (vector<1xf32>) -> vector<1xf32>27// CHECK: scf.yield %[[VAL_6]] : vector<1xf32>28// CHECK: }29// CHECK: vector.transfer_write %[[SCF]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>30 scf.for %i = %lb to %ub step %step {31 %r0 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>32 %u0 = "val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>33 vector.transfer_write %u0, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>34 }35 return36}37 38module attributes {transform.with_named_sequence} {39 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {40 %0 = transform.structured.match ops{["func.func"]} in %arg141 : (!transform.any_op) -> !transform.any_op42 transform.structured.hoist_redundant_vector_transfers %043 : (!transform.any_op) -> !transform.any_op44 transform.yield45 }46}47 48// -----49 50// Similar as the example above, but hoisting is no longer safe. That's due to51// an extra xfer_write inside the loop.52 53// CHECK-LABEL: func.func @negative_hoist_basic_vector_xfer_pair_extra_write(54// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,55// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,56// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,57// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index,58// CHECK-SAME: %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {59func.func @negative_hoist_basic_vector_xfer_pair_extra_write(60 %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {61 %c0 = arith.constant 0 : index62 %pad = arith.constant 0.0 : f3263 64// CHECK: %[[C0:.*]] = arith.constant 0 : index65// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f3266// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {67// CHECK: vector.transfer_write %[[IN]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>68// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>69// CHECK: %[[USE:.*]] = "val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>70// CHECK: vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>71// CHECK: }72 73 scf.for %i = %lb to %ub step %step {74 vector.transfer_write %in, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>75 76 %r0 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>77 %u0 = "val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>78 vector.transfer_write %u0, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>79 }80 return81}82 83module attributes {transform.with_named_sequence} {84 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {85 %0 = transform.structured.match ops{["func.func"]} in %arg186 : (!transform.any_op) -> !transform.any_op87 transform.structured.hoist_redundant_vector_transfers %088 : (!transform.any_op) -> !transform.any_op89 transform.yield90 }91}92 93// -----94 95// Similar as the example above, but hoisting is no longer safe. That's due to96// an extra xfer_write into _an alias_ of the %mem Op that is used by the97// original xfer pair.98 99// CHECK-LABEL: func.func @negative_hoist_basic_vector_xfer_pair_extra_write_into_alias(100// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,101// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,102// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,103// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index,104// CHECK-SAME: %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {105func.func @negative_hoist_basic_vector_xfer_pair_extra_write_into_alias(106 %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {107 %c0 = arith.constant 0 : index108 %pad = arith.constant 0.0 : f32109 110// CHECK: %[[C0:.*]] = arith.constant 0 : index111// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32112// CHECK: %[[SV:.*]] = memref.subview %[[MEM]][0, 0] [1, 1] [1, 1] : memref<?x?xf32> to memref<1x1xf32, strided<[?, 1]>>113// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {114// CHECK: vector.transfer_write %[[IN]], %[[SV]][%[[C0]], %[[C0]]] {{.*}} : vector<1xf32>, memref<1x1xf32, strided<[?, 1]>>115// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>116// CHECK: %[[USE:.*]] = "val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>117// CHECK: vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>118// CHECK: }119 120 %sv = memref.subview %mem[0, 0][1, 1][1, 1] : memref<?x?xf32> to memref<1x1xf32, strided<[?, 1]>>121 scf.for %i = %lb to %ub step %step {122 vector.transfer_write %in, %sv[%c0, %c0] : vector<1xf32>, memref<1x1xf32, strided<[?, 1]>>123 124 %r0 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>125 %u0 = "val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>126 vector.transfer_write %u0, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>127 }128 return129}130 131module attributes {transform.with_named_sequence} {132 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {133 %0 = transform.structured.match ops{["func.func"]} in %arg1134 : (!transform.any_op) -> !transform.any_op135 transform.structured.hoist_redundant_vector_transfers %0136 : (!transform.any_op) -> !transform.any_op137 transform.yield138 }139}140 141// -----142 143// Similar as the example above, but the memory access is done via144// memref.assume_alignment. Hoisting is safe as the only users of the145// "allignment" Op are the xfer Ops within the loop that we want to hoist.146 147// CHECK-LABEL: func.func @hoist_basic_vector_xfer_pair_with_assume_align(148// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,149// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,150// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,151// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index,152// CHECK-SAME: %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {153func.func @hoist_basic_vector_xfer_pair_with_assume_align(154 %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {155 %c0 = arith.constant 0 : index156 %pad = arith.constant 0.0 : f32157 158// CHECK: %[[C0:.*]] = arith.constant 0 : index159// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32160// CHECK: %[[AA:.*]] = memref.assume_alignment %[[MEM]], 4 : memref<?x?xf32>161// CHECK: %[[READ:.*]] = vector.transfer_read %[[AA]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>162// CHECK: %[[SCF:.*]] = scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[INIT:.*]] = %[[READ]]) -> (vector<1xf32>) {163// CHECK: %[[USE:.*]] = "val_use"(%[[INIT]]) : (vector<1xf32>) -> vector<1xf32>164// CHECK: }165// CHECK: vector.transfer_write %[[SCF]], %[[AA]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>166 167 %aa = memref.assume_alignment %mem, 4 : memref<?x?xf32>168 scf.for %i = %lb to %ub step %step {169 %r0 = vector.transfer_read %aa[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>170 %u0 = "val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>171 vector.transfer_write %u0, %aa[%c0, %c0] : vector<1xf32>, memref<?x?xf32>172 }173 return174}175 176module attributes {transform.with_named_sequence} {177 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {178 %0 = transform.structured.match ops{["func.func"]} in %arg1179 : (!transform.any_op) -> !transform.any_op180 transform.structured.hoist_redundant_vector_transfers %0181 : (!transform.any_op) -> !transform.any_op182 transform.yield183 }184}185 186// -----187 188// Similar as the example above, but hoisting is not safe due to extra memory189// access inside the loop via the original memref.190 191// CHECK-LABEL: func.func @negative_hoist_basic_vector_xfer_pair_with_assume_align(192// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,193// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,194// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,195// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index,196// CHECK-SAME: %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {197func.func @negative_hoist_basic_vector_xfer_pair_with_assume_align(198 %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {199 %c0 = arith.constant 0 : index200 %pad = arith.constant 0.0 : f32201 202// CHECK: %[[C0:.*]] = arith.constant 0 : index203// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32204// CHECK: %[[AA:.*]] = memref.assume_alignment %[[MEM]], 4 : memref<?x?xf32>205// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {206// CHECK: %[[READ:.*]] = vector.transfer_read %[[AA]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>207// CHECK: "mem_use"(%[[MEM]])208// CHECK: vector.transfer_write %[[READ]], %[[AA]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>209// CHECK: }210 211 %aa = memref.assume_alignment %mem, 4 : memref<?x?xf32>212 scf.for %i = %lb to %ub step %step {213 %r0 = vector.transfer_read %aa[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>214 "mem_use"(%mem) : (memref<?x?xf32>) -> ()215 vector.transfer_write %r0, %aa[%c0, %c0] : vector<1xf32>, memref<?x?xf32>216 }217 return218}219 220module attributes {transform.with_named_sequence} {221 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {222 %0 = transform.structured.match ops{["func.func"]} in %arg1223 : (!transform.any_op) -> !transform.any_op224 transform.structured.hoist_redundant_vector_transfers %0225 : (!transform.any_op) -> !transform.any_op226 transform.yield227 }228}229 230// -----231 232///----------------------------------------------------------------------------------------233/// Tests for vector.transfer_read + vector.transfer_write pairs234///235/// * Nested in double loops236// * Indices depend on induction variables237///----------------------------------------------------------------------------------------238 239// CHECK-LABEL: func @mem_use_outside240// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,241// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,242// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,243// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)244func.func @mem_use_outside(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {245 %pad = arith.constant 0.0 : f32246 247// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32248// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {249// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[I]], %[[I]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>250// CHECK: %[[SCF:.*]] = scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[VAL_5:.*]] = %[[READ]]) -> (vector<1xf32>) {251// CHECK: %[[USE:.*]] = "val_use"(%[[VAL_5]]) : (vector<1xf32>) -> vector<1xf32>252// CHECK: scf.yield %[[USE]] : vector<1xf32>253// CHECK: }254// CHECK: vector.transfer_write %[[SCF]], %[[MEM]][%[[I]], %[[I]]] : vector<1xf32>, memref<?x?xf32>255// CHECK: "mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()256// CHECK: }257 scf.for %i = %lb to %ub step %step {258 scf.for %j = %lb to %ub step %step {259 %read = vector.transfer_read %mem[%i, %i], %pad: memref<?x?xf32>, vector<1xf32>260 %use = "val_use"(%read) : (vector<1xf32>) -> vector<1xf32>261 vector.transfer_write %use, %mem[%i, %i] : vector<1xf32>, memref<?x?xf32>262 }263 }264 "mem_use"(%mem) : (memref<?x?xf32>) -> ()265 return266}267 268module attributes {transform.with_named_sequence} {269 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {270 %0 = transform.structured.match ops{["func.func"]} in %arg1271 : (!transform.any_op) -> !transform.any_op272 transform.structured.hoist_redundant_vector_transfers %0273 : (!transform.any_op) -> !transform.any_op274 transform.yield275 }276}277 278// -----279 280// CHECK-LABEL: func @mem_use_inside_outer_loop281// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,282// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,283// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,284// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)285func.func @mem_use_inside_outer_loop(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {286 %pad = arith.constant 0.0 : f32287 288// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32289// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {290// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]]{{\[}}%[[I]], %[[I]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>291// CHECK: %[[SCF:.*]] = scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[VAL_5:.*]] = %[[READ]]) -> (vector<1xf32>) {292// CHECK: %[[USE:.*]] = "val_use"(%[[VAL_5]]) : (vector<1xf32>) -> vector<1xf32>293// CHECK: scf.yield %[[USE]] : vector<1xf32>294// CHECK: }295// CHECK: vector.transfer_write %[[SCF]], %[[MEM]]{{\[}}%[[I]], %[[I]]] : vector<1xf32>, memref<?x?xf32>296// CHECK: "mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()297// CHECK: }298 scf.for %i = %lb to %ub step %step {299 scf.for %j = %lb to %ub step %step {300 %read = vector.transfer_read %mem[%i, %i], %pad: memref<?x?xf32>, vector<1xf32>301 %use = "val_use"(%read) : (vector<1xf32>) -> vector<1xf32>302 vector.transfer_write %use, %mem[%i, %i] : vector<1xf32>, memref<?x?xf32>303 }304 "mem_use"(%mem) : (memref<?x?xf32>) -> ()305 }306 return307}308 309module attributes {transform.with_named_sequence} {310 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {311 %0 = transform.structured.match ops{["func.func"]} in %arg1312 : (!transform.any_op) -> !transform.any_op313 transform.structured.hoist_redundant_vector_transfers %0314 : (!transform.any_op) -> !transform.any_op315 transform.yield316 }317}318 319// -----320 321///----------------------------------------------------------------------------------------322/// Tests for vector.transfer_read + vector.transfer_write pairs323///324/// * Nested in double loops325// * Indices are constant326///----------------------------------------------------------------------------------------327 328// CHECK-LABEL: func @negative_mem_use_inside_inner_loop_before_write329// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,330// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,331// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,332// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)333func.func @negative_mem_use_inside_inner_loop_before_write(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {334 %c0 = arith.constant 0 : index335 %pad = arith.constant 0.0 : f32336 337// CHECK: %[[C0:.*]] = arith.constant 0 : index338// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32339// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {340// CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {341// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>342// CHECK: %[[USE:.*]] = "val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>343// CHECK: "mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()344// CHECK: vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>345// CHECK: }346// CHECK: }347 scf.for %i = %lb to %ub step %step {348 scf.for %j = %lb to %ub step %step {349 %read = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>350 %use = "val_use"(%read) : (vector<1xf32>) -> vector<1xf32>351 "mem_use"(%mem) : (memref<?x?xf32>) -> ()352 vector.transfer_write %use, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>353 }354 }355 return356}357 358module attributes {transform.with_named_sequence} {359 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {360 %0 = transform.structured.match ops{["func.func"]} in %arg1361 : (!transform.any_op) -> !transform.any_op362 transform.structured.hoist_redundant_vector_transfers %0363 : (!transform.any_op) -> !transform.any_op364 transform.yield365 }366}367 368// -----369 370// CHECK-LABEL: func @negative_mem_use_inside_inner_loop_after_write371// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,372// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,373// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,374// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)375func.func @negative_mem_use_inside_inner_loop_after_write(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {376 %c0 = arith.constant 0 : index377 %pad = arith.constant 0.0 : f32378 379// CHECK: %[[C0:.*]] = arith.constant 0 : index380// CHECK: %[[PAD:.*]] = arith.constant 0.000000e+00 : f32381// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {382// CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {383// CHECK: %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>384// CHECK: %[[USE:.*]] = "val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>385// CHECK: vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>386// CHECK: "mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()387// CHECK: }388// CHECK: }389 scf.for %i = %lb to %ub step %step {390 scf.for %j = %lb to %ub step %step {391 %r3 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>392 %u3 = "val_use"(%r3) : (vector<1xf32>) -> vector<1xf32>393 vector.transfer_write %u3, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>394 "mem_use"(%mem) : (memref<?x?xf32>) -> ()395 }396 }397 return398}399 400module attributes {transform.with_named_sequence} {401 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {402 %0 = transform.structured.match ops{["func.func"]} in %arg1403 : (!transform.any_op) -> !transform.any_op404 transform.structured.hoist_redundant_vector_transfers %0405 : (!transform.any_op) -> !transform.any_op406 transform.yield407 }408}409 410// -----411 412// CHECK-LABEL: func @negative_mem_use_inside_inner_loop_before_read413// CHECK-SAME: %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,414// CHECK-SAME: %[[LB:[a-zA-Z0-9]+]]: index,415// CHECK-SAME: %[[UB:[a-zA-Z0-9]+]]: index,416// CHECK-SAME: %[[STEP:[a-zA-Z0-9]+]]: index)417func.func @negative_mem_use_inside_inner_loop_before_read(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {418 %c0 = arith.constant 0 : index419 %pad = arith.constant 0.0 : f32420 421// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {422// CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {423// CHECK: "mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()424// CHECK: vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<1xf32>425// CHECK: "val_use"(%{{.*}}) : (vector<1xf32>) -> vector<1xf32>426// CHECK: vector.transfer_write %{{.*}} : vector<1xf32>, memref<?x?xf32>427// CHECK: }428// CHECK: }429 scf.for %i = %lb to %ub step %step {430 scf.for %j = %lb to %ub step %step {431 "mem_use"(%mem) : (memref<?x?xf32>) -> ()432 %read = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>433 %use = "val_use"(%read) : (vector<1xf32>) -> vector<1xf32>434 vector.transfer_write %use, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>435 }436 }437 return438}439 440module attributes {transform.with_named_sequence} {441 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {442 %0 = transform.structured.match ops{["func.func"]} in %arg1443 : (!transform.any_op) -> !transform.any_op444 transform.structured.hoist_redundant_vector_transfers %0445 : (!transform.any_op) -> !transform.any_op446 transform.yield447 }448}449 450// -----451 452///----------------------------------------------------------------------------------------453/// Other tests454///455/// TODO: Document456///----------------------------------------------------------------------------------------457 458// CHECK-LABEL: func @hoist_vector_transfer_pairs_disjoint(459// CHECK-SAME: %[[MEMREF0:[a-zA-Z0-9]*]]: memref<?x?xf32>,460// CHECK-SAME: %[[MEMREF1:[a-zA-Z0-9]*]]: memref<?x?xf32>,461// CHECK-SAME: %[[MEMREF2:[a-zA-Z0-9]*]]: memref<?x?xf32>,462// CHECK-SAME: %[[MEMREF3:[a-zA-Z0-9]*]]: memref<?x?xf32>,463// CHECK-SAME: %[[VAL:[a-zA-Z0-9]*]]: index,464// CHECK-SAME: %[[LB:[a-zA-Z0-9]*]]: index,465// CHECK-SAME: %[[UB:[a-zA-Z0-9]*]]: index,466// CHECK-SAME: %[[STEP:[a-zA-Z0-9]*]]: index,467// CHECK-SAME: %[[RANDOM:[a-zA-Z0-9]*]]: index,468// CHECK-SAME: %[[CMP:[a-zA-Z0-9]*]]: i1469func.func @hoist_vector_transfer_pairs_disjoint(470 %memref0: memref<?x?xf32>, %memref1: memref<?x?xf32>,471 %memref2: memref<?x?xf32>, %memref3: memref<?x?xf32>, %val: index, %lb : index, %ub : index,472 %step: index, %random_index : index, %cmp: i1) {473 %c0 = arith.constant 0 : index474 %c1 = arith.constant 1 : index475 %c3 = arith.constant 3 : index476 %cst = arith.constant 0.0 : f32477 478// CHECK: vector.transfer_read %[[MEMREF2]]{{.*}} : memref<?x?xf32>, vector<3xf32>479// CHECK: vector.transfer_read %[[MEMREF2]]{{.*}} : memref<?x?xf32>, vector<3xf32>480// CHECK: vector.transfer_read %[[MEMREF3]]{{.*}} : memref<?x?xf32>, vector<4xf32>481// CHECK: vector.transfer_read %[[MEMREF3]]{{.*}} : memref<?x?xf32>, vector<4xf32>482// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) ->483// CHECK-SAME: (vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) {484// CHECK: scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) ->485// CHECK-SAME: (vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>) {486// CHECK: vector.transfer_read %[[MEMREF1]]{{.*}} : memref<?x?xf32>, vector<2xf32>487// CHECK: vector.transfer_read %[[MEMREF1]]{{.*}} : memref<?x?xf32>, vector<2xf32>488// CHECK: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>489// CHECK: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>490// CHECK: "some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32>491// CHECK: "some_use"(%{{.*}}) : (vector<3xf32>) -> vector<3xf32>492// CHECK: "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>493// CHECK: "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>494// CHECK: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>495// CHECK: "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>496// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF1]]{{.*}} : vector<2xf32>, memref<?x?xf32>497// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF1]]{{.*}} : vector<2xf32>, memref<?x?xf32>498// CHECK: scf.yield {{.*}} : vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>499// CHECK: }500// CHECK: scf.yield {{.*}} : vector<3xf32>, vector<3xf32>, vector<4xf32>, vector<4xf32>501// CHECK: }502// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF3]]{{.*}} : vector<4xf32>, memref<?x?xf32>503// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF3]]{{.*}} : vector<4xf32>, memref<?x?xf32>504// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF2]]{{.*}} : vector<3xf32>, memref<?x?xf32>505// CHECK: vector.transfer_write %{{.*}}, %[[MEMREF2]]{{.*}} : vector<3xf32>, memref<?x?xf32>506 scf.for %i = %lb to %ub step %step {507 scf.for %j = %lb to %ub step %step {508 %r00 = vector.transfer_read %memref1[%c0, %c0], %cst: memref<?x?xf32>, vector<2xf32>509 %r01 = vector.transfer_read %memref1[%c0, %c1], %cst: memref<?x?xf32>, vector<2xf32>510 %r20 = vector.transfer_read %memref2[%c0, %c0], %cst: memref<?x?xf32>, vector<3xf32>511 %r21 = vector.transfer_read %memref2[%c0, %c3], %cst: memref<?x?xf32>, vector<3xf32>512 %r30 = vector.transfer_read %memref3[%c0, %random_index], %cst: memref<?x?xf32>, vector<4xf32>513 %r31 = vector.transfer_read %memref3[%c1, %random_index], %cst: memref<?x?xf32>, vector<4xf32>514 %r10 = vector.transfer_read %memref0[%i, %i], %cst: memref<?x?xf32>, vector<2xf32>515 %r11 = vector.transfer_read %memref0[%random_index, %random_index], %cst: memref<?x?xf32>, vector<2xf32>516 %u00 = "some_use"(%r00) : (vector<2xf32>) -> vector<2xf32>517 %u01 = "some_use"(%r01) : (vector<2xf32>) -> vector<2xf32>518 %u20 = "some_use"(%r20) : (vector<3xf32>) -> vector<3xf32>519 %u21 = "some_use"(%r21) : (vector<3xf32>) -> vector<3xf32>520 %u30 = "some_use"(%r30) : (vector<4xf32>) -> vector<4xf32>521 %u31 = "some_use"(%r31) : (vector<4xf32>) -> vector<4xf32>522 %u10 = "some_use"(%r10) : (vector<2xf32>) -> vector<2xf32>523 %u11 = "some_use"(%r11) : (vector<2xf32>) -> vector<2xf32>524 vector.transfer_write %u00, %memref1[%c0, %c0] : vector<2xf32>, memref<?x?xf32>525 vector.transfer_write %u01, %memref1[%c0, %c1] : vector<2xf32>, memref<?x?xf32>526 vector.transfer_write %u20, %memref2[%c0, %c0] : vector<3xf32>, memref<?x?xf32>527 vector.transfer_write %u21, %memref2[%c0, %c3] : vector<3xf32>, memref<?x?xf32>528 vector.transfer_write %u30, %memref3[%c0, %random_index] : vector<4xf32>, memref<?x?xf32>529 vector.transfer_write %u31, %memref3[%c1, %random_index] : vector<4xf32>, memref<?x?xf32>530 vector.transfer_write %u10, %memref0[%i, %i] : vector<2xf32>, memref<?x?xf32>531 vector.transfer_write %u11, %memref0[%random_index, %random_index] : vector<2xf32>, memref<?x?xf32>532 }533 }534 return535}536 537module attributes {transform.with_named_sequence} {538 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {539 %0 = transform.structured.match ops{["func.func"]} in %arg1540 : (!transform.any_op) -> !transform.any_op541 transform.structured.hoist_redundant_vector_transfers %0542 : (!transform.any_op) -> !transform.any_op543 transform.yield544 }545}546 547// -----548 549// CHECK-LABEL: func @hoist_vector_transfer_pairs_in_affine_loops(550// CHECK-SAME: %[[MEMREF0:[a-zA-Z0-9]+]]: memref<64x64xi32>,551// CHECK-SAME: %[[MEMREF1:[a-zA-Z0-9]+]]: memref<64x64xi32>,552// CHECK-SAME: %[[MEMREF2:[a-zA-Z0-9]+]]: memref<64x64xi32>) {553// CHECK: %[[C0:.*]] = arith.constant 0 : i32554// CHECK: affine.for %[[I:.*]] = 0 to 64 {555// CHECK: affine.for %[[J:.*]] = 0 to 64 step 16 {556// CHECK: %[[R0:.*]] = vector.transfer_read %[[MEMREF2]][%[[I]], %[[J]]], %[[C0]] : memref<64x64xi32>, vector<16xi32>557// CHECK: %[[R:.*]] = affine.for %[[K:.*]] = 0 to 64 iter_args(%[[ACC:.*]] = %[[R0]]) -> (vector<16xi32>) {558// CHECK: %[[AV:.*]] = vector.transfer_read %[[MEMREF0]][%[[I]], %[[K]]], %[[C0]] {{.*}}: memref<64x64xi32>, vector<16xi32>559// CHECK: %[[BV:.*]] = vector.transfer_read %[[MEMREF1]][%[[K]], %[[J]]], %[[C0]] {{.*}}: memref<64x64xi32>, vector<16xi32>560// CHECK: %[[T0:.*]] = arith.muli %[[AV]], %[[BV]] : vector<16xi32>561// CHECK: %[[T1:.*]] = arith.addi %[[ACC]], %[[T0]] : vector<16xi32>562// CHECK: affine.yield %[[T1]] : vector<16xi32>563// CHECK: }564// CHECK: vector.transfer_write %[[R]], %[[MEMREF2]][%[[I]], %[[J]]] : vector<16xi32>, memref<64x64xi32>565// CHECK: }566// CHECK: }567func.func @hoist_vector_transfer_pairs_in_affine_loops(%memref0: memref<64x64xi32>, %memref1: memref<64x64xi32>, %memref2: memref<64x64xi32>) {568 %c0_i32 = arith.constant 0 : i32569 affine.for %arg3 = 0 to 64 {570 affine.for %arg4 = 0 to 64 step 16 {571 affine.for %arg5 = 0 to 64 {572 %0 = vector.transfer_read %memref0[%arg3, %arg5], %c0_i32 {permutation_map = affine_map<(d0, d1) -> (0)>} : memref<64x64xi32>, vector<16xi32>573 %1 = vector.transfer_read %memref1[%arg5, %arg4], %c0_i32 : memref<64x64xi32>, vector<16xi32>574 %2 = vector.transfer_read %memref2[%arg3, %arg4], %c0_i32 : memref<64x64xi32>, vector<16xi32>575 %3 = arith.muli %0, %1 : vector<16xi32>576 %4 = arith.addi %2, %3 : vector<16xi32>577 vector.transfer_write %4, %memref2[%arg3, %arg4] : vector<16xi32>, memref<64x64xi32>578 }579 }580 }581 return582}583 584module attributes {transform.with_named_sequence} {585 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {586 %0 = transform.structured.match ops{["func.func"]} in %arg1587 : (!transform.any_op) -> !transform.any_op588 transform.structured.hoist_redundant_vector_transfers %0589 : (!transform.any_op) -> !transform.any_op590 transform.yield591 }592}593 594// -----595 596// CHECK-LABEL: func.func @hoist_vector_transfer_read(597// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index598// CHECK-DAG: %[[C128:.+]] = arith.constant 128 : index599// CHECK-DAG: %[[C1024:.+]] = arith.constant 1024 : index600// CHECK-DAG: %[[CST:.+]] = arith.constant 0.000000e+00 : f32601// CHECK: %[[ALLOC:.+]] = memref.alloc() : memref<32x64xf32>602// CHECK: %[[ALLOC_0:.+]] = memref.alloc() : memref<32x128xf32>603// CHECK: %[[CAST:.+]] = memref.cast %[[ALLOC_0]] : memref<32x128xf32> to memref<32x128xf32, strided<[128, 1],604// CHECK-SAME: offset: ?>>605// CHECK: %[[D0:.+]] = vector.transfer_read %[[ALLOC]][%[[C0]], %[[C0]]], %[[CST]] {in_bounds = [true, true]} :606// CHECK-SAME: memref<32x64xf32>, vector<32x64xf32>607// CHECK: scf.for %[[ARG0:.+]] = %[[C0]] to %[[C1024]] step %[[C128]] {608// CHECK: %[[D1:.+]] = vector.transfer_read %[[ALLOC_0]][%[[C0]], %[[C0]]], %[[CST]] {in_bounds = [true, true]}609// CHECK-SAME: : memref<32x128xf32>, vector<32x128xf32>610// CHECK: "some_use"(%[[D0]], %[[D1]], %[[CAST]]) : (vector<32x64xf32>, vector<32x128xf32>, memref<32x128xf32,611// CHECK-SAME: strided<[128, 1], offset: ?>>) -> ()612// CHECK: }613// CHECK: memref.dealloc %[[ALLOC]] : memref<32x64xf32>614// CHECK: return615func.func @hoist_vector_transfer_read() {616 %c0 = arith.constant 0 : index617 %c128 = arith.constant 128 : index618 %c1024 = arith.constant 1024 : index619 %cst_2 = arith.constant 0.000000e+00 : f32620 %memref0 = memref.alloc() : memref<32x64xf32>621 %memref2 = memref.alloc() : memref<32x128xf32>622 %subview2 = memref.subview %memref2[%c0, %c0] [32, 128] [1, 1]: memref<32x128xf32> to memref<32x128xf32, strided<[128, 1], offset: ?>>623 scf.for %arg0 = %c0 to %c1024 step %c128 {624 %2 = vector.transfer_read %memref2[%c0, %c0], %cst_2 {in_bounds = [true, true]} : memref<32x128xf32>, vector<32x128xf32>625 %3 = vector.transfer_read %memref0[%c0, %c0], %cst_2 {in_bounds = [true, true]} : memref<32x64xf32>, vector<32x64xf32>626 "some_use"(%3, %2, %subview2) : (vector<32x64xf32>, vector<32x128xf32>, memref<32x128xf32, strided<[128, 1], offset: ?>>) -> ()627 }628 memref.dealloc %memref0 : memref<32x64xf32>629 return630}631 632module attributes {transform.with_named_sequence} {633 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {634 %0 = transform.structured.match ops{["func.func"]} in %arg1635 : (!transform.any_op) -> !transform.any_op636 transform.structured.hoist_redundant_vector_transfers %0637 : (!transform.any_op) -> !transform.any_op638 transform.yield639 }640}641 642// -----643 644// The transfers in this test case cannot be hoisted and replaced by a vector645// iter_arg because they do not match.646 647// CHECK-LABEL: func.func @non_matching_transfers(648// CHECK: scf.for {{.*}} {649// CHECK: vector.transfer_read650// CHECK: vector.transfer_write651// CHECK: }652func.func @non_matching_transfers(%m: memref<6x1x7x32xf32>) {653 %c0 = arith.constant 0 : index654 %c1024 = arith.constant 1024 : index655 %c128 = arith.constant 128 : index656 %cst = arith.constant dense<5.5> : vector<6x7x32xf32>657 %cst_0 = arith.constant 0.0 : f32658 scf.for %iv = %c0 to %c1024 step %c128 {659 %read = vector.transfer_read %m[%c0, %c0, %c0, %c0], %cst_0 {in_bounds = [true, true, true], permutation_map = affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>} : memref<6x1x7x32xf32>, vector<6x7x32xf32>660 %added = arith.addf %read, %cst : vector<6x7x32xf32>661 %bc = vector.broadcast %added : vector<6x7x32xf32> to vector<1x6x7x32xf32>662 %tr = vector.transpose %bc, [1, 0, 2, 3] : vector<1x6x7x32xf32> to vector<6x1x7x32xf32>663 vector.transfer_write %tr, %m[%c0, %c0, %c0, %c0] {in_bounds = [true, true, true, true]} : vector<6x1x7x32xf32>, memref<6x1x7x32xf32>664 }665 return666}667 668module attributes {transform.with_named_sequence} {669 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {670 %0 = transform.structured.match ops{["func.func"]} in %arg1671 : (!transform.any_op) -> !transform.any_op672 transform.structured.hoist_redundant_vector_transfers %0673 : (!transform.any_op) -> !transform.any_op674 transform.yield675 }676}677 678// -----679 680// CHECK-LABEL: func.func @no_hoisting_unknown_bound_loop681func.func @no_hoisting_unknown_bound_loop(%memref0: memref<20xi32>, %lb: index, %ub: index) {682 %c0_i32 = arith.constant 0 : i32683 %c0 = arith.constant 0 : index684 %c1 = arith.constant 1 : index685 686 // %lb and %ub are unbounded, so do not hoist.687 // CHECK: scf.for {{.*}} {688 // CHECK-NEXT: vector.transfer_read689 // CHECK-NEXT: "test.some_use"690 scf.for %arg2 = %lb to %ub step %c1 {691 %read = vector.transfer_read %memref0[%c0], %c0_i32 {in_bounds = [true]} : memref<20xi32>, vector<4xi32>692 "test.some_use"(%read) : (vector<4xi32>) ->()693 }694 return695}696 697module attributes {transform.with_named_sequence} {698 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {699 %0 = transform.structured.match ops{["func.func"]} in %arg1700 : (!transform.any_op) -> !transform.any_op701 transform.structured.hoist_redundant_vector_transfers %0 { verify_non_zero_trip }702 : (!transform.any_op) -> !transform.any_op703 transform.yield704 }705}706 707// -----708 709// CHECK-LABEL: func.func @no_hoisting_possibly_zero_trip_loop710func.func @no_hoisting_possibly_zero_trip_loop(%memref0: memref<20xi32>, %lb: index, %ub: index) {711 %c0_i32 = arith.constant 0 : i32712 %c0 = arith.constant 0 : index713 %c1 = arith.constant 1 : index714 715 // %lb_0 is in range [%lb, 8], and %ub_0 is in range [4, %ub].716 // Since %lb_0 could be greater than %ub_0, do not hoist.717 %lb_0 = affine.min affine_map<(d0) -> (d0, 8)>(%lb)718 %ub_0 = affine.max affine_map<(d0) -> (d0, 4)>(%ub)719 720 // CHECK: scf.for {{.*}} {721 // CHECK-NEXT: vector.transfer_read722 // CHECK-NEXT: "test.some_use"723 scf.for %arg2 = %lb_0 to %ub_0 step %c1 {724 %read = vector.transfer_read %memref0[%c0], %c0_i32 {in_bounds = [true]} : memref<20xi32>, vector<4xi32>725 "test.some_use"(%read) : (vector<4xi32>) ->()726 }727 return728}729 730module attributes {transform.with_named_sequence} {731 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {732 %0 = transform.structured.match ops{["func.func"]} in %arg1733 : (!transform.any_op) -> !transform.any_op734 transform.structured.hoist_redundant_vector_transfers %0 { verify_non_zero_trip }735 : (!transform.any_op) -> !transform.any_op736 transform.yield737 }738}739 740// -----741 742// CHECK-LABEL: func.func @no_hoisting_possibly_zero_trip_loop_eq_lb_and_ub743func.func @no_hoisting_possibly_zero_trip_loop_eq_lb_and_ub(%memref0: memref<20xi32>, %lb: index, %ub: index) {744 %c0_i32 = arith.constant 0 : i32745 %c0 = arith.constant 0 : index746 %c1 = arith.constant 1 : index747 748 // %lb_0 is in range [%lb, 8], and %ub_0 is in range [8, %ub].749 // Since %lb_0 could be equal to %ub_0, do not hoist.750 %lb_0 = affine.min affine_map<(d0) -> (d0, 8)>(%lb)751 %ub_0 = affine.max affine_map<(d0) -> (d0, 8)>(%ub)752 753 // CHECK: scf.for {{.*}} {754 // CHECK-NEXT: vector.transfer_read755 // CHECK-NEXT: "test.some_use"756 scf.for %arg2 = %lb_0 to %ub_0 step %c1 {757 %read = vector.transfer_read %memref0[%c0], %c0_i32 {in_bounds = [true]} : memref<20xi32>, vector<4xi32>758 "test.some_use"(%read) : (vector<4xi32>) ->()759 }760 return761}762 763module attributes {transform.with_named_sequence} {764 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {765 %0 = transform.structured.match ops{["func.func"]} in %arg1766 : (!transform.any_op) -> !transform.any_op767 transform.structured.hoist_redundant_vector_transfers %0 { verify_non_zero_trip }768 : (!transform.any_op) -> !transform.any_op769 transform.yield770 }771}772 773// -----774 775// CHECK-LABEL: func.func @hoisting_non_zero_trip_loop776func.func @hoisting_non_zero_trip_loop(%memref0: memref<20xi32>, %lb: index, %ub: index) {777 %c0_i32 = arith.constant 0 : i32778 %c0 = arith.constant 0 : index779 %c1 = arith.constant 1 : index780 781 // %lb_0 is in range [%lb, 4], and %ub_0 is in range [8, %ub].782 // Since %lb_0 is guaranteed to be less than %ub_0, hoisting is possible.783 %lb_0 = affine.min affine_map<(d0) -> (d0, 4)>(%lb)784 %ub_0 = affine.max affine_map<(d0) -> (d0, 8)>(%ub)785 786 // CHECK: vector.transfer_read787 // CHECK: scf.for {{.*}} {788 // CHECK-NEXT: "test.some_use"789 scf.for %arg2 = %lb_0 to %ub_0 step %c1 {790 %read = vector.transfer_read %memref0[%c0], %c0_i32 {in_bounds = [true]} : memref<20xi32>, vector<4xi32>791 "test.some_use"(%read) : (vector<4xi32>) ->()792 }793 return794}795 796module attributes {transform.with_named_sequence} {797 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {798 %0 = transform.structured.match ops{["func.func"]} in %arg1799 : (!transform.any_op) -> !transform.any_op800 transform.structured.hoist_redundant_vector_transfers %0 { verify_non_zero_trip }801 : (!transform.any_op) -> !transform.any_op802 transform.yield803 }804}805 806// -----807 808// Regression test - `vector.transfer_read` below should not be hoisted.809// Indeed, %collapse_shape (written to by `vector.transfer_write`) and %alloca810// (read by `vector.transfer_read`) alias.811 812// CHECK-LABEL: func.func @no_hoisting_collapse_shape813// CHECK: scf.for {{.*}} {814// CHECK: vector.transfer_write {{.*}} : vector<4xi32>, memref<4xi32>815// CHECK-NEXT: vector.transfer_read {{.*}} : memref<1x4x1xi32>, vector<1x4x1xi32>816// CHECK-NEXT: vector.transfer_write {{.*}} : vector<1x4x1xi32>, memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>817// CHECK-NEXT: }818 819func.func @no_hoisting_collapse_shape(%in_0: memref<1x20x1xi32>, %1: memref<9x1xi32>, %vec: vector<4xi32>) {820 %c0_i32 = arith.constant 0 : i32821 %c0 = arith.constant 0 : index822 %c4 = arith.constant 4 : index823 %c20 = arith.constant 20 : index824 %alloca = memref.alloca() {alignment = 64 : i64} : memref<1x4x1xi32>825 scf.for %arg0 = %c0 to %c20 step %c4 {826 %subview = memref.subview %in_0[0, %arg0, 0] [1, 4, 1] [1, 1, 1] : memref<1x20x1xi32> to memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>827 %collapse_shape = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x4x1xi32> into memref<4xi32>828 vector.transfer_write %vec, %collapse_shape[%c0] {in_bounds = [true]} : vector<4xi32>, memref<4xi32>829 %read = vector.transfer_read %alloca[%c0, %c0, %c0], %c0_i32 {in_bounds = [true, true, true]} : memref<1x4x1xi32>, vector<1x4x1xi32>830 vector.transfer_write %read, %subview[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x4x1xi32>, memref<1x4x1xi32, strided<[20, 1, 1], offset: ?>>831 }832 return833}834 835module attributes {transform.with_named_sequence} {836 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {837 %0 = transform.structured.match ops{["func.func"]} in %arg1838 : (!transform.any_op) -> !transform.any_op839 transform.structured.hoist_redundant_vector_transfers %0840 : (!transform.any_op) -> !transform.any_op841 transform.yield842 }843}844 845// -----846 847// Regression test - `vector.transfer_read` below should not be hoisted.848// Indeed, %collapse_shape (read by `vector.transfer_read`) and %alloca849// (written to by `vector.transfer_write`) alias.850 851// CHECK-LABEL: func.func @no_hoisting_collapse_shape_2852// CHECK: scf.for {{.*}} {853// CHECK: vector.transfer_write854// CHECK: vector.transfer_read855 856func.func @no_hoisting_collapse_shape_2(%vec: vector<1x12x1xi32>) {857 %c0_i32 = arith.constant 0 : i32858 %c0 = arith.constant 0 : index859 %c4 = arith.constant 4 : index860 %c20 = arith.constant 20 : index861 %alloca = memref.alloca() {alignment = 64 : i64} : memref<1x12x1xi32>862 scf.for %arg0 = %c0 to %c20 step %c4 {863 %collapse_shape = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x12x1xi32> into memref<12xi32>864 vector.transfer_write %vec, %alloca[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x12x1xi32>, memref<1x12x1xi32>865 %read = vector.transfer_read %collapse_shape[%c0], %c0_i32 {in_bounds = [true]} : memref<12xi32>, vector<12xi32>866 "test.some_use"(%read) : (vector<12xi32>) ->()867 }868 return869}870 871module attributes {transform.with_named_sequence} {872 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {873 %0 = transform.structured.match ops{["func.func"]} in %arg1874 : (!transform.any_op) -> !transform.any_op875 transform.structured.hoist_redundant_vector_transfers %0876 : (!transform.any_op) -> !transform.any_op877 transform.yield878 }879}880 881// -----882 883// Regression test - hoisting the following `vector.transfer_{read|write}` pair884// would not be safe:885// %lhs = vector.transfer_read %collapsed_1[%c0]886// vector.transfer_write %op, %collapsed_1[%c0]887// That's because the following `vector.transfer_read` reads from the same888// memory (i.e. `%collapsed_1` and `%collapsed_2` alias):889// %acc = vector.transfer_read %collapsed_2[%c0]890 891// CHECK-LABEL: func.func @no_hoisting_write_to_buffer892// CHECK: scf.for {{.*}} {893// CHECK: vector.transfer_read {{.*}} : memref<2xi32>, vector<1xi32>894// CHECK-NEXT: vector.transfer_read {{.*}} : memref<2xi32>, vector<1xi32>895// CHECK-NEXT: vector.outerproduct {{.*}} : vector<1xi32>, i32896// CHECK-NEXT: vector.transfer_write {{.*}} : vector<1xi32>, memref<2xi32>897// CHECK-NEXT: }898 899func.func @no_hoisting_write_to_buffer(%rhs: i32, %arg1: vector<1xi32>) {900 %c0_i32 = arith.constant 0 : i32901 %c0 = arith.constant 0 : index902 %c1 = arith.constant 1 : index903 %c4 = arith.constant 4 : index904 %c20 = arith.constant 20 : index905 %alloca = memref.alloca() {alignment = 64 : i64} : memref<1x1x2xi32>906 %cast = memref.cast %alloca : memref<1x1x2xi32> to memref<1x1x2xi32>907 %collapsed_1 = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x1x2xi32> into memref<2xi32>908 scf.for %_ = %c0 to %c20 step %c4 {909 %collapsed_2 = memref.collapse_shape %alloca [[0, 1, 2]] : memref<1x1x2xi32> into memref<2xi32>910 %lhs = vector.transfer_read %collapsed_1[%c0], %c0_i32 {in_bounds = [true]} : memref<2xi32>, vector<1xi32>911 %acc = vector.transfer_read %collapsed_2[%c0], %c0_i32 {in_bounds = [true]} : memref<2xi32>, vector<1xi32>912 %op = vector.outerproduct %lhs, %rhs, %acc {kind = #vector.kind<add>} : vector<1xi32>, i32913 vector.transfer_write %op, %collapsed_1[%c0] {in_bounds = [true]} : vector<1xi32>, memref<2xi32>914 }915 return916}917 918module attributes {transform.with_named_sequence} {919 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {920 %0 = transform.structured.match ops{["func.func"]} in %arg1921 : (!transform.any_op) -> !transform.any_op922 transform.structured.hoist_redundant_vector_transfers %0923 : (!transform.any_op) -> !transform.any_op924 transform.yield925 }926}927 928// -----929 930// Test that we can hoist out 1-D read-write pairs whose indices are dynamic values.931 932// CHECK: #[[$MAP1:.+]] = affine_map<()[s0] -> (s0 + 1)>933// CHECK: #[[$MAP4:.+]] = affine_map<()[s0] -> (s0 + 4)>934 935// CHECK-LABEL: func.func @hoist_vector_transfer_pairs_disjoint_dynamic936// CHECK-SAME: (%[[BUFFER:.+]]: memref<?x?xf32>, %{{.+}}: index, %{{.+}}: index, %{{.+}}: index, %[[I0:.+]]: index)937 938// CHECK: %[[PLUS1:.+]] = affine.apply #[[$MAP1]]()[%[[I0]]]939// CHECK: %[[PLUS4:.+]] = affine.apply #[[$MAP4]]()[%[[I0]]]940// CHECK: %2 = vector.transfer_read %[[BUFFER]][%[[I0]], %[[I0]]]941// CHECK: %3 = vector.transfer_read %[[BUFFER]][%[[PLUS1]], %[[I0]]]942// CHECK: %4 = vector.transfer_read %[[BUFFER]][%[[PLUS1]], %[[PLUS4]]]943// CHECK-COUNT-2: scf.for %{{.+}} = {{.+}} -> (vector<4xf32>, vector<4xf32>, vector<4xf32>)944// CHECK-COUNT-3: "some_use"945// CHECK-COUNT-2: scf.yield {{.+}} : vector<4xf32>, vector<4xf32>, vector<4xf32>946// CHECK: vector.transfer_write %{{.+}}, %[[BUFFER]][%[[PLUS1]], %[[PLUS4]]]947// CHECK: vector.transfer_write %{{.+}}, %[[BUFFER]][%[[PLUS1]], %[[I0]]]948// CHECK: vector.transfer_write %{{.+}}, %[[BUFFER]][%[[I0]], %[[I0]]]949 950func.func @hoist_vector_transfer_pairs_disjoint_dynamic(951 %buffer: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %i0 : index) {952 %cst = arith.constant 0.0 : f32953 %i1 = affine.apply affine_map<(d0) -> (d0 + 1)>(%i0)954 %i2 = affine.apply affine_map<(d0) -> (d0 + 4)>(%i0)955 956 scf.for %i = %lb to %ub step %step {957 scf.for %j = %lb to %ub step %step {958 %r0 = vector.transfer_read %buffer[%i0, %i0], %cst: memref<?x?xf32>, vector<4xf32>959 // Disjoint leading dim960 %r1 = vector.transfer_read %buffer[%i1, %i0], %cst: memref<?x?xf32>, vector<4xf32>961 // Non-overlap trailing dim962 %r2 = vector.transfer_read %buffer[%i1, %i2], %cst: memref<?x?xf32>, vector<4xf32>963 %u0 = "some_use"(%r0) : (vector<4xf32>) -> vector<4xf32>964 %u1 = "some_use"(%r1) : (vector<4xf32>) -> vector<4xf32>965 %u2 = "some_use"(%r2) : (vector<4xf32>) -> vector<4xf32>966 vector.transfer_write %u0, %buffer[%i0, %i0] : vector<4xf32>, memref<?x?xf32>967 vector.transfer_write %u1, %buffer[%i1, %i0] : vector<4xf32>, memref<?x?xf32>968 vector.transfer_write %u2, %buffer[%i1, %i2] : vector<4xf32>, memref<?x?xf32>969 }970 }971 return972}973 974module attributes {transform.with_named_sequence} {975 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {976 %0 = transform.structured.match ops{["func.func"]} in %arg1977 : (!transform.any_op) -> !transform.any_op978 transform.structured.hoist_redundant_vector_transfers %0979 : (!transform.any_op) -> !transform.any_op980 transform.yield981 }982}983 984// -----985 986// Test that we cannot hoist out read-write pairs whose indices are overlapping.987 988// CHECK-LABEL: func.func @hoist_vector_transfer_pairs_overlapping_dynamic989// CHECK-COUNT-2: scf.for990// CHECK-COUNT-2: vector.transfer_read991// CHECK-COUNT-2: vector.transfer_write992 993func.func @hoist_vector_transfer_pairs_overlapping_dynamic(994 %buffer: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %i0 : index) {995 %cst = arith.constant 0.0 : f32996 %i1 = affine.apply affine_map<(d0) -> (d0 + 3)>(%i0)997 998 scf.for %i = %lb to %ub step %step {999 scf.for %j = %lb to %ub step %step {1000 %r0 = vector.transfer_read %buffer[%i0, %i0], %cst: memref<?x?xf32>, vector<4xf32>1001 // Overlapping range with the above1002 %r1 = vector.transfer_read %buffer[%i0, %i1], %cst: memref<?x?xf32>, vector<4xf32>1003 %u0 = "some_use"(%r0) : (vector<4xf32>) -> vector<4xf32>1004 %u1 = "some_use"(%r1) : (vector<4xf32>) -> vector<4xf32>1005 vector.transfer_write %u0, %buffer[%i0, %i0] : vector<4xf32>, memref<?x?xf32>1006 vector.transfer_write %u1, %buffer[%i0, %i1] : vector<4xf32>, memref<?x?xf32>1007 }1008 }1009 return1010}1011 1012module attributes {transform.with_named_sequence} {1013 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {1014 %0 = transform.structured.match ops{["func.func"]} in %arg11015 : (!transform.any_op) -> !transform.any_op1016 transform.structured.hoist_redundant_vector_transfers %01017 : (!transform.any_op) -> !transform.any_op1018 transform.yield1019 }1020}1021 1022// -----1023 1024// Test that we can hoist out 2-D read-write pairs whose indices are dynamic values.1025 1026// CHECK-LABEL: func.func @hoist_vector_transfer_pairs_disjoint_dynamic1027// CHECK-COUNT-3: vector.transfer_read1028// CHECK-COUNT-2: %{{.+}}:3 = scf.for {{.+}} -> (vector<16x8xf32>, vector<16x8xf32>, vector<16x8xf32>)1029// CHECK-COUNT-2: scf.yield {{.+}} : vector<16x8xf32>, vector<16x8xf32>, vector<16x8xf32>1030// CHECK-COUNT-3: vector.transfer_write1031// CHECK: return1032 1033func.func @hoist_vector_transfer_pairs_disjoint_dynamic(1034 %buffer: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %i0 : index, %i1 : index) {1035 %cst = arith.constant 0.0 : f321036 %i2 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16)>(%i1)1037 %i3 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16 + 8)>(%i1)1038 %i4 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16 + 16)>(%i1)1039 1040 scf.for %i = %lb to %ub step %step {1041 scf.for %j = %lb to %ub step %step {1042 %r0 = vector.transfer_read %buffer[%i0, %i2], %cst: memref<?x?xf32>, vector<16x8xf32>1043 %r1 = vector.transfer_read %buffer[%i0, %i3], %cst: memref<?x?xf32>, vector<16x8xf32>1044 %r2 = vector.transfer_read %buffer[%i0, %i4], %cst: memref<?x?xf32>, vector<16x8xf32>1045 %u0 = "some_use"(%r0) : (vector<16x8xf32>) -> vector<16x8xf32>1046 %u1 = "some_use"(%r1) : (vector<16x8xf32>) -> vector<16x8xf32>1047 %u2 = "some_use"(%r2) : (vector<16x8xf32>) -> vector<16x8xf32>1048 vector.transfer_write %u2, %buffer[%i0, %i4] : vector<16x8xf32>, memref<?x?xf32>1049 vector.transfer_write %u1, %buffer[%i0, %i3] : vector<16x8xf32>, memref<?x?xf32>1050 vector.transfer_write %u0, %buffer[%i0, %i2] : vector<16x8xf32>, memref<?x?xf32>1051 }1052 }1053 return1054}1055 1056module attributes {transform.with_named_sequence} {1057 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {1058 %0 = transform.structured.match ops{["func.func"]} in %arg11059 : (!transform.any_op) -> !transform.any_op1060 transform.structured.hoist_redundant_vector_transfers %01061 : (!transform.any_op) -> !transform.any_op1062 transform.yield1063 }1064}1065 1066// -----1067 1068// Test hoisting of vector.extract/vector.broadcast pairs1069 1070// CHECK-LABEL: func.func @hoist_vector_broadcasts1071// CHECK-SAME: (%{{.+}}: index, %{{.+}}: index, %{{.+}}: index, %[[VEC:.+]]: vector<3x4xf32>) -> vector<3x4xf32> {1072// CHECK: %[[EXTRACT:.+]] = vector.extract %[[VEC]][0] : vector<4xf32> from vector<3x4xf32>1073// CHECK-NEXT: %[[LOOP:.+]] = scf.for {{.*}} {1074// CHECK-NEXT: %[[USE:.+]] = "some_use"({{.*}}) : (vector<4xf32>) -> vector<4xf32>1075// CHECK-NEXT: scf.yield %[[USE]] : vector<4xf32>1076// CHECK-NEXT: }1077// CHECK-NEXT: %[[BCAST:.+]] = vector.broadcast %[[LOOP]] : vector<4xf32> to vector<3x4xf32>1078// CHECK-NEXT: return %[[BCAST]] : vector<3x4xf32>1079 1080func.func @hoist_vector_broadcasts(%lb : index, %ub : index, %step : index, %vec : vector<3x4xf32>) -> vector<3x4xf32> {1081 %bcast_vec = scf.for %arg0 = %lb to %ub step %step iter_args(%iarg = %vec) -> vector<3x4xf32> {1082 %extract = vector.extract %iarg[0] : vector<4xf32> from vector<3x4xf32>1083 %use = "some_use"(%extract) : (vector<4xf32>) -> vector<4xf32>1084 %broadcast = vector.broadcast %use : vector<4xf32> to vector<3x4xf32>1085 scf.yield %broadcast : vector<3x4xf32>1086 }1087 return %bcast_vec : vector<3x4xf32>1088}1089 1090module attributes {transform.with_named_sequence} {1091 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {1092 %0 = transform.structured.match ops{["func.func"]} in %arg11093 : (!transform.any_op) -> !transform.any_op1094 transform.structured.hoist_redundant_vector_broadcasts %01095 : (!transform.any_op) -> !transform.any_op1096 transform.yield1097 }1098}1099 1100// -----1101 1102// Test hoisting of vector.extract/vector.broadcast pairs with dynamic position1103 1104// CHECK-LABEL: func.func @hoist_vector_broadcasts1105// CHECK-SAME: (%{{.+}}: index, %{{.+}}: index, %{{.+}}: index, %[[VEC:.+]]: vector<3x4xf32>, %[[POS:.+]]: index) -> vector<3x4xf32> {1106// CHECK: %[[EXTRACT:.+]] = vector.extract %[[VEC]][%[[POS]]] : vector<4xf32> from vector<3x4xf32>1107// CHECK-NEXT: %[[LOOP:.+]] = scf.for {{.*}} {1108// CHECK-NEXT: %[[USE:.+]] = "some_use"({{.*}}) : (vector<4xf32>) -> vector<4xf32>1109// CHECK-NEXT: scf.yield %[[USE]] : vector<4xf32>1110// CHECK-NEXT: }1111// CHECK-NEXT: %[[BCAST:.+]] = vector.broadcast %[[LOOP]] : vector<4xf32> to vector<3x4xf32>1112// CHECK-NEXT: return %[[BCAST]] : vector<3x4xf32>1113 1114func.func @hoist_vector_broadcasts_dynamic(%lb : index, %ub : index, %step : index, %vec : vector<3x4xf32>, %pos: index) -> vector<3x4xf32> {1115 %bcast_vec = scf.for %arg0 = %lb to %ub step %step iter_args(%iarg = %vec) -> vector<3x4xf32> {1116 %extract = vector.extract %iarg[%pos] : vector<4xf32> from vector<3x4xf32>1117 %use = "some_use"(%extract) : (vector<4xf32>) -> vector<4xf32>1118 %broadcast = vector.broadcast %use : vector<4xf32> to vector<3x4xf32>1119 scf.yield %broadcast : vector<3x4xf32>1120 }1121 return %bcast_vec : vector<3x4xf32>1122}1123 1124module attributes {transform.with_named_sequence} {1125 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {1126 %0 = transform.structured.match ops{["func.func"]} in %arg11127 : (!transform.any_op) -> !transform.any_op1128 transform.structured.hoist_redundant_vector_broadcasts %01129 : (!transform.any_op) -> !transform.any_op1130 transform.yield1131 }1132}1133 1134// -----1135 1136// Test hoisting of vector.extract/vector.broadcast pairs with multiple iter_args1137 1138// CHECK-LABEL: func.func @hoist_vector_broadcasts_multiple1139// CHECK-SAME: (%{{.+}}: index, %{{.+}}: index, %{{.+}}: index, %[[VEC1:.+]]: vector<3x4xf32>,1140// CHECK-SAME: %[[VEC2:.+]]: vector<3x5xf32>) -> (vector<3x4xf32>, vector<3x5xf32>) {1141// CHECK-DAG: %[[EXTRACT1:.+]] = vector.extract %[[VEC1]][0] : vector<4xf32> from vector<3x4xf32>1142// CHECK-DAG: %[[EXTRACT2:.+]] = vector.extract %[[VEC2]][1] : vector<5xf32> from vector<3x5xf32>1143// CHECK-NEXT: %[[LOOP:.+]]:2 = scf.for {{.*}} {1144// CHECK-DAG: %[[USE1:.+]] = "some_use1"({{.*}}) : (vector<4xf32>) -> vector<4xf32>1145// CHECK-DAG: %[[USE2:.+]] = "some_use2"({{.*}}) : (vector<5xf32>) -> vector<5xf32>1146// CHECK-NEXT: scf.yield %[[USE1]], %[[USE2]] : vector<4xf32>, vector<5xf32>1147// CHECK-NEXT: }1148// CHECK-DAG: %[[BCAST1:.+]] = vector.broadcast %[[LOOP]]#0 : vector<4xf32> to vector<3x4xf32>1149// CHECK-DAG: %[[BCAST2:.+]] = vector.broadcast %[[LOOP]]#1 : vector<5xf32> to vector<3x5xf32>1150// CHECK-NEXT: return %[[BCAST1]], %[[BCAST2]] : vector<3x4xf32>, vector<3x5xf32>1151 1152func.func @hoist_vector_broadcasts_multiple(%lb : index, %ub : index, %step : index, %vec1 : vector<3x4xf32>, %vec2 : vector<3x5xf32>) -> (vector<3x4xf32>, vector<3x5xf32>) {1153 %bcast_vec:2 = scf.for %arg0 = %lb to %ub step %step iter_args(%iarg = %vec1, %iarg2 = %vec2) -> (vector<3x4xf32>, vector<3x5xf32>) {1154 %extract1 = vector.extract %iarg[0] : vector<4xf32> from vector<3x4xf32>1155 %extract2 = vector.extract %iarg2[1] : vector<5xf32> from vector<3x5xf32>1156 %use1 = "some_use1"(%extract1) : (vector<4xf32>) -> vector<4xf32>1157 %use2 = "some_use2"(%extract2) : (vector<5xf32>) -> vector<5xf32>1158 %broadcast1 = vector.broadcast %use1 : vector<4xf32> to vector<3x4xf32>1159 %broadcast2 = vector.broadcast %use2 : vector<5xf32> to vector<3x5xf32>1160 scf.yield %broadcast1, %broadcast2 : vector<3x4xf32>,vector<3x5xf32>1161 }1162 return %bcast_vec#0, %bcast_vec#1 : vector<3x4xf32>, vector<3x5xf32>1163}1164 1165module attributes {transform.with_named_sequence} {1166 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {1167 %0 = transform.structured.match ops{["func.func"]} in %arg11168 : (!transform.any_op) -> !transform.any_op1169 transform.structured.hoist_redundant_vector_broadcasts %01170 : (!transform.any_op) -> !transform.any_op1171 transform.yield1172 }1173}1174