359 lines · plain
1// RUN: mlir-opt %s -transform-interpreter -split-input-file -verify-diagnostics | FileCheck %s2 3func.func @conv1d_nwc_wcf_dyn_ch_dim(%input: memref<4x6x?xf32>, %filter: memref<1x?x8xf32>, %output: memref<4x2x8xf32>) {4 // expected-error @+1 {{Attempted to vectorize, but failed}}5 linalg.conv_1d_nwc_wcf6 {dilations = dense<1> : tensor<1xi64>, strides = dense<3> : tensor<1xi64>}7 ins(%input, %filter : memref<4x6x?xf32>, memref<1x?x8xf32>)8 outs(%output : memref<4x2x8xf32>)9 return10}11 12module attributes {transform.with_named_sequence} {13 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {14 %0 = transform.structured.match ops{["linalg.conv_1d_nwc_wcf"]} in %arg1 : (!transform.any_op) -> !transform.any_op15 transform.structured.vectorize %0 : !transform.any_op16 transform.yield17 }18}19 20// -----21 22// Masked vectorisation of 1D depthwise CW convs is not yet supported23 24func.func @depthwise_conv1d_ncw_cw(%input: memref<3x?x4xf32>, %filter: memref<?x1xf32>, %output: memref<3x?x4xf32>) {25 // expected-error @+1 {{Attempted to vectorize, but failed}}26 linalg.depthwise_conv_1d_ncw_cw27 {dilations = dense<2> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>}28 ins(%input, %filter : memref<3x?x4xf32>, memref<?x1xf32>)29 outs(%output : memref<3x?x4xf32>)30 return31}32 33module attributes {transform.with_named_sequence} {34 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {35 %0 = transform.structured.match ops{["linalg.depthwise_conv_1d_ncw_cw"]} in %arg1 : (!transform.any_op) -> !transform.any_op36 transform.structured.vectorize %0 vector_sizes [3, 4, 5, 1] : !transform.any_op37 transform.yield38 }39}40 41// -----42 43func.func @depthwise_conv1d_nwc_wc_dyn_w_dim(%input: memref<3x?x4xf32>, %filter: memref<?x4xf32>, %output: memref<3x?x4xf32>) {44 // expected-error @+1 {{Attempted to vectorize, but failed}}45 linalg.depthwise_conv_1d_nwc_wc46 {dilations = dense<2> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>}47 ins(%input, %filter : memref<3x?x4xf32>, memref<?x4xf32>)48 outs(%output : memref<3x?x4xf32>)49 return50}51 52module attributes {transform.with_named_sequence} {53 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {54 %0 = transform.structured.match ops{["linalg.depthwise_conv_1d_nwc_wc"]} in %arg1 : (!transform.any_op) -> !transform.any_op55 transform.structured.vectorize %0 vector_sizes [3, 2, 4, 2] : !transform.any_op56 transform.yield57 }58}59 60// -----61 62func.func @depthwise_conv1d_nwc_wc_dyn_ch_dim(%input: memref<3x5x?xf32>, %filter: memref<2x?xf32>, %output: memref<3x2x?xf32>) {63 // expected-error @+1 {{Attempted to vectorize, but failed}}64 linalg.depthwise_conv_1d_nwc_wc65 ins(%input, %filter : memref<3x5x?xf32>, memref<2x?xf32>)66 outs(%output : memref<3x2x?xf32>)67 return68}69 70module attributes {transform.with_named_sequence} {71 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {72 %0 = transform.structured.match ops{["linalg.depthwise_conv_1d_nwc_wc"]} in %arg1 : (!transform.any_op) -> !transform.any_op73 transform.structured.vectorize %0 : !transform.any_op74 transform.yield75 }76}77 78// -----79 80func.func @depthwise_conv1d_nwc_wc_dyn_w_dim(%input: memref<3x?x3xf32>, %filter: memref<2x3xf32>, %output: memref<3x?x3xf32>) {81 // expected-error @+1 {{Attempted to vectorize, but failed}}82 linalg.depthwise_conv_1d_nwc_wc83 ins(%input, %filter : memref<3x?x3xf32>, memref<2x3xf32>)84 outs(%output : memref<3x?x3xf32>)85 return86}87 88module attributes {transform.with_named_sequence} {89 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {90 %0 = transform.structured.match ops{["linalg.depthwise_conv_1d_nwc_wc"]} in %arg1 : (!transform.any_op) -> !transform.any_op91 transform.structured.vectorize %0 : !transform.any_op92 transform.yield93 }94}95 96// -----97 98func.func @conv1d_dyn_w_dim(%input: tensor<?xf32>, %filter: tensor<4xf32>, %output: tensor<?xf32>) -> tensor<?xf32> {99 // expected-error @+1 {{Attempted to vectorize, but failed}}100 %0 = linalg.conv_1d ins(%input, %filter : tensor<?xf32>, tensor<4xf32>)101 outs(%output : tensor<?xf32>) -> tensor<?xf32>102 return %0 : tensor<?xf32>103}104 105module attributes {transform.with_named_sequence} {106 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {107 %0 = transform.structured.match ops{["linalg.conv_1d"]} in %arg1 : (!transform.any_op) -> !transform.any_op108 transform.structured.vectorize %0 : !transform.any_op109 transform.yield110 }111}112 113// -----114 115func.func @conv2d_nchw_fchw(%input: tensor<1x5x8x8xf32>, %filter: tensor<4x5x3x3xf32>, %output: tensor<1x4x6x6xf32>) {116 // expected-error @+1 {{Attempted to vectorize, but failed}}117 linalg.conv_2d_nchw_fchw {dilations = dense<1> : vector<2xi64>, strides = dense<1> : vector<2xi64>} ins(%input, %filter : tensor<1x5x8x8xf32>, tensor<4x5x3x3xf32>) outs(%output : tensor<1x4x6x6xf32>) -> tensor<1x4x6x6xf32>118 return119}120 121module attributes {transform.with_named_sequence} {122 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {123 %0 = transform.structured.match ops{["linalg.conv_2d_nchw_fchw"]} in %arg1 : (!transform.any_op) -> !transform.any_op124 transform.structured.vectorize %0 : !transform.any_op125 transform.yield126 }127}128 129// -----130 131func.func @conv2d_nhwc_fhwc(%input: tensor<1x8x8x5xf32>, %filter: tensor<4x3x3x5xf32>, %output: tensor<1x6x6x4xf32>) {132 // expected-error @+1 {{Attempted to vectorize, but failed}}133 linalg.conv_2d_nhwc_fhwc {dilations = dense<1> : vector<2xi64>, strides = dense<1> : vector<2xi64>} ins(%input, %filter : tensor<1x8x8x5xf32>, tensor<4x3x3x5xf32>) outs(%output : tensor<1x6x6x4xf32>) -> tensor<1x6x6x4xf32>134 return135}136 137 138module attributes {transform.with_named_sequence} {139 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {140 %0 = transform.structured.match ops{["linalg.conv_2d_nhwc_fhwc"]} in %arg1 : (!transform.any_op) -> !transform.any_op141 transform.structured.vectorize %0 : !transform.any_op142 transform.yield143 }144}145 146// -----147 148func.func @conv3d_ncdhw_fcdhw(%input: tensor<1x5x8x8x8xf32>, %filter: tensor<4x5x3x3x3xf32>, %output: tensor<1x4x6x6x6xf32>) {149 // expected-error @+1 {{Attempted to vectorize, but failed}}150 linalg.conv_3d_ncdhw_fcdhw {dilations = dense<1> : vector<3xi64>, strides = dense<1> : vector<3xi64>} ins(%input, %filter : tensor<1x5x8x8x8xf32>, tensor<4x5x3x3x3xf32>) outs(%output : tensor<1x4x6x6x6xf32>) -> tensor<1x4x6x6x6xf32>151 return152}153 154module attributes {transform.with_named_sequence} {155 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {156 %0 = transform.structured.match ops{["linalg.conv_3d_ncdhw_fcdhw"]} in %arg1 : (!transform.any_op) -> !transform.any_op157 transform.structured.vectorize %0 : !transform.any_op158 transform.yield159 }160}161 162// -----163 164func.func @test_pack_no_vectorize_dynamic_shape(%arg0: tensor<?xf32>, %arg1: tensor<4x16xf32>) -> tensor<4x16xf32> {165 %pad = arith.constant 0.000000e+00 : f32166 // expected-error @+1 {{Attempted to vectorize, but failed}}167 %pack = linalg.pack %arg0 padding_value(%pad : f32) inner_dims_pos = [0] inner_tiles = [16] into %arg1 : tensor<?xf32> -> tensor<4x16xf32>168 return %pack : tensor<4x16xf32>169}170 171module attributes {transform.with_named_sequence} {172 transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {173 %0 = transform.structured.match ops{["linalg.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op174 transform.structured.vectorize %0 : !transform.any_op175 transform.yield176 }177}178 179// -----180 181func.func @linalg_reduce_scalable_leading_dim(%input: tensor<?x?xf32>,182 %acc: tensor<?xf32>) -> tensor<?xf32> {183 184 // expected-error @+1 {{Attempted to vectorize, but failed}}185 %0 = linalg.reduce ins(%input : tensor<?x?xf32>) outs(%acc : tensor<?xf32>) dimensions = [0]186 (%in: f32, %init: f32) {187 %0 = arith.addf %in, %init : f32188 linalg.yield %0 : f32189 }190 return %0 : tensor<?xf32>191}192 193module attributes {transform.with_named_sequence} {194 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {195 %0 = transform.structured.match ops{["linalg.reduce"]} in %arg1 : (!transform.any_op) -> !transform.any_op196 transform.structured.vectorize %0 vector_sizes [[4], 1] : !transform.any_op197 transform.yield198 }199}200 201// -----202 203func.func @linalg_generic_reduction_scalable_leading_dim(%input: tensor<?x?xf32>,204 %acc: tensor<?xf32>) -> tensor<?xf32> {205 206 // expected-error @+1 {{Attempted to vectorize, but failed}}207 %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,208 affine_map<(d0, d1) -> (d1)>],209 iterator_types = ["reduction", "parallel"] }210 ins(%input : tensor<?x?xf32>)211 outs(%acc : tensor<?xf32>) {212 ^bb(%in: f32, %out: f32) :213 %0 = arith.addf %in, %out : f32214 linalg.yield %0 : f32215 } -> tensor<?xf32>216 return %0 : tensor<?xf32>217}218 219module attributes {transform.with_named_sequence} {220 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {221 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op222 transform.structured.vectorize %0 vector_sizes [[4], 1] : !transform.any_op223 transform.yield224 }225}226 227// -----228 229func.func @linalg_matvec_scalable_two_dims(%A: memref<?x?xf32>, %B: memref<?xf32>, %C: memref<?xf32>) {230 // expected-error @+1 {{Attempted to vectorize, but failed}}231 linalg.matvec ins(%A, %B: memref<?x?xf32>, memref<?xf32>)232 outs(%C: memref<?xf32>)233 return234}235 236module attributes {transform.with_named_sequence} {237 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {238 %matmul = transform.structured.match ops{["linalg.matvec"]} in %arg1 : (!transform.any_op) -> !transform.any_op239 transform.structured.vectorize %matmul vector_sizes [[4], [4]] : !transform.any_op240 transform.yield241 }242}243 244// -----245 246func.func @linalg_matmul_scalable_leading_parallel_dim(%A: memref<?x?xf32>, %B: memref<?x?xf32>, %C: memref<?x?xf32>) {247 // expected-error @+1 {{Attempted to vectorize, but failed}}248 linalg.matmul ins(%A, %B: memref<?x?xf32>, memref<?x?xf32>)249 outs(%C: memref<?x?xf32>)250 return251}252 253module attributes {transform.with_named_sequence} {254 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {255 %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op256 transform.structured.vectorize %matmul vector_sizes [[8], 16, 4] : !transform.any_op257 transform.yield258 }259}260 261// -----262 263func.func @linalg_matmul_scalable_trailing_reduction_dim(%A: memref<?x?xf32>, %B: memref<?x?xf32>, %C: memref<?x?xf32>) {264 // expected-error @+1 {{Attempted to vectorize, but failed}}265 linalg.matmul ins(%A, %B: memref<?x?xf32>, memref<?x?xf32>)266 outs(%C: memref<?x?xf32>)267 return268}269 270module attributes {transform.with_named_sequence} {271 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {272 %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op273 transform.structured.vectorize %matmul vector_sizes [8, 16, [4]] : !transform.any_op274 transform.yield275 }276}277 278// -----279 280func.func @linalg_generic_matmul_scalable_two_trailing_dims(%A: tensor<?x64xf32>, %B: tensor<64x?xf32>,281 %C: tensor<?x?xf32>) -> tensor<?x?xf32> {282 283 // expected-error @+1 {{Attempted to vectorize, but failed}}284 %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>,285 affine_map<(d0, d1, d2) -> (d2, d1)>,286 affine_map<(d0, d1, d2) -> (d0, d1)>],287 iterator_types = ["parallel", "parallel", "reduction"] }288 ins(%A, %B : tensor<?x64xf32>, tensor<64x?xf32>)289 outs(%C: tensor<?x?xf32>) {290 ^bb(%in1: f32, %in2: f32, %out: f32) :291 %0 = arith.mulf %in1, %in2 : f32292 %1 = arith.addf %0, %out : f32293 linalg.yield %1 : f32294 } -> tensor<?x?xf32>295 return %0 : tensor<?x?xf32>296}297 298module attributes {transform.with_named_sequence} {299 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {300 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op301 transform.structured.vectorize %0 vector_sizes [2, [4], [4]] : !transform.any_op302 transform.yield303 }304}305 306// -----307 308// Padding with non-zero low pad values is not supported, unless the corresponding309// result dim is 1. Here `%l0` being a non-zero low pad applied to a310// non-unit result dimension makes this case unsupported.311func.func @tensor_pad_non_zero_low_pad(312 %0 : tensor<?x?xf32>, %h0 : index, %h1 : index, %l0 : index)313 -> tensor<2x4xf32> {314 // expected-error @+3 {{Attempted to vectorize, but failed}}315 %cst = arith.constant 42.43 : f32316 %c0 = arith.constant 0 : index317 %1 = tensor.pad %0 low[%l0, %c0] high[%h0, %h1] {318 ^bb0(%hh1: index, %hh2: index):319 tensor.yield %cst : f32320 } : tensor<?x?xf32> to tensor<2x4xf32>321 return %1: tensor<2x4xf32>322}323 324module attributes {transform.with_named_sequence} {325 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {326 %0 = transform.structured.match ops{["tensor.pad"]} in %arg1327 : (!transform.any_op) -> !transform.any_op328 transform.structured.vectorize %0 vector_sizes [2, 4] : !transform.any_op329 transform.yield330 }331}332 333// -----334 335// With dynamically shaped source, the vectorizer infers the vector size for336// xfer Ops from the destination tensor and, conservatively, assumes337// out-of-bounds accesses. Out-of-bounds accesses require a pad value, but338// that's impossible to recover in this example. Hence no vectorization.339 340// TODO: Use diagnostics once we can vectorize tensor.insert_slice with341// transform.structured.vectorize342 343// CHECK-LABEL: @insert_dynamic_slice_unknown_pad344// CHECK-NOT: vector345// CHECK: tensor.insert_slice346func.func @insert_dynamic_slice_unknown_pad(%arg0: tensor<1x?x3xf32>, %arg1: tensor<9x8x7x1x2x3xf32>, %size: index) -> tensor<9x8x7x1x2x3xf32> {347 %res = tensor.insert_slice %arg0 into %arg1[0, 0, 0, 0, 0, 0] [1, 1, 1, 1, %size, 3][1, 1, 1, 1, 1, 1] : tensor<1x?x3xf32> into tensor<9x8x7x1x2x3xf32>348 return %res : tensor<9x8x7x1x2x3xf32>349}350 351module attributes {transform.with_named_sequence} {352 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {353 %0 = transform.structured.match ops{["tensor.insert_slice"]} in %arg1 : (!transform.any_op) -> !transform.any_op354 %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op355 %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op356 transform.yield357 }358}359