833 lines · plain
1// RUN: mlir-opt -finalize-memref-to-llvm %s -split-input-file | FileCheck --check-prefixes=ALL,CHECK %s2// RUN: mlir-opt -finalize-memref-to-llvm='index-bitwidth=32' %s -split-input-file | FileCheck --check-prefix=CHECK32 %s3 4// Same below, but using the `ConvertToLLVMPatternInterface` entry point5// and the generic `convert-to-llvm` pass. This produces slightly different IR6// because the conversion target is set up differently.7// RUN: mlir-opt --convert-to-llvm="filter-dialects=memref" --split-input-file %s | FileCheck --check-prefixes=ALL,CHECK-INTERFACE %s8 9// TODO: In some (all?) cases, CHECK and CHECK-INTERFACE outputs are identical.10// Use a common prefix instead (e.g. ALL).11 12// CHECK-LABEL: func @view(13// CHECK: %[[ARG0F:.*]]: index, %[[ARG1F:.*]]: index, %[[ARG2F:.*]]: index14// CHECK-INTERFACE-LABEL: func @view(15// CHECK-INTERFACE-NOT: memref.alloc16// CHECK-INTERFACE-NOT: memref.view17func.func @view(%arg0 : index, %arg1 : index, %arg2 : index) {18 // CHECK-DAG: %[[ARG2:.*]] = builtin.unrealized_conversion_cast %[[ARG2F]]19 // CHECK-DAG: %[[ARG0:.*]] = builtin.unrealized_conversion_cast %[[ARG0F]]20 // CHECK-DAG: %[[ARG1:.*]] = builtin.unrealized_conversion_cast %[[ARG1F]]21 // CHECK: llvm.mlir.constant(2048 : index) : i6422 // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>23 %0 = memref.alloc() : memref<2048xi8>24 25 // Test two dynamic sizes.26 // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>27 // CHECK: %[[BASE_PTR:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>28 // CHECK: %[[SHIFTED_BASE_PTR:.*]] = llvm.getelementptr %[[BASE_PTR]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr, i829 // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>30 // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i6431 // CHECK: llvm.insertvalue %[[C0]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>32 // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>33 // CHECK: llvm.mlir.constant(1 : index) : i6434 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>35 // CHECK: llvm.insertvalue %[[ARG0]], %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>36 // CHECK: llvm.mul %{{.*}}, %[[ARG1]]37 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>38 %1 = memref.view %0[%arg2][%arg0, %arg1] : memref<2048xi8> to memref<?x?xf32>39 40 // Test one dynamic size.41 // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>42 // CHECK: %[[BASE_PTR_2:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>43 // CHECK: %[[SHIFTED_BASE_PTR_2:.*]] = llvm.getelementptr %[[BASE_PTR_2]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr, i844 // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR_2]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>45 // CHECK: %[[C0_2:.*]] = llvm.mlir.constant(0 : index) : i6446 // CHECK: llvm.insertvalue %[[C0_2]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>47 // CHECK: llvm.insertvalue %[[ARG1]], %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>48 // CHECK: llvm.mlir.constant(1 : index) : i6449 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>50 // CHECK: llvm.mlir.constant(4 : index) : i6451 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>52 // CHECK: llvm.mul %{{.*}}, %[[ARG1]]53 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>54 %3 = memref.view %0[%arg2][%arg1] : memref<2048xi8> to memref<4x?xf32>55 56 // Test static sizes.57 // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>58 // CHECK: %[[BASE_PTR_3:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>59 // CHECK: %[[SHIFTED_BASE_PTR_3:.*]] = llvm.getelementptr %[[BASE_PTR_3]][%[[ARG2]]] : (!llvm.ptr, i64) -> !llvm.ptr, i860 // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR_3]], %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>61 // CHECK: %[[C0_3:.*]] = llvm.mlir.constant(0 : index) : i6462 // CHECK: llvm.insertvalue %[[C0_3]], %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>63 // CHECK: llvm.mlir.constant(4 : index) : i6464 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>65 // CHECK: llvm.mlir.constant(1 : index) : i6466 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>67 // CHECK: llvm.mlir.constant(64 : index) : i6468 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>69 // CHECK: llvm.mlir.constant(4 : index) : i6470 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>71 %5 = memref.view %0[%arg2][] : memref<2048xi8> to memref<64x4xf32>72 73 // Test view memory space.74 // CHECK: llvm.mlir.constant(2048 : index) : i6475 // CHECK: llvm.mlir.poison : !llvm.struct<(ptr<4>, ptr<4>, i64, array<1 x i64>, array<1 x i64>)>76 %6 = memref.alloc() : memref<2048xi8, 4>77 78 // CHECK: llvm.mlir.poison : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)>79 // CHECK: %[[BASE_PTR_4:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<1 x i64>, array<1 x i64>)>80 // CHECK: %[[SHIFTED_BASE_PTR_4:.*]] = llvm.getelementptr %[[BASE_PTR_4]][%[[ARG2]]] : (!llvm.ptr<4>, i64) -> !llvm.ptr<4>, i881 // CHECK: llvm.insertvalue %[[SHIFTED_BASE_PTR_4]], %{{.*}}[1] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)>82 // CHECK: %[[C0_4:.*]] = llvm.mlir.constant(0 : index) : i6483 // CHECK: llvm.insertvalue %[[C0_4]], %{{.*}}[2] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)>84 // CHECK: llvm.mlir.constant(4 : index) : i6485 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)>86 // CHECK: llvm.mlir.constant(1 : index) : i6487 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)>88 // CHECK: llvm.mlir.constant(64 : index) : i6489 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)>90 // CHECK: llvm.mlir.constant(4 : index) : i6491 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr<4>, ptr<4>, i64, array<2 x i64>, array<2 x i64>)>92 %7 = memref.view %6[%arg2][] : memref<2048xi8, 4> to memref<64x4xf32, 4>93 94 return95}96 97// -----98 99// CHECK-LABEL: func @view_empty_memref(100// CHECK: %[[ARG0:.*]]: index,101// CHECK: %[[ARG1:.*]]: memref<0xi8>)102 103// CHECK-INTERFACE-LABEL: func @view_empty_memref(104// CHECK-INTERFACE: %[[ARG0:.*]]: index,105// CHECK-INTERFACE: %[[ARG1:.*]]: memref<0xi8>)106func.func @view_empty_memref(%offset: index, %mem: memref<0xi8>) {107 108 // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>109 // CHECK: llvm.mlir.constant(0 : index) : i64110 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>111 // CHECK: llvm.mlir.constant(4 : index) : i64112 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>113 // CHECK: llvm.mlir.constant(1 : index) : i64114 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>115 // CHECK: llvm.mlir.constant(0 : index) : i64116 // CHECK: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>117 // CHECK: llvm.mlir.constant(4 : index) : i64118 // CHECK: = llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>119 120 // CHECK-INTERFACE: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>121 // CHECK-INTERFACE: llvm.mlir.constant(0 : index) : i64122 // CHECK-INTERFACE: llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>123 // CHECK-INTERFACE: llvm.mlir.constant(4 : index) : i64124 // CHECK-INTERFACE: llvm.insertvalue %{{.*}}, %{{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>125 // CHECK-INTERFACE: llvm.mlir.constant(1 : index) : i64126 // CHECK-INTERFACE: llvm.insertvalue %{{.*}}, %{{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>127 // CHECK-INTERFACE: llvm.mlir.constant(0 : index) : i64128 // CHECK-INTERFACE: llvm.insertvalue %{{.*}}, %{{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>129 // CHECK-INTERFACE: llvm.mlir.constant(4 : index) : i64130 // CHECK-INTERFACE: = llvm.insertvalue %{{.*}}, %{{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>131 %0 = memref.view %mem[%offset][] : memref<0xi8> to memref<0x4xf32>132 133 return134}135 136// -----137 138// ALL-LABEL: func.func @view_memref_as_rank0(139// ALL-SAME: %[[OFFSET:.*]]: index,140// ALL-SAME: %[[MEM:.*]]: memref<2xi8>) {141func.func @view_memref_as_rank0(%offset: index, %mem: memref<2xi8>) {142 143 // ALL: builtin.unrealized_conversion_cast %[[OFFSET]] : index to i64144 // ALL: builtin.unrealized_conversion_cast %[[MEM]] : memref<2xi8> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>145 // ALL: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64)>146 // ALL: llvm.extractvalue %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>147 // ALL: llvm.insertvalue %{{.*}}, %{{.*}}[0] : !llvm.struct<(ptr, ptr, i64)>148 // ALL: llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>149 // ALL: llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, i8150 // ALL: llvm.insertvalue %{{.*}}, %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64)>151 // ALL: llvm.mlir.constant(0 : index) : i64152 // ALL: llvm.insertvalue %{{.*}}, %{{.*}}[2] : !llvm.struct<(ptr, ptr, i64)>153 %memref_view_bf16 = memref.view %mem[%offset][] : memref<2xi8> to memref<bf16>154 155 return156}157 158// -----159 160// Subviews needs to be expanded outside of the memref-to-llvm pass.161// CHECK-LABEL: func @subview(162// CHECK: %[[MEMREF:.*]]: memref<{{.*}}>,163// CHECK: %[[ARG0:[a-zA-Z0-9]*]]: index,164// CHECK: %[[ARG1:[a-zA-Z0-9]*]]: index,165// CHECK: %[[ARG2:.*]]: index)166// CHECK32-LABEL: func @subview(167// CHECK32: %[[MEMREF:.*]]: memref<{{.*}}>,168// CHECK32: %[[ARG0:[a-zA-Z0-9]*]]: index,169// CHECK32: %[[ARG1:[a-zA-Z0-9]*]]: index,170// CHECK32: %[[ARG2:.*]]: index)171// CHECK-INTERFACE-LABEL: func @subview(172func.func @subview(%0 : memref<64x4xf32, strided<[4, 1], offset: 0>>, %arg0 : index, %arg1 : index, %arg2 : index) {173 // CHECK: memref.subview %[[MEMREF]][%[[ARG0]], %[[ARG1]]] [%[[ARG0]], %[[ARG1]]]174 // CHECK32: memref.subview %[[MEMREF]][%[[ARG0]], %[[ARG1]]] [%[[ARG0]], %[[ARG1]]] [%[[ARG0]], %[[ARG1]]]175 // CHECK-INTERFACE: memref.subview176 %1 = memref.subview %0[%arg0, %arg1][%arg0, %arg1][%arg0, %arg1] :177 memref<64x4xf32, strided<[4, 1], offset: 0>>178 to memref<?x?xf32, strided<[?, ?], offset: ?>>179 return180}181 182// -----183 184// CHECK-LABEL: func @assume_alignment(185// CHECK-INTERFACE-LABEL: func @assume_alignment(186func.func @assume_alignment(%0 : memref<4x4xf16>) {187 // CHECK: %[[PTR:.*]] = llvm.extractvalue %[[MEMREF:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>188 // CHECK-NEXT: %[[TRUE:.*]] = llvm.mlir.constant(true) : i1189 // CHECK-NEXT: %[[ALIGN:.*]] = llvm.mlir.constant(16 : index) : i64190 // CHECK-NEXT: llvm.intr.assume %[[TRUE]] ["align"(%[[PTR]], %[[ALIGN]] : !llvm.ptr, i64)] : i1191 // CHECK-INTERFACE: llvm.intr.assume192 %1 = memref.assume_alignment %0, 16 : memref<4x4xf16>193 return194}195 196// -----197 198// ALL-LABEL: func @distinct_objects199// ALL-SAME: (%[[ARG0:.*]]: memref<?xf16>, %[[ARG1:.*]]: memref<?xf32>, %[[ARG2:.*]]: memref<?xf64>)200func.func @distinct_objects(%arg0: memref<?xf16>, %arg1: memref<?xf32>, %arg2: memref<?xf64>) -> (memref<?xf16>, memref<?xf32>, memref<?xf64>) {201// ALL-DAG: %[[CAST_0:.*]] = builtin.unrealized_conversion_cast %[[ARG0]] : memref<?xf16> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>202// ALL-DAG: %[[CAST_1:.*]] = builtin.unrealized_conversion_cast %[[ARG1]] : memref<?xf32> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>203// ALL-DAG: %[[CAST_2:.*]] = builtin.unrealized_conversion_cast %[[ARG2]] : memref<?xf64> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>204// ALL: %[[PTR_0:.*]] = llvm.extractvalue %[[CAST_0]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>205// ALL: %[[PTR_1:.*]] = llvm.extractvalue %[[CAST_1]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>206// ALL: %[[PTR_2:.*]] = llvm.extractvalue %[[CAST_2]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>207// ALL: %[[TRUE:.*]] = llvm.mlir.constant(true) : i1208// ALL: llvm.intr.assume %[[TRUE]] ["separate_storage"(%[[PTR_0]], %[[PTR_1]] : !llvm.ptr, !llvm.ptr)] : i1209// ALL: llvm.intr.assume %[[TRUE]] ["separate_storage"(%[[PTR_0]], %[[PTR_2]] : !llvm.ptr, !llvm.ptr)] : i1210// ALL: llvm.intr.assume %[[TRUE]] ["separate_storage"(%[[PTR_1]], %[[PTR_2]] : !llvm.ptr, !llvm.ptr)] : i1211 %1, %2, %3 = memref.distinct_objects %arg0, %arg1, %arg2 : memref<?xf16>, memref<?xf32>, memref<?xf64>212 return %1, %2, %3 : memref<?xf16>, memref<?xf32>, memref<?xf64>213}214 215// -----216 217// ALL-LABEL: func @distinct_objects_noop218// ALL-SAME: (%[[ARG0:.*]]: memref<?xf16>)219func.func @distinct_objects_noop(%arg0: memref<?xf16>) -> memref<?xf16> {220// 1-operand version is noop221// ALL-NEXT: return %[[ARG0]]222 %1 = memref.distinct_objects %arg0 : memref<?xf16>223 return %1 : memref<?xf16>224}225 226// -----227 228// CHECK-LABEL: func @assume_alignment_w_offset229// CHECK-INTERFACE-LABEL: func @assume_alignment_w_offset230func.func @assume_alignment_w_offset(%0 : memref<4x4xf16, strided<[?, ?], offset: ?>>) {231 // CHECK-DAG: %[[PTR:.*]] = llvm.extractvalue %[[MEMREF:.*]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>232 // CHECK-DAG: %[[OFFSET:.*]] = llvm.extractvalue %[[MEMREF]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>233 // CHECK-DAG: %[[BUFF_ADDR:.*]] = llvm.getelementptr %[[PTR]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, f16234 // CHECK-DAG: %[[TRUE:.*]] = llvm.mlir.constant(true) : i1235 // CHECK-DAG: %[[ALIGN:.*]] = llvm.mlir.constant(16 : index) : i64236 // CHECK-NEXT: llvm.intr.assume %[[TRUE]] ["align"(%[[BUFF_ADDR]], %[[ALIGN]] : !llvm.ptr, i64)] : i1237 // CHECK-INTERFACE: llvm.intr.assume238 %1 = memref.assume_alignment %0, 16 : memref<4x4xf16, strided<[?, ?], offset: ?>>239 return240}241// -----242 243// CHECK-LABEL: func @dim_of_unranked244// CHECK32-LABEL: func @dim_of_unranked245// CHECK-INTERFACE-LABEL: func @dim_of_unranked246func.func @dim_of_unranked(%unranked: memref<*xi32>) -> index {247 %c0 = arith.constant 0 : index248 %dim = memref.dim %unranked, %c0 : memref<*xi32>249 return %dim : index250}251// CHECK: %[[UNRANKED_DESC:.*]] = builtin.unrealized_conversion_cast252 253// CHECK: %[[RANKED_DESC:.*]] = llvm.extractvalue %[[UNRANKED_DESC]][1]254// CHECK-SAME: : !llvm.struct<(i64, ptr)>255 256// CHECK: %[[OFFSET_PTR:.*]] = llvm.getelementptr %[[RANKED_DESC]]{{\[}}257// CHECK-SAME: 0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, ptr, i64)>258 259// CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : index) : i64260// CHECK: %[[INDEX_INC:.*]] = llvm.add %[[C1]], %{{.*}} : i64261 262// CHECK: %[[SIZE_PTR:.*]] = llvm.getelementptr %[[OFFSET_PTR]]{{\[}}263// CHECK-SAME: %[[INDEX_INC]]] : (!llvm.ptr, i64) -> !llvm.ptr264 265// CHECK: %[[SIZE:.*]] = llvm.load %[[SIZE_PTR]] : !llvm.ptr -> i64266 267// CHECK32: %[[SIZE:.*]] = llvm.load %{{.*}} : !llvm.ptr -> i32268 269// CHECK-INTERFACE-NOT: memref.dim270 271// -----272 273// CHECK-LABEL: func @address_space(274// CHECK-INTERFACE-LABEL: func @address_space(275func.func @address_space(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>, 7>) {276 // CHECK: %[[MEMORY:.*]] = llvm.call @malloc(%{{.*}})277 // CHECK: %[[CAST:.*]] = llvm.addrspacecast %[[MEMORY]] : !llvm.ptr to !llvm.ptr<5>278 // CHECK: llvm.insertvalue %[[CAST]], %{{[[:alnum:]]+}}[0]279 // CHECK: llvm.insertvalue %[[CAST]], %{{[[:alnum:]]+}}[1]280 // CHECK-INTERFACE: llvm.call @malloc281 // CHECK-INTERFACE: llvm.addrspacecast282 %0 = memref.alloc() : memref<32xf32, affine_map<(d0) -> (d0)>, 5>283 %1 = arith.constant 7 : index284 // CHECK: llvm.load %{{.*}} : !llvm.ptr<5> -> f32285 // CHECK-INTERFACE: llvm.load286 %2 = memref.load %0[%1] : memref<32xf32, affine_map<(d0) -> (d0)>, 5>287 func.return288}289 290// -----291 292// CHECK-LABEL: func @transpose293// CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>294// CHECK: llvm.insertvalue {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>295// CHECK: llvm.insertvalue {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>296// CHECK: llvm.insertvalue {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>297// CHECK: llvm.extractvalue {{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>298// CHECK: llvm.insertvalue {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>299// CHECK: llvm.extractvalue {{.*}}[4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>300// CHECK: llvm.insertvalue {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>301// CHECK: llvm.extractvalue {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>302// CHECK: llvm.insertvalue {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>303// CHECK: llvm.extractvalue {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>304// CHECK: llvm.insertvalue {{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>305// CHECK: llvm.extractvalue {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>306// CHECK: llvm.insertvalue {{.*}}[3, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>307// CHECK: llvm.extractvalue {{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>308// CHECK: llvm.insertvalue {{.*}}[4, 2] : !llvm.struct<(ptr, ptr, i64, array<3 x i64>, array<3 x i64>)>309// CHECK-INTERFACE-LABEL: func @transpose310// CHECK-INTERFACE-NOT: memref.transpose311func.func @transpose(%arg0: memref<?x?x?xf32, strided<[?, ?, 1], offset: ?>>) {312 %0 = memref.transpose %arg0 (i, j, k) -> (k, i, j) : memref<?x?x?xf32, strided<[?, ?, 1], offset: ?>> to memref<?x?x?xf32, strided<[1, ?, ?], offset: ?>>313 return314}315 316// -----317 318// CHECK: llvm.mlir.global external @gv0() {addr_space = 0 : i32} : !llvm.array<2 x f32> {319// CHECK-NEXT: %0 = llvm.mlir.undef : !llvm.array<2 x f32>320// CHECK-NEXT: llvm.return %0 : !llvm.array<2 x f32>321// CHECK-NEXT: }322// CHECK-INTERFACE: llvm.mlir.global external323memref.global @gv0 : memref<2xf32> = uninitialized324 325// CHECK: llvm.mlir.global private @gv1() {addr_space = 0 : i32} : !llvm.array<2 x f32>326// CHECK-INTERFACE: llvm.mlir.global private327memref.global "private" @gv1 : memref<2xf32>328 329// CHECK: llvm.mlir.global external @gv2(dense<{{\[\[}}0.000000e+00, 1.000000e+00, 2.000000e+00], [3.000000e+00, 4.000000e+00, 5.000000e+00]]> : tensor<2x3xf32>) {addr_space = 0 : i32} : !llvm.array<2 x array<3 x f32>>330// CHECK-INTERFACE: llvm.mlir.global external331memref.global @gv2 : memref<2x3xf32> = dense<[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]>332 333// Test 1D memref.334// CHECK-LABEL: func @get_gv0_memref335// CHECK-INTERFACE-LABEL: func @get_gv0_memref336func.func @get_gv0_memref() {337 %0 = memref.get_global @gv0 : memref<2xf32>338 // CHECK: %[[DIM:.*]] = llvm.mlir.constant(2 : index) : i64339 // CHECK: %[[STRIDE:.*]] = llvm.mlir.constant(1 : index) : i64340 // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv0 : !llvm.ptr341 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<2 x f32>342 // CHECK: %[[DEADBEEF:.*]] = llvm.mlir.constant(3735928559 : index) : i64343 // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr344 // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>345 // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>346 // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>347 // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64348 // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>349 // CHECK: llvm.insertvalue %[[DIM]], {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>350 // CHECK: llvm.insertvalue %[[STRIDE]], {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>351 // CHECK-INTERFACE: llvm.mlir.addressof352 // CHECK-INTERFACE: llvm.getelementptr353 return354}355 356// Test 2D memref.357// CHECK-LABEL: func @get_gv2_memref358// CHECK-INTERFACE-LABEL: func @get_gv2_memref359func.func @get_gv2_memref() {360 // CHECK: %[[DIM0:.*]] = llvm.mlir.constant(2 : index) : i64361 // CHECK: %[[DIM1:.*]] = llvm.mlir.constant(3 : index) : i64362 // CHECK: %[[STRIDE1:.*]] = llvm.mlir.constant(1 : index) : i64363 // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv2 : !llvm.ptr364 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0, 0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<2 x array<3 x f32>>365 // CHECK: %[[DEADBEEF:.*]] = llvm.mlir.constant(3735928559 : index) : i64366 // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr367 // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>368 // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>369 // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>370 // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64371 // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>372 // CHECK: llvm.insertvalue %[[DIM0]], {{.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>373 // CHECK: llvm.insertvalue %[[DIM1]], {{.*}}[3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>374 // CHECK: llvm.insertvalue %[[DIM1]], {{.*}}[4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>375 // CHECK: llvm.insertvalue %[[STRIDE1]], {{.*}}[4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>376 // CHECK-INTERFACE: llvm.mlir.addressof377 // CHECK-INTERFACE: llvm.getelementptr378 379 %0 = memref.get_global @gv2 : memref<2x3xf32>380 return381}382 383// Test scalar memref.384// CHECK: llvm.mlir.global external @gv3(1.000000e+00 : f32) {addr_space = 0 : i32} : f32385memref.global @gv3 : memref<f32> = dense<1.0>386 387// CHECK-LABEL: func @get_gv3_memref388// CHECK-INTERFACE-LABEL: func @get_gv3_memref389func.func @get_gv3_memref() {390 // CHECK: %[[ADDR:.*]] = llvm.mlir.addressof @gv3 : !llvm.ptr391 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ADDR]][0] : (!llvm.ptr) -> !llvm.ptr, f32392 // CHECK: %[[DEADBEEF:.*]] = llvm.mlir.constant(3735928559 : index) : i64393 // CHECK: %[[DEADBEEFPTR:.*]] = llvm.inttoptr %[[DEADBEEF]] : i64 to !llvm.ptr394 // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64)>395 // CHECK: llvm.insertvalue %[[DEADBEEFPTR]], {{.*}}[0] : !llvm.struct<(ptr, ptr, i64)>396 // CHECK: llvm.insertvalue %[[GEP]], {{.*}}[1] : !llvm.struct<(ptr, ptr, i64)>397 // CHECK: %[[OFFSET:.*]] = llvm.mlir.constant(0 : index) : i64398 // CHECK: llvm.insertvalue %[[OFFSET]], {{.*}}[2] : !llvm.struct<(ptr, ptr, i64)>399 // CHECK-INTERFACE: llvm.mlir.addressof400 // CHECK-INTERFACE: llvm.getelementptr401 %0 = memref.get_global @gv3 : memref<f32>402 return403}404 405// Test scalar memref with an alignment.406// CHECK: llvm.mlir.global private @gv4(1.000000e+00 : f32) {addr_space = 0 : i32, alignment = 64 : i64} : f32407// CHECK-INTERFACE: llvm.mlir.global private408memref.global "private" @gv4 : memref<f32> = dense<1.0> {alignment = 64}409 410// -----411 412// Expand shapes need to be expanded outside of the memref-to-llvm pass.413// CHECK-LABEL: func @expand_shape_static(414// CHECK-SAME: %[[ARG:.*]]: memref<{{.*}}>)415// CHECK-INTERFACE-LABEL: func @expand_shape_static(416func.func @expand_shape_static(%arg0: memref<3x4x5xf32>) -> memref<1x3x4x1x5xf32> {417 // CHECK: memref.expand_shape %[[ARG]] {{\[}}[0, 1], [2], [3, 4]] output_shape [1, 3, 4, 1, 5]418 // CHECK-INTERFACE: memref.expand_shape419 // Reshapes that expand a contiguous tensor with some 1's.420 %0 = memref.expand_shape %arg0 [[0, 1], [2], [3, 4]] output_shape [1, 3, 4, 1, 5]421 : memref<3x4x5xf32> into memref<1x3x4x1x5xf32>422 return %0 : memref<1x3x4x1x5xf32>423}424 425// -----426 427// Collapse shapes need to be expanded outside of the memref-to-llvm pass.428// CHECK-LABEL: func @collapse_shape_static429// CHECK-SAME: %[[ARG:.*]]: memref<1x3x4x1x5xf32>) -> memref<3x4x5xf32> {430// CHECK-INTERFACE-LABEL: func @collapse_shape_static431func.func @collapse_shape_static(%arg0: memref<1x3x4x1x5xf32>) -> memref<3x4x5xf32> {432 // CHECK: memref.collapse_shape %[[ARG]]433 // CHECK-INTERFACE: memref.collapse_shape434 %0 = memref.collapse_shape %arg0 [[0, 1], [2], [3, 4]] :435 memref<1x3x4x1x5xf32> into memref<3x4x5xf32>436 return %0 : memref<3x4x5xf32>437}438 439// -----440 441// CHECK-LABEL: func @rank_of_unranked442// CHECK32-LABEL: func @rank_of_unranked443// CHECK-INTERFACE-LABEL: func @rank_of_unranked444func.func @rank_of_unranked(%unranked: memref<*xi32>) {445 %rank = memref.rank %unranked : memref<*xi32>446 return447}448// CHECK: %[[UNRANKED_DESC:.*]] = builtin.unrealized_conversion_cast449// CHECK-NEXT: llvm.extractvalue %[[UNRANKED_DESC]][0] : !llvm.struct<(i64, ptr)>450// CHECK32: llvm.extractvalue %{{.*}}[0] : !llvm.struct<(i32, ptr)>451// CHECK-INTERFACE-NOT: memref.rank452 453// -----454 455// CHECK-LABEL: func @rank_of_ranked456// CHECK32-LABEL: func @rank_of_ranked457// CHECK-INTERFACE-LABEL: func @rank_of_ranked458func.func @rank_of_ranked(%ranked: memref<?xi32>) {459 %rank = memref.rank %ranked : memref<?xi32>460 return461}462// CHECK: llvm.mlir.constant(1 : index) : i64463// CHECK32: llvm.mlir.constant(1 : index) : i32464// CHECK-INTERFACE: llvm.mlir.constant(1 : index) : i64465 466// -----467 468// CHECK-LABEL: func @atomic_rmw469// CHECK-INTERFACE-LABEL: func @atomic_rmw470func.func @atomic_rmw(%I : memref<10xi32>, %ival : i32, %F : memref<10xf32>, %fval : f32, %i : index) {471 memref.atomic_rmw assign %fval, %F[%i] : (f32, memref<10xf32>) -> f32472 // CHECK: llvm.atomicrmw xchg %{{.*}}, %{{.*}} acq_rel473 memref.atomic_rmw addi %ival, %I[%i] : (i32, memref<10xi32>) -> i32474 // CHECK: llvm.atomicrmw add %{{.*}}, %{{.*}} acq_rel475 memref.atomic_rmw maxs %ival, %I[%i] : (i32, memref<10xi32>) -> i32476 // CHECK: llvm.atomicrmw max %{{.*}}, %{{.*}} acq_rel477 memref.atomic_rmw mins %ival, %I[%i] : (i32, memref<10xi32>) -> i32478 // CHECK: llvm.atomicrmw min %{{.*}}, %{{.*}} acq_rel479 memref.atomic_rmw maxu %ival, %I[%i] : (i32, memref<10xi32>) -> i32480 // CHECK: llvm.atomicrmw umax %{{.*}}, %{{.*}} acq_rel481 memref.atomic_rmw minu %ival, %I[%i] : (i32, memref<10xi32>) -> i32482 // CHECK: llvm.atomicrmw umin %{{.*}}, %{{.*}} acq_rel483 memref.atomic_rmw addf %fval, %F[%i] : (f32, memref<10xf32>) -> f32484 // CHECK: llvm.atomicrmw fadd %{{.*}}, %{{.*}} acq_rel485 memref.atomic_rmw maximumf %fval, %F[%i] : (f32, memref<10xf32>) -> f32486 // CHECK: llvm.atomicrmw fmaximum %{{.*}}, %{{.*}} acq_rel487 memref.atomic_rmw maxnumf %fval, %F[%i] : (f32, memref<10xf32>) -> f32488 // CHECK: llvm.atomicrmw fmax %{{.*}}, %{{.*}} acq_rel489 memref.atomic_rmw minimumf %fval, %F[%i] : (f32, memref<10xf32>) -> f32490 // CHECK: llvm.atomicrmw fminimum %{{.*}}, %{{.*}} acq_rel491 memref.atomic_rmw minnumf %fval, %F[%i] : (f32, memref<10xf32>) -> f32492 // CHECK: llvm.atomicrmw fmin %{{.*}}, %{{.*}} acq_rel493 memref.atomic_rmw ori %ival, %I[%i] : (i32, memref<10xi32>) -> i32494 // CHECK: llvm.atomicrmw _or %{{.*}}, %{{.*}} acq_rel495 memref.atomic_rmw andi %ival, %I[%i] : (i32, memref<10xi32>) -> i32496 // CHECK: llvm.atomicrmw _and %{{.*}}, %{{.*}} acq_rel497 memref.atomic_rmw xori %ival, %I[%i] : (i32, memref<10xi32>) -> i32498 // CHECK: llvm.atomicrmw _xor %{{.*}}, %{{.*}} acq_rel499 // CHECK-INTERFACE-COUNT-14: llvm.atomicrmw500 return501}502 503// -----504 505func.func @atomic_rmw_with_offset(%I : memref<10xi32, strided<[1], offset: 5>>, %ival : i32, %i : index) {506 memref.atomic_rmw andi %ival, %I[%i] : (i32, memref<10xi32, strided<[1], offset: 5>>) -> i32507 return508}509// CHECK-LABEL: func @atomic_rmw_with_offset510// CHECK-SAME: %[[ARG0:.+]]: memref<10xi32, strided<[1], offset: 5>>511// CHECK-SAME: %[[ARG1:.+]]: i32512// CHECK-SAME: %[[ARG2:.+]]: index513// CHECK-DAG: %[[MEMREF_STRUCT:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] : memref<10xi32, strided<[1], offset: 5>> to !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>514// CHECK-DAG: %[[INDEX:.+]] = builtin.unrealized_conversion_cast %[[ARG2]] : index to i64515// CHECK: %[[BASE_PTR:.+]] = llvm.extractvalue %[[MEMREF_STRUCT]][1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>516// CHECK: %[[OFFSET:.+]] = llvm.mlir.constant(5 : index) : i64517// CHECK: %[[OFFSET_PTR:.+]] = llvm.getelementptr %[[BASE_PTR]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32518// CHECK: %[[PTR:.+]] = llvm.getelementptr %[[OFFSET_PTR]][%[[INDEX]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32519// CHECK: llvm.atomicrmw _and %[[PTR]], %[[ARG1]] acq_rel520 521// CHECK-INTERFACE-LABEL: func @atomic_rmw_with_offset522// CHECK-INTERFACE: llvm.atomicrmw523 524// -----525 526// CHECK-LABEL: func @generic_atomic_rmw527// CHECK-INTERFACE-LABEL: func @generic_atomic_rmw528llvm.func @generic_atomic_rmw() {529 %I = "test.foo"() : () -> (memref<10xi32>)530 %i = "test.foo"() : () -> (index)531 %x = memref.generic_atomic_rmw %I[%i] : memref<10xi32> {532 ^bb0(%old_value : i32):533 memref.atomic_yield %old_value : i32534 }535 llvm.return536}537// CHECK: %[[INIT:.*]] = llvm.load %{{.*}} : !llvm.ptr -> i32538// CHECK-NEXT: llvm.br ^bb1(%[[INIT]] : i32)539// CHECK-NEXT: ^bb1(%[[LOADED:.*]]: i32):540// CHECK-NEXT: %[[PAIR:.*]] = llvm.cmpxchg %{{.*}}, %[[LOADED]], %[[LOADED]]541// CHECK-SAME: acq_rel monotonic : !llvm.ptr, i32542// CHECK-NEXT: %[[NEW:.*]] = llvm.extractvalue %[[PAIR]][0]543// CHECK-NEXT: %[[OK:.*]] = llvm.extractvalue %[[PAIR]][1]544// CHECK-NEXT: llvm.cond_br %[[OK]], ^bb2, ^bb1(%[[NEW]] : i32)545 546// CHECK-INTERFACE: llvm.cmpxchg547 548// -----549 550// CHECK-LABEL: func @generic_atomic_rmw_in_alloca_scope551// CHECK-INTERFACE-LABEL: func @generic_atomic_rmw_in_alloca_scope552llvm.func @generic_atomic_rmw_in_alloca_scope() {553 %c1 = arith.constant 1 : index554 %alloc = memref.alloc() : memref<2x3xi32>555 memref.alloca_scope {556 %0 = memref.generic_atomic_rmw %alloc[%c1, %c1] : memref<2x3xi32> {557 ^bb0(%arg0: i32):558 memref.atomic_yield %arg0 : i32559 }560 }561 llvm.return562}563// CHECK: %[[STACK_SAVE:.*]] = llvm.intr.stacksave : !llvm.ptr564// CHECK-NEXT: llvm.br ^bb1565// CHECK: ^bb1:566// CHECK: %[[INIT:.*]] = llvm.load %[[BUF:.*]] : !llvm.ptr -> i32567// CHECK-NEXT: llvm.br ^bb2(%[[INIT]] : i32)568// CHECK-NEXT: ^bb2(%[[LOADED:.*]]: i32):569// CHECK-NEXT: %[[PAIR:.*]] = llvm.cmpxchg %[[BUF]], %[[LOADED]], %[[LOADED]]570// CHECK-SAME: acq_rel monotonic : !llvm.ptr, i32571// CHECK-NEXT: %[[NEW:.*]] = llvm.extractvalue %[[PAIR]][0]572// CHECK-NEXT: %[[OK:.*]] = llvm.extractvalue %[[PAIR]][1]573// CHECK-NEXT: llvm.cond_br %[[OK]], ^bb3, ^bb2(%[[NEW]] : i32)574// CHECK-NEXT: ^bb3:575// CHECK-NEXT: llvm.intr.stackrestore %[[STACK_SAVE]] : !llvm.ptr576// CHECK-NEXT: llvm.br ^bb4577// CHECK-NEXT: ^bb4:578// CHECK-NEXT: return579 580// CHECK-INTERFACE: llvm.cmpxchg581 582// -----583 584// CHECK-LABEL: func @memref_copy_ranked585// CHECK-INTERFACE-LABEL: func @memref_copy_ranked586func.func @memref_copy_ranked() {587 %0 = memref.alloc() : memref<2xf32>588 // CHECK: llvm.mlir.constant(2 : index) : i64589 // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>590 %1 = memref.cast %0 : memref<2xf32> to memref<?xf32>591 %2 = memref.alloc() : memref<2xf32>592 // CHECK: llvm.mlir.constant(2 : index) : i64593 // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>594 %3 = memref.cast %2 : memref<2xf32> to memref<?xf32>595 memref.copy %1, %3 : memref<?xf32> to memref<?xf32>596 // CHECK: [[ONE:%.*]] = llvm.mlir.constant(1 : index) : i64597 // CHECK: [[EXTRACT0:%.*]] = llvm.extractvalue {{%.*}}[3, 0] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>598 // CHECK: [[MUL:%.*]] = llvm.mul [[ONE]], [[EXTRACT0]] : i64599 // CHECK: [[NULL:%.*]] = llvm.mlir.zero : !llvm.ptr600 // CHECK: [[GEP:%.*]] = llvm.getelementptr [[NULL]][1] : (!llvm.ptr) -> !llvm.ptr, f32601 // CHECK: [[PTRTOINT:%.*]] = llvm.ptrtoint [[GEP]] : !llvm.ptr to i64602 // CHECK: [[SIZE:%.*]] = llvm.mul [[MUL]], [[PTRTOINT]] : i64603 // CHECK: [[EXTRACT1P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>604 // CHECK: [[EXTRACT1O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>605 // CHECK: [[GEP1:%.*]] = llvm.getelementptr [[EXTRACT1P]][[[EXTRACT1O]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32606 // CHECK: [[EXTRACT2P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>607 // CHECK: [[EXTRACT2O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>608 // CHECK: [[GEP2:%.*]] = llvm.getelementptr [[EXTRACT2P]][[[EXTRACT2O]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32609 // CHECK: "llvm.intr.memcpy"([[GEP2]], [[GEP1]], [[SIZE]]) <{isVolatile = false}>610 // CHECK-INTERFACE: llvm.intr.memcpy611 return612}613 614 615// -----616 617// CHECK-LABEL: func @memref_copy_contiguous618// CHECK-INTERFACE-LABEL: func @memref_copy_contiguous619func.func @memref_copy_contiguous(%in: memref<16x4xi32>, %offset: index) {620 %buf = memref.alloc() : memref<1x2xi32>621 %sub = memref.subview %in[%offset, 0] [1, 2] [1, 1] : memref<16x4xi32> to memref<1x2xi32, strided<[4, 1], offset: ?>>622 memref.copy %sub, %buf : memref<1x2xi32, strided<[4, 1], offset: ?>> to memref<1x2xi32>623 // Skip the memref descriptor of the alloc.624 // CHECK: llvm.insertvalue {{%.*}}, {{%.*}}[4, 1]625 // Get the memref for the subview.626 // CHECK: %[[SUBVIEW:.*]] = memref.subview %{{.*}}[%{{.*}}, 0] [1, 2] [1, 1] : memref<16x4xi32> to memref<1x2xi32, strided<[4, 1], offset: ?>>627 // CHECK: %[[DESC:.*]] = builtin.unrealized_conversion_cast %[[SUBVIEW]] : memref<1x2xi32, strided<[4, 1], offset: ?>> to !llvm.struct<(ptr628 // CHECK: [[EXTRACT0:%.*]] = llvm.extractvalue %[[DESC]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>629 // CHECK: [[MUL1:%.*]] = llvm.mul {{.*}}, [[EXTRACT0]] : i64630 // CHECK: [[EXTRACT1:%.*]] = llvm.extractvalue %[[DESC]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>631 // CHECK: [[MUL2:%.*]] = llvm.mul [[MUL1]], [[EXTRACT1]] : i64632 // CHECK: [[NULL:%.*]] = llvm.mlir.zero : !llvm.ptr633 // CHECK: [[GEP:%.*]] = llvm.getelementptr [[NULL]][1] : (!llvm.ptr) -> !llvm.ptr, i32634 // CHECK: [[PTRTOINT:%.*]] = llvm.ptrtoint [[GEP]] : !llvm.ptr to i64635 // CHECK: [[SIZE:%.*]] = llvm.mul [[MUL2]], [[PTRTOINT]] : i64636 // CHECK: [[EXTRACT1P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>637 // CHECK: [[EXTRACT1O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>638 // CHECK: [[GEP1:%.*]] = llvm.getelementptr [[EXTRACT1P]][[[EXTRACT1O]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32639 // CHECK: [[EXTRACT2P:%.*]] = llvm.extractvalue {{%.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>640 // CHECK: [[EXTRACT2O:%.*]] = llvm.extractvalue {{%.*}}[2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>641 // CHECK: [[GEP2:%.*]] = llvm.getelementptr [[EXTRACT2P]][[[EXTRACT2O]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32642 // CHECK: "llvm.intr.memcpy"([[GEP2]], [[GEP1]], [[SIZE]]) <{isVolatile = false}>643 // CHECK-INTERFACE: llvm.intr.memcpy644 return645}646 647// -----648 649// CHECK-LABEL: func @memref_copy_0d_offset650// CHECK-INTERFACE-LABEL: func @memref_copy_0d_offset651func.func @memref_copy_0d_offset(%in: memref<2xi32>) {652 %buf = memref.alloc() : memref<i32>653 %sub = memref.subview %in[1] [1] [1] : memref<2xi32> to memref<1xi32, strided<[1], offset: 1>>654 %scalar = memref.collapse_shape %sub [] : memref<1xi32, strided<[1], offset: 1>> into memref<i32, strided<[], offset: 1>>655 memref.copy %scalar, %buf : memref<i32, strided<[], offset: 1>> to memref<i32>656 // CHECK: llvm.intr.memcpy657 // CHECK-INTERFACE: llvm.intr.memcpy658 return659}660 661// -----662 663// CHECK-LABEL: func @memref_copy_noncontiguous664// CHECK-INTERFACE-LABEL: func @memref_copy_noncontiguous665func.func @memref_copy_noncontiguous(%in: memref<16x2xi32>, %offset: index) {666 %buf = memref.alloc() : memref<2x1xi32>667 %sub = memref.subview %in[%offset, 0] [2, 1] [1, 1] : memref<16x2xi32> to memref<2x1xi32, strided<[2, 1], offset: ?>>668 memref.copy %sub, %buf : memref<2x1xi32, strided<[2, 1], offset: ?>> to memref<2x1xi32>669 // CHECK: llvm.call @memrefCopy670 // CHECK-INTERFACE: llvm.call @memrefCopy671 return672}673 674// -----675 676// CHECK-LABEL: func @memref_copy_unranked677// CHECK-INTERFACE-LABEL: func @memref_copy_unranked678func.func @memref_copy_unranked() {679 %0 = memref.alloc() : memref<2xi1>680 // CHECK: llvm.mlir.constant(2 : index) : i64681 // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>682 %1 = memref.cast %0 : memref<2xi1> to memref<*xi1>683 %2 = memref.alloc() : memref<2xi1>684 // CHECK: llvm.mlir.constant(2 : index) : i64685 // CHECK: llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>686 %3 = memref.cast %2 : memref<2xi1> to memref<*xi1>687 memref.copy %1, %3 : memref<*xi1> to memref<*xi1>688 // CHECK: [[ONE:%.*]] = llvm.mlir.constant(1 : index) : i64689 // CHECK: [[ALLOCA:%.*]] = llvm.alloca [[ONE]] x !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)> : (i64) -> !llvm.ptr690 // CHECK: llvm.store {{%.*}}, [[ALLOCA]] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>, !llvm.ptr691 // CHECK: [[RANK:%.*]] = llvm.mlir.constant(1 : index) : i64692 // CHECK: [[UNDEF:%.*]] = llvm.mlir.poison : !llvm.struct<(i64, ptr)>693 // CHECK: [[INSERT:%.*]] = llvm.insertvalue [[RANK]], [[UNDEF]][0] : !llvm.struct<(i64, ptr)>694 // CHECK: [[INSERT2:%.*]] = llvm.insertvalue [[ALLOCA]], [[INSERT]][1] : !llvm.struct<(i64, ptr)>695 // CHECK: [[STACKSAVE:%.*]] = llvm.intr.stacksave : !llvm.ptr696 // CHECK: [[RANK2:%.*]] = llvm.mlir.constant(1 : index) : i64697 // CHECK: [[ALLOCA2:%.*]] = llvm.alloca [[RANK2]] x !llvm.struct<(i64, ptr)> : (i64) -> !llvm.ptr698 // CHECK: llvm.store {{%.*}}, [[ALLOCA2]] : !llvm.struct<(i64, ptr)>, !llvm.ptr699 // CHECK: [[ALLOCA3:%.*]] = llvm.alloca [[RANK2]] x !llvm.struct<(i64, ptr)> : (i64) -> !llvm.ptr700 // CHECK: llvm.store [[INSERT2]], [[ALLOCA3]] : !llvm.struct<(i64, ptr)>, !llvm.ptr701 // CHECK: [[SIZEPTR:%.*]] = llvm.getelementptr {{%.*}}[1] : (!llvm.ptr) -> !llvm.ptr, i1702 // CHECK: [[SIZE:%.*]] = llvm.ptrtoint [[SIZEPTR]] : !llvm.ptr to i64703 // CHECK: llvm.call @memrefCopy([[SIZE]], [[ALLOCA2]], [[ALLOCA3]]) : (i64, !llvm.ptr, !llvm.ptr) -> ()704 // CHECK: llvm.intr.stackrestore [[STACKSAVE]]705 // CHECK-INTERFACE: llvm.call @memrefCopy706 return707}708 709// -----710 711// CHECK-LABEL: func @extract_aligned_pointer_as_index712// CHECK-INTERFACE-LABEL: func @extract_aligned_pointer_as_index713func.func @extract_aligned_pointer_as_index(%m: memref<?xf32>) -> index {714 %0 = memref.extract_aligned_pointer_as_index %m: memref<?xf32> -> index715 // CHECK: %[[E:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>716 // CHECK: %[[I64:.*]] = llvm.ptrtoint %[[E]] : !llvm.ptr to i64717 // CHECK: %[[R:.*]] = builtin.unrealized_conversion_cast %[[I64]] : i64 to index718 // CHECK-INTERFACE-NOT: memref.extract_aligned_pointer_as_index719 720 // CHECK: return %[[R:.*]] : index721 return %0: index722}723 724// -----725 726// CHECK-LABEL: func @extract_aligned_pointer_as_index_unranked727// CHECK-INTERFACE-LABEL: func @extract_aligned_pointer_as_index_unranked728func.func @extract_aligned_pointer_as_index_unranked(%m: memref<*xf32>) -> index {729 %0 = memref.extract_aligned_pointer_as_index %m: memref<*xf32> -> index730 // CHECK: %[[PTR:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(i64, ptr)>731 // CHECK: %[[ALIGNED_FIELD:.*]] = llvm.getelementptr %[[PTR]][1] : (!llvm.ptr) -> !llvm.ptr, !llvm.ptr732 // CHECK: %[[ALIGNED_PTR:.*]] = llvm.load %[[ALIGNED_FIELD]] : !llvm.ptr -> !llvm.ptr733 // CHECK: %[[I64:.*]] = llvm.ptrtoint %[[ALIGNED_PTR]] : !llvm.ptr to i64734 // CHECK: %[[R:.*]] = builtin.unrealized_conversion_cast %[[I64]] : i64 to index735 // CHECK-INTERFACE-NOT: memref.extract_aligned_pointer_as_index736 737 // CHECK: return %[[R]] : index738 return %0: index739}740 741// -----742 743// CHECK-LABEL: func @extract_strided_metadata(744// CHECK-SAME: %[[ARG:.*]]: memref745// CHECK: %[[MEM_DESC:.*]] = builtin.unrealized_conversion_cast %[[ARG]] : memref<?x?xf32, strided<[?, ?], offset: ?>> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>746// CHECK: %[[BASE:.*]] = llvm.extractvalue %[[MEM_DESC]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>747// CHECK: %[[ALIGNED_BASE:.*]] = llvm.extractvalue %[[MEM_DESC]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>748// CHECK: %[[DESC:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64)>749// CHECK: %[[DESC0:.*]] = llvm.insertvalue %[[BASE]], %[[DESC]][0] : !llvm.struct<(ptr, ptr, i64)>750// CHECK: %[[DESC1:.*]] = llvm.insertvalue %[[ALIGNED_BASE]], %[[DESC0]][1] : !llvm.struct<(ptr, ptr, i64)>751// CHECK: %[[OFF0:.*]] = llvm.mlir.constant(0 : index) : i64752// CHECK: %[[BASE_BUFFER_DESC:.*]] = llvm.insertvalue %[[OFF0]], %[[DESC1]][2] : !llvm.struct<(ptr, ptr, i64)>753// CHECK: %[[OFFSET:.*]] = llvm.extractvalue %[[MEM_DESC]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>754// CHECK: %[[SIZE0:.*]] = llvm.extractvalue %[[MEM_DESC]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>755// CHECK: %[[SIZE1:.*]] = llvm.extractvalue %[[MEM_DESC]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>756// CHECK: %[[STRIDE0:.*]] = llvm.extractvalue %[[MEM_DESC]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>757// CHECK: %[[STRIDE1:.*]] = llvm.extractvalue %[[MEM_DESC]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>758 759// CHECK-INTERFACE-LABEL: func @extract_strided_metadata760// CHECK-INTERFACE-NOT: memref.extract_strided_metadata761 762func.func @extract_strided_metadata(763 %ref: memref<?x?xf32, strided<[?,?], offset: ?>>) {764 765 %base, %offset, %sizes:2, %strides:2 =766 memref.extract_strided_metadata %ref : memref<?x?xf32, strided<[?,?], offset: ?>>767 -> memref<f32>, index,768 index, index,769 index, index770 771 return772}773 774// -----775 776// CHECK-LABEL: func @load_non_temporal(777// CHECK-INTERFACE-LABEL: func @load_non_temporal(778func.func @load_non_temporal(%arg0 : memref<32xf32, affine_map<(d0) -> (d0)>>) {779 %1 = arith.constant 7 : index780 // CHECK: llvm.load %{{.*}} {nontemporal} : !llvm.ptr -> f32781 // CHECK-INTERFACE: llvm.load782 %2 = memref.load %arg0[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>>783 func.return784}785 786// -----787 788// CHECK-LABEL: func @load_with_alignment(789// CHECK-INTERFACE-LABEL: func @load_with_alignment(790func.func @load_with_alignment(%arg0 : memref<32xf32>, %arg1 : index) {791 // CHECK: llvm.load %{{.*}} {alignment = 32 : i64} : !llvm.ptr -> f32792 // CHECK-INTERFACE: llvm.load793 %1 = memref.load %arg0[%arg1] {alignment = 32} : memref<32xf32>794 func.return795}796 797// -----798 799// CHECK-LABEL: func @store_non_temporal(800// CHECK-INTERFACE-LABEL: func @store_non_temporal(801func.func @store_non_temporal(%input : memref<32xf32, affine_map<(d0) -> (d0)>>, %output : memref<32xf32, affine_map<(d0) -> (d0)>>) {802 %1 = arith.constant 7 : index803 %2 = memref.load %input[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>>804 // CHECK: llvm.store %{{.*}}, %{{.*}} {nontemporal} : f32, !llvm.ptr805 // CHECK-INTERFACE: llvm.store806 memref.store %2, %output[%1] {nontemporal = true} : memref<32xf32, affine_map<(d0) -> (d0)>>807 func.return808}809 810// -----811 812// CHECK-LABEL: func @store_with_alignment(813// CHECK-INTERFACE-LABEL: func @store_with_alignment(814func.func @store_with_alignment(%arg0 : memref<32xf32>, %arg1 : f32, %arg2 : index) {815 // CHECK: llvm.store %{{.*}}, %{{.*}} {alignment = 32 : i64} : f32, !llvm.ptr816 // CHECK-INTERFACE: llvm.store817 memref.store %arg1, %arg0[%arg2] {alignment = 32} : memref<32xf32>818 func.return819}820 821// -----822 823// Ensure unconvertable memory space not cause a crash824 825// CHECK-LABEL: @alloca_unconvertable_memory_space826// CHECK-INTERFACE-LABEL: @alloca_unconvertable_memory_space827func.func @alloca_unconvertable_memory_space() {828 // CHECK: memref.alloca829 // CHECK-INTERFACE: memref.alloca830 %alloca = memref.alloca() : memref<1x32x33xi32, #spirv.storage_class<StorageBuffer>>831 func.return832}833