751 lines · plain
1// RUN: mlir-opt -allow-unregistered-dialect %s | FileCheck %s2// Verify the printed output can be parsed.3// RUN: mlir-opt -allow-unregistered-dialect %s | mlir-opt -allow-unregistered-dialect | FileCheck %s4// Verify the generic form can be parsed.5// RUN: mlir-opt -allow-unregistered-dialect -mlir-print-op-generic %s | mlir-opt -allow-unregistered-dialect | FileCheck %s6 7// CHECK-LABEL: func @ext_packed_fp8_s8// CHECK: amdgpu.ext_packed_fp8 {{.*}} vector<4xf8E4M3FNUZ> to f329func.func @ext_packed_fp8_s(%v: vector<4xf8E4M3FNUZ>) -> f32 {10 %ret = amdgpu.ext_packed_fp8 %v[0] : vector<4xf8E4M3FNUZ> to f3211 func.return %ret : f3212}13 14// CHECK-LABEL: func @ext_packed_fp8_v15// CHECK: amdgpu.ext_packed_fp8 {{.*}} vector<4xf8E4M3FNUZ> to vector<2xf3216func.func @ext_packed_fp8_v(%v: vector<4xf8E4M3FNUZ>) -> vector<2xf32> {17 %ret = amdgpu.ext_packed_fp8 %v[0] : vector<4xf8E4M3FNUZ> to vector<2xf32>18 func.return %ret : vector<2xf32>19}20 21// CHECK-LABEL: func @packed_trunc_2xfp822// CHECK: amdgpu.packed_trunc_2xfp823func.func @packed_trunc_2xfp8(%v1: f32, %v2: f32, %others: vector<4xf8E5M2FNUZ>, %stoch: i32) -> vector<4xf8E5M2FNUZ> {24 %ret = amdgpu.packed_trunc_2xfp8 %v1, %v2 into %others[word 1] : f32 to vector<4xf8E5M2FNUZ> into vector<4xf8E5M2FNUZ>25 func.return %ret : vector<4xf8E5M2FNUZ>26}27 28// CHECK-LABEL: func @packed_stoch_round_fp829// CHECK: amdgpu.packed_stoch_round_fp830func.func @packed_stoch_round_fp8(%v1: f32, %stoch: i32, %others: vector<4xf8E5M2FNUZ>) -> vector<4xf8E5M2FNUZ> {31 %ret = amdgpu.packed_stoch_round_fp8 %v1 + %stoch into %others[2] : f32 to vector<4xf8E5M2FNUZ> into vector<4xf8E5M2FNUZ>32 func.return %ret : vector<4xf8E5M2FNUZ>33}34 35// CHECK-LABEL: func.func @scaled_ext_full_f8e4m3_f3236// CHECK: amdgpu.scaled_ext_packed37func.func @scaled_ext_full_f8e4m3_f32(%v: vector<4xf8E4M3FN>, %scale: f32) -> vector<2xf32> {38 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E4M3FN> to vector<2xf32>39 func.return %ret : vector<2xf32>40}41 42// CHECK-LABEL: func.func @scaled_ext_full_f8e4m3_f1643// CHECK: amdgpu.scaled_ext_packed44func.func @scaled_ext_full_f8e4m3_f16(%v: vector<4xf8E4M3FN>, %scale: f32) -> vector<2xf16> {45 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E4M3FN> to vector<2xf16>46 func.return %ret : vector<2xf16>47}48 49// CHECK-LABEL: func.func @scaled_ext_full_f8e4m3_bf1650// CHECK: amdgpu.scaled_ext_packed51func.func @scaled_ext_full_f8e4m3_bf16(%v: vector<4xf8E4M3FN>, %scale: f32) -> vector<2xbf16> {52 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E4M3FN> to vector<2xbf16>53 func.return %ret : vector<2xbf16>54}55 56// CHECK-LABEL: func.func @scaled_ext_half_f8e4m3_f3257// CHECK: amdgpu.scaled_ext_packed58func.func @scaled_ext_half_f8e4m3_f32(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xf32> {59 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xf32>60 func.return %ret : vector<2xf32>61}62 63// CHECK-LABEL: func.func @scaled_ext_half_f8e4m3_f1664// CHECK: amdgpu.scaled_ext_packed65func.func @scaled_ext_half_f8e4m3_f16(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xf16> {66 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xf16>67 func.return %ret : vector<2xf16>68}69 70// CHECK-LABEL: func.func @scaled_ext_half_f8e4m3_bf1671// CHECK: amdgpu.scaled_ext_packed72func.func @scaled_ext_half_f8e4m3_bf16(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xbf16> {73 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xbf16>74 func.return %ret : vector<2xbf16>75}76 77// CHECK-LABEL: func.func @scaled_ext_scalar_f8e4m3_f3278// CHECK: amdgpu.scaled_ext_packed79func.func @scaled_ext_scalar_f8e4m3_f32(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xf32> {80 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xf32>81 func.return %ret : vector<2xf32>82}83 84// CHECK-LABEL: func.func @scaled_ext_scalar_f8e4m3_f1685// CHECK: amdgpu.scaled_ext_packed86func.func @scaled_ext_scalar_f8e4m3_f16(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xf16> {87 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xf16>88 func.return %ret : vector<2xf16>89}90 91// CHECK-LABEL: func.func @scaled_ext_scalar_f8e4m3_bf1692// CHECK: amdgpu.scaled_ext_packed93func.func @scaled_ext_scalar_f8e4m3_bf16(%v: vector<2xf8E4M3FN>, %scale: f32) -> vector<2xbf16> {94 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E4M3FN> to vector<2xbf16>95 func.return %ret : vector<2xbf16>96}97 98// CHECK-LABEL: func.func @scaled_ext_full_f8e5m2_f3299// CHECK: amdgpu.scaled_ext_packed100func.func @scaled_ext_full_f8e5m2_f32(%v: vector<4xf8E5M2>, %scale: f32) -> vector<2xf32> {101 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E5M2> to vector<2xf32>102 func.return %ret : vector<2xf32>103}104 105// CHECK-LABEL: func.func @scaled_ext_full_f8e5m2_f16106// CHECK: amdgpu.scaled_ext_packed107func.func @scaled_ext_full_f8e5m2_f16(%v: vector<4xf8E5M2>, %scale: f32) -> vector<2xf16> {108 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E5M2> to vector<2xf16>109 func.return %ret : vector<2xf16>110}111 112// CHECK-LABEL: func.func @scaled_ext_full_f8e5m2_bf16113// CHECK: amdgpu.scaled_ext_packed114func.func @scaled_ext_full_f8e5m2_bf16(%v: vector<4xf8E5M2>, %scale: f32) -> vector<2xbf16> {115 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf8E5M2> to vector<2xbf16>116 func.return %ret : vector<2xbf16>117}118 119// CHECK-LABEL: func.func @scaled_ext_half_f8e5m2_f32120// CHECK: amdgpu.scaled_ext_packed121func.func @scaled_ext_half_f8e5m2_f32(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xf32> {122 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xf32>123 func.return %ret : vector<2xf32>124}125 126// CHECK-LABEL: func.func @scaled_ext_half_f8e5m2_f16127// CHECK: amdgpu.scaled_ext_packed128func.func @scaled_ext_half_f8e5m2_f16(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xf16> {129 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xf16>130 func.return %ret : vector<2xf16>131}132 133// CHECK-LABEL: func.func @scaled_ext_half_f8e5m2_bf16134// CHECK: amdgpu.scaled_ext_packed135func.func @scaled_ext_half_f8e5m2_bf16(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xbf16> {136 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xbf16>137 func.return %ret : vector<2xbf16>138}139 140// CHECK-LABEL: func.func @scaled_ext_scalar_f8e5m2_f32141// CHECK: amdgpu.scaled_ext_packed142func.func @scaled_ext_scalar_f8e5m2_f32(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xf32> {143 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xf32>144 func.return %ret : vector<2xf32>145}146 147// CHECK-LABEL: func.func @scaled_ext_scalar_f8e5m2_f16148// CHECK: amdgpu.scaled_ext_packed149func.func @scaled_ext_scalar_f8e5m2_f16(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xf16> {150 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xf16>151 func.return %ret : vector<2xf16>152}153 154// CHECK-LABEL: func.func @scaled_ext_scalar_f8e5m2_bf16155// CHECK: amdgpu.scaled_ext_packed156func.func @scaled_ext_scalar_f8e5m2_bf16(%v: vector<2xf8E5M2>, %scale: f32) -> vector<2xbf16> {157 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf8E5M2> to vector<2xbf16>158 func.return %ret : vector<2xbf16>159}160 161// CHECK-LABEL: func.func @scaled_ext_full_f4e2m1_f32162// CHECK: amdgpu.scaled_ext_packed163func.func @scaled_ext_full_f4e2m1_f32(%v: vector<8xf4E2M1FN>, %scale: f32) -> vector<2xf32> {164 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<8xf4E2M1FN> to vector<2xf32>165 func.return %ret : vector<2xf32>166}167 168// CHECK-LABEL: func.func @scaled_ext_full_f4e2m1_f16169// CHECK: amdgpu.scaled_ext_packed170func.func @scaled_ext_full_f4e2m1_f16(%v: vector<8xf4E2M1FN>, %scale: f32) -> vector<2xf16> {171 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<8xf4E2M1FN> to vector<2xf16>172 func.return %ret : vector<2xf16>173}174 175// CHECK-LABEL: func.func @scaled_ext_full_f4e2m1_bf16176// CHECK: amdgpu.scaled_ext_packed177func.func @scaled_ext_full_f4e2m1_bf16(%v: vector<8xf4E2M1FN>, %scale: f32) -> vector<2xbf16> {178 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<8xf4E2M1FN> to vector<2xbf16>179 func.return %ret : vector<2xbf16>180}181 182// CHECK-LABEL: func.func @scaled_ext_half_f4e2m1_f32183// CHECK: amdgpu.scaled_ext_packed184func.func @scaled_ext_half_f4e2m1_f32(%v: vector<8xf4E2M1FN>, %scale: f32) -> vector<2xf32> {185 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<8xf4E2M1FN> to vector<2xf32>186 func.return %ret : vector<2xf32>187}188 189// CHECK-LABEL: func.func @scaled_ext_half_f4e2m1_f16190// CHECK: amdgpu.scaled_ext_packed191func.func @scaled_ext_half_f4e2m1_f16(%v: vector<4xf4E2M1FN>, %scale: f32) -> vector<2xf16> {192 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf4E2M1FN> to vector<2xf16>193 func.return %ret : vector<2xf16>194}195 196// CHECK-LABEL: func.func @scaled_ext_half_f4e2m1_bf16197// CHECK: amdgpu.scaled_ext_packed198func.func @scaled_ext_half_f4e2m1_bf16(%v: vector<4xf4E2M1FN>, %scale: f32) -> vector<2xbf16> {199 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<4xf4E2M1FN> to vector<2xbf16>200 func.return %ret : vector<2xbf16>201}202 203// CHECK-LABEL: func.func @scaled_ext_scalar_f4e2m1_f32204// CHECK: amdgpu.scaled_ext_packed205func.func @scaled_ext_scalar_f4e2m1_f32(%v: vector<2xf4E2M1FN>, %scale: f32) -> vector<2xf32> {206 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf4E2M1FN> to vector<2xf32>207 func.return %ret : vector<2xf32>208}209 210// CHECK-LABEL: func.func @scaled_ext_scalar_f4e2m1_f16211// CHECK: amdgpu.scaled_ext_packed212func.func @scaled_ext_scalar_f4e2m1_f16(%v: vector<2xf4E2M1FN>, %scale: f32) -> vector<2xf16> {213 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf4E2M1FN> to vector<2xf16>214 func.return %ret : vector<2xf16>215}216 217// CHECK-LABEL: func.func @scaled_ext_scalar_f4e2m1_bf16218// CHECK: amdgpu.scaled_ext_packed219func.func @scaled_ext_scalar_f4e2m1_bf16(%v: vector<2xf4E2M1FN>, %scale: f32) -> vector<2xbf16> {220 %ret = amdgpu.scaled_ext_packed %v[0], %scale : vector<2xf4E2M1FN> to vector<2xbf16>221 func.return %ret : vector<2xbf16>222}223 224// CHECK-LABEL: func.func @scaled_ext_packed816_fp4225func.func @scaled_ext_packed816_fp4(%v: vector<8xf4E2M1FN>, %scale: vector<4xf8E8M0FNU>) -> (vector<8xf16>, vector<8xbf16>, vector<8xf32>) {226 // CHECK: amdgpu.scaled_ext_packed816227 %ret0 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<8xf4E2M1FN>, vector<4xf8E8M0FNU> -> vector<8xf16>228 // CHECK: amdgpu.scaled_ext_packed816229 %ret1 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<8xf4E2M1FN>, vector<4xf8E8M0FNU> -> vector<8xbf16>230 // CHECK: amdgpu.scaled_ext_packed816231 %ret2 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<8xf4E2M1FN>, vector<4xf8E8M0FNU> -> vector<8xf32>232 func.return %ret0, %ret1, %ret2 : vector<8xf16>, vector<8xbf16>, vector<8xf32>233}234 235// CHECK-LABEL: func.func @scaled_ext_packed816_fp8236func.func @scaled_ext_packed816_fp8(%v: vector<8xf8E4M3FN>, %scale: vector<4xf8E8M0FNU>) -> (vector<8xf16>, vector<8xbf16>, vector<8xf32>) {237 // CHECK: amdgpu.scaled_ext_packed816238 %ret0 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<8xf8E4M3FN>, vector<4xf8E8M0FNU> -> vector<8xf16>239 // CHECK: amdgpu.scaled_ext_packed816240 %ret1 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<8xf8E4M3FN>, vector<4xf8E8M0FNU> -> vector<8xbf16>241 // CHECK: amdgpu.scaled_ext_packed816242 %ret2 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<8xf8E4M3FN>, vector<4xf8E8M0FNU> -> vector<8xf32>243 func.return %ret0, %ret1, %ret2 : vector<8xf16>, vector<8xbf16>, vector<8xf32>244}245 246// CHECK-LABEL: func.func @scaled_ext_packed816_bf8247func.func @scaled_ext_packed816_bf8(%v: vector<8xf8E5M2>, %scale: vector<4xf8E8M0FNU>) -> (vector<8xf16>, vector<8xbf16>, vector<8xf32>) {248 // CHECK: amdgpu.scaled_ext_packed816249 %ret0 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<8xf8E5M2>, vector<4xf8E8M0FNU> -> vector<8xf16>250 // CHECK: amdgpu.scaled_ext_packed816251 %ret1 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<8xf8E5M2>, vector<4xf8E8M0FNU> -> vector<8xbf16>252 // CHECK: amdgpu.scaled_ext_packed816253 %ret2 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<8xf8E5M2>, vector<4xf8E8M0FNU> -> vector<8xf32>254 func.return %ret0, %ret1, %ret2 : vector<8xf16>, vector<8xbf16>, vector<8xf32>255}256 257// CHECK-LABEL: func.func @scaled_ext_packed816_fp6258func.func @scaled_ext_packed816_fp6(%v: vector<16xf6E2M3FN>, %scale: vector<4xf8E8M0FNU>) -> (vector<16xf16>, vector<16xbf16>, vector<16xf32>) {259 // CHECK: amdgpu.scaled_ext_packed816260 %ret0 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<16xf6E2M3FN>, vector<4xf8E8M0FNU> -> vector<16xf16>261 // CHECK: amdgpu.scaled_ext_packed816262 %ret1 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<16xf6E2M3FN>, vector<4xf8E8M0FNU> -> vector<16xbf16>263 // CHECK: amdgpu.scaled_ext_packed816264 %ret2 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<16xf6E2M3FN>, vector<4xf8E8M0FNU> -> vector<16xf32>265 func.return %ret0, %ret1, %ret2 : vector<16xf16>, vector<16xbf16>, vector<16xf32>266}267 268// CHECK-LABEL: func.func @scaled_ext_packed816_bf16269func.func @scaled_ext_packed816_bf16(%v: vector<16xf6E3M2FN>, %scale: vector<4xf8E8M0FNU>) -> (vector<16xf16>, vector<16xbf16>, vector<16xf32>) {270 // CHECK: amdgpu.scaled_ext_packed816271 %ret0 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<16xf6E3M2FN>, vector<4xf8E8M0FNU> -> vector<16xf16>272 // CHECK: amdgpu.scaled_ext_packed816273 %ret1 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<16xf6E3M2FN>, vector<4xf8E8M0FNU> -> vector<16xbf16>274 // CHECK: amdgpu.scaled_ext_packed816275 %ret2 = amdgpu.scaled_ext_packed816 %v scale(%scale) blockSize(32) firstScaleLane(0) firstScaleByte(0) : vector<16xf6E3M2FN>, vector<4xf8E8M0FNU> -> vector<16xf32>276 func.return %ret0, %ret1, %ret2 : vector<16xf16>, vector<16xbf16>, vector<16xf32>277}278 279// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_f32280// CHECK: amdgpu.packed_scaled_trunc281func.func @packed_scaled_trunc_f8e4m3_f32(%v: vector<2xf32>, %scale: f32) -> vector<4xf8E4M3FN> {282 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf32> to vector<4xf8E4M3FN>283 func.return %ret : vector<4xf8E4M3FN>284}285 286// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_f32287// CHECK: amdgpu.packed_scaled_trunc288func.func @packed_scaled_trunc_into_f8e4m3_f32(%v: vector<2xf32>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {289 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf32> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>290 func.return %ret : vector<4xf8E4M3FN>291}292 293// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_f16294// CHECK: amdgpu.packed_scaled_trunc295func.func @packed_scaled_trunc_f8e4m3_f16(%v: vector<2xf16>, %scale: f32) -> vector<4xf8E4M3FN> {296 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf16> to vector<4xf8E4M3FN>297 func.return %ret : vector<4xf8E4M3FN>298}299 300// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_f16301// CHECK: amdgpu.packed_scaled_trunc302func.func @packed_scaled_trunc_into_f8e4m3_f16(%v: vector<2xf16>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {303 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf16> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>304 func.return %ret : vector<4xf8E4M3FN>305}306 307// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_bf16308// CHECK: amdgpu.packed_scaled_trunc309func.func @packed_scaled_trunc_f8e4m3_bf16(%v: vector<2xbf16>, %scale: f32) -> vector<4xf8E4M3FN> {310 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xbf16> to vector<4xf8E4M3FN>311 func.return %ret : vector<4xf8E4M3FN>312}313 314// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_bf16315// CHECK: amdgpu.packed_scaled_trunc316func.func @packed_scaled_trunc_into_f8e4m3_bf16(%v: vector<2xbf16>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {317 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xbf16> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>318 func.return %ret : vector<4xf8E4M3FN>319}320 321// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_f32322// CHECK: amdgpu.packed_scaled_trunc323func.func @packed_scaled_trunc_f8e5m2_f32(%v: vector<2xf32>, %scale: f32) -> vector<4xf8E5M2> {324 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf32> to vector<4xf8E5M2>325 func.return %ret : vector<4xf8E5M2>326}327 328// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_f32329// CHECK: amdgpu.packed_scaled_trunc330func.func @packed_scaled_trunc_into_f8e5m2_f32(%v: vector<2xf32>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {331 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf32> to vector<4xf8E5M2> into vector<4xf8E5M2>332 func.return %ret : vector<4xf8E5M2>333}334 335// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_f16336// CHECK: amdgpu.packed_scaled_trunc337func.func @packed_scaled_trunc_f8e5m2_f16(%v: vector<2xf16>, %scale: f32) -> vector<4xf8E5M2> {338 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf16> to vector<4xf8E5M2>339 func.return %ret : vector<4xf8E5M2>340}341 342// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_f16343// CHECK: amdgpu.packed_scaled_trunc344func.func @packed_scaled_trunc_into_f8e5m2_f16(%v: vector<2xf16>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {345 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf16> to vector<4xf8E5M2> into vector<4xf8E5M2>346 func.return %ret : vector<4xf8E5M2>347}348 349// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_bf16350// CHECK: amdgpu.packed_scaled_trunc351func.func @packed_scaled_trunc_f8e5m2_bf16(%v: vector<2xbf16>, %scale: f32) -> vector<4xf8E5M2> {352 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xbf16> to vector<4xf8E5M2>353 func.return %ret : vector<4xf8E5M2>354}355 356// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_bf16357// CHECK: amdgpu.packed_scaled_trunc358func.func @packed_scaled_trunc_into_f8e5m2_bf16(%v: vector<2xbf16>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {359 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xbf16> to vector<4xf8E5M2> into vector<4xf8E5M2>360 func.return %ret : vector<4xf8E5M2>361}362 363// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_f32364// CHECK: amdgpu.packed_scaled_trunc365func.func @packed_scaled_trunc_f4e2m1_f32(%v: vector<2xf32>, %scale: f32) -> vector<8xf4E2M1FN> {366 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf32> to vector<8xf4E2M1FN>367 func.return %ret : vector<8xf4E2M1FN>368}369 370// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_f32371// CHECK: amdgpu.packed_scaled_trunc372func.func @packed_scaled_trunc_into_f4e2m1_f32(%v: vector<2xf32>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {373 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf32> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>374 func.return %ret : vector<8xf4E2M1FN>375}376 377// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_f16378// CHECK: amdgpu.packed_scaled_trunc379func.func @packed_scaled_trunc_f4e2m1_f16(%v: vector<2xf16>, %scale: f32) -> vector<8xf4E2M1FN> {380 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf16> to vector<8xf4E2M1FN>381 func.return %ret : vector<8xf4E2M1FN>382}383 384// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_f16385// CHECK: amdgpu.packed_scaled_trunc386func.func @packed_scaled_trunc_into_f4e2m1_f16(%v: vector<2xf16>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {387 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf16> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>388 func.return %ret : vector<8xf4E2M1FN>389}390 391// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_bf16392// CHECK: amdgpu.packed_scaled_trunc393func.func @packed_scaled_trunc_f4e2m1_bf16(%v: vector<2xbf16>, %scale: f32) -> vector<8xf4E2M1FN> {394 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xbf16> to vector<8xf4E2M1FN>395 func.return %ret : vector<8xf4E2M1FN>396}397 398// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_bf16399// CHECK: amdgpu.packed_scaled_trunc400func.func @packed_scaled_trunc_into_f4e2m1_bf16(%v: vector<2xbf16>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {401 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xbf16> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>402 func.return %ret : vector<8xf4E2M1FN>403}404 405// CHECK-LABEL: func @fat_raw_buffer_cast_easy406// CHECK: amdgpu.fat_raw_buffer_cast407func.func @fat_raw_buffer_cast_easy(%m: memref<8xi32>) -> memref<8xi32, #amdgpu.address_space<fat_raw_buffer>> {408 %ret = amdgpu.fat_raw_buffer_cast %m : memref<8xi32> to memref<8xi32, #amdgpu.address_space<fat_raw_buffer>>409 func.return %ret : memref<8xi32, #amdgpu.address_space<fat_raw_buffer>>410}411 412// CHECK-LABEL: func @fat_raw_buffer_cast413// CHECK: amdgpu.fat_raw_buffer_cast414// CHECK-SAME: validBytes(%{{[^)]*}})415// CHECK-SAME: cacheSwizzleStride(%{{[^)]*}})416// CHECK-SAME: boundsCheck(false)417// CHECK-SAME: resetOffset418func.func @fat_raw_buffer_cast(%m: memref<8xi32, strided<[1], offset: ?>>, %validBytes: i64, %cacheSwizzle: i14) -> memref<8xi32, #amdgpu.address_space<fat_raw_buffer>> {419 %ret = amdgpu.fat_raw_buffer_cast %m validBytes(%validBytes) cacheSwizzleStride(%cacheSwizzle) boundsCheck(false) resetOffset420 : memref<8xi32, strided<[1], offset: ?>> to memref<8xi32, #amdgpu.address_space<fat_raw_buffer>>421 func.return %ret : memref<8xi32, #amdgpu.address_space<fat_raw_buffer>>422}423 424// CHECK-LABEL: func @fat_raw_buffer_cast_dynamic_1d_reset_offset425// CHECK: amdgpu.fat_raw_buffer_cast426func.func @fat_raw_buffer_cast_dynamic_1d_reset_offset(%m: memref<?xi32, strided<[1], offset: ?>>) -> memref<?xi32, #amdgpu.address_space<fat_raw_buffer>> {427 %ret = amdgpu.fat_raw_buffer_cast %m resetOffset428 : memref<?xi32, strided<[1], offset: ?>> to memref<?xi32, #amdgpu.address_space<fat_raw_buffer>>429 func.return %ret : memref<?xi32, #amdgpu.address_space<fat_raw_buffer>>430}431 432// CHECK-LABEL: func @fat_raw_buffer_cast_dynamic_0d_reset_offset433// CHECK: %[[ret:.+]] = amdgpu.fat_raw_buffer_cast434// CHECK: return %[[ret]]435func.func @fat_raw_buffer_cast_dynamic_0d_reset_offset(%m: memref<i32, strided<[], offset: ?>>) -> memref<i32, #amdgpu.address_space<fat_raw_buffer>> {436 %ret = amdgpu.fat_raw_buffer_cast %m resetOffset437 : memref<i32, strided<[], offset: ?>> to memref<i32, #amdgpu.address_space<fat_raw_buffer>>438 func.return %ret : memref<i32, #amdgpu.address_space<fat_raw_buffer>>439}440 441// CHECK-LABEL: func @fat_raw_buffer_cast_static_shape_2d_reset_offset442// CHECK: %[[ret:.+]] = amdgpu.fat_raw_buffer_cast443// CHECK: return %[[ret]]444func.func @fat_raw_buffer_cast_static_shape_2d_reset_offset(%m: memref<4x4xi32, strided<[4, 1], offset: ?>>) -> memref<4x4xi32, #amdgpu.address_space<fat_raw_buffer>> {445 %ret = amdgpu.fat_raw_buffer_cast %m resetOffset446 : memref<4x4xi32, strided<[4, 1], offset: ?>> to memref<4x4xi32, #amdgpu.address_space<fat_raw_buffer>>447 func.return %ret : memref<4x4xi32, #amdgpu.address_space<fat_raw_buffer>>448}449 450// CHECK-LABEL: func @fat_raw_buffer_cast_dynamic_2d_reset_offset451// CHECK: %[[ret:.+]] = amdgpu.fat_raw_buffer_cast452// CHECK: return %[[ret]]453func.func @fat_raw_buffer_cast_dynamic_2d_reset_offset(%m: memref<?x?xi32, strided<[?, 1], offset: ?>>) -> memref<?x?xi32, strided<[?, 1]>, #amdgpu.address_space<fat_raw_buffer>> {454 %ret = amdgpu.fat_raw_buffer_cast %m resetOffset455 : memref<?x?xi32, strided<[?, 1], offset: ?>> to memref<?x?xi32, strided<[?, 1]>, #amdgpu.address_space<fat_raw_buffer>>456 func.return %ret : memref<?x?xi32, strided<[?, 1]>, #amdgpu.address_space<fat_raw_buffer>>457}458 459// CHECK-LABEL: func @fat_raw_buffer_cast_noncontiguous_2d_reset_offset460// CHECK: %[[ret:.+]] = amdgpu.fat_raw_buffer_cast461// CHECK: return %[[ret]]462func.func @fat_raw_buffer_cast_noncontiguous_2d_reset_offset(%m: memref<4x4xi32, strided<[8, 1], offset: ?>>) -> memref<4x4xi32, strided<[8, 1]>, #amdgpu.address_space<fat_raw_buffer>> {463 %ret = amdgpu.fat_raw_buffer_cast %m resetOffset464 : memref<4x4xi32, strided<[8, 1], offset: ?>> to memref<4x4xi32, strided<[8, 1]>, #amdgpu.address_space<fat_raw_buffer>>465 func.return %ret : memref<4x4xi32, strided<[8, 1]>, #amdgpu.address_space<fat_raw_buffer>>466}467 468// CHECK-LABEL: func @raw_buffer_load_f32_from_rank_1469func.func @raw_buffer_load_f32_from_rank_1(%src : memref<128xf32>, %offset : i32, %idx0 : i32) -> f32 {470 // CHECK: amdgpu.raw_buffer_load {indexOffset = 1 : i32} %{{.*}}[{{.*}}] sgprOffset %{{.*}} : memref<128xf32>, i32 -> f32471 %0 = amdgpu.raw_buffer_load {boundsCheck = true, indexOffset = 1 : i32} %src[%idx0] sgprOffset %offset : memref<128xf32>, i32 -> f32472 func.return %0 : f32473}474 475// CHECK-LABEL: func @raw_buffer_load_f32_from_rank_4476func.func @raw_buffer_load_f32_from_rank_4(%src : memref<128x64x32x16xf32>, %offset : i32, %idx0 : i32, %idx1 : i32, %idx2 : i32, %idx3 : i32) -> f32 {477 // CHECK: amdgpu.raw_buffer_load {indexOffset = 1 : i32} %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] sgprOffset %{{.*}} : memref<128x64x32x16xf32>, i32, i32, i32, i32 -> f32478 %0 = amdgpu.raw_buffer_load {boundsCheck = true, indexOffset = 1 : i32} %src[%idx0, %idx1, %idx2, %idx3] sgprOffset %offset : memref<128x64x32x16xf32>, i32, i32, i32, i32 -> f32479 func.return %0 : f32480}481 482// CHECK-LABEL: func @raw_buffer_load_scalar483func.func @raw_buffer_load_scalar(%src : memref<f32>) -> f32 {484 // CHECK: amdgpu.raw_buffer_load {indexOffset = 1 : i32} %{{.*}}[] : memref<f32> -> f32485 %0 = amdgpu.raw_buffer_load {indexOffset = 1 : i32} %src[] : memref<f32> -> f32486 func.return %0 : f32487}488 489// CHECK-LABEL: func @raw_buffer_load_4xf32_from_rank_4490func.func @raw_buffer_load_4xf32_from_rank_4(%src : memref<128x64x32x16xf32>, %offset : i32, %idx0 : i32, %idx1 : i32, %idx2 : i32, %idx3 : i32) -> vector<4xf32> {491 // CHECK: amdgpu.raw_buffer_load {indexOffset = 1 : i32} %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] sgprOffset %{{.*}} : memref<128x64x32x16xf32>, i32, i32, i32, i32 -> vector<4xf32>492 %0 = amdgpu.raw_buffer_load {boundsCheck = true, indexOffset = 1 : i32} %src[%idx0, %idx1, %idx2, %idx3] sgprOffset %offset : memref<128x64x32x16xf32>, i32, i32, i32, i32 -> vector<4xf32>493 func.return %0 : vector<4xf32>494}495 496// CHECK-LABEL: func @raw_buffer_store_f32_to_rank_1497func.func @raw_buffer_store_f32_to_rank_1(%value : f32, %dst : memref<128xf32>, %offset : i32, %idx0 : i32) {498 // CHECK: amdgpu.raw_buffer_store {indexOffset = 1 : i32} %{{.*}} -> %{{.*}}[{{.*}}] sgprOffset %{{.*}} : f32 -> memref<128xf32>, i32499 amdgpu.raw_buffer_store {boundsCheck = true, indexOffset = 1 : i32} %value -> %dst[%idx0] sgprOffset %offset : f32 -> memref<128xf32>, i32500 func.return501}502 503// CHECK-LABEL: func @raw_buffer_store_f32_to_rank_4504func.func @raw_buffer_store_f32_to_rank_4(%value : f32, %dst : memref<128x64x32x16xf32>, %offset : i32, %idx0 : i32, %idx1 : i32, %idx2 : i32, %idx3 : i32) {505 // CHECK: amdgpu.raw_buffer_store {indexOffset = 1 : i32} %{{.*}} -> %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] sgprOffset %{{.*}} : f32 -> memref<128x64x32x16xf32>, i32, i32, i32, i32506 amdgpu.raw_buffer_store {boundsCheck = true, indexOffset = 1 : i32} %value -> %dst[%idx0, %idx1, %idx2, %idx3] sgprOffset %offset : f32 -> memref<128x64x32x16xf32>, i32, i32, i32, i32507 func.return508}509 510// CHECK-LABEL: func @raw_buffer_store_4xf32_to_rank_4511func.func @raw_buffer_store_4xf32_to_rank_4(%value : vector<4xf32>, %dst : memref<128x64x32x16xf32>, %offset : i32, %idx0 : i32, %idx1 : i32, %idx2 : i32, %idx3 : i32) {512 // CHECK: amdgpu.raw_buffer_store {indexOffset = 1 : i32} %{{.*}} -> %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] sgprOffset %{{.*}} : vector<4xf32> -> memref<128x64x32x16xf32>, i32, i32, i32, i32513 amdgpu.raw_buffer_store {boundsCheck = true, indexOffset = 1 : i32} %value -> %dst[%idx0, %idx1, %idx2, %idx3] sgprOffset %offset : vector<4xf32> -> memref<128x64x32x16xf32>, i32, i32, i32, i32514 func.return515}516 517// CHECK-LABEL: func @raw_buffer_store_scalar518func.func @raw_buffer_store_scalar(%value : f32, %dst : memref<f32>) {519 // CHECK: amdgpu.raw_buffer_store {indexOffset = 1 : i32} %{{.*}} -> %{{.*}}[] : f32 -> memref<f32>520 amdgpu.raw_buffer_store {indexOffset = 1 : i32} %value -> %dst[] : f32 -> memref<f32>521 func.return522}523 524// CHECK-LABEL: func @raw_buffer_atomic_fadd_f32_to_rank_1525func.func @raw_buffer_atomic_fadd_f32_to_rank_1(%value : f32, %dst : memref<128xf32>, %offset : i32, %idx0 : i32) {526 // CHECK: amdgpu.raw_buffer_atomic_fadd {indexOffset = 1 : i32} %{{.*}} -> %{{.*}}[{{.*}}] sgprOffset %{{.*}} : f32 -> memref<128xf32>, i32527 amdgpu.raw_buffer_atomic_fadd {boundsCheck = true, indexOffset = 1 : i32} %value -> %dst[%idx0] sgprOffset %offset : f32 -> memref<128xf32>, i32528 func.return529}530 531// CHECK-LABEL: func @raw_buffer_atomic_fadd_f32_to_rank_4532func.func @raw_buffer_atomic_fadd_f32_to_rank_4(%value : f32, %dst : memref<128x64x32x16xf32>, %offset : i32, %idx0 : i32, %idx1 : i32, %idx2 : i32, %idx3 : i32) {533 // CHECK: amdgpu.raw_buffer_atomic_fadd {indexOffset = 1 : i32} %{{.*}} -> %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] sgprOffset %{{.*}} : f32 -> memref<128x64x32x16xf32>, i32, i32, i32, i32534 amdgpu.raw_buffer_atomic_fadd {boundsCheck = true, indexOffset = 1 : i32} %value -> %dst[%idx0, %idx1, %idx2, %idx3] sgprOffset %offset : f32 -> memref<128x64x32x16xf32>, i32, i32, i32, i32535 func.return536}537 538// CHECK-LABEL: func @raw_buffer_atomic_cmpswap_f32539func.func @raw_buffer_atomic_cmpswap_f32(%src : f32, %cmp : f32, %dst : memref<128x64x32x16xf32>, %offset : i32, %idx0 : i32, %idx1 : i32, %idx2 : i32, %idx3 : i32) {540 // CHECK: amdgpu.raw_buffer_atomic_cmpswap {indexOffset = 1 : i32} %{{.*}}, %{{.*}} -> %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] sgprOffset %{{.*}} : f32 -> memref<128x64x32x16xf32>, i32, i32, i32, i32541 amdgpu.raw_buffer_atomic_cmpswap {boundsCheck = true, indexOffset = 1 : i32} %src, %cmp -> %dst[%idx0, %idx1, %idx2, %idx3] sgprOffset %offset : f32 -> memref<128x64x32x16xf32>, i32, i32, i32, i32542 func.return543}544 545// CHECK-LABEL: func @lds_barrier546func.func @lds_barrier() {547 // CHECK: amdgpu.lds_barrier548 amdgpu.lds_barrier549 func.return550}551 552// CHECK-LABEL: func @sched_barrier553func.func @sched_barrier() {554 // CHECK: amdgpu.sched_barrier allow = <none>555 amdgpu.sched_barrier allow = <none>556 // CHECK: amdgpu.sched_barrier allow = <valu|all_vmem>557 amdgpu.sched_barrier allow = <valu|all_vmem>558 func.return559}560 561// CHECK-LABEL: func @mfma562func.func @mfma(%arg0 : vector<4xf16>, %arg1 : vector<4xf32>) -> vector<4xf32> {563 // CHECK: amdgpu.mfma 16x16x16564 %0 = amdgpu.mfma 16x16x16 %arg0 * %arg0 + %arg1 { abid = 0 : i32, cbsz = 0 : i32 } blgp = none : vector<4xf16>, vector<4xf16>, vector<4xf32>565 func.return %0 : vector<4xf32>566}567 568// CHECK-LABEL: func @mfma_with_blocks569func.func @mfma_with_blocks(%arg0 : f32, %arg1 : vector<32xf32>) -> vector<32xf32> {570 // CHECK: amdgpu.mfma 32x32x1571 %0 = amdgpu.mfma 32x32x1 %arg0 * %arg0 + %arg1 { abid = 1 : i32, cbsz = 1 : i32, blocks = 2 : i32 } blgp = bcast_second_32 : f32, f32, vector<32xf32>572 func.return %0 : vector<32xf32>573}574 575// CHECK-LABEL: func @wmma_f16_16x16x16_f16576func.func @wmma_f16_16x16x16_f16(%arg0 : vector<16xf16>, %arg1 : vector<8xf16>) -> vector<8xf16> {577 // CHECK: amdgpu.wmma 16x16x16578 %0 = amdgpu.wmma 16x16x16 %arg0 * %arg0 + %arg1 : vector<16xf16>, vector<16xf16>, vector<8xf16>579 func.return %0 : vector<8xf16>580}581 582// CHECK-LABEL: func @wmma_i32_16x16x32_i4583func.func @wmma_i32_16x16x32_i4(%arg0 : vector<16xi4>, %arg1 : vector<8xi32>) -> vector<8xi32> {584 // CHECK: amdgpu.wmma 16x16x32585 %0 = amdgpu.wmma 16x16x32 %arg0 * %arg0 + %arg1 : vector<16xi4>, vector<16xi4>, vector<8xi32>586 func.return %0 : vector<8xi32>587}588 589// CHECK-LABEL: func @wmma_f32_16x16x4_f32590func.func @wmma_f32_16x16x4_f32(%arg0 : vector<2xf32>, %arg1 : vector<8xf32>) -> vector<8xf32> {591 // CHECK: amdgpu.wmma 16x16x4592 %0 = amdgpu.wmma 16x16x4 %arg0 * %arg0 + %arg1 : vector<2xf32>, vector<2xf32>, vector<8xf32>593 func.return %0 : vector<8xf32>594}595 596// CHECK-LABEL: func @wmma_f32_16x16x64_f8597func.func @wmma_f32_16x16x64_f8(%arg0 : vector<32xf8E4M3FN>, %arg1 : vector<8xf32>) -> vector<8xf32> {598 // CHECK: amdgpu.wmma 16x16x64599 %0 = amdgpu.wmma 16x16x64 %arg0 * %arg0 + %arg1 : vector<32xf8E4M3FN>, vector<32xf8E4M3FN>, vector<8xf32>600 func.return %0 : vector<8xf32>601}602 603// CHECK-LABEL: func @wmma_f32_16x16x64_bf8604func.func @wmma_f32_16x16x64_bf8(%arg0 : vector<32xf8E5M2>, %arg1 : vector<8xf32>) -> vector<8xf32> {605 // CHECK: amdgpu.wmma 16x16x64606 %0 = amdgpu.wmma 16x16x64 %arg0 * %arg0 + %arg1 : vector<32xf8E5M2>, vector<32xf8E5M2>, vector<8xf32>607 func.return %0 : vector<8xf32>608}609 610// CHECK-LABEL: func @wmma_f16_16x16x64_bf8611func.func @wmma_f16_16x16x64_bf8(%arg0 : vector<32xf8E5M2>, %arg1 : vector<8xf16>) -> vector<8xf16> {612 // CHECK: amdgpu.wmma 16x16x64613 %0 = amdgpu.wmma 16x16x64 %arg0 * %arg0 + %arg1 : vector<32xf8E5M2>, vector<32xf8E5M2>, vector<8xf16>614 func.return %0 : vector<8xf16>615}616 617// CHECK-LABEL: func @wmma_f16_16x16x64_f8618func.func @wmma_f16_16x16x64_f8(%arg0 : vector<32xf8E4M3FN>, %arg1 : vector<8xf16>) -> vector<8xf16> {619 // CHECK: amdgpu.wmma 16x16x64620 %0 = amdgpu.wmma 16x16x64 %arg0 * %arg0 + %arg1 : vector<32xf8E4M3FN>, vector<32xf8E4M3FN>, vector<8xf16>621 func.return %0 : vector<8xf16>622}623 624// CHECK-LABEL: func @swizzle_bitmode625func.func @swizzle_bitmode(%arg0 : f32) -> f32 {626 // CHECK: amdgpu.swizzle_bitmode627 %0 = amdgpu.swizzle_bitmode %arg0 1 2 4 : f32628 func.return %0 : f32629}630 631// CHECK-LABEL: func @permlane16_swap632func.func @permlane16_swap(%arg0 : f32) -> f32 {633 // CHECK: amdgpu.permlane_swap634 %0 = amdgpu.permlane_swap %arg0 16 : f32635 func.return %0 : f32636}637 638// CHECK-LABEL: func @permlane32_swap639func.func @permlane32_swap(%arg0 : f32) -> f32 {640 // CHECK: amdgpu.permlane_swap641 %0 = amdgpu.permlane_swap %arg0 32 : f32642 func.return %0 : f32643}644 645// CHECK-LABEL: func @scaled_mfma646func.func @scaled_mfma(%arg0 : f8E8M0FNU, %arg1 : vector<32xf6E2M3FN>, %arg2 : vector<16xf32>) -> vector<16xf32> {647 // CHECK: amdgpu.scaled_mfma 32x32x64648 %0 = amdgpu.scaled_mfma 32x32x64 (%arg0[0] * %arg1) * (%arg0[1] * %arg1) + %arg2 : f8E8M0FNU, vector<32xf6E2M3FN>, f8E8M0FNU, vector<32xf6E2M3FN>, vector<16xf32>649 func.return %0 : vector<16xf32>650}651 652// CHECK-LABEL: func @transpose_load653func.func @transpose_load(%idx1 : index, %idx2 : index, %mem : memref<128x32xf16, 3>) -> vector<4xf16> {654 // CHECK: amdgpu.transpose_load655 %0 = amdgpu.transpose_load %mem[%idx1, %idx2] : memref<128x32xf16, 3> -> vector<4xf16>656 func.return %0 : vector<4xf16>657}658 659// CHECK-LABEL: func @gather_to_lds660func.func @gather_to_lds(%idx1 : index, %idx2 : index, %mem1 : memref<32xf16>, %mem2 : memref<32x32xf16>, %smem1 : memref<32xf16, #gpu.address_space<workgroup>>, %smem2 : memref<32x32xf16, #gpu.address_space<workgroup>>, %smem3 : memref<?x?xf16, strided<[?, 1]>, #gpu.address_space<workgroup>>) {661 // CHECK: amdgpu.gather_to_lds %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}, %{{.*}}]662 // CHECK: amdgpu.gather_to_lds %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}]663 // CHECK: amdgpu.gather_to_lds %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}, %{{.*}}]664 // CHECK: amdgpu.gather_to_lds %{{.*}}[%{{.*}}], %{{.*}}[%{{.*}}, %{{.*}}]665 amdgpu.gather_to_lds %mem2[%idx1, %idx2], %smem2[%idx1, %idx2] : vector<2xf16>, memref<32x32xf16>, memref<32x32xf16, #gpu.address_space<workgroup>>666 amdgpu.gather_to_lds %mem2[%idx1, %idx2], %smem1[%idx1] : vector<2xf16>, memref<32x32xf16>, memref<32xf16, #gpu.address_space<workgroup>>667 amdgpu.gather_to_lds %mem1[%idx1], %smem2[%idx1, %idx2] : vector<2xf16>, memref<32xf16>, memref<32x32xf16, #gpu.address_space<workgroup>>668 amdgpu.gather_to_lds %mem1[%idx1], %smem3[%idx1, %idx2] : vector<2xf16>, memref<32xf16>, memref<?x?xf16, strided<[?, 1]>, #gpu.address_space<workgroup>>669 func.return670}671 672// CHECK-LABEL: func @memory_counter_wait673func.func @memory_counter_wait() {674 // CHECK: amdgpu.memory_counter_wait load(1) store(2) ds(3) exp(4)675 // CHECK: amdgpu.memory_counter_wait load(4) store(2) ds(3) exp(1)676 // CHECK: amdgpu.memory_counter_wait load(1)677 // CHECK: amdgpu.memory_counter_wait store(2)678 // CHECK: amdgpu.memory_counter_wait ds(3)679 // CHECK: amdgpu.memory_counter_wait exp(4)680 amdgpu.memory_counter_wait load(1) store(2) ds(3) exp(4)681 amdgpu.memory_counter_wait exp(1) store(2) ds(3) load(4)682 amdgpu.memory_counter_wait load(1)683 amdgpu.memory_counter_wait store(2)684 amdgpu.memory_counter_wait ds(3)685 amdgpu.memory_counter_wait exp(4)686 func.return687}688 689// CHECK-LABEL: func @make_dma_base690// CHECK-SAME: (%[[IDX:.+]]: index, %[[MEM:.+]]: memref<8xi32>, %[[SMEM:.+]]: memref<8xi32, #gpu.address_space<workgroup>>)691func.func @make_dma_base(%idx: index, %mem: memref<8xi32>, %smem: memref<8xi32, #gpu.address_space<workgroup>>) {692 // CHECK: amdgpu.make_dma_base %[[MEM]][%[[IDX]]], %[[SMEM]][%[[IDX]]] : memref<8xi32>, memref<8xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_base<i32>693 amdgpu.make_dma_base %mem[%idx], %smem[%idx] : memref<8xi32>, memref<8xi32, #gpu.address_space<workgroup>> -> !amdgpu.tdm_base<i32>694 695 // CHECK: amdgpu.make_dma_base %[[SMEM]][%[[IDX]]], %[[MEM]][%[[IDX]]] : memref<8xi32, #gpu.address_space<workgroup>>, memref<8xi32> -> !amdgpu.tdm_base<i32>696 amdgpu.make_dma_base %smem[%idx], %mem[%idx] : memref<8xi32, #gpu.address_space<workgroup>>, memref<8xi32> -> !amdgpu.tdm_base<i32>697 func.return698}699 700// CHECK-LABEL: func @make_dma_descriptor701// CHECK-SAME: (%[[BASE:.+]]: !amdgpu.tdm_base<i32>, %[[BARRIER:.+]]: memref<8xi32, #gpu.address_space<workgroup>>, %[[IDX:.+]]: index)702func.func @make_dma_descriptor(%base: !amdgpu.tdm_base<i32>, %barrier: memref<8xi32, #gpu.address_space<workgroup>>, %idx: index) {703 704 // CHECK: amdgpu.make_dma_descriptor %[[BASE]]705 amdgpu.make_dma_descriptor %base706 // CHECK-SAME: globalSize [0]707 globalSize [0]708 // CHECK-SAME: globalStride [1]709 globalStride [1]710 // CHECK-SAME: sharedSize [0] : !amdgpu.tdm_base<i32> -> !amdgpu.tdm_descriptor711 sharedSize [0] : !amdgpu.tdm_base<i32> -> !amdgpu.tdm_descriptor712 713 // CHECK: amdgpu.make_dma_descriptor %[[BASE]]714 amdgpu.make_dma_descriptor %base715 // CHECK-SAME: globalSize [0]716 globalSize [0]717 // CHECK-SAME: globalStride [1]718 globalStride [1]719 // CHECK-SAME: sharedSize [0]720 sharedSize [0]721 // CHECK-SAME: padShared(%[[IDX]] every %[[IDX]])722 padShared(%idx every %idx)723 : !amdgpu.tdm_base<i32> -> !amdgpu.tdm_descriptor724 725 // CHECK: amdgpu.make_dma_descriptor %[[BASE]]726 amdgpu.make_dma_descriptor %base727 // CHECK-SAME: globalSize [0]728 globalSize [0]729 // CHECK-SAME: globalStride [1]730 globalStride [1]731 // CHECK-SAME: sharedSize [0]732 sharedSize [0]733 // CHECK-SAME: atomicBarrier(%[[BARRIER]][%[[IDX]]] : memref<8xi32, #gpu.address_space<workgroup>>)734 atomicBarrier(%barrier[%idx] : memref<8xi32, #gpu.address_space<workgroup>>)735 : !amdgpu.tdm_base<i32> -> !amdgpu.tdm_descriptor736 737 // CHECK: amdgpu.make_dma_descriptor %[[BASE]]738 amdgpu.make_dma_descriptor %base739 // CHECK-SAME: globalSize [0]740 globalSize [0]741 // CHECK-SAME: globalStride [1]742 globalStride [1]743 // CHECK-SAME: sharedSize [0]744 sharedSize [0]745 // CHECK-SAME: iterate %[[IDX]], %[[IDX]], %[[IDX]]746 iterate %idx, %idx, %idx747 : !amdgpu.tdm_base<i32> -> !amdgpu.tdm_descriptor748 749 func.return750}751