1402 lines · plain
1// RUN: mlir-opt %s -split-input-file -verify-diagnostics | FileCheck %s2 3func.func @rocdl_special_regs() -> i32 {4 // CHECK-LABEL: rocdl_special_regs5 // CHECK: rocdl.workitem.id.x : i326 %0 = rocdl.workitem.id.x : i327 // CHECK: rocdl.workitem.id.y : i328 %1 = rocdl.workitem.id.y : i329 // CHECK: rocdl.workitem.id.z : i3210 %2 = rocdl.workitem.id.z : i3211 // CHECK: rocdl.workgroup.id.x : i3212 %3 = rocdl.workgroup.id.x : i3213 // CHECK: rocdl.workgroup.id.y : i3214 %4 = rocdl.workgroup.id.y : i3215 // CHECK: rocdl.workgroup.id.z : i3216 %5 = rocdl.workgroup.id.z : i3217 // CHECK: rocdl.cluster.id.x : i3218 %6 = rocdl.cluster.id.x : i3219 // CHECK: rocdl.cluster.id.y : i3220 %7 = rocdl.cluster.id.y : i3221 // CHECK: rocdl.cluster.id.z : i3222 %8 = rocdl.cluster.id.z : i3223 // CHECK: rocdl.workgroup.dim.x : i3224 %9 = rocdl.workgroup.dim.x : i3225 // CHECK: rocdl.workgroup.dim.y : i3226 %10 = rocdl.workgroup.dim.y : i3227 // CHECK: rocdl.workgroup.dim.z : i3228 %11 = rocdl.workgroup.dim.z : i3229 // CHECK: rocdl.grid.dim.x : i3230 %12 = rocdl.grid.dim.x : i3231 // CHECK: rocdl.grid.dim.y : i3232 %13 = rocdl.grid.dim.y : i3233 // CHECK: rocdl.grid.dim.z : i3234 %14 = rocdl.grid.dim.z : i3235 llvm.return %0 : i3236}37 38func.func @rocdl.fmed3.scalar(%a: f32, %b: f32, %c: f32) -> f32 {39 // CHECK-LABEL: rocdl.fmed3.scalar40 // CHECK: %0 = rocdl.fmed3 %arg0, %arg1, %arg2 : f3241 %0 = rocdl.fmed3 %a, %b, %c : f3242 llvm.return %0 : f3243}44 45func.func @rocdl.fmed3.vector(%a: vector<4xf16>, %b: vector<4xf16>, %c: vector<4xf16>) -> vector<4xf16> {46 // CHECK-LABEL: rocdl.fmed3.vector47 // CHECK: %0 = rocdl.fmed3 %arg0, %arg1, %arg2 : vector<4xf16>48 %0 = rocdl.fmed3 %a, %b, %c : vector<4xf16>49 llvm.return %0 : vector<4xf16>50}51 52func.func @rocdl.barrier() {53 // CHECK: rocdl.barrier54 rocdl.barrier55 llvm.return56}57 58func.func @rocdl.sched_barrier() {59 // CHECK: rocdl.sched.barrier60 rocdl.sched.barrier 061 llvm.return62}63 64func.func @rocdl_sched_group_barrier() {65 // CHECK: rocdl.sched.group.barrier66 rocdl.sched.group.barrier 8, 1, 067 llvm.return68}69 70func.func @rocdl_iglp_opt() {71 // CHECK: rocdl.iglp.opt72 rocdl.iglp.opt 073 llvm.return74}75 76func.func @rocdl.setprio() {77 // CHECK: rocdl.s.setprio78 rocdl.s.setprio 079 llvm.return80}81 82func.func @rocdl.xdlops(%arg0 : f32, %arg1 : f32,83 %arg2 : vector<32xf32>, %arg3 : i32,84 %arg4 : vector<16xf32>, %arg5 : vector<4xf32>,85 %arg6 : vector<4xf16>, %arg7 : vector<32xi32>,86 %arg8 : vector<16xi32>, %arg9 : vector<4xi32>,87 %arg10 : vector<2xi16>, %arg11 : vector<4xi16>,88 %arg12 : vector<4xf64>, %arg13 : f64,89 %arg14 : i64, %arg15 : vector<2xf32>,90 %arg16: vector<8xbf16>, %arg17 : vector<8xf16>) {91 // CHECK-LABEL: rocdl.xdlops92 // CHECK: rocdl.mfma.f32.32x32x1f32 {{.*}} : (f32, f32, vector<32xf32>, i32, i32, i32) -> vector<32xf32>93 %r0 = rocdl.mfma.f32.32x32x1f32 %arg0, %arg1, %arg2, %arg3, %arg3, %arg3 :94 (f32, f32, vector<32xf32>,95 i32, i32, i32) -> vector<32xf32>96 97 // CHECK: rocdl.mfma.f32.16x16x1f32 {{.*}} : (f32, f32, vector<16xf32>, i32, i32, i32) -> vector<16xf32>98 %r1 = rocdl.mfma.f32.16x16x1f32 %arg0, %arg1, %arg4, %arg3, %arg3, %arg3 :99 (f32, f32, vector<16xf32>,100 i32, i32, i32) -> vector<16xf32>101 102 // CHECK: rocdl.mfma.f32.4x4x1f32 {{.*}} : (f32, f32, vector<4xf32>, i32, i32, i32) -> vector<4xf32>103 %r2 = rocdl.mfma.f32.4x4x1f32 %arg0, %arg1, %arg5, %arg3, %arg3, %arg3 :104 (f32, f32, vector<4xf32>,105 i32, i32, i32) -> vector<4xf32>106 107 // CHECK: rocdl.mfma.f32.32x32x2f32 {{.*}} : (f32, f32, vector<16xf32>, i32, i32, i32) -> vector<16xf32>108 %r3= rocdl.mfma.f32.32x32x2f32 %arg0, %arg1, %arg4, %arg3, %arg3, %arg3 :109 (f32, f32, vector<16xf32>,110 i32, i32, i32) -> vector<16xf32>111 112 // CHECK: rocdl.mfma.f32.16x16x4f32 {{.*}} : (f32, f32, vector<4xf32>, i32, i32, i32) -> vector<4xf32>113 %r4 = rocdl.mfma.f32.16x16x4f32 %arg0, %arg1, %arg5, %arg3, %arg3, %arg3 :114 (f32, f32, vector<4xf32>,115 i32, i32, i32) -> vector<4xf32>116 117 // CHECK: rocdl.mfma.f32.32x32x4f16 {{.*}} : (vector<4xf16>, vector<4xf16>, vector<32xf32>, i32, i32, i32) -> vector<32xf32>118 %r5 = rocdl.mfma.f32.32x32x4f16 %arg6, %arg6, %arg2, %arg3, %arg3, %arg3 :119 (vector<4xf16>, vector<4xf16>, vector<32xf32>,120 i32, i32, i32) -> vector<32xf32>121 122 // CHECK: rocdl.mfma.f32.16x16x4f16 {{.*}} : (vector<4xf16>, vector<4xf16>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>123 %r6 = rocdl.mfma.f32.16x16x4f16 %arg6, %arg6, %arg4, %arg3, %arg3, %arg3 :124 (vector<4xf16>, vector<4xf16>, vector<16xf32>,125 i32, i32, i32) -> vector<16xf32>126 127 // CHECK: rocdl.mfma.f32.4x4x4f16 {{.*}} : (vector<4xf16>, vector<4xf16>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>128 %r7 = rocdl.mfma.f32.4x4x4f16 %arg6, %arg6, %arg5, %arg3, %arg3, %arg3 :129 (vector<4xf16>, vector<4xf16>, vector<4xf32>,130 i32, i32, i32) -> vector<4xf32>131 132 // CHECK: rocdl.mfma.f32.32x32x8f16 {{.*}} : (vector<4xf16>, vector<4xf16>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>133 %r8 = rocdl.mfma.f32.32x32x8f16 %arg6, %arg6, %arg4, %arg3, %arg3, %arg3 :134 (vector<4xf16>, vector<4xf16>, vector<16xf32>,135 i32, i32, i32) -> vector<16xf32>136 137 // CHECK: rocdl.mfma.f32.16x16x16f16 {{.*}} : (vector<4xf16>, vector<4xf16>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>138 %r9 = rocdl.mfma.f32.16x16x16f16 %arg6, %arg6, %arg5, %arg3, %arg3, %arg3 :139 (vector<4xf16>, vector<4xf16>, vector<4xf32>,140 i32, i32, i32) -> vector<4xf32>141 142 // CHECK: rocdl.mfma.i32.32x32x4i8 {{.*}} : (i32, i32, vector<32xi32>, i32, i32, i32) -> vector<32xi32>143 %r10 = rocdl.mfma.i32.32x32x4i8 %arg3, %arg3, %arg7, %arg3, %arg3, %arg3 :144 (i32, i32, vector<32xi32>,145 i32, i32, i32) -> vector<32xi32>146 147 // CHECK: rocdl.mfma.i32.16x16x4i8 {{.*}} : (i32, i32, vector<16xi32>, i32, i32, i32) -> vector<16xi32>148 %r11 = rocdl.mfma.i32.16x16x4i8 %arg3, %arg3, %arg8, %arg3, %arg3, %arg3 :149 (i32, i32, vector<16xi32>,150 i32, i32, i32) -> vector<16xi32>151 152 // CHECK: rocdl.mfma.i32.4x4x4i8 {{.*}} : (i32, i32, vector<4xi32>, i32, i32, i32) -> vector<4xi32>153 %r12 = rocdl.mfma.i32.4x4x4i8 %arg3, %arg3, %arg9, %arg3, %arg3, %arg3 :154 (i32, i32, vector<4xi32>,155 i32, i32, i32) -> vector<4xi32>156 157 // CHECK: rocdl.mfma.i32.32x32x8i8 {{.*}} : (i32, i32, vector<16xi32>, i32, i32, i32) -> vector<16xi32>158 %r13 = rocdl.mfma.i32.32x32x8i8 %arg3, %arg3, %arg8, %arg3, %arg3, %arg3 :159 (i32, i32, vector<16xi32>,160 i32, i32, i32) -> vector<16xi32>161 162 // CHECK: rocdl.mfma.i32.16x16x16i8 {{.*}} : (i32, i32, vector<4xi32>, i32, i32, i32) -> vector<4xi32>163 %r14 = rocdl.mfma.i32.16x16x16i8 %arg3, %arg3, %arg9, %arg3, %arg3, %arg3 :164 (i32, i32, vector<4xi32>,165 i32, i32, i32) -> vector<4xi32>166 167 // CHECK: rocdl.mfma.f32.32x32x2bf16 {{.*}} : (vector<2xi16>, vector<2xi16>, vector<32xf32>, i32, i32, i32) -> vector<32xf32>168 %r15 = rocdl.mfma.f32.32x32x2bf16 %arg10, %arg10, %arg2, %arg3, %arg3, %arg3 :169 (vector<2xi16>, vector<2xi16>, vector<32xf32>,170 i32, i32, i32) -> vector<32xf32>171 172 // CHECK: rocdl.mfma.f32.16x16x2bf16 {{.*}} : (vector<2xi16>, vector<2xi16>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>173 %r16 = rocdl.mfma.f32.16x16x2bf16 %arg10, %arg10, %arg4, %arg3, %arg3, %arg3 :174 (vector<2xi16>, vector<2xi16>, vector<16xf32>,175 i32, i32, i32) -> vector<16xf32>176 177 // CHECK: rocdl.mfma.f32.4x4x2bf16 {{.*}} : (vector<2xi16>, vector<2xi16>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>178 %r17 = rocdl.mfma.f32.4x4x2bf16 %arg10, %arg10, %arg5, %arg3, %arg3, %arg3 :179 (vector<2xi16>, vector<2xi16>, vector<4xf32>,180 i32, i32, i32) -> vector<4xf32>181 182 // CHECK: rocdl.mfma.f32.32x32x4bf16 {{.*}} : (vector<2xi16>, vector<2xi16>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>183 %r18 = rocdl.mfma.f32.32x32x4bf16 %arg10, %arg10, %arg4, %arg3, %arg3, %arg3 :184 (vector<2xi16>, vector<2xi16>, vector<16xf32>,185 i32, i32, i32) -> vector<16xf32>186 187 // CHECK: rocdl.mfma.f32.16x16x8bf16 {{.*}} : (vector<2xi16>, vector<2xi16>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>188 %r19 = rocdl.mfma.f32.16x16x8bf16 %arg10, %arg10, %arg5, %arg3, %arg3, %arg3 :189 (vector<2xi16>, vector<2xi16>, vector<4xf32>,190 i32, i32, i32) -> vector<4xf32>191 192 193 // CHECK: rocdl.mfma.f32.32x32x4bf16.1k {{.*}} : (vector<4xi16>, vector<4xi16>, vector<32xf32>, i32, i32, i32) -> vector<32xf32>194 %r20 = rocdl.mfma.f32.32x32x4bf16.1k %arg11, %arg11, %arg2, %arg3, %arg3, %arg3 :195 (vector<4xi16>, vector<4xi16>, vector<32xf32>,196 i32, i32, i32) -> vector<32xf32>197 198 // CHECK: rocdl.mfma.f32.16x16x4bf16.1k {{.*}} : (vector<4xi16>, vector<4xi16>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>199 %r21 = rocdl.mfma.f32.16x16x4bf16.1k %arg11, %arg11, %arg4, %arg3, %arg3, %arg3 :200 (vector<4xi16>, vector<4xi16>, vector<16xf32>,201 i32, i32, i32) -> vector<16xf32>202 203 // CHECK: rocdl.mfma.f32.4x4x4bf16.1k {{.*}} : (vector<4xi16>, vector<4xi16>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>204 %r22 = rocdl.mfma.f32.4x4x4bf16.1k %arg11, %arg11, %arg5, %arg3, %arg3, %arg3 :205 (vector<4xi16>, vector<4xi16>, vector<4xf32>,206 i32, i32, i32) -> vector<4xf32>207 208 // CHECK: rocdl.mfma.f32.32x32x8bf16.1k {{.*}} : (vector<4xi16>, vector<4xi16>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>209 %r23 = rocdl.mfma.f32.32x32x8bf16.1k %arg11, %arg11, %arg4, %arg3, %arg3, %arg3 :210 (vector<4xi16>, vector<4xi16>, vector<16xf32>,211 i32, i32, i32) -> vector<16xf32>212 213 // CHECK: rocdl.mfma.f32.16x16x16bf16.1k {{.*}} : (vector<4xi16>, vector<4xi16>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>214 %r24 = rocdl.mfma.f32.16x16x16bf16.1k %arg11, %arg11, %arg5, %arg3, %arg3, %arg3 :215 (vector<4xi16>, vector<4xi16>, vector<4xf32>,216 i32, i32, i32) -> vector<4xf32>217 218 // CHECK: rocdl.mfma.f64.16x16x4f64 {{.*}} : (f64, f64, vector<4xf64>, i32, i32, i32) -> vector<4xf64>219 %r25 = rocdl.mfma.f64.16x16x4f64 %arg13, %arg13, %arg12, %arg3, %arg3, %arg3 :220 (f64, f64, vector<4xf64>,221 i32, i32, i32) -> vector<4xf64>222 223 // CHECK: rocdl.mfma.f64.4x4x4f64 {{.*}} : (f64, f64, f64, i32, i32, i32) -> f64224 %r26 = rocdl.mfma.f64.4x4x4f64 %arg13, %arg13, %arg13, %arg3, %arg3, %arg3 :225 (f64, f64, f64,226 i32, i32, i32) -> f64227 228 // CHECK: rocdl.mfma.i32.16x16x32.i8 {{.*}} : (i64, i64, vector<4xi32>, i32, i32, i32) -> vector<4xi32>229 %r27 = rocdl.mfma.i32.16x16x32.i8 %arg14, %arg14, %arg9, %arg3, %arg3, %arg3 :230 (i64, i64, vector<4xi32>,231 i32, i32, i32) -> vector<4xi32>232 233 // CHECK: rocdl.mfma.i32.32x32x16.i8 {{.*}} : (i64, i64, vector<16xi32>, i32, i32, i32) -> vector<16xi32>234 %r28 = rocdl.mfma.i32.32x32x16.i8 %arg14, %arg14, %arg8, %arg3, %arg3, %arg3 :235 (i64, i64, vector<16xi32>,236 i32, i32, i32) -> vector<16xi32>237 238 // CHECK: rocdl.mfma.f32.16x16x8.xf32 {{.*}} : (vector<2xf32>, vector<2xf32>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>239 %r29 = rocdl.mfma.f32.16x16x8.xf32 %arg15, %arg15, %arg5, %arg3, %arg3, %arg3 :240 (vector<2xf32>, vector<2xf32>, vector<4xf32>,241 i32, i32, i32) -> vector<4xf32>242 243 // CHECK: rocdl.mfma.f32.32x32x4.xf32 {{.*}} : (vector<2xf32>, vector<2xf32>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>244 %r30 = rocdl.mfma.f32.32x32x4.xf32 %arg15, %arg15, %arg4, %arg3, %arg3, %arg3 :245 (vector<2xf32>, vector<2xf32>, vector<16xf32>,246 i32, i32, i32) -> vector<16xf32>247 248 // CHECK: rocdl.mfma.f32.16x16x32.bf16 {{.*}} : (vector<8xbf16>, vector<8xbf16>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>249 %r31 = rocdl.mfma.f32.16x16x32.bf16 %arg16, %arg16, %arg5, %arg3, %arg3, %arg3 :250 (vector<8xbf16>, vector<8xbf16>, vector<4xf32>,251 i32, i32, i32) -> vector<4xf32>252 253 // CHECK: rocdl.mfma.i32.16x16x64.i8 {{.*}} : (vector<4xi32>, vector<4xi32>, vector<4xi32>, i32, i32, i32) -> vector<4xi32>254 %r32 = rocdl.mfma.i32.16x16x64.i8 %arg9, %arg9, %arg9, %arg3, %arg3, %arg3 :255 (vector<4xi32>, vector<4xi32>, vector<4xi32>,256 i32, i32, i32) -> vector<4xi32>257 258 // CHECK: rocdl.mfma.f32.16x16x32.f16 {{.*}} : (vector<8xf16>, vector<8xf16>, vector<4xf32>, i32, i32, i32) -> vector<4xi32>259 %r33 = rocdl.mfma.f32.16x16x32.f16 %arg17, %arg17, %arg5, %arg3, %arg3, %arg3 :260 (vector<8xf16>, vector<8xf16>, vector<4xf32>,261 i32, i32, i32) -> vector<4xi32>262 263 // CHECK: rocdl.mfma.f32.32x32x16.bf16 {{.*}} : (vector<8xbf16>, vector<8xbf16>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>264 %r34 = rocdl.mfma.f32.32x32x16.bf16 %arg16, %arg16, %arg4, %arg3, %arg3, %arg3 :265 (vector<8xbf16>, vector<8xbf16>, vector<16xf32>,266 i32, i32, i32) -> vector<16xf32>267 268 // CHECK: rocdl.mfma.i32.32x32x32.i8 {{.*}} : (vector<4xi32>, vector<4xi32>, vector<16xi32>, i32, i32, i32) -> vector<16xi32>269 %r35 = rocdl.mfma.i32.32x32x32.i8 %arg9, %arg9, %arg8, %arg3, %arg3, %arg3 :270 (vector<4xi32>, vector<4xi32>, vector<16xi32>,271 i32, i32, i32) -> vector<16xi32>272 273 // CHECK: rocdl.mfma.f32.32x32x16.f16 {{.*}} : (vector<8xf16>, vector<8xf16>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>274 %r36 = rocdl.mfma.f32.32x32x16.f16 %arg17, %arg17, %arg4, %arg3, %arg3, %arg3 :275 (vector<8xf16>, vector<8xf16>, vector<16xf32>,276 i32, i32, i32) -> vector<16xf32>277 278 llvm.return279}280 281 282llvm.func @rocdl.smfmac(%arg0 : i32,283 %arg1 : vector<4 x f16>,284 %arg2 : vector<8 x f16>,285 %arg3 : vector<4 x f32>,286 %arg4 : vector<16 x f32>,287 %arg5 : vector<4 x i16>,288 %arg6 : vector<8 x i16>,289 %arg7 : vector<2xi32>,290 %arg8 : vector<4xi32>,291 %arg9 : vector<16xi32>) -> vector<4 x f32> {292 %csti32 = llvm.mlir.constant(42 : i32) : i32293 294 // CHECK-LABEL: rocdl.smfmac295 // CHECK: rocdl.smfmac.f32.16x16x32.f16 %{{.*}} : (vector<4xf16>, vector<8xf16>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>296 %r0 = rocdl.smfmac.f32.16x16x32.f16 %arg1, %arg2, %arg3, %csti32, %csti32, %csti32 :297 (vector<4xf16>, vector<8xf16>, vector<4xf32>,298 i32, i32, i32) -> vector<4xf32>299 300 // CHECK: rocdl.smfmac.f32.32x32x16.f16 %{{.*}} : (vector<4xf16>, vector<8xf16>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>301 %r1 = rocdl.smfmac.f32.32x32x16.f16 %arg1, %arg2, %arg4, %csti32, %csti32, %csti32 :302 (vector<4xf16>, vector<8xf16>, vector<16xf32>,303 i32, i32, i32) -> vector<16xf32>304 305 // CHECK: rocdl.smfmac.f32.16x16x32.bf16 %{{.*}} : (vector<4xi16>, vector<8xi16>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>306 %r2 = rocdl.smfmac.f32.16x16x32.bf16 %arg5, %arg6, %arg3, %csti32, %csti32, %csti32 :307 (vector<4xi16>, vector<8xi16>, vector<4xf32>,308 i32, i32, i32) -> vector<4xf32>309 310 // CHECK: rocdl.smfmac.f32.32x32x16.bf16 %{{.*}} : (vector<4xi16>, vector<8xi16>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>311 %r3 = rocdl.smfmac.f32.32x32x16.bf16 %arg5, %arg6, %arg4, %csti32, %csti32, %csti32 :312 (vector<4xi16>, vector<8xi16>, vector<16xf32>,313 i32, i32, i32) -> vector<16xf32>314 315 // CHECK: rocdl.smfmac.i32.16x16x64.i8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<4xi32>, i32, i32, i32) -> vector<4xi32>316 %r4 = rocdl.smfmac.i32.16x16x64.i8 %arg7, %arg8, %arg8, %csti32, %csti32, %csti32 :317 (vector<2xi32>, vector<4xi32>, vector<4xi32>,318 i32, i32, i32) -> vector<4xi32>319 320 // CHECK: rocdl.smfmac.i32.32x32x32.i8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<16xi32>, i32, i32, i32) -> vector<16xi32>321 %r5 = rocdl.smfmac.i32.32x32x32.i8 %arg7, %arg8, %arg9, %csti32, %csti32, %csti32 :322 (vector<2xi32>, vector<4xi32>, vector<16xi32>,323 i32, i32, i32) -> vector<16xi32>324 325 // CHECK: rocdl.smfmac.f32.16x16x64.bf8.bf8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>326 %r6 = rocdl.smfmac.f32.16x16x64.bf8.bf8 %arg7, %arg8, %arg3, %csti32, %csti32, %csti32 :327 (vector<2xi32>, vector<4xi32>, vector<4xf32>,328 i32, i32, i32) -> vector<4xf32>329 330 // CHECK: rocdl.smfmac.f32.16x16x64.bf8.fp8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>331 %r7 = rocdl.smfmac.f32.16x16x64.bf8.fp8 %arg7, %arg8, %arg3, %csti32, %csti32, %csti32 :332 (vector<2xi32>, vector<4xi32>, vector<4xf32>,333 i32, i32, i32) -> vector<4xf32>334 335 // CHECK: rocdl.smfmac.f32.16x16x64.fp8.bf8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>336 %r8 = rocdl.smfmac.f32.16x16x64.fp8.bf8 %arg7, %arg8, %arg3, %csti32, %csti32, %csti32 :337 (vector<2xi32>, vector<4xi32>, vector<4xf32>,338 i32, i32, i32) -> vector<4xf32>339 340 // CHECK: rocdl.smfmac.f32.16x16x64.fp8.fp8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>341 %r9 = rocdl.smfmac.f32.16x16x64.fp8.fp8 %arg7, %arg8, %arg3, %csti32, %csti32, %csti32 :342 (vector<2xi32>, vector<4xi32>, vector<4xf32>,343 i32, i32, i32) -> vector<4xf32>344 345 // CHECK: rocdl.smfmac.f32.32x32x32.bf8.bf8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>346 %r10 = rocdl.smfmac.f32.32x32x32.bf8.bf8 %arg7, %arg8, %arg4, %csti32, %csti32, %csti32 :347 (vector<2xi32>, vector<4xi32>, vector<16xf32>,348 i32, i32, i32) -> vector<16xf32>349 350 // CHECK: rocdl.smfmac.f32.32x32x32.bf8.fp8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>351 %r11 = rocdl.smfmac.f32.32x32x32.bf8.fp8 %arg7, %arg8, %arg4, %csti32, %csti32, %csti32 :352 (vector<2xi32>, vector<4xi32>, vector<16xf32>,353 i32, i32, i32) -> vector<16xf32>354 355 // CHECK: rocdl.smfmac.f32.32x32x32.fp8.bf8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>356 %r12 = rocdl.smfmac.f32.32x32x32.fp8.bf8 %arg7, %arg8, %arg4, %csti32, %csti32, %csti32 :357 (vector<2xi32>, vector<4xi32>, vector<16xf32>,358 i32, i32, i32) -> vector<16xf32>359 360 // CHECK: rocdl.smfmac.f32.32x32x32.fp8.fp8 %{{.*}} : (vector<2xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>361 %r13 = rocdl.smfmac.f32.32x32x32.fp8.fp8 %arg7, %arg8, %arg4, %csti32, %csti32, %csti32 :362 (vector<2xi32>, vector<4xi32>, vector<16xf32>,363 i32, i32, i32) -> vector<16xf32>364 365 llvm.return %r0 : vector<4 x f32>366}367 368llvm.func @rocdl.mfma.scale.f32.32x32x64.f8f6f4(%arg0 : i32,369 %arg1 : vector<16 x f32>, %arg2 : vector<8xi32>,370 %arg3 : vector<6xi32>, %arg4 : vector<4xi32>) {371 %cst0 = llvm.mlir.constant(0 : i32) : i32372 %cst1 = llvm.mlir.constant(1 : i32) : i32373 %cst2 = llvm.mlir.constant(2 : i32) : i32374 %cst3 = llvm.mlir.constant(3 : i32) : i32375 %cst4 = llvm.mlir.constant(4 : i32) : i32376 377 // CHECK-LABEL: rocdl.mfma.scale.f32.32x32x64.f8f6f4378 // fp8 * fp8379 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>380 %r00 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, %cst0, %cst0, %cst0, %arg0, %cst0, %arg0 :381 (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>382 383 // fp8 * bf8384 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>385 %r01 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, %cst0, %cst1, %cst0, %arg0, %cst0, %arg0 :386 (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>387 388 // fp8 * fp6389 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>390 %r02 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, %cst0, %cst2, %cst0, %arg0, %cst0, %arg0 :391 (vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>392 393 // fp8 * bf6394 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>395 %r03 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, %cst0, %cst3, %cst0, %arg0, %cst0, %arg0 :396 (vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>397 398 // fp8 * fp4399 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>400 %r04 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg4, %arg1, %cst0, %cst4, %cst0, %arg0, %cst0, %arg0 :401 (vector<8xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>402 403 // bf8 * fp8404 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>405 %r10 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, %cst1, %cst0, %cst0, %arg0, %cst0, %arg0 :406 (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>407 408 // bf8 * bf8409 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>410 %r11 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg2, %arg1, %cst1, %cst1, %cst0, %arg0, %cst0, %arg0 :411 (vector<8xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>412 413 // bf8 * fp6414 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>415 %r12 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, %cst1, %cst2, %cst0, %arg0, %cst0, %arg0 :416 (vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>417 418 // bf8 * bf6419 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>420 %r13 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg3, %arg1, %cst1, %cst3, %cst0, %arg0, %cst0, %arg0 :421 (vector<8xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>422 423 // bf8 * fp4424 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<8xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>425 %r14 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg2, %arg4, %arg1, %cst1, %cst4, %cst0, %arg0, %cst0, %arg0 :426 (vector<8xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>427 428 // fp6 * fp8429 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>430 %r20 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, %cst2, %cst0, %cst0, %arg0, %cst0, %arg0 :431 (vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>432 433 // fp6 * bf8434 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>435 %r21 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, %cst2, %cst1, %cst0, %arg0, %cst0, %arg0 :436 (vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>437 438 // fp6 * fp6439 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>440 %r22 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, %cst2, %cst2, %cst0, %arg0, %cst0, %arg0 :441 (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>442 443 // fp6 * bf6444 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>445 %r23 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, %cst2, %cst3, %cst0, %arg0, %cst0, %arg0 :446 (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>447 448 // fp6 * fp4449 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>450 %r24 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg4, %arg1, %cst2, %cst4, %cst0, %arg0, %cst0, %arg0 :451 (vector<6xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>452 453 // bf6 * fp8454 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>455 %r30 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, %cst3, %cst0, %cst0, %arg0, %cst0, %arg0 :456 (vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>457 458 // bf6 * bf8459 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>460 %r31 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg2, %arg1, %cst3, %cst1, %cst0, %arg0, %cst0, %arg0 :461 (vector<6xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>462 463 // bf6 * fp6464 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>465 %r32 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, %cst3, %cst2, %cst0, %arg0, %cst0, %arg0 :466 (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>467 468 // bf6 * bf6469 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>470 %r33 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg3, %arg1, %cst3, %cst3, %cst0, %arg0, %cst0, %arg0 :471 (vector<6xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>472 473 // bf6 * fp4474 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<6xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>475 %r34 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg3, %arg4, %arg1, %cst3, %cst4, %cst0, %arg0, %cst0, %arg0 :476 (vector<6xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>477 478 // fp4 * fp8479 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<4xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>480 %r40 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg2, %arg1, %cst4, %cst0, %cst0, %arg0, %cst0, %arg0 :481 (vector<4xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>482 483 // fp4 * bf8484 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<4xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>485 %r41 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg2, %arg1, %cst4, %cst1, %cst0, %arg0, %cst0, %arg0 :486 (vector<4xi32>, vector<8xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>487 488 // fp4 * fp6489 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<4xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>490 %r42 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg3, %arg1, %cst4, %cst2, %cst0, %arg0, %cst0, %arg0 :491 (vector<4xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>492 493 // fp4 * bf6494 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<4xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>495 %r43 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg3, %arg1, %cst4, %cst3, %cst0, %arg0, %cst0, %arg0 :496 (vector<4xi32>, vector<6xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>497 498 // fp4 * fp4499 // CHECK: rocdl.mfma.scale.f32.32x32x64.f8f6f4 {{.*}} : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>500 %r44 = rocdl.mfma.scale.f32.32x32x64.f8f6f4 %arg4, %arg4, %arg1, %cst4, %cst4, %cst0, %arg0, %cst0, %arg0 :501 (vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32, i32, i32, i32, i32) -> vector<16xf32>502 503 llvm.return504}505 506llvm.func @rocdl.mfma.scale.f32.16x16x128.f8f6f4(%arg0 : i32,507 %arg1 : vector<4 x f32>, %arg2 : vector<8xi32>,508 %arg3 : vector<6xi32>, %arg4 : vector<4xi32>) {509 %cst0 = llvm.mlir.constant(0 : i32) : i32510 %cst1 = llvm.mlir.constant(1 : i32) : i32511 %cst2 = llvm.mlir.constant(2 : i32) : i32512 %cst3 = llvm.mlir.constant(3 : i32) : i32513 %cst4 = llvm.mlir.constant(4 : i32) : i32514 515 // CHECK-LABEL: rocdl.mfma.scale.f32.16x16x128.f8f6f4516 // fp8 * fp8517 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>518 %r00 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, %cst0, %cst0, %cst0, %arg0, %cst0, %arg0 :519 (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>520 521 // fp8 * bf8522 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>523 %r01 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, %cst0, %cst1, %cst0, %arg0, %cst0, %arg0 :524 (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>525 526 // fp8 * fp6527 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>528 %r02 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, %cst0, %cst2, %cst0, %arg0, %cst0, %arg0 :529 (vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>530 531 // fp8 * bf6532 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>533 %r03 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, %cst0, %cst3, %cst0, %arg0, %cst0, %arg0 :534 (vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>535 536 // fp8 * fp4537 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>538 %r04 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg4, %arg1, %cst0, %cst4, %cst0, %arg0, %cst0, %arg0 :539 (vector<8xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>540 541 // bf8 * fp8542 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>543 %r10 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, %cst1, %cst0, %cst0, %arg0, %cst0, %arg0 :544 (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>545 546 // bf8 * bf8547 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>548 %r11 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg2, %arg1, %cst1, %cst1, %cst0, %arg0, %cst0, %arg0 :549 (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>550 551 // bf8 * fp6552 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>553 %r12 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, %cst1, %cst2, %cst0, %arg0, %cst0, %arg0 :554 (vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>555 556 // bf8 * bf6557 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>558 %r13 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg3, %arg1, %cst1, %cst3, %cst0, %arg0, %cst0, %arg0 :559 (vector<8xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>560 561 // bf8 * fp4562 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<8xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>563 %r14 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg2, %arg4, %arg1, %cst1, %cst4, %cst0, %arg0, %cst0, %arg0 :564 (vector<8xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>565 566 // fp6 * fp8567 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>568 %r20 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, %cst2, %cst0, %cst0, %arg0, %cst0, %arg0 :569 (vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>570 571 // fp6 * bf8572 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>573 %r21 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, %cst2, %cst1, %cst0, %arg0, %cst0, %arg0 :574 (vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>575 576 // fp6 * fp6577 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>578 %r22 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, %cst2, %cst2, %cst0, %arg0, %cst0, %arg0 :579 (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>580 581 // fp6 * bf6582 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>583 %r23 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, %cst2, %cst3, %cst0, %arg0, %cst0, %arg0 :584 (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>585 586 // fp6 * fp4587 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>588 %r24 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg4, %arg1, %cst2, %cst4, %cst0, %arg0, %cst0, %arg0 :589 (vector<6xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>590 591 // bf6 * fp8592 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>593 %r30 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, %cst3, %cst0, %cst0, %arg0, %cst0, %arg0 :594 (vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>595 596 // bf6 * bf8597 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>598 %r31 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg2, %arg1, %cst3, %cst1, %cst0, %arg0, %cst0, %arg0 :599 (vector<6xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>600 601 // bf6 * fp6602 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>603 %r32 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, %cst3, %cst2, %cst0, %arg0, %cst0, %arg0 :604 (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>605 606 // bf6 * bf6607 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>608 %r33 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg3, %arg1, %cst3, %cst3, %cst0, %arg0, %cst0, %arg0 :609 (vector<6xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>610 611 // bf6 * fp4612 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<6xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>613 %r34 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg3, %arg4, %arg1, %cst3, %cst4, %cst0, %arg0, %cst0, %arg0 :614 (vector<6xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>615 616 // fp4 * fp8617 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<4xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>618 %r40 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg2, %arg1, %cst4, %cst0, %cst0, %arg0, %cst0, %arg0 :619 (vector<4xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>620 621 // fp4 * bf8622 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<4xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>623 %r41 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg2, %arg1, %cst4, %cst1, %cst0, %arg0, %cst0, %arg0 :624 (vector<4xi32>, vector<8xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>625 626 // fp4 * fp6627 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<4xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>628 %r42 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg3, %arg1, %cst4, %cst2, %cst0, %arg0, %cst0, %arg0 :629 (vector<4xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>630 631 // fp4 * bf6632 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<4xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>633 %r43 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg3, %arg1, %cst4, %cst3, %cst0, %arg0, %cst0, %arg0 :634 (vector<4xi32>, vector<6xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>635 636 // fp4 * fp4637 // CHECK: rocdl.mfma.scale.f32.16x16x128.f8f6f4 {{.*}} : (vector<4xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>638 %r44 = rocdl.mfma.scale.f32.16x16x128.f8f6f4 %arg4, %arg4, %arg1, %cst4, %cst4, %cst0, %arg0, %cst0, %arg0 :639 (vector<4xi32>, vector<4xi32>, vector<4xf32>, i32, i32, i32, i32, i32, i32) -> vector<4xf32>640 641 llvm.return642}643 644llvm.func @rocdl.ds.read.tr(%ptr : !llvm.ptr<3>) -> vector<4xf16> {645 // CHECK-LABEL: rocdl.ds.read.tr646 // CHECK: rocdl.ds.read.tr4.b64 {{.*}} : <3> -> vector<2xi32>647 %r0 = rocdl.ds.read.tr4.b64 %ptr : !llvm.ptr<3> -> vector<2xi32>648 // CHECK: rocdl.ds.read.tr6.b96 {{.*}} : <3> -> vector<3xi32>649 %r1 = rocdl.ds.read.tr6.b96 %ptr : !llvm.ptr<3> -> vector<3xi32>650 // CHECK: rocdl.ds.read.tr8.b64 {{.*}} : <3> -> vector<2xi32>651 %r2 = rocdl.ds.read.tr8.b64 %ptr : !llvm.ptr<3> -> vector<2xi32>652 // CHECK: rocdl.ds.read.tr16.b64 {{.*}} : <3> -> vector<4xf16>653 %r3 = rocdl.ds.read.tr16.b64 %ptr : !llvm.ptr<3> -> vector<4xf16>654 // CHECK: rocdl.ds.read.tr16.b64 {{.*}} : <3> -> vector<4xbf16>655 %r4 = rocdl.ds.read.tr16.b64 %ptr : !llvm.ptr<3> -> vector<4xbf16>656 llvm.return %r3 : vector<4xf16>657}658 659llvm.func @rocdl.load.tr.ops(%gl_ptr : !llvm.ptr<1>, %ds_ptr : !llvm.ptr<3>) {660 // CHECK-LABEL: @rocdl.load.tr.ops661 // CHECK-SAME: (%[[GL_PTR:.+]]: !llvm.ptr<1>, %[[DS_OTR:.+]]: !llvm.ptr<3>)662 // CHECK: rocdl.global.load.tr4.b64 %[[GL_PTR]] : !llvm.ptr<1> -> vector<2xi32>663 // CHECK: rocdl.global.load.tr.b64 %[[GL_PTR]] : !llvm.ptr<1> -> vector<2xi32>664 // CHECK: rocdl.global.load.tr6.b96 %[[GL_PTR]] : !llvm.ptr<1> -> vector<3xi32>665 // CHECK: rocdl.global.load.tr.b128 %[[GL_PTR]] : !llvm.ptr<1> -> vector<8xi16>666 // CHECK: rocdl.global.load.tr.b128 %[[GL_PTR]] : !llvm.ptr<1> -> vector<8xf16>667 // CHECK: rocdl.global.load.tr.b128 %[[GL_PTR]] : !llvm.ptr<1> -> vector<8xbf16>668 // CHECK: rocdl.ds.load.tr4.b64 %[[DS_OTR]] : !llvm.ptr<3> -> vector<2xi32>669 // CHECK: rocdl.ds.load.tr8.b64 %[[DS_OTR]] : !llvm.ptr<3> -> vector<2xi32>670 // CHECK: rocdl.ds.load.tr6.b96 %[[DS_OTR]] : !llvm.ptr<3> -> vector<3xi32>671 // CHECK: rocdl.ds.load.tr16.b128 %[[DS_OTR]] : !llvm.ptr<3> -> vector<8xi16>672 // CHECK: rocdl.ds.load.tr16.b128 %[[DS_OTR]] : !llvm.ptr<3> -> vector<8xf16>673 // CHECK: rocdl.ds.load.tr16.b128 %[[DS_OTR]] : !llvm.ptr<3> -> vector<8xbf16>674 // CHECK: llvm.return675 676 rocdl.global.load.tr4.b64 %gl_ptr : !llvm.ptr<1> -> vector<2xi32>677 rocdl.global.load.tr.b64 %gl_ptr : !llvm.ptr<1> -> vector<2xi32>678 rocdl.global.load.tr6.b96 %gl_ptr : !llvm.ptr<1> -> vector<3xi32>679 rocdl.global.load.tr.b128 %gl_ptr : !llvm.ptr<1> -> vector<8xi16>680 rocdl.global.load.tr.b128 %gl_ptr : !llvm.ptr<1> -> vector<8xf16>681 rocdl.global.load.tr.b128 %gl_ptr : !llvm.ptr<1> -> vector<8xbf16>682 683 rocdl.ds.load.tr4.b64 %ds_ptr : !llvm.ptr<3> -> vector<2xi32>684 rocdl.ds.load.tr8.b64 %ds_ptr : !llvm.ptr<3> -> vector<2xi32>685 rocdl.ds.load.tr6.b96 %ds_ptr : !llvm.ptr<3> -> vector<3xi32>686 rocdl.ds.load.tr16.b128 %ds_ptr : !llvm.ptr<3> -> vector<8xi16>687 rocdl.ds.load.tr16.b128 %ds_ptr : !llvm.ptr<3> -> vector<8xf16>688 rocdl.ds.load.tr16.b128 %ds_ptr : !llvm.ptr<3> -> vector<8xbf16>689 llvm.return690}691 692llvm.func @rocdl.load.to.lds(%src : !llvm.ptr<7>, %dst: !llvm.ptr<3>) {693 // CHECK-LABEL @rocdl.load.to.lds694 //CHECK: rocdl.load.to.lds %{{.*}}, %{{.*}}, 4, 0, 0 : <7>695 rocdl.load.to.lds %src, %dst, 4, 0, 0 : <7>696 llvm.return697}698 699llvm.func @rocdl.global.load.lds(%src : !llvm.ptr<1>, %dst: !llvm.ptr<3>) {700 // CHECK-LABEL @rocdl.global.load.lds701 //CHECK: rocdl.global.load.lds %{{.*}}, %{{.*}}, 4, 0, 0702 rocdl.global.load.lds %src, %dst, 4, 0, 0703 llvm.return704}705 706llvm.func @rocdl.global.load.async.to.lds(%src : !llvm.ptr<1>, %dst: !llvm.ptr<3>) {707 // CHECK-LABEL @rocdl.global.load.async.to.lds708 // CHECK: rocdl.global.load.async.to.lds.b8 %{{.*}}, %{{.*}}, 0, 0709 // CHECK: rocdl.global.load.async.to.lds.b32 %{{.*}}, %{{.*}}, 0, 0710 // CHECK: rocdl.global.load.async.to.lds.b64 %{{.*}}, %{{.*}}, 0, 0711 // CHECK: rocdl.global.load.async.to.lds.b128 %{{.*}}, %{{.*}}, 0, 0712 rocdl.global.load.async.to.lds.b8 %src, %dst, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>713 rocdl.global.load.async.to.lds.b32 %src, %dst, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>714 rocdl.global.load.async.to.lds.b64 %src, %dst, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>715 rocdl.global.load.async.to.lds.b128 %src, %dst, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>716 llvm.return717}718 719llvm.func @rocdl.cluster.load.async.to.lds(%src : !llvm.ptr<1>, %dst: !llvm.ptr<3>) {720 // CHECK-LABEL @rocdl.cluster.load.async.to.lds721 // CHECK: rocdl.cluster.load.async.to.lds.b8 %{{.*}}, %{{.*}}, 0, 0, 0722 // CHECK: rocdl.cluster.load.async.to.lds.b32 %{{.*}}, %{{.*}}, 0, 0, 0723 // CHECK: rocdl.cluster.load.async.to.lds.b64 %{{.*}}, %{{.*}}, 0, 0, 0724 // CHECK: rocdl.cluster.load.async.to.lds.b128 %{{.*}}, %{{.*}}, 0, 0, 0725 rocdl.cluster.load.async.to.lds.b8 %src, %dst, 0, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>726 rocdl.cluster.load.async.to.lds.b32 %src, %dst, 0, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>727 rocdl.cluster.load.async.to.lds.b64 %src, %dst, 0, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>728 rocdl.cluster.load.async.to.lds.b128 %src, %dst, 0, 0, 0 : !llvm.ptr<1>, !llvm.ptr<3>729 llvm.return730}731 732 733// CHECK-LABEL @rocdl.tensor.load.to.lds734llvm.func @rocdl.tensor.load.to.lds(%dgroup0 : vector<4xi32>, %dgroup1 : vector<8xi32>,735 %dgroup2 : vector<4xi32>, %dgroup3 : vector<4xi32>) {736 // CHECK: rocdl.tensor.load.to.lds %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} cachepolicy 0 : vector<4xi32>, vector<8xi32>737 rocdl.tensor.load.to.lds %dgroup0, %dgroup1, %dgroup2, %dgroup3 cachepolicy 0 : vector<4xi32>, vector<8xi32>738 llvm.return739}740 741// CHECK-LABEL @rocdl.tensor.store.from.lds742llvm.func @rocdl.tensor.store.from.lds(%dgroup0 : vector<4xi32>, %dgroup1 : vector<8xi32>,743 %dgroup2 : vector<4xi32>, %dgroup3 : vector<4xi32>) {744 // CHECK: rocdl.tensor.store.from.lds %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} cachepolicy 0 : vector<4xi32>, vector<8xi32>745 rocdl.tensor.store.from.lds %dgroup0, %dgroup1, %dgroup2, %dgroup3 cachepolicy 0 : vector<4xi32>, vector<8xi32>746 llvm.return747}748 749// CHECK-LABEL @rocdl.tensor.load.to.lds.d2750llvm.func @rocdl.tensor.load.to.lds.d2(%dgroup0 : vector<4xi32>, %dgroup1 : vector<8xi32>) {751 // CHECK: rocdl.tensor.load.to.lds.d2 %{{.*}}, %{{.*}} cachepolicy 0 : vector<4xi32>, vector<8xi32>752 rocdl.tensor.load.to.lds.d2 %dgroup0, %dgroup1 cachepolicy 0 : vector<4xi32>, vector<8xi32>753 llvm.return754}755 756// CHECK-LABEL @rocdl.tensor.store.from.lds.d2757llvm.func @rocdl.tensor.store.from.lds.d2(%dgroup0 : vector<4xi32>, %dgroup1 : vector<8xi32>) {758 // CHECK: rocdl.tensor.store.from.lds.d2 %{{.*}}, %{{.*}} cachepolicy 0 : vector<4xi32>, vector<8xi32>759 rocdl.tensor.store.from.lds.d2 %dgroup0, %dgroup1 cachepolicy 0 : vector<4xi32>, vector<8xi32>760 llvm.return761}762 763llvm.func @rocdl.make.buffer.rsrc(%ptr : !llvm.ptr,764 %stride : i16,765 %numRecords : i64,766 %flags : i32) -> !llvm.ptr<8> {767 // CHECK-LABEL: rocdl.make.buffer.rsrc768 // CHECK: %{{.*}} = rocdl.make.buffer.rsrc %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : !llvm.ptr to <8>769 %rsrc = rocdl.make.buffer.rsrc %ptr, %stride, %numRecords, %flags : !llvm.ptr to !llvm.ptr<8>770 llvm.return %rsrc : !llvm.ptr<8>771}772 773llvm.func @rocdl.raw.ptr.buffer.f32(%rsrc : !llvm.ptr<8>,774 %offset : i32, %soffset : i32,775 %aux : i32, %vdata1 : f32,776 %vdata2 : vector<2xf32>, %vdata4 : vector<4xf32>) {777 // CHECK-LABEL: rocdl.raw.ptr.buffer.f32778 // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}} %{{.*}}, %{{.*}} : f32779 // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}} %{{.*}}, %{{.*}} : vector<2xf32>780 // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.load %{{.*}}, %{{.*}} %{{.*}}, %{{.*}} : vector<4xf32>781 782 // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : f32783 // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : vector<2xf32>784 // CHECK: rocdl.raw.ptr.buffer.store %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : vector<4xf32>785 786 // CHECK: rocdl.raw.ptr.buffer.atomic.fadd %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : f32787 // CHECK: rocdl.raw.ptr.buffer.atomic.fmax %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : f32788 789 %r1 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, %aux : f32790 %r2 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, %aux : vector<2xf32>791 %r4 = rocdl.raw.ptr.buffer.load %rsrc, %offset, %soffset, %aux : vector<4xf32>792 793 rocdl.raw.ptr.buffer.store %vdata1, %rsrc, %offset, %soffset, %aux : f32794 rocdl.raw.ptr.buffer.store %vdata2, %rsrc, %offset, %soffset, %aux : vector<2xf32>795 rocdl.raw.ptr.buffer.store %vdata4, %rsrc, %offset, %offset, %aux : vector<4xf32>796 797 rocdl.raw.ptr.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, %aux : f32798 rocdl.raw.ptr.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, %aux : f32799 800 llvm.return801}802 803llvm.func @rocdl.raw.ptr.buffer.load.lds(%rsrc : !llvm.ptr<8>, %dstLds : !llvm.ptr<3>,804 %size: i32, %voffset : i32, %soffset : i32, %offset : i32,805 %aux : i32) {806 // CHECK-LABEL: rocdl.raw.ptr.buffer.load.lds807 // CHECK: rocdl.raw.ptr.buffer.load.lds %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}808 rocdl.raw.ptr.buffer.load.lds %rsrc, %dstLds, %size, %voffset, %soffset, %offset, %aux809 810 llvm.return811}812 813llvm.func @rocdl.raw.ptr.buffer.i32(%rsrc : !llvm.ptr<8>,814 %offset : i32, %soffset : i32,815 %aux : i32, %vdata1 : i32,816 %vdata2 : vector<2xi32>, %vdata4 : vector<4xi32>) {817 // CHECK-LABEL: rocdl.raw.ptr.buffer.i32818 // CHECK: rocdl.raw.ptr.buffer.atomic.smax %{{.*}}, %{{.*}}, %{{.*}} %{{.*}}, %{{.*}} : i32819 // CHECK: rocdl.raw.ptr.buffer.atomic.umin %{{.*}}, %{{.*}}, %{{.*}} %{{.*}}, %{{.*}} : i32820 // CHECK: %{{.*}} = rocdl.raw.ptr.buffer.atomic.cmpswap %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : i32821 822 rocdl.raw.ptr.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, %aux : i32823 rocdl.raw.ptr.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, %aux : i32824 %val = rocdl.raw.ptr.buffer.atomic.cmpswap %vdata1, %vdata1, %rsrc, %offset, %soffset, %aux : i32825 llvm.return826}827 828// -----829 830llvm.func @rocdl.raw.buffer.f32(%rsrc : vector<4xi32>,831 %offset : i32, %soffset : i32,832 %aux : i32, %vdata1 : f32,833 %vdata2 : vector<2xf32>, %vdata4 : vector<4xf32>) {834 // CHECK-LABEL: rocdl.raw.buffer.f32835 // CHECK: %{{.*}} = rocdl.raw.buffer.load %{{.*}} %{{.*}} %{{.*}} %{{.*}} : f32836 // CHECK: %{{.*}} = rocdl.raw.buffer.load %{{.*}} %{{.*}} %{{.*}} %{{.*}} : vector<2xf32>837 // CHECK: %{{.*}} = rocdl.raw.buffer.load %{{.*}} %{{.*}} %{{.*}} %{{.*}} : vector<4xf32>838 839 // CHECK: rocdl.raw.buffer.store %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : f32840 // CHECK: rocdl.raw.buffer.store %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : vector<2xf32>841 // CHECK: rocdl.raw.buffer.store %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : vector<4xf32>842 843 // CHECK: rocdl.raw.buffer.atomic.fadd %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : f32844 845 %r1 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, %aux : f32846 %r2 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, %aux : vector<2xf32>847 %r4 = rocdl.raw.buffer.load %rsrc, %offset, %soffset, %aux : vector<4xf32>848 849 rocdl.raw.buffer.store %vdata1, %rsrc, %offset, %soffset, %aux : f32850 rocdl.raw.buffer.store %vdata2, %rsrc, %offset, %soffset, %aux : vector<2xf32>851 rocdl.raw.buffer.store %vdata4, %rsrc, %offset, %offset, %aux : vector<4xf32>852 853 rocdl.raw.buffer.atomic.fadd %vdata1, %rsrc, %offset, %soffset, %aux : f32854 rocdl.raw.buffer.atomic.fmax %vdata1, %rsrc, %offset, %soffset, %aux : f32855 856 llvm.return857}858 859 860llvm.func @rocdl.raw.buffer.i32(%rsrc : vector<4xi32>,861 %offset : i32, %soffset : i32,862 %aux : i32, %vdata1 : i32,863 %vdata2 : vector<2xi32>, %vdata4 : vector<4xi32>) {864 // CHECK-LABEL: rocdl.raw.buffer.i32865 // CHECK: rocdl.raw.buffer.atomic.smax %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : i32866 // CHECK: rocdl.raw.buffer.atomic.umin %{{.*}} %{{.*}} %{{.*}} %{{.*}} %{{.*}} : i32867 // CHECK: %{{.*}} = rocdl.raw.buffer.atomic.cmpswap(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : i32, vector<4xi32>868 869 rocdl.raw.buffer.atomic.smax %vdata1, %rsrc, %offset, %soffset, %aux : i32870 rocdl.raw.buffer.atomic.umin %vdata1, %rsrc, %offset, %soffset, %aux : i32871 %val = rocdl.raw.buffer.atomic.cmpswap(%vdata1, %vdata1, %rsrc, %offset, %soffset, %aux) : i32, vector<4xi32>872 llvm.return873}874 875llvm.func @rocdl_8bit_floats(%source: i32, %source_half: f16, %source_bfloat: bf16, %stoch: i32) -> i32 {876// CHECK-LABEL: @rocdl_8bit_floats877// CHECK: rocdl.cvt.f32.bf8878// CHECK: rocdl.cvt.f32.fp8879// CHECK: rocdl.cvt.scalef32.f32.bf8880// CHECK: rocdl.cvt.scalef32.f32.fp8881// CHECK: rocdl.cvt.scalef32.pk.f16.bf8882// CHECK: rocdl.cvt.scalef32.pk.f16.fp8883// CHECK: rocdl.cvt.scalef32.pk.bf16.bf8884// CHECK: rocdl.cvt.scalef32.pk.bf16.fp8885// CHECK: rocdl.cvt.scalef32.f16.fp8886// CHECK: rocdl.cvt.scalef32.f16.bf8887// CHECK: rocdl.cvt.pk.bf8.f32888// CHECK: rocdl.cvt.pk.fp8.f32889// CHECK: rocdl.cvt.pk.f32.bf8890// CHECK: rocdl.cvt.pk.f32.fp8891// CHECK: rocdl.cvt.sr.bf8.f32892// CHECK: rocdl.cvt.sr.fp8.f32893// CHECK: rocdl.cvt.scalef32.sr.fp8.f32894// CHECK: rocdl.cvt.scalef32.sr.fp8.f16895// CHECK: rocdl.cvt.scalef32.sr.fp8.bf16896// CHECK: rocdl.cvt.sr.bf8.f32897// CHECK: rocdl.cvt.scalef32.sr.bf8.f32898// CHECK: rocdl.cvt.scalef32.sr.bf8.f16899// CHECK: rocdl.cvt.scalef32.sr.bf8.bf16900// CHECK: rocdl.cvt.scalef32.pk.f32.fp8901// CHECK: rocdl.cvt.scalef32.pk.f32.bf8902 %c4 = llvm.mlir.constant(1.0 : f32) : f32903 %v1 = rocdl.cvt.f32.bf8 %source[0] : f32904 %v2 = rocdl.cvt.f32.fp8 %source[0] : f32905 %v1_scaled = rocdl.cvt.scalef32.f32.bf8 %source[0], %c4 : f32906 %v2_scaled = rocdl.cvt.scalef32.f32.fp8 %source[0], %c4 : f32907 %v3_scaled = rocdl.cvt.scalef32.pk.f16.bf8 %source[false], %c4 : vector<2xf16>908 %v4_scaled = rocdl.cvt.scalef32.pk.f16.fp8 %source[false], %c4 : vector<2xf16>909 %v3_scaled_bf16 = rocdl.cvt.scalef32.pk.bf16.bf8 %source[false], %c4 : vector<2xbf16>910 %v4_scaled_bf16 = rocdl.cvt.scalef32.pk.bf16.fp8 %source[false], %c4 : vector<2xbf16>911 %v5 = rocdl.cvt.scalef32.f16.fp8 %source[0], %c4 -> %v3_scaled[false] : vector<2xf16>912 %v6 = rocdl.cvt.scalef32.f16.bf8 %source[0], %c4 -> %v3_scaled[false] : vector<2xf16>913 %source2 = rocdl.cvt.pk.bf8.f32 %v1, %v2 -> %source[false] : i32914 %source3 = rocdl.cvt.pk.fp8.f32 %v1, %v2 -> %source2[false] : i32915 %source2_ext = rocdl.cvt.pk.f32.bf8 %source[false] : vector<2xf32>916 %source3_ext = rocdl.cvt.pk.f32.fp8 %source[false] : vector<2xf32>917 %source4 = rocdl.cvt.sr.bf8.f32 %v1, %stoch -> %source3[2] : i32918 %source5 = rocdl.cvt.sr.fp8.f32 %v2, %stoch -> %source4[3] : i32919 %source5_scaled = rocdl.cvt.scalef32.sr.fp8.f32 %v2, %stoch, %c4 -> %source4[3] : i32920 %source5_scaled_half = rocdl.cvt.scalef32.sr.fp8.f16 %source_half, %stoch, %c4 -> %source4[3] : i32921 %source5_scaled_bfloat = rocdl.cvt.scalef32.sr.fp8.bf16 %source_bfloat, %stoch, %c4 -> %source4[3] : i32922 %source6 = rocdl.cvt.sr.bf8.f32 %v1, %stoch -> %source3[3] : i32923 %source6_scaled = rocdl.cvt.scalef32.sr.bf8.f32 %v2, %stoch, %c4 -> %source3[3] : i32924 %source6_scaled_half = rocdl.cvt.scalef32.sr.bf8.f16 %source_half, %stoch, %c4 -> %source3[3] : i32925 %source6_scaled_bfloat = rocdl.cvt.scalef32.sr.bf8.bf16 %source_bfloat, %stoch, %c4 -> %source3[3] : i32926 %source7_scaled = rocdl.cvt.scalef32.pk.f32.fp8 %source[false], %c4 : vector<2xf32>927 %source8_scaled = rocdl.cvt.scalef32.pk.f32.bf8 %source[false], %c4 : vector<2xf32>928 llvm.return %source5 : i32929}930 931llvm.func @rocdl_8bit_packed_v2i16(%sourceA: f32, %sourceB: f32, %old: vector<2xi16>) -> vector<2xi16> {932// CHECK-LABEL: @rocdl_8bit_packed_v2i16933// CHECK: rocdl.cvt.scalef32.pk.fp8.f32934 %c0 = llvm.mlir.constant(1.0 : f32) : f32935 %source_scaled = rocdl.cvt.scalef32.pk.fp8.f32 %sourceA, %sourceB, %c0 -> %old[false] : vector<2xi16>936 %source2_scaled = rocdl.cvt.scalef32.pk.bf8.f32 %sourceA, %sourceB, %c0 -> %old[false] : vector<2xi16>937 llvm.return %source_scaled : vector<2xi16>938}939 940llvm.func @rocdl_v2f16_v2i16(%source: vector<2xf16>, %source2: vector<2xbf16>, %old: vector<2xi16>) -> vector<2xi16> {941// CHECK-LABEL: @rocdl_v2f16_v2i16942// CHECK: rocdl.cvt.scalef32.pk.fp8.f16943 %c0 = llvm.mlir.constant(1.0 : f32) : f32944 %source_scaled = rocdl.cvt.scalef32.pk.fp8.f16 %source, %c0 -> %old[false] : vector<2xi16>945 %source2_scaled = rocdl.cvt.scalef32.pk.fp8.bf16 %source2, %c0 -> %old[false] : vector<2xi16>946 %source3_scaled = rocdl.cvt.scalef32.pk.bf8.f16 %source, %c0 -> %old[false] : vector<2xi16>947 %source4_scaled = rocdl.cvt.scalef32.pk.bf8.bf16 %source2, %c0 -> %old[false] : vector<2xi16>948 llvm.return %source_scaled : vector<2xi16>949}950 951// CHECK-LABEL: @rocdl_6_bit_floats952// CHECK-SAME: (%[[V32F6:.+]]: vector<6xi32>, %[[V16F32:.+]]: vector<16xf32>, %[[V32F32:.+]]: vector<32xf32>, %[[V32F16:.+]]: vector<32xf16>, %[[V32BF16:.+]]: vector<32xbf16>, %[[SEED:.+]]: i32, %[[SCALE:.+]]: f32)953llvm.func @rocdl_6_bit_floats(954 %v32f6: vector<6xi32>, %v16f32: vector<16xf32>, %v32f32: vector<32xf32>,955 %v32f16: vector<32xf16>, %v32bf16: vector<32xbf16>, %seed: i32,956 %scale: f32) {957 // CHECK-NEXT: rocdl.cvt.scalef32.2xpk16.bf6.f32 %[[V16F32]], %[[V16F32]], %[[SCALE]]958 %f32_to_bf6 = rocdl.cvt.scalef32.2xpk16.bf6.f32 %v16f32, %v16f32, %scale : vector<6xi32>959 // CHECK-NEXT: rocdl.cvt.scalef32.2xpk16.fp6.f32 %[[V16F32]], %[[V16F32]], %[[SCALE]]960 %f32_to_fp6 = rocdl.cvt.scalef32.2xpk16.fp6.f32 %v16f32, %v16f32, %scale : vector<6xi32>961 // CHECK-NEXT: rocdl.cvt.scalef32.pk32.bf6.f16 %[[V32F16]], %[[SCALE]]962 %f16_to_bf6 = rocdl.cvt.scalef32.pk32.bf6.f16 %v32f16, %scale : vector<6xi32>963 // CHECK-NEXT: rocdl.cvt.scalef32.pk32.fp6.f16 %[[V32F16]], %[[SCALE]]964 %f16_to_fp6 = rocdl.cvt.scalef32.pk32.fp6.f16 %v32f16, %scale : vector<6xi32>965 // CHECK-NEXT: rocdl.cvt.scalef32.pk32.bf6.bf16 %[[V32BF16]], %[[SCALE]]966 %bf16_to_bf6 = rocdl.cvt.scalef32.pk32.bf6.bf16 %v32bf16, %scale : vector<6xi32>967 // CHECK-NEXT: rocdl.cvt.scalef32.pk32.fp6.bf16 %[[V32BF16]], %[[SCALE]]968 %bf16_to_fp6 = rocdl.cvt.scalef32.pk32.fp6.bf16 %v32bf16, %scale : vector<6xi32>969 970 // CHECK-NEXT: rocdl.cvt.scalef32.pk32.f32.bf6 %[[V32F6]], %[[SCALE]]971 %bf6_to_f32 = rocdl.cvt.scalef32.pk32.f32.bf6 %v32f6, %scale : vector<32xf32>972 // CHECK-NEXT: rocdl.cvt.scalef32.pk32.f32.fp6 %[[V32F6]], %[[SCALE]]973 %fp6_to_f32 = rocdl.cvt.scalef32.pk32.f32.fp6 %v32f6, %scale : vector<32xf32>974 // CHECK-NEXT: rocdl.cvt.scalef32.pk32.f16.bf6 %[[V32F6]], %[[SCALE]]975 %bf6_to_f16 = rocdl.cvt.scalef32.pk32.f16.bf6 %v32f6, %scale : vector<32xf16>976 // CHECK-NEXT: rocdl.cvt.scalef32.pk32.f16.fp6 %[[V32F6]], %[[SCALE]]977 %fp6_to_f16 = rocdl.cvt.scalef32.pk32.f16.fp6 %v32f6, %scale : vector<32xf16>978 // CHECK-NEXT: rocdl.cvt.scalef32.pk32.bf16.bf6 %[[V32F6]], %[[SCALE]]979 %bf6_to_bf16 = rocdl.cvt.scalef32.pk32.bf16.bf6 %v32f6, %scale : vector<32xbf16>980 // CHECK-NEXT: rocdl.cvt.scalef32.pk32.bf16.fp6 %[[V32F6]], %[[SCALE]]981 %fp6_to_bf16 = rocdl.cvt.scalef32.pk32.bf16.fp6 %v32f6, %scale : vector<32xbf16>982 983 // CHECK-NEXT: rocdl.cvt.scalef32.sr.pk32.bf6.f32 %[[V32F32]], %[[SEED]], %[[SCALE]]984 %f32_to_bf6_sr = rocdl.cvt.scalef32.sr.pk32.bf6.f32 %v32f32, %seed, %scale : vector<6xi32>985 // CHECK-NEXT: rocdl.cvt.scalef32.sr.pk32.fp6.f32 %[[V32F32]], %[[SEED]], %[[SCALE]]986 %f32_to_fp6_sr = rocdl.cvt.scalef32.sr.pk32.fp6.f32 %v32f32, %seed, %scale : vector<6xi32>987 // CHECK-NEXT: rocdl.cvt.scalef32.sr.pk32.bf6.f16 %[[V32F16]], %[[SEED]], %[[SCALE]]988 %f16_to_bf6_sr = rocdl.cvt.scalef32.sr.pk32.bf6.f16 %v32f16, %seed, %scale : vector<6xi32>989 // CHECK-NEXT: rocdl.cvt.scalef32.sr.pk32.fp6.f16 %[[V32F16]], %[[SEED]], %[[SCALE]]990 %f16_to_fp6_sr = rocdl.cvt.scalef32.sr.pk32.fp6.f16 %v32f16, %seed, %scale : vector<6xi32>991 // CHECK-NEXT: rocdl.cvt.scalef32.sr.pk32.bf6.bf16 %[[V32BF16]], %[[SEED]], %[[SCALE]]992 %bf16_to_bf6_sr = rocdl.cvt.scalef32.sr.pk32.bf6.bf16 %v32bf16, %seed, %scale : vector<6xi32>993 // CHECK-NEXT: rocdl.cvt.scalef32.sr.pk32.fp6.bf16 %[[V32BF16]], %[[SEED]], %[[SCALE]]994 %bf16_to_fp6_sr = rocdl.cvt.scalef32.sr.pk32.fp6.bf16 %v32bf16, %seed, %scale : vector<6xi32>995 996 llvm.return997}998 999// CHECK-LABEL: @rocdl_4_bit_floats1000// CHECK-SAME: (%[[V8F4:.+]]: i32, %[[F32:.+]]: f32, %[[V2F32:.+]]: vector<2xf32>, %[[V2F16:.+]]: vector<2xf16>, %[[V2BF16:.+]]: vector<2xbf16>, %[[SEED:.+]]: i32, %[[SCALE:.+]]: f32)1001llvm.func @rocdl_4_bit_floats(1002 %v8f4: i32, %f32: f32, %v2f32: vector<2xf32>, %v2f16: vector<2xf16>,1003 %v2bf16: vector<2xbf16>, %seed: i32, %scale: f32) {1004 1005 // CHECK-NEXT: rocdl.cvt.scalef32.pk.fp4.f32 %[[F32]], %[[F32]], %[[SCALE]] -> %[[V8F4]][0]1006 %f32_to_fp4 = rocdl.cvt.scalef32.pk.fp4.f32 %f32, %f32, %scale -> %v8f4[0] : i321007 // CHECK-NEXT: rocdl.cvt.scalef32.pk.fp4.f16 %[[V2F16]], %[[SCALE]] -> %[[V8F4]][1]1008 %f16_to_fp4 = rocdl.cvt.scalef32.pk.fp4.f16 %v2f16, %scale -> %v8f4[1] : i321009 // CHECK-NEXT: rocdl.cvt.scalef32.pk.fp4.bf16 %[[V2BF16]], %[[SCALE]] -> %[[V8F4]][0]1010 %bf16_to_fp4 = rocdl.cvt.scalef32.pk.fp4.bf16 %v2bf16, %scale -> %v8f4[0] : i321011 1012 // CHECK-NEXT: rocdl.cvt.scalef32.pk.f32.fp4 %[[V8F4]][0], %[[SCALE]]1013 %fp4_to_f32 = rocdl.cvt.scalef32.pk.f32.fp4 %v8f4[0], %scale : vector<2xf32>1014 // CHECK-NEXT: rocdl.cvt.scalef32.pk.f16.fp4 %[[V8F4]][1], %[[SCALE]]1015 %fp4_to_f16 = rocdl.cvt.scalef32.pk.f16.fp4 %v8f4[1], %scale : vector<2xf16>1016 // CHECK-NEXT: rocdl.cvt.scalef32.pk.bf16.fp4 %[[V8F4]][0], %[[SCALE]]1017 %fp4_to_bf16 = rocdl.cvt.scalef32.pk.bf16.fp4 %v8f4[0], %scale : vector<2xbf16>1018 1019 // CHECK-NEXT: rocdl.cvt.scalef32.sr.pk.fp4.f32 %[[V2F32]], %[[SEED]], %[[SCALE]] -> %[[V8F4]][0]1020 %f32_to_fp4_sr = rocdl.cvt.scalef32.sr.pk.fp4.f32 %v2f32, %seed, %scale -> %v8f4[0] : i321021 // CHECK-NEXT: rocdl.cvt.scalef32.sr.pk.fp4.f16 %[[V2F16]], %[[SEED]], %[[SCALE]] -> %[[V8F4]][1]1022 %f16_to_fp4_sr = rocdl.cvt.scalef32.sr.pk.fp4.f16 %v2f16, %seed, %scale -> %v8f4[1] : i321023 // CHECK-NEXT: rocdl.cvt.scalef32.sr.pk.fp4.bf16 %[[V2BF16]], %[[SEED]], %[[SCALE]] -> %[[V8F4]][0]1024 %bf16_to_fp4_sr = rocdl.cvt.scalef32.sr.pk.fp4.bf16 %v2bf16, %seed, %scale -> %v8f4[0] : i321025 1026 llvm.return1027}1028 1029llvm.func @rocdl.s.waitcnt() {1030 // CHECK-LABEL: rocdl.s.waitcnt1031 // CHECK: rocdl.s.waitcnt 01032 rocdl.s.waitcnt 01033 llvm.return1034}1035 1036llvm.func @rocdl.s.sleep() {1037 // CHECK-LABEL: rocdl.s.sleep1038 // CHECK: rocdl.s.sleep 01039 rocdl.s.sleep 01040 llvm.return1041}1042 1043llvm.func @rocdl.s.barrier() {1044 // CHECK-LABEL: rocdl.s.barrier1045 // CHECK: rocdl.s.barrier1046 rocdl.s.barrier1047 llvm.return1048}1049 1050llvm.func @rocdl.s.barrier.init(%ptr : !llvm.ptr<3>) {1051 // CHECK-LABEL: rocdl.s.barrier.init1052 // CHECK: rocdl.s.barrier.init %[[PTR:.+]], 11053 rocdl.s.barrier.init %ptr, 11054 llvm.return1055}1056 1057llvm.func @rocdl.s.barrier.signal() {1058 // CHECK-LABEL: rocdl.s.barrier.signal1059 // CHECK: rocdl.s.barrier.signal -11060 rocdl.s.barrier.signal -11061 llvm.return1062}1063 1064llvm.func @rocdl.s.barrier.signal.var(%ptr : !llvm.ptr<3>) {1065 // CHECK-LABEL: rocdl.s.barrier.signal.var1066 // CHECK: rocdl.s.barrier.signal.var %[[PTR:.+]], 11067 rocdl.s.barrier.signal.var %ptr, 11068 llvm.return1069}1070 1071llvm.func @rocdl.s.barrier.join(%ptr : !llvm.ptr<3>) {1072 // CHECK-LABEL: rocdl.s.barrier.join1073 // CHECK: rocdl.s.barrier.join %[[PTR:.+]]1074 rocdl.s.barrier.join %ptr1075 llvm.return1076}1077 1078llvm.func @rocdl.s.barrier.leave() {1079 // CHECK-LABEL: rocdl.s.barrier.leave1080 // CHECK: rocdl.s.barrier.leave 11081 rocdl.s.barrier.leave 11082 llvm.return1083}1084 1085llvm.func @rocdl.s.barrier.wait() {1086 // CHECK-LABEL: rocdl.s.barrier.wait1087 // CHECK: rocdl.s.barrier.wait -11088 rocdl.s.barrier.wait -11089 llvm.return1090}1091 1092llvm.func @rocdl.s.barrier.signal.isfirst() {1093 // CHECK-LABEL: rocdl.s.barrier.signal.isfirst1094 // CHECK: rocdl.s.barrier.signal.isfirst 11095 %0 = rocdl.s.barrier.signal.isfirst 1 : i11096 llvm.return1097}1098 1099llvm.func @rocdl.s.get.barrier.state() {1100 // CHECK-LABEL: rocdl.s.get.barrier.state1101 // CHECK: rocdl.s.get.barrier.state 11102 %0 = rocdl.s.get.barrier.state 1 : i321103 llvm.return1104}1105 1106llvm.func @rocdl.s.get.named.barrier.state(%ptr : !llvm.ptr<3>) {1107 // CHECK-LABEL: rocdl.s.get.named.barrier.state1108 // CHECK: rocdl.s.get.named.barrier.state %[[PTR:.+]]1109 %0 = rocdl.s.get.named.barrier.state %ptr : i321110 llvm.return1111}1112 1113llvm.func @rocdl.s.wait.dscnt() {1114 // CHECK-LABEL: rocdl.s.wait.dscnt1115 // CHECK: rocdl.s.wait.dscnt 01116 rocdl.s.wait.dscnt 01117 llvm.return1118}1119 1120llvm.func @rocdl.s.wait.loadcnt() {1121 // CHECK-LABEL: rocdl.s.wait.loadcnt1122 // CHECK: rocdl.s.wait.loadcnt 01123 rocdl.s.wait.loadcnt 01124 llvm.return1125}1126 1127llvm.func @rocdl.s.wait.storecnt() {1128 // CHECK-LABEL: rocdl.s.wait.storecnt1129 // CHECK: rocdl.s.wait.storecnt 01130 rocdl.s.wait.storecnt 01131 llvm.return1132}1133 1134llvm.func @rocdl.s.wait.expcnt() {1135 // CHECK-LABEL: rocdl.s.wait.expcnt1136 // CHECK: rocdl.s.wait.expcnt 01137 rocdl.s.wait.expcnt 01138 llvm.return1139}1140 1141llvm.func @rocdl.s.wait.asynccnt() {1142 // CHECK-LABEL: rocdl.s.wait.asynccnt1143 // CHECK: rocdl.s.wait.asynccnt 01144 rocdl.s.wait.asynccnt 01145 llvm.return1146}1147 1148llvm.func @rocdl.s.wait.tensorcnt() {1149 // CHECK-LABEL: rocdl.s.wait.tensorcnt1150 // CHECK: rocdl.s.wait.tensorcnt 01151 rocdl.s.wait.tensorcnt 01152 llvm.return1153}1154 1155// -----1156 1157llvm.func @rocdl.readfirstlane(%src : f32) -> f32 {1158 // CHECK-LABEL: rocdl.readfirstlane1159 // CHECK: rocdl.readfirstlane %{{.*}} : f321160 %ret = rocdl.readfirstlane %src : f321161 llvm.return %ret : f321162}1163 1164llvm.func @rocdl.readlane(%src : f32) -> f32 {1165 %cst0 = llvm.mlir.constant(0 : i32) : i321166 1167 // CHECK-LABEL: rocdl.readlane1168 // CHECK: rocdl.readlane %{{.*}} %{{.*}}1169 %ret = rocdl.readlane %src, %cst0 : (f32, i32) -> f321170 llvm.return %ret : f321171}1172 1173// -----1174 1175llvm.func @rocdl.permlanex16(%src : f32) -> f32 {1176 %cst0 = llvm.mlir.constant(-1 : i32) : i321177 // CHECK-LABEL: rocdl.permlanex161178 // CHECK: rocdl.permlanex16 %{{.*}} %{{.*}}1179 %ret = rocdl.permlanex16 %src, %src, %cst0, %cst0, 0, -1 : f32, i321180 llvm.return %ret : f321181}1182 1183// -----1184 1185llvm.func @rocdl.permlane16.swap(%src : i32) -> !llvm.struct<(i32, i32)> {1186 // CHECK-LABEL: rocdl.permlane16.swap1187 // CHECK: rocdl.permlane16.swap %{{.*}} %{{.*}}1188 %res = rocdl.permlane16.swap %src, %src, 0, -1 : (i32, i32) -> !llvm.struct<(i32, i32)>1189 llvm.return %res : !llvm.struct<(i32, i32)>1190}1191 1192llvm.func @rocdl.permlane32.swap(%src : i32) -> !llvm.struct<(i32, i32)> {1193 // CHECK-LABEL: rocdl.permlane32.swap1194 // CHECK: rocdl.permlane32.swap %{{.*}} %{{.*}}1195 %res = rocdl.permlane32.swap %src, %src, 0, -1 : (i32, i32) -> !llvm.struct<(i32, i32)>1196 llvm.return %res : !llvm.struct<(i32, i32)>1197}1198 1199// -----1200 1201// CHECK-LABEL: rocdl.cvt.scale.pk81202llvm.func @rocdl.cvt.scale.pk8(%i32: i32, %v2xi32: vector<2xi32>, %scale: i32) {1203 1204 // CHECK: rocdl.cvt.scale.pk8.f16.fp41205 %0 = rocdl.cvt.scale.pk8.f16.fp4 %i32, %scale[0] : vector<8xf16>1206 // CHECK: rocdl.cvt.scale.pk8.bf16.fp41207 %1 = rocdl.cvt.scale.pk8.bf16.fp4 %i32, %scale[0] : vector<8xbf16>1208 // CHECK: rocdl.cvt.scale.pk8.f32.fp41209 %2 = rocdl.cvt.scale.pk8.f32.fp4 %i32, %scale[0] : vector<8xf32>1210 1211 // CHECK: rocdl.cvt.scale.pk8.f16.fp81212 %3 = rocdl.cvt.scale.pk8.f16.fp8 %v2xi32, %scale[0] : vector<8xf16>1213 // CHECK: rocdl.cvt.scale.pk8.bf16.fp81214 %4 = rocdl.cvt.scale.pk8.bf16.fp8 %v2xi32, %scale[0] : vector<8xbf16>1215 // CHECK: rocdl.cvt.scale.pk8.f32.fp81216 %5 = rocdl.cvt.scale.pk8.f32.fp8 %v2xi32, %scale[0] : vector<8xf32>1217 1218 // CHECK: rocdl.cvt.scale.pk8.f16.bf81219 %6 = rocdl.cvt.scale.pk8.f16.bf8 %v2xi32, %scale[0] : vector<8xf16>1220 // CHECK: rocdl.cvt.scale.pk8.bf16.bf81221 %7 = rocdl.cvt.scale.pk8.bf16.bf8 %v2xi32, %scale[0] : vector<8xbf16>1222 // CHECK: rocdl.cvt.scale.pk8.f32.bf81223 %8 = rocdl.cvt.scale.pk8.f32.bf8 %v2xi32, %scale[0] : vector<8xf32>1224 1225 llvm.return1226}1227 1228// -----1229 1230// CHECK-LABEL: rocdl.cvt.scalef32.pk81231llvm.func @rocdl.cvt.scalef32.pk8(%v8xf32: vector<8xf32>,1232 %v8xf16: vector<8xf16>,1233 %v8xbf16: vector<8xbf16>,1234 %scale: f32) {1235 1236 // CHECK: rocdl.cvt.scalef32.pk8.fp8.f321237 %0 = rocdl.cvt.scalef32.pk8.fp8.f32 %v8xf32, %scale : vector<2xi32>1238 // CHECK: rocdl.cvt.scalef32.pk8.bf8.f321239 %1 = rocdl.cvt.scalef32.pk8.bf8.f32 %v8xf32, %scale : vector<2xi32>1240 // CHECK: rocdl.cvt.scalef32.pk8.fp4.f321241 %2 = rocdl.cvt.scalef32.pk8.fp4.f32 %v8xf32, %scale : i321242 1243 // CHECK: rocdl.cvt.scalef32.pk8.fp8.f161244 %3 = rocdl.cvt.scalef32.pk8.fp8.f16 %v8xf16, %scale : vector<2xi32>1245 // CHECK: rocdl.cvt.scalef32.pk8.bf8.f161246 %4 = rocdl.cvt.scalef32.pk8.bf8.f16 %v8xf16, %scale : vector<2xi32>1247 // CHECK: rocdl.cvt.scalef32.pk8.fp4.f161248 %5 = rocdl.cvt.scalef32.pk8.fp4.f16 %v8xf16, %scale : i321249 1250 // CHECK: rocdl.cvt.scalef32.pk8.fp8.bf161251 %6 = rocdl.cvt.scalef32.pk8.fp8.bf16 %v8xbf16, %scale : vector<2xi32>1252 // CHECK: rocdl.cvt.scalef32.pk8.bf8.bf161253 %7 = rocdl.cvt.scalef32.pk8.bf8.bf16 %v8xbf16, %scale : vector<2xi32>1254 // CHECK: rocdl.cvt.scalef32.pk8.fp4.bf161255 %8 = rocdl.cvt.scalef32.pk8.fp4.bf16 %v8xbf16, %scale : i321256 1257 llvm.return1258}1259 1260// -----1261 1262// CHECK-LABEL: rocdl.cvt.scalef32.sr.pk81263llvm.func @rocdl.cvt.scalef32.sr.pk8(%v8xf32: vector<8xf32>,1264 %v8xf16: vector<8xf16>,1265 %v8xbf16: vector<8xbf16>,1266 %seed: i32,1267 %scale: f32) {1268 1269 // CHECK: rocdl.cvt.scalef32.sr.pk8.fp8.f321270 %0 = rocdl.cvt.scalef32.sr.pk8.fp8.f32 %v8xf32, %seed, %scale : vector<2xi32>1271 // CHECK: rocdl.cvt.scalef32.sr.pk8.bf8.f321272 %1 = rocdl.cvt.scalef32.sr.pk8.bf8.f32 %v8xf32, %seed, %scale : vector<2xi32>1273 // CHECK: rocdl.cvt.scalef32.sr.pk8.fp4.f321274 %2 = rocdl.cvt.scalef32.sr.pk8.fp4.f32 %v8xf32, %seed, %scale : i321275 1276 // CHECK: rocdl.cvt.scalef32.sr.pk8.fp8.f161277 %3 = rocdl.cvt.scalef32.sr.pk8.fp8.f16 %v8xf16, %seed, %scale : vector<2xi32>1278 // CHECK: rocdl.cvt.scalef32.sr.pk8.bf8.f161279 %4 = rocdl.cvt.scalef32.sr.pk8.bf8.f16 %v8xf16, %seed, %scale : vector<2xi32>1280 // CHECK: rocdl.cvt.scalef32.sr.pk8.fp4.f161281 %5 = rocdl.cvt.scalef32.sr.pk8.fp4.f16 %v8xf16, %seed, %scale : i321282 1283 // CHECK: rocdl.cvt.scalef32.sr.pk8.fp8.bf161284 %6 = rocdl.cvt.scalef32.sr.pk8.fp8.bf16 %v8xbf16, %seed, %scale : vector<2xi32>1285 // CHECK: rocdl.cvt.scalef32.sr.pk8.bf8.bf161286 %7 = rocdl.cvt.scalef32.sr.pk8.bf8.bf16 %v8xbf16, %seed, %scale : vector<2xi32>1287 // CHECK: rocdl.cvt.scalef32.sr.pk8.fp4.bf161288 %8 = rocdl.cvt.scalef32.sr.pk8.fp4.bf16 %v8xbf16, %seed, %scale : i321289 1290 llvm.return1291}1292 1293// -----1294 1295// CHECK-LABEL: rocdl.cvt.scale.pk161296llvm.func @rocdl.cvt.scale.pk16(%v3xi32: vector<3xi32>, %scale:i32) {1297 1298 // CHECK: rocdl.cvt.scale.pk16.f16.fp61299 %0 = rocdl.cvt.scale.pk16.f16.fp6 %v3xi32, %scale[0] : vector<16xf16>1300 // CHECK: rocdl.cvt.scale.pk16.bf16.fp61301 %1 = rocdl.cvt.scale.pk16.bf16.fp6 %v3xi32, %scale[0] : vector<16xbf16>1302 // CHECK: rocdl.cvt.scale.pk16.f32.fp61303 %2 = rocdl.cvt.scale.pk16.f32.fp6 %v3xi32, %scale[0] : vector<16xf32>1304 1305 // CHECK: rocdl.cvt.scale.pk16.f16.bf61306 %3 = rocdl.cvt.scale.pk16.f16.bf6 %v3xi32, %scale[0] : vector<16xf16>1307 // CHECK: rocdl.cvt.scale.pk16.bf16.bf61308 %4 = rocdl.cvt.scale.pk16.bf16.bf6 %v3xi32, %scale[0] : vector<16xbf16>1309 // CHECK: rocdl.cvt.scale.pk16.f32.bf61310 %5 = rocdl.cvt.scale.pk16.f32.bf6 %v3xi32, %scale[0] : vector<16xf32>1311 1312 llvm.return1313}1314 1315// -----1316 1317// CHECK-LABEL: rocdl.cvt.scalef32.pk161318llvm.func @rocdl.cvt.scalef32.pk16(%v16xf32: vector<16xf32>,1319 %v16xf16: vector<16xf16>,1320 %v16xbf16: vector<16xbf16>,1321 %scale: f32) {1322 1323 // CHECK: rocdl.cvt.scalef32.pk16.fp6.f161324 %0 = rocdl.cvt.scalef32.pk16.fp6.f16 %v16xf16, %scale : vector<3xi32>1325 // CHECK: rocdl.cvt.scalef32.pk16.fp6.bf161326 %1 = rocdl.cvt.scalef32.pk16.fp6.bf16 %v16xbf16, %scale : vector<3xi32>1327 // CHECK: rocdl.cvt.scalef32.pk16.fp6.f321328 %2 = rocdl.cvt.scalef32.pk16.fp6.f32 %v16xf32, %scale : vector<3xi32>1329 1330 // CHECK: rocdl.cvt.scalef32.pk16.bf6.f161331 %3 = rocdl.cvt.scalef32.pk16.bf6.f16 %v16xf16, %scale : vector<3xi32>1332 // CHECK: rocdl.cvt.scalef32.pk16.bf6.bf161333 %4 = rocdl.cvt.scalef32.pk16.bf6.bf16 %v16xbf16, %scale : vector<3xi32>1334 // CHECK: rocdl.cvt.scalef32.pk16.bf6.f321335 %5 = rocdl.cvt.scalef32.pk16.bf6.f32 %v16xf32, %scale : vector<3xi32>1336 1337 llvm.return1338}1339 1340// -----1341 1342// CHECK-LABEL: rocdl.cvt.scalef32.sr.pk161343llvm.func @rocdl.cvt.scalef32.sr.pk16(%v16xf32: vector<16xf32>,1344 %v16xf16: vector<16xf16>,1345 %v16xbf16: vector<16xbf16>,1346 %seed: i32,1347 %scale: f32) {1348 1349 // CHECK: rocdl.cvt.scalef32.sr.pk16.fp6.f161350 %0 = rocdl.cvt.scalef32.sr.pk16.fp6.f16 %v16xf16, %seed, %scale : vector<3xi32>1351 // CHECK: rocdl.cvt.scalef32.sr.pk16.fp6.bf161352 %1 = rocdl.cvt.scalef32.sr.pk16.fp6.bf16 %v16xbf16, %seed, %scale : vector<3xi32>1353 // CHECK: rocdl.cvt.scalef32.sr.pk16.fp6.f321354 %2 = rocdl.cvt.scalef32.sr.pk16.fp6.f32 %v16xf32, %seed, %scale : vector<3xi32>1355 1356 // CHECK: rocdl.cvt.scalef32.sr.pk16.bf6.f161357 %3 = rocdl.cvt.scalef32.sr.pk16.bf6.f16 %v16xf16, %seed, %scale : vector<3xi32>1358 // CHECK: rocdl.cvt.scalef32.sr.pk16.bf6.bf161359 %4 = rocdl.cvt.scalef32.sr.pk16.bf6.bf16 %v16xbf16, %seed, %scale : vector<3xi32>1360 // CHECK: rocdl.cvt.scalef32.sr.pk16.bf6.f321361 %5 = rocdl.cvt.scalef32.sr.pk16.bf6.f32 %v16xf32, %seed, %scale : vector<3xi32>1362 1363 llvm.return1364}1365 1366// -----1367 1368// CHECK-LABEL: @rocdl_wmma_scale_ops1369llvm.func @rocdl_wmma_scale_ops(%a_f8: vector<8xi32>, %a_f4: vector<4xi32>, %c_f32: vector<4xf32>, %c16_f32: vector<16xf32>,1370 %scale_i32: i32, %scale_i64: i64) {1371 // CHECK: rocdl.wmma.scale.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>1372 %r0 = rocdl.wmma.scale.f32.16x16x128.f8f6f4 %a_f8, %a_f8, %c_f32, %scale_i32, %scale_i32 : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i32, i32) -> vector<4xf32>1373 1374 // CHECK: rocdl.wmma.scale16.f32.16x16x128.f8f6f4 %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i64, i64) -> vector<4xf32>1375 %r1 = rocdl.wmma.scale16.f32.16x16x128.f8f6f4 %a_f8, %a_f8, %c_f32, %scale_i64, %scale_i64 : (vector<8xi32>, vector<8xi32>, vector<4xf32>, i64, i64) -> vector<4xf32>1376 1377 // CHECK: rocdl.wmma.scale.f32.32x16x128.f4 %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>1378 %r2 = rocdl.wmma.scale.f32.32x16x128.f4 %a_f4, %a_f4, %c16_f32, %scale_i32, %scale_i32 : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i32, i32) -> vector<16xf32>1379 1380 // CHECK: rocdl.wmma.scale16.f32.32x16x128.f4 %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}} : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i64, i64) -> vector<16xf32>1381 %r3 = rocdl.wmma.scale16.f32.32x16x128.f4 %a_f4, %a_f4, %c16_f32, %scale_i64, %scale_i64 : (vector<4xi32>, vector<4xi32>, vector<16xf32>, i64, i64) -> vector<16xf32>1382 1383 llvm.return1384}1385 1386// -----1387 1388// expected-error@below {{attribute attached to unexpected op}}1389func.func private @expected_llvm_func() attributes { rocdl.kernel }1390 1391// -----1392 1393// Just check these don't emit errors.1394gpu.module @module_1 [#rocdl.target<O = 1, chip = "gfx900", abi = "500", link = ["my_device_lib.bc"], flags = {fast, daz, unsafe_math}>] {1395}1396 1397gpu.module @module_2 [#rocdl.target<chip = "gfx900">, #rocdl.target<chip = "gfx90a">] {1398}1399 1400gpu.module @module_3 [#rocdl.target<O = 1, chip = "gfx900", abi = "600", link = ["my_device_lib.bc"], flags = {fast, daz, unsafe_math}>] {1401}1402