536 lines · plain
1// RUN: mlir-opt %s -convert-amdgpu-to-rocdl=chipset=gfx950 | FileCheck %s2 3// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_f324// CHECK-DAG: [[ZERO:%.+]] = llvm.mlir.zero : vector<2xi16>5// CHECK-DAG: [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i326// CHECK-DAG: [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i327// CHECK: [[ELEM0:%.+]] = llvm.extractelement %arg0{{\[}}[[C0]] : i32] : vector<2xf32>8// CHECK: [[ELEM1:%.+]] = llvm.extractelement %arg0{{\[}}[[C1]] : i32] : vector<2xf32>9// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f32 [[ELEM0]], [[ELEM1]], %arg1 -> [[ZERO]][false] : vector<2xi16>10// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>11// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>12// CHECK: return [[CAST]] : vector<4xf8E4M3FN>13func.func @packed_scaled_trunc_f8e4m3_f32(%v: vector<2xf32>, %scale: f32) -> vector<4xf8E4M3FN> {14 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf32> to vector<4xf8E4M3FN>15 func.return %ret : vector<4xf8E4M3FN>16}17 18// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_f32_vec119// CHECK-DAG: [[ZERO_I16:%.+]] = llvm.mlir.zero : vector<2xi16>20// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i3221// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf32>22// CHECK: [[ZERO_F32:%.+]] = llvm.mlir.zero : vector<2xf32>23// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F32]]{{\[}}[[C0_I32]] : i32] : vector<2xf32>24// CHECK-DAG: [[C0_I32_2:%.+]] = llvm.mlir.constant(0 : i32) : i3225// CHECK-DAG: [[C1_I32:%.+]] = llvm.mlir.constant(1 : i32) : i3226// CHECK: [[ELEM0:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C0_I32_2]] : i32] : vector<2xf32>27// CHECK: [[ELEM1:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C1_I32]] : i32] : vector<2xf32>28// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f32 [[ELEM0]], [[ELEM1]], %arg1 -> [[ZERO_I16]][false] : vector<2xi16>29// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>30// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>31// CHECK: return [[CAST]] : vector<4xf8E4M3FN>32func.func @packed_scaled_trunc_f8e4m3_f32_vec1(%v: vector<1xf32>, %scale: f32) -> vector<4xf8E4M3FN> {33 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xf32> to vector<4xf8E4M3FN>34 func.return %ret : vector<4xf8E4M3FN>35}36 37// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_f3238// CHECK-DAG: [[EXISTING_CAST_TO_I8:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E4M3FN> to vector<4xi8>39// CHECK-DAG: [[EXISTING_BITCAST_TO_I16:%.+]] = llvm.bitcast [[EXISTING_CAST_TO_I8]] : vector<4xi8> to vector<2xi16>40// CHECK-DAG: [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i3241// CHECK-DAG: [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i3242// CHECK: [[ELEM0:%.+]] = llvm.extractelement %arg0{{\[}}[[C0]] : i32] : vector<2xf32>43// CHECK: [[ELEM1:%.+]] = llvm.extractelement %arg0{{\[}}[[C1]] : i32] : vector<2xf32>44// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f32 [[ELEM0]], [[ELEM1]], %arg2 -> [[EXISTING_BITCAST_TO_I16]][false] : vector<2xi16>45// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>46// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>47// CHECK: return [[CAST]] : vector<4xf8E4M3FN>48func.func @packed_scaled_trunc_into_f8e4m3_f32(%v: vector<2xf32>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {49 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf32> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>50 func.return %ret : vector<4xf8E4M3FN>51}52 53// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_f32_vec154// CHECK-DAG: [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E4M3FN> to vector<4xi8>55// CHECK-DAG: [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<4xi8> to vector<2xi16>56// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i3257// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf32>58// CHECK: [[ZERO_F32:%.+]] = llvm.mlir.zero : vector<2xf32>59// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F32]]{{\[}}[[C0_I32]] : i32] : vector<2xf32>60// CHECK-DAG: [[C0_I32_2:%.+]] = llvm.mlir.constant(0 : i32) : i3261// CHECK-DAG: [[C1_I32:%.+]] = llvm.mlir.constant(1 : i32) : i3262// CHECK: [[ELEM0:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C0_I32_2]] : i32] : vector<2xf32>63// CHECK: [[ELEM1:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C1_I32]] : i32] : vector<2xf32>64// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f32 [[ELEM0]], [[ELEM1]], %arg2 -> [[EXISTING_BITCAST]][false] : vector<2xi16>65// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>66// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>67// CHECK: return [[CAST]] : vector<4xf8E4M3FN>68func.func @packed_scaled_trunc_into_f8e4m3_f32_vec1(%v: vector<1xf32>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {69 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xf32> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>70 func.return %ret : vector<4xf8E4M3FN>71}72 73// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_f1674// CHECK-DAG: [[ZERO:%.+]] = llvm.mlir.zero : vector<2xi16>75// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f16 %arg0, %arg1 -> [[ZERO]][false] : vector<2xi16>76// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>77// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>78// CHECK: return [[CAST]] : vector<4xf8E4M3FN>79func.func @packed_scaled_trunc_f8e4m3_f16(%v: vector<2xf16>, %scale: f32) -> vector<4xf8E4M3FN> {80 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf16> to vector<4xf8E4M3FN>81 func.return %ret : vector<4xf8E4M3FN>82}83 84// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_f16_vec185// CHECK-DAG: [[ZERO_I16:%.+]] = llvm.mlir.zero : vector<2xi16>86// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i3287// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf16>88// CHECK: [[ZERO_F16:%.+]] = llvm.mlir.zero : vector<2xf16>89// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F16]]{{\[}}[[C0_I32]] : i32] : vector<2xf16>90// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f16 [[INSERT]], %arg1 -> [[ZERO_I16]][false] : vector<2xi16>91// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>92// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>93// CHECK: return [[CAST]] : vector<4xf8E4M3FN>94func.func @packed_scaled_trunc_f8e4m3_f16_vec1(%v: vector<1xf16>, %scale: f32) -> vector<4xf8E4M3FN> {95 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xf16> to vector<4xf8E4M3FN>96 func.return %ret : vector<4xf8E4M3FN>97}98 99// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_f16100// CHECK-DAG: [[EXISTING_CAST_TO_I8:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E4M3FN> to vector<4xi8>101// CHECK-DAG: [[EXISTING_BITCAST_TO_I16:%.+]] = llvm.bitcast [[EXISTING_CAST_TO_I8]] : vector<4xi8> to vector<2xi16>102// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f16 %arg0, %arg2 -> [[EXISTING_BITCAST_TO_I16]][false] : vector<2xi16>103// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>104// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>105// CHECK: return [[CAST]] : vector<4xf8E4M3FN>106func.func @packed_scaled_trunc_into_f8e4m3_f16(%v: vector<2xf16>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {107 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf16> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>108 func.return %ret : vector<4xf8E4M3FN>109}110 111// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_f16_vec1112// CHECK-DAG: [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E4M3FN> to vector<4xi8>113// CHECK-DAG: [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<4xi8> to vector<2xi16>114// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32115// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf16>116// CHECK: [[ZERO_F16:%.+]] = llvm.mlir.zero : vector<2xf16>117// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F16]]{{\[}}[[C0_I32]] : i32] : vector<2xf16>118// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.f16 [[INSERT]], %arg2 -> [[EXISTING_BITCAST]][false] : vector<2xi16>119// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>120// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>121// CHECK: return [[CAST]] : vector<4xf8E4M3FN>122func.func @packed_scaled_trunc_into_f8e4m3_f16_vec1(%v: vector<1xf16>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {123 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xf16> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>124 func.return %ret : vector<4xf8E4M3FN>125}126 127// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_bf16128// CHECK-DAG: [[ZERO:%.+]] = llvm.mlir.zero : vector<2xi16>129// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.bf16 %arg0, %arg1 -> [[ZERO]][false] : vector<2xi16>130// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>131// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>132// CHECK: return [[CAST]] : vector<4xf8E4M3FN>133func.func @packed_scaled_trunc_f8e4m3_bf16(%v: vector<2xbf16>, %scale: f32) -> vector<4xf8E4M3FN> {134 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xbf16> to vector<4xf8E4M3FN>135 func.return %ret : vector<4xf8E4M3FN>136}137 138// CHECK-LABEL: func.func @packed_scaled_trunc_f8e4m3_bf16_vec1139// CHECK-DAG: [[ZERO_I16:%.+]] = llvm.mlir.zero : vector<2xi16>140// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32141// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xbf16>142// CHECK: [[ZERO_BF16:%.+]] = llvm.mlir.zero : vector<2xbf16>143// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_BF16]]{{\[}}[[C0_I32]] : i32] : vector<2xbf16>144// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.bf16 [[INSERT]], %arg1 -> [[ZERO_I16]][false] : vector<2xi16>145// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>146// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>147// CHECK: return [[CAST]] : vector<4xf8E4M3FN>148func.func @packed_scaled_trunc_f8e4m3_bf16_vec1(%v: vector<1xbf16>, %scale: f32) -> vector<4xf8E4M3FN> {149 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xbf16> to vector<4xf8E4M3FN>150 func.return %ret : vector<4xf8E4M3FN>151}152 153// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_bf16154// CHECK-DAG: [[EXISTING_CAST_TO_I8:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E4M3FN> to vector<4xi8>155// CHECK-DAG: [[EXISTING_BITCAST_TO_I16:%.+]] = llvm.bitcast [[EXISTING_CAST_TO_I8]] : vector<4xi8> to vector<2xi16>156// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.bf16 %arg0, %arg2 -> [[EXISTING_BITCAST_TO_I16]][false] : vector<2xi16>157// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>158// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>159// CHECK: return [[CAST]] : vector<4xf8E4M3FN>160func.func @packed_scaled_trunc_into_f8e4m3_bf16(%v: vector<2xbf16>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {161 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xbf16> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>162 func.return %ret : vector<4xf8E4M3FN>163}164 165// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e4m3_bf16_vec1166// CHECK-DAG: [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E4M3FN> to vector<4xi8>167// CHECK-DAG: [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<4xi8> to vector<2xi16>168// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32169// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xbf16>170// CHECK: [[ZERO_BF16:%.+]] = llvm.mlir.zero : vector<2xbf16>171// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_BF16]]{{\[}}[[C0_I32]] : i32] : vector<2xbf16>172// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp8.bf16 [[INSERT]], %arg2 -> [[EXISTING_BITCAST]][false] : vector<2xi16>173// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>174// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E4M3FN>175// CHECK: return [[CAST]] : vector<4xf8E4M3FN>176func.func @packed_scaled_trunc_into_f8e4m3_bf16_vec1(%v: vector<1xbf16>, %existing: vector<4xf8E4M3FN>, %scale: f32) -> vector<4xf8E4M3FN> {177 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xbf16> to vector<4xf8E4M3FN> into vector<4xf8E4M3FN>178 func.return %ret : vector<4xf8E4M3FN>179}180 181// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_f32182// CHECK-DAG: [[ZERO:%.+]] = llvm.mlir.zero : vector<2xi16>183// CHECK-DAG: [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32184// CHECK-DAG: [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32185// CHECK: [[ELEM0:%.+]] = llvm.extractelement %arg0{{\[}}[[C0]] : i32] : vector<2xf32>186// CHECK: [[ELEM1:%.+]] = llvm.extractelement %arg0{{\[}}[[C1]] : i32] : vector<2xf32>187// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f32 [[ELEM0]], [[ELEM1]], %arg1 -> [[ZERO]][false] : vector<2xi16>188// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>189// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>190// CHECK: return [[CAST]] : vector<4xf8E5M2>191func.func @packed_scaled_trunc_f8e5m2_f32(%v: vector<2xf32>, %scale: f32) -> vector<4xf8E5M2> {192 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf32> to vector<4xf8E5M2>193 func.return %ret : vector<4xf8E5M2>194}195 196// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_f32_vec1197// CHECK-DAG: [[ZERO_I16:%.+]] = llvm.mlir.zero : vector<2xi16>198// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32199// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf32>200// CHECK: [[ZERO_F32:%.+]] = llvm.mlir.zero : vector<2xf32>201// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F32]]{{\[}}[[C0_I32]] : i32] : vector<2xf32>202// CHECK-DAG: [[C0_I32_2:%.+]] = llvm.mlir.constant(0 : i32) : i32203// CHECK-DAG: [[C1_I32:%.+]] = llvm.mlir.constant(1 : i32) : i32204// CHECK: [[ELEM0:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C0_I32_2]] : i32] : vector<2xf32>205// CHECK: [[ELEM1:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C1_I32]] : i32] : vector<2xf32>206// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f32 [[ELEM0]], [[ELEM1]], %arg1 -> [[ZERO_I16]][false] : vector<2xi16>207// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>208// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>209// CHECK: return [[CAST]] : vector<4xf8E5M2>210func.func @packed_scaled_trunc_f8e5m2_f32_vec1(%v: vector<1xf32>, %scale: f32) -> vector<4xf8E5M2> {211 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xf32> to vector<4xf8E5M2>212 func.return %ret : vector<4xf8E5M2>213}214 215// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_f32216// CHECK-DAG: [[EXISTING_CAST_TO_I8:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E5M2> to vector<4xi8>217// CHECK-DAG: [[EXISTING_BITCAST_TO_I16:%.+]] = llvm.bitcast [[EXISTING_CAST_TO_I8]] : vector<4xi8> to vector<2xi16>218// CHECK-DAG: [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32219// CHECK-DAG: [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32220// CHECK: [[ELEM0:%.+]] = llvm.extractelement %arg0{{\[}}[[C0]] : i32] : vector<2xf32>221// CHECK: [[ELEM1:%.+]] = llvm.extractelement %arg0{{\[}}[[C1]] : i32] : vector<2xf32>222// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f32 [[ELEM0]], [[ELEM1]], %arg2 -> [[EXISTING_BITCAST_TO_I16]][false] : vector<2xi16>223// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>224// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>225// CHECK: return [[CAST]] : vector<4xf8E5M2>226func.func @packed_scaled_trunc_into_f8e5m2_f32(%v: vector<2xf32>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {227 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf32> to vector<4xf8E5M2> into vector<4xf8E5M2>228 func.return %ret : vector<4xf8E5M2>229}230 231// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_f32_vec1232// CHECK-DAG: [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E5M2> to vector<4xi8>233// CHECK-DAG: [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<4xi8> to vector<2xi16>234// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32235// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf32>236// CHECK: [[ZERO_F32:%.+]] = llvm.mlir.zero : vector<2xf32>237// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F32]]{{\[}}[[C0_I32]] : i32] : vector<2xf32>238// CHECK-DAG: [[C0_I32_2:%.+]] = llvm.mlir.constant(0 : i32) : i32239// CHECK-DAG: [[C1_I32:%.+]] = llvm.mlir.constant(1 : i32) : i32240// CHECK: [[ELEM0:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C0_I32_2]] : i32] : vector<2xf32>241// CHECK: [[ELEM1:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C1_I32]] : i32] : vector<2xf32>242// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f32 [[ELEM0]], [[ELEM1]], %arg2 -> [[EXISTING_BITCAST]][false] : vector<2xi16>243// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>244// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>245// CHECK: return [[CAST]] : vector<4xf8E5M2>246func.func @packed_scaled_trunc_into_f8e5m2_f32_vec1(%v: vector<1xf32>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {247 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xf32> to vector<4xf8E5M2> into vector<4xf8E5M2>248 func.return %ret : vector<4xf8E5M2>249}250 251// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_f16252// CHECK-DAG: [[ZERO:%.+]] = llvm.mlir.zero : vector<2xi16>253// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f16 %arg0, %arg1 -> [[ZERO]][false] : vector<2xi16>254// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>255// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>256// CHECK: return [[CAST]] : vector<4xf8E5M2>257func.func @packed_scaled_trunc_f8e5m2_f16(%v: vector<2xf16>, %scale: f32) -> vector<4xf8E5M2> {258 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf16> to vector<4xf8E5M2>259 func.return %ret : vector<4xf8E5M2>260}261 262// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_f16_vec1263// CHECK-DAG: [[ZERO_I16:%.+]] = llvm.mlir.zero : vector<2xi16>264// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32265// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf16>266// CHECK: [[ZERO_F16:%.+]] = llvm.mlir.zero : vector<2xf16>267// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F16]]{{\[}}[[C0_I32]] : i32] : vector<2xf16>268// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f16 [[INSERT]], %arg1 -> [[ZERO_I16]][false] : vector<2xi16>269// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>270// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>271// CHECK: return [[CAST]] : vector<4xf8E5M2>272func.func @packed_scaled_trunc_f8e5m2_f16_vec1(%v: vector<1xf16>, %scale: f32) -> vector<4xf8E5M2> {273 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xf16> to vector<4xf8E5M2>274 func.return %ret : vector<4xf8E5M2>275}276 277// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_f16278// CHECK-DAG: [[EXISTING_CAST_TO_I8:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E5M2> to vector<4xi8>279// CHECK-DAG: [[EXISTING_BITCAST_TO_I16:%.+]] = llvm.bitcast [[EXISTING_CAST_TO_I8]] : vector<4xi8> to vector<2xi16>280// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f16 %arg0, %arg2 -> [[EXISTING_BITCAST_TO_I16]][false] : vector<2xi16>281// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>282// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>283// CHECK: return [[CAST]] : vector<4xf8E5M2>284func.func @packed_scaled_trunc_into_f8e5m2_f16(%v: vector<2xf16>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {285 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf16> to vector<4xf8E5M2> into vector<4xf8E5M2>286 func.return %ret : vector<4xf8E5M2>287}288 289// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_f16_vec1290// CHECK-DAG: [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E5M2> to vector<4xi8>291// CHECK-DAG: [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<4xi8> to vector<2xi16>292// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32293// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf16>294// CHECK: [[ZERO_F16:%.+]] = llvm.mlir.zero : vector<2xf16>295// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F16]]{{\[}}[[C0_I32]] : i32] : vector<2xf16>296// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.f16 [[INSERT]], %arg2 -> [[EXISTING_BITCAST]][false] : vector<2xi16>297// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>298// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>299// CHECK: return [[CAST]] : vector<4xf8E5M2>300func.func @packed_scaled_trunc_into_f8e5m2_f16_vec1(%v: vector<1xf16>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {301 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xf16> to vector<4xf8E5M2> into vector<4xf8E5M2>302 func.return %ret : vector<4xf8E5M2>303}304 305// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_bf16306// CHECK-DAG: [[ZERO:%.+]] = llvm.mlir.zero : vector<2xi16>307// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.bf16 %arg0, %arg1 -> [[ZERO]][false] : vector<2xi16>308// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>309// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>310// CHECK: return [[CAST]] : vector<4xf8E5M2>311func.func @packed_scaled_trunc_f8e5m2_bf16(%v: vector<2xbf16>, %scale: f32) -> vector<4xf8E5M2> {312 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xbf16> to vector<4xf8E5M2>313 func.return %ret : vector<4xf8E5M2>314}315 316// CHECK-LABEL: func.func @packed_scaled_trunc_f8e5m2_bf16_vec1317// CHECK-DAG: [[ZERO_I16:%.+]] = llvm.mlir.zero : vector<2xi16>318// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32319// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xbf16>320// CHECK: [[ZERO_BF16:%.+]] = llvm.mlir.zero : vector<2xbf16>321// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_BF16]]{{\[}}[[C0_I32]] : i32] : vector<2xbf16>322// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.bf16 [[INSERT]], %arg1 -> [[ZERO_I16]][false] : vector<2xi16>323// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>324// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>325// CHECK: return [[CAST]] : vector<4xf8E5M2>326func.func @packed_scaled_trunc_f8e5m2_bf16_vec1(%v: vector<1xbf16>, %scale: f32) -> vector<4xf8E5M2> {327 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xbf16> to vector<4xf8E5M2>328 func.return %ret : vector<4xf8E5M2>329}330 331// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_bf16332// CHECK-DAG: [[EXISTING_CAST_TO_I8:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E5M2> to vector<4xi8>333// CHECK-DAG: [[EXISTING_BITCAST_TO_I16:%.+]] = llvm.bitcast [[EXISTING_CAST_TO_I8]] : vector<4xi8> to vector<2xi16>334// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.bf16 %arg0, %arg2 -> [[EXISTING_BITCAST_TO_I16]][false] : vector<2xi16>335// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>336// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>337// CHECK: return [[CAST]] : vector<4xf8E5M2>338func.func @packed_scaled_trunc_into_f8e5m2_bf16(%v: vector<2xbf16>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {339 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xbf16> to vector<4xf8E5M2> into vector<4xf8E5M2>340 func.return %ret : vector<4xf8E5M2>341}342 343// CHECK-LABEL: func.func @packed_scaled_trunc_into_f8e5m2_bf16_vec1344// CHECK-DAG: [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<4xf8E5M2> to vector<4xi8>345// CHECK-DAG: [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<4xi8> to vector<2xi16>346// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32347// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xbf16>348// CHECK: [[ZERO_BF16:%.+]] = llvm.mlir.zero : vector<2xbf16>349// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_BF16]]{{\[}}[[C0_I32]] : i32] : vector<2xbf16>350// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.bf8.bf16 [[INSERT]], %arg2 -> [[EXISTING_BITCAST]][false] : vector<2xi16>351// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : vector<2xi16> to vector<4xi8>352// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<4xi8> to vector<4xf8E5M2>353// CHECK: return [[CAST]] : vector<4xf8E5M2>354func.func @packed_scaled_trunc_into_f8e5m2_bf16_vec1(%v: vector<1xbf16>, %existing: vector<4xf8E5M2>, %scale: f32) -> vector<4xf8E5M2> {355 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xbf16> to vector<4xf8E5M2> into vector<4xf8E5M2>356 func.return %ret : vector<4xf8E5M2>357}358 359// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_f32360// CHECK-DAG: [[ZERO:%.+]] = llvm.mlir.zero : i32361// CHECK-DAG: [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32362// CHECK-DAG: [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32363// CHECK: [[ELEM0:%.+]] = llvm.extractelement %arg0{{\[}}[[C0]] : i32] : vector<2xf32>364// CHECK: [[ELEM1:%.+]] = llvm.extractelement %arg0{{\[}}[[C1]] : i32] : vector<2xf32>365// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f32 [[ELEM0]], [[ELEM1]], %arg1 -> [[ZERO]][0] : i32366// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>367// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>368// CHECK: return [[CAST]] : vector<8xf4E2M1FN>369func.func @packed_scaled_trunc_f4e2m1_f32(%v: vector<2xf32>, %scale: f32) -> vector<8xf4E2M1FN> {370 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf32> to vector<8xf4E2M1FN>371 func.return %ret : vector<8xf4E2M1FN>372}373 374// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_f32_vec1375// CHECK-DAG: [[ZERO_I32:%.+]] = llvm.mlir.zero : i32376// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32377// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf32>378// CHECK: [[ZERO_F32:%.+]] = llvm.mlir.zero : vector<2xf32>379// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F32]]{{\[}}[[C0_I32]] : i32] : vector<2xf32>380// CHECK-DAG: [[C0_I32_2:%.+]] = llvm.mlir.constant(0 : i32) : i32381// CHECK-DAG: [[C1_I32:%.+]] = llvm.mlir.constant(1 : i32) : i32382// CHECK: [[ELEM0:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C0_I32_2]] : i32] : vector<2xf32>383// CHECK: [[ELEM1:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C1_I32]] : i32] : vector<2xf32>384// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f32 [[ELEM0]], [[ELEM1]], %arg1 -> [[ZERO_I32]][0] : i32385// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>386// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>387// CHECK: return [[CAST]] : vector<8xf4E2M1FN>388func.func @packed_scaled_trunc_f4e2m1_f32_vec1(%v: vector<1xf32>, %scale: f32) -> vector<8xf4E2M1FN> {389 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xf32> to vector<8xf4E2M1FN>390 func.return %ret : vector<8xf4E2M1FN>391}392 393// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_f32394// CHECK-DAG: [[BITCAST_I4:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<8xf4E2M1FN> to vector<8xi4>395// CHECK-DAG: [[BITCAST_I32:%.+]] = llvm.bitcast [[BITCAST_I4]] : vector<8xi4> to i32396// CHECK-DAG: [[C0:%.+]] = llvm.mlir.constant(0 : i32) : i32397// CHECK-DAG: [[C1:%.+]] = llvm.mlir.constant(1 : i32) : i32398// CHECK: [[ELEM0:%.+]] = llvm.extractelement %arg0{{\[}}[[C0]] : i32] : vector<2xf32>399// CHECK: [[ELEM1:%.+]] = llvm.extractelement %arg0{{\[}}[[C1]] : i32] : vector<2xf32>400// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f32 [[ELEM0]], [[ELEM1]], %arg2 -> [[BITCAST_I32]][0] : i32401// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>402// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>403// CHECK: return [[CAST]] : vector<8xf4E2M1FN>404func.func @packed_scaled_trunc_into_f4e2m1_f32(%v: vector<2xf32>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {405 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf32> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>406 func.return %ret : vector<8xf4E2M1FN>407}408 409// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_f32_vec1410// CHECK-DAG: [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<8xf4E2M1FN> to vector<8xi4>411// CHECK-DAG: [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<8xi4> to i32412// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32413// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf32>414// CHECK: [[ZERO_F32:%.+]] = llvm.mlir.zero : vector<2xf32>415// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F32]]{{\[}}[[C0_I32]] : i32] : vector<2xf32>416// CHECK-DAG: [[C0_I32_2:%.+]] = llvm.mlir.constant(0 : i32) : i32417// CHECK-DAG: [[C1_I32:%.+]] = llvm.mlir.constant(1 : i32) : i32418// CHECK: [[ELEM0:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C0_I32_2]] : i32] : vector<2xf32>419// CHECK: [[ELEM1:%.+]] = llvm.extractelement [[INSERT]]{{\[}}[[C1_I32]] : i32] : vector<2xf32>420// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f32 [[ELEM0]], [[ELEM1]], %arg2 -> [[EXISTING_BITCAST]][0] : i32421// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>422// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>423// CHECK: return [[CAST]] : vector<8xf4E2M1FN>424func.func @packed_scaled_trunc_into_f4e2m1_f32_vec1(%v: vector<1xf32>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {425 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xf32> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>426 func.return %ret : vector<8xf4E2M1FN>427}428 429// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_f16430// CHECK-DAG: [[ZERO:%.+]] = llvm.mlir.zero : i32431// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f16 %arg0, %arg1 -> [[ZERO]][0] : i32432// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>433// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>434// CHECK: return [[CAST]] : vector<8xf4E2M1FN>435func.func @packed_scaled_trunc_f4e2m1_f16(%v: vector<2xf16>, %scale: f32) -> vector<8xf4E2M1FN> {436 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xf16> to vector<8xf4E2M1FN>437 func.return %ret : vector<8xf4E2M1FN>438}439 440// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_f16_vec1441// CHECK-DAG: [[ZERO_I32:%.+]] = llvm.mlir.zero : i32442// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32443// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf16>444// CHECK: [[ZERO_F16:%.+]] = llvm.mlir.zero : vector<2xf16>445// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F16]]{{\[}}[[C0_I32]] : i32] : vector<2xf16>446// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f16 [[INSERT]], %arg1 -> [[ZERO_I32]][0] : i32447// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>448// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>449// CHECK: return [[CAST]] : vector<8xf4E2M1FN>450func.func @packed_scaled_trunc_f4e2m1_f16_vec1(%v: vector<1xf16>, %scale: f32) -> vector<8xf4E2M1FN> {451 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xf16> to vector<8xf4E2M1FN>452 func.return %ret : vector<8xf4E2M1FN>453}454 455// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_f16456// CHECK-DAG: [[BITCAST_I4:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<8xf4E2M1FN> to vector<8xi4>457// CHECK-DAG: [[BITCAST_I32:%.+]] = llvm.bitcast [[BITCAST_I4]] : vector<8xi4> to i32458// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f16 %arg0, %arg2 -> [[BITCAST_I32]][0] : i32459// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>460// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>461// CHECK: return [[CAST]] : vector<8xf4E2M1FN>462func.func @packed_scaled_trunc_into_f4e2m1_f16(%v: vector<2xf16>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {463 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xf16> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>464 func.return %ret : vector<8xf4E2M1FN>465}466 467// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_f16_vec1468// CHECK-DAG: [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<8xf4E2M1FN> to vector<8xi4>469// CHECK-DAG: [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<8xi4> to i32470// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32471// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xf16>472// CHECK: [[ZERO_F16:%.+]] = llvm.mlir.zero : vector<2xf16>473// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_F16]]{{\[}}[[C0_I32]] : i32] : vector<2xf16>474// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.f16 [[INSERT]], %arg2 -> [[EXISTING_BITCAST]][0] : i32475// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>476// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>477// CHECK: return [[CAST]] : vector<8xf4E2M1FN>478func.func @packed_scaled_trunc_into_f4e2m1_f16_vec1(%v: vector<1xf16>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {479 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xf16> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>480 func.return %ret : vector<8xf4E2M1FN>481}482 483// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_bf16484// CHECK-DAG: [[ZERO:%.+]] = llvm.mlir.zero : i32485// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.bf16 %arg0, %arg1 -> [[ZERO]][0] : i32486// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>487// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>488// CHECK: return [[CAST]] : vector<8xf4E2M1FN>489func.func @packed_scaled_trunc_f4e2m1_bf16(%v: vector<2xbf16>, %scale: f32) -> vector<8xf4E2M1FN> {490 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<2xbf16> to vector<8xf4E2M1FN>491 func.return %ret : vector<8xf4E2M1FN>492}493 494// CHECK-LABEL: func.func @packed_scaled_trunc_f4e2m1_bf16_vec1495// CHECK-DAG: [[ZERO_I32:%.+]] = llvm.mlir.zero : i32496// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32497// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xbf16>498// CHECK: [[ZERO_BF16:%.+]] = llvm.mlir.zero : vector<2xbf16>499// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_BF16]]{{\[}}[[C0_I32]] : i32] : vector<2xbf16>500// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.bf16 [[INSERT]], %arg1 -> [[ZERO_I32]][0] : i32501// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>502// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>503// CHECK: return [[CAST]] : vector<8xf4E2M1FN>504func.func @packed_scaled_trunc_f4e2m1_bf16_vec1(%v: vector<1xbf16>, %scale: f32) -> vector<8xf4E2M1FN> {505 %ret = amdgpu.packed_scaled_trunc %v into undef[0], %scale : vector<1xbf16> to vector<8xf4E2M1FN>506 func.return %ret : vector<8xf4E2M1FN>507}508 509// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_bf16510// CHECK-DAG: [[BITCAST_I4:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<8xf4E2M1FN> to vector<8xi4>511// CHECK-DAG: [[BITCAST_I32:%.+]] = llvm.bitcast [[BITCAST_I4]] : vector<8xi4> to i32512// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.bf16 %arg0, %arg2 -> [[BITCAST_I32]][0] : i32513// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>514// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>515// CHECK: return [[CAST]] : vector<8xf4E2M1FN>516func.func @packed_scaled_trunc_into_f4e2m1_bf16(%v: vector<2xbf16>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {517 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<2xbf16> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>518 func.return %ret : vector<8xf4E2M1FN>519}520 521// CHECK-LABEL: func.func @packed_scaled_trunc_into_f4e2m1_bf16_vec1522// CHECK-DAG: [[EXISTING_CAST:%.+]] = builtin.unrealized_conversion_cast %arg1 : vector<8xf4E2M1FN> to vector<8xi4>523// CHECK-DAG: [[EXISTING_BITCAST:%.+]] = llvm.bitcast [[EXISTING_CAST]] : vector<8xi4> to i32524// CHECK-DAG: [[C0_I32:%.+]] = llvm.mlir.constant(0 : i32) : i32525// CHECK: [[EXTRACT:%.+]] = llvm.extractelement %arg0{{\[}}[[C0_I32]] : i32] : vector<1xbf16>526// CHECK: [[ZERO_BF16:%.+]] = llvm.mlir.zero : vector<2xbf16>527// CHECK: [[INSERT:%.+]] = llvm.insertelement [[EXTRACT]], [[ZERO_BF16]]{{\[}}[[C0_I32]] : i32] : vector<2xbf16>528// CHECK: [[CVT:%.+]] = rocdl.cvt.scalef32.pk.fp4.bf16 [[INSERT]], %arg2 -> [[EXISTING_BITCAST]][0] : i32529// CHECK: [[BITCAST:%.+]] = llvm.bitcast [[CVT]] : i32 to vector<8xi4>530// CHECK: [[CAST:%.+]] = builtin.unrealized_conversion_cast [[BITCAST]] : vector<8xi4> to vector<8xf4E2M1FN>531// CHECK: return [[CAST]] : vector<8xf4E2M1FN>532func.func @packed_scaled_trunc_into_f4e2m1_bf16_vec1(%v: vector<1xbf16>, %existing: vector<8xf4E2M1FN>, %scale: f32) -> vector<8xf4E2M1FN> {533 %ret = amdgpu.packed_scaled_trunc %v into %existing[0], %scale : vector<1xbf16> to vector<8xf4E2M1FN> into vector<8xf4E2M1FN>534 func.return %ret : vector<8xf4E2M1FN>535}536