1245 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s --mtriple=x86_64-unknown-unknown -mattr=avx10.2 | FileCheck %s3 4define <8 x bfloat> @fma_123_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z) {5; CHECK-LABEL: fma_123_v8bf16:6; CHECK: # %bb.0:7; CHECK-NEXT: vfmadd213bf16 %xmm2, %xmm1, %xmm08; CHECK-NEXT: retq9 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z)10 ret <8 x bfloat> %a11}12 13define <8 x bfloat> @fma_213_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z) {14; CHECK-LABEL: fma_213_v8bf16:15; CHECK: # %bb.0:16; CHECK-NEXT: vfmadd213bf16 %xmm2, %xmm1, %xmm017; CHECK-NEXT: retq18 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %x, <8 x bfloat> %z)19 ret <8 x bfloat> %a20}21 22define <8 x bfloat> @fma_231_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z) {23; CHECK-LABEL: fma_231_v8bf16:24; CHECK: # %bb.0:25; CHECK-NEXT: vfmadd231bf16 %xmm1, %xmm2, %xmm026; CHECK-NEXT: retq27 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %z, <8 x bfloat> %x)28 ret <8 x bfloat> %a29}30 31define <8 x bfloat> @fma_321_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z) {32; CHECK-LABEL: fma_321_v8bf16:33; CHECK: # %bb.0:34; CHECK-NEXT: vfmadd231bf16 %xmm1, %xmm2, %xmm035; CHECK-NEXT: retq36 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %y, <8 x bfloat> %x)37 ret <8 x bfloat> %a38}39 40define <8 x bfloat> @fma_132_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z) {41; CHECK-LABEL: fma_132_v8bf16:42; CHECK: # %bb.0:43; CHECK-NEXT: vfmadd213bf16 %xmm1, %xmm2, %xmm044; CHECK-NEXT: retq45 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %z, <8 x bfloat> %y)46 ret <8 x bfloat> %a47}48 49define <8 x bfloat> @fma_312_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z) {50; CHECK-LABEL: fma_312_v8bf16:51; CHECK: # %bb.0:52; CHECK-NEXT: vfmadd213bf16 %xmm1, %xmm2, %xmm053; CHECK-NEXT: retq54 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %x, <8 x bfloat> %y)55 ret <8 x bfloat> %a56}57 58define <8 x bfloat> @fma_load_123_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp) {59; CHECK-LABEL: fma_load_123_v8bf16:60; CHECK: # %bb.0:61; CHECK-NEXT: vfmadd213bf16 (%rdi), %xmm1, %xmm062; CHECK-NEXT: retq63 %z = load <8 x bfloat>, ptr %zp64 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z)65 ret <8 x bfloat> %a66}67 68define <8 x bfloat> @fma_load_213_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp) {69; CHECK-LABEL: fma_load_213_v8bf16:70; CHECK: # %bb.0:71; CHECK-NEXT: vfmadd213bf16 (%rdi), %xmm1, %xmm072; CHECK-NEXT: retq73 %z = load <8 x bfloat>, ptr %zp74 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %x, <8 x bfloat> %z)75 ret <8 x bfloat> %a76}77 78define <8 x bfloat> @fma_load_231_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp) {79; CHECK-LABEL: fma_load_231_v8bf16:80; CHECK: # %bb.0:81; CHECK-NEXT: vfmadd231bf16 (%rdi), %xmm1, %xmm082; CHECK-NEXT: retq83 %z = load <8 x bfloat>, ptr %zp84 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %z, <8 x bfloat> %x)85 ret <8 x bfloat> %a86}87 88define <8 x bfloat> @fma_load_321_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp) {89; CHECK-LABEL: fma_load_321_v8bf16:90; CHECK: # %bb.0:91; CHECK-NEXT: vfmadd231bf16 (%rdi), %xmm1, %xmm092; CHECK-NEXT: retq93 %z = load <8 x bfloat>, ptr %zp94 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %y, <8 x bfloat> %x)95 ret <8 x bfloat> %a96}97 98define <8 x bfloat> @fma_load_132_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp) {99; CHECK-LABEL: fma_load_132_v8bf16:100; CHECK: # %bb.0:101; CHECK-NEXT: vfmadd132bf16 (%rdi), %xmm1, %xmm0102; CHECK-NEXT: retq103 %z = load <8 x bfloat>, ptr %zp104 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %z, <8 x bfloat> %y)105 ret <8 x bfloat> %a106}107 108define <8 x bfloat> @fma_load_312_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp) {109; CHECK-LABEL: fma_load_312_v8bf16:110; CHECK: # %bb.0:111; CHECK-NEXT: vfmadd132bf16 (%rdi), %xmm1, %xmm0112; CHECK-NEXT: retq113 %z = load <8 x bfloat>, ptr %zp114 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %x, <8 x bfloat> %y)115 ret <8 x bfloat> %a116}117 118define <8 x bfloat> @fma_mask_123_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {119; CHECK-LABEL: fma_mask_123_v8bf16:120; CHECK: # %bb.0:121; CHECK-NEXT: kmovd %edi, %k1122; CHECK-NEXT: vfmadd132bf16 %xmm1, %xmm2, %xmm0 {%k1}123; CHECK-NEXT: retq124 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z)125 %b = bitcast i8 %mask to <8 x i1>126 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x127 ret <8 x bfloat> %c128}129 130define <8 x bfloat> @fma_mask_213_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {131; CHECK-LABEL: fma_mask_213_v8bf16:132; CHECK: # %bb.0:133; CHECK-NEXT: kmovd %edi, %k1134; CHECK-NEXT: vfmadd213bf16 %xmm2, %xmm1, %xmm0 {%k1}135; CHECK-NEXT: retq136 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %x, <8 x bfloat> %z)137 %b = bitcast i8 %mask to <8 x i1>138 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x139 ret <8 x bfloat> %c140}141 142define <8 x bfloat> @fma_mask_231_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {143; CHECK-LABEL: fma_mask_231_v8bf16:144; CHECK: # %bb.0:145; CHECK-NEXT: kmovd %edi, %k1146; CHECK-NEXT: vfmadd231bf16 %xmm2, %xmm1, %xmm0 {%k1}147; CHECK-NEXT: retq148 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %z, <8 x bfloat> %x)149 %b = bitcast i8 %mask to <8 x i1>150 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x151 ret <8 x bfloat> %c152}153 154define <8 x bfloat> @fma_mask_321_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {155; CHECK-LABEL: fma_mask_321_v8bf16:156; CHECK: # %bb.0:157; CHECK-NEXT: kmovd %edi, %k1158; CHECK-NEXT: vfmadd231bf16 %xmm1, %xmm2, %xmm0 {%k1}159; CHECK-NEXT: retq160 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %y, <8 x bfloat> %x)161 %b = bitcast i8 %mask to <8 x i1>162 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x163 ret <8 x bfloat> %c164}165 166define <8 x bfloat> @fma_mask_132_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {167; CHECK-LABEL: fma_mask_132_v8bf16:168; CHECK: # %bb.0:169; CHECK-NEXT: kmovd %edi, %k1170; CHECK-NEXT: vfmadd132bf16 %xmm2, %xmm1, %xmm0 {%k1}171; CHECK-NEXT: retq172 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %z, <8 x bfloat> %y)173 %b = bitcast i8 %mask to <8 x i1>174 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x175 ret <8 x bfloat> %c176}177 178define <8 x bfloat> @fma_mask_312_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {179; CHECK-LABEL: fma_mask_312_v8bf16:180; CHECK: # %bb.0:181; CHECK-NEXT: kmovd %edi, %k1182; CHECK-NEXT: vfmadd213bf16 %xmm1, %xmm2, %xmm0 {%k1}183; CHECK-NEXT: retq184 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %x, <8 x bfloat> %y)185 %b = bitcast i8 %mask to <8 x i1>186 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x187 ret <8 x bfloat> %c188}189 190define <8 x bfloat> @fma_maskz_123_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {191; CHECK-LABEL: fma_maskz_123_v8bf16:192; CHECK: # %bb.0:193; CHECK-NEXT: kmovd %edi, %k1194; CHECK-NEXT: vfmadd213bf16 %xmm2, %xmm1, %xmm0 {%k1} {z}195; CHECK-NEXT: retq196 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z)197 %b = bitcast i8 %mask to <8 x i1>198 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer199 ret <8 x bfloat> %c200}201 202define <8 x bfloat> @fma_maskz_213_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {203; CHECK-LABEL: fma_maskz_213_v8bf16:204; CHECK: # %bb.0:205; CHECK-NEXT: kmovd %edi, %k1206; CHECK-NEXT: vfmadd213bf16 %xmm2, %xmm1, %xmm0 {%k1} {z}207; CHECK-NEXT: retq208 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %x, <8 x bfloat> %z)209 %b = bitcast i8 %mask to <8 x i1>210 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer211 ret <8 x bfloat> %c212}213 214define <8 x bfloat> @fma_maskz_231_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {215; CHECK-LABEL: fma_maskz_231_v8bf16:216; CHECK: # %bb.0:217; CHECK-NEXT: kmovd %edi, %k1218; CHECK-NEXT: vfmadd231bf16 %xmm1, %xmm2, %xmm0 {%k1} {z}219; CHECK-NEXT: retq220 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %z, <8 x bfloat> %x)221 %b = bitcast i8 %mask to <8 x i1>222 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer223 ret <8 x bfloat> %c224}225 226define <8 x bfloat> @fma_maskz_321_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {227; CHECK-LABEL: fma_maskz_321_v8bf16:228; CHECK: # %bb.0:229; CHECK-NEXT: kmovd %edi, %k1230; CHECK-NEXT: vfmadd231bf16 %xmm1, %xmm2, %xmm0 {%k1} {z}231; CHECK-NEXT: retq232 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %y, <8 x bfloat> %x)233 %b = bitcast i8 %mask to <8 x i1>234 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer235 ret <8 x bfloat> %c236}237 238define <8 x bfloat> @fma_maskz_132_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {239; CHECK-LABEL: fma_maskz_132_v8bf16:240; CHECK: # %bb.0:241; CHECK-NEXT: kmovd %edi, %k1242; CHECK-NEXT: vfmadd213bf16 %xmm1, %xmm2, %xmm0 {%k1} {z}243; CHECK-NEXT: retq244 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %z, <8 x bfloat> %y)245 %b = bitcast i8 %mask to <8 x i1>246 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer247 ret <8 x bfloat> %c248}249 250define <8 x bfloat> @fma_maskz_312_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z, i8 %mask) {251; CHECK-LABEL: fma_maskz_312_v8bf16:252; CHECK: # %bb.0:253; CHECK-NEXT: kmovd %edi, %k1254; CHECK-NEXT: vfmadd213bf16 %xmm1, %xmm2, %xmm0 {%k1} {z}255; CHECK-NEXT: retq256 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %x, <8 x bfloat> %y)257 %b = bitcast i8 %mask to <8 x i1>258 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer259 ret <8 x bfloat> %c260}261 262define <8 x bfloat> @fma_mask_load_123_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {263; CHECK-LABEL: fma_mask_load_123_v8bf16:264; CHECK: # %bb.0:265; CHECK-NEXT: kmovd %esi, %k1266; CHECK-NEXT: vfmadd213bf16 (%rdi), %xmm1, %xmm0 {%k1}267; CHECK-NEXT: retq268 %z = load <8 x bfloat>, ptr %zp269 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z)270 %b = bitcast i8 %mask to <8 x i1>271 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x272 ret <8 x bfloat> %c273}274 275define <8 x bfloat> @fma_mask_load_213_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {276; CHECK-LABEL: fma_mask_load_213_v8bf16:277; CHECK: # %bb.0:278; CHECK-NEXT: kmovd %esi, %k1279; CHECK-NEXT: vfmadd213bf16 (%rdi), %xmm1, %xmm0 {%k1}280; CHECK-NEXT: retq281 %z = load <8 x bfloat>, ptr %zp282 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %x, <8 x bfloat> %z)283 %b = bitcast i8 %mask to <8 x i1>284 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x285 ret <8 x bfloat> %c286}287 288define <8 x bfloat> @fma_mask_load_231_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {289; CHECK-LABEL: fma_mask_load_231_v8bf16:290; CHECK: # %bb.0:291; CHECK-NEXT: kmovd %esi, %k1292; CHECK-NEXT: vfmadd231bf16 (%rdi), %xmm1, %xmm0 {%k1}293; CHECK-NEXT: retq294 %z = load <8 x bfloat>, ptr %zp295 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %z, <8 x bfloat> %x)296 %b = bitcast i8 %mask to <8 x i1>297 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x298 ret <8 x bfloat> %c299}300 301define <8 x bfloat> @fma_mask_load_321_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {302; CHECK-LABEL: fma_mask_load_321_v8bf16:303; CHECK: # %bb.0:304; CHECK-NEXT: kmovd %esi, %k1305; CHECK-NEXT: vfmadd231bf16 (%rdi), %xmm1, %xmm0 {%k1}306; CHECK-NEXT: retq307 %z = load <8 x bfloat>, ptr %zp308 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %y, <8 x bfloat> %x)309 %b = bitcast i8 %mask to <8 x i1>310 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x311 ret <8 x bfloat> %c312}313 314define <8 x bfloat> @fma_mask_load_132_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {315; CHECK-LABEL: fma_mask_load_132_v8bf16:316; CHECK: # %bb.0:317; CHECK-NEXT: kmovd %esi, %k1318; CHECK-NEXT: vfmadd132bf16 (%rdi), %xmm1, %xmm0 {%k1}319; CHECK-NEXT: retq320 %z = load <8 x bfloat>, ptr %zp321 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %z, <8 x bfloat> %y)322 %b = bitcast i8 %mask to <8 x i1>323 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x324 ret <8 x bfloat> %c325}326 327define <8 x bfloat> @fma_mask_load_312_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {328; CHECK-LABEL: fma_mask_load_312_v8bf16:329; CHECK: # %bb.0:330; CHECK-NEXT: kmovd %esi, %k1331; CHECK-NEXT: vfmadd132bf16 (%rdi), %xmm1, %xmm0 {%k1}332; CHECK-NEXT: retq333 %z = load <8 x bfloat>, ptr %zp334 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %x, <8 x bfloat> %y)335 %b = bitcast i8 %mask to <8 x i1>336 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> %x337 ret <8 x bfloat> %c338}339 340define <8 x bfloat> @fma_maskz_load_123_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {341; CHECK-LABEL: fma_maskz_load_123_v8bf16:342; CHECK: # %bb.0:343; CHECK-NEXT: kmovd %esi, %k1344; CHECK-NEXT: vfmadd213bf16 (%rdi), %xmm1, %xmm0 {%k1} {z}345; CHECK-NEXT: retq346 %z = load <8 x bfloat>, ptr %zp347 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, <8 x bfloat> %z)348 %b = bitcast i8 %mask to <8 x i1>349 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer350 ret <8 x bfloat> %c351}352 353define <8 x bfloat> @fma_maskz_load_213_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {354; CHECK-LABEL: fma_maskz_load_213_v8bf16:355; CHECK: # %bb.0:356; CHECK-NEXT: kmovd %esi, %k1357; CHECK-NEXT: vfmadd213bf16 (%rdi), %xmm1, %xmm0 {%k1} {z}358; CHECK-NEXT: retq359 %z = load <8 x bfloat>, ptr %zp360 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %x, <8 x bfloat> %z)361 %b = bitcast i8 %mask to <8 x i1>362 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer363 ret <8 x bfloat> %c364}365 366define <8 x bfloat> @fma_maskz_load_231_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {367; CHECK-LABEL: fma_maskz_load_231_v8bf16:368; CHECK: # %bb.0:369; CHECK-NEXT: kmovd %esi, %k1370; CHECK-NEXT: vfmadd231bf16 (%rdi), %xmm1, %xmm0 {%k1} {z}371; CHECK-NEXT: retq372 %z = load <8 x bfloat>, ptr %zp373 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %y, <8 x bfloat> %z, <8 x bfloat> %x)374 %b = bitcast i8 %mask to <8 x i1>375 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer376 ret <8 x bfloat> %c377}378 379define <8 x bfloat> @fma_maskz_load_321_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {380; CHECK-LABEL: fma_maskz_load_321_v8bf16:381; CHECK: # %bb.0:382; CHECK-NEXT: kmovd %esi, %k1383; CHECK-NEXT: vfmadd231bf16 (%rdi), %xmm1, %xmm0 {%k1} {z}384; CHECK-NEXT: retq385 %z = load <8 x bfloat>, ptr %zp386 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %y, <8 x bfloat> %x)387 %b = bitcast i8 %mask to <8 x i1>388 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer389 ret <8 x bfloat> %c390}391 392define <8 x bfloat> @fma_maskz_load_132_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {393; CHECK-LABEL: fma_maskz_load_132_v8bf16:394; CHECK: # %bb.0:395; CHECK-NEXT: kmovd %esi, %k1396; CHECK-NEXT: vfmadd132bf16 (%rdi), %xmm1, %xmm0 {%k1} {z}397; CHECK-NEXT: retq398 %z = load <8 x bfloat>, ptr %zp399 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %x, <8 x bfloat> %z, <8 x bfloat> %y)400 %b = bitcast i8 %mask to <8 x i1>401 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer402 ret <8 x bfloat> %c403}404 405define <8 x bfloat> @fma_maskz_load_312_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y, ptr %zp, i8 %mask) {406; CHECK-LABEL: fma_maskz_load_312_v8bf16:407; CHECK: # %bb.0:408; CHECK-NEXT: kmovd %esi, %k1409; CHECK-NEXT: vfmadd132bf16 (%rdi), %xmm1, %xmm0 {%k1} {z}410; CHECK-NEXT: retq411 %z = load <8 x bfloat>, ptr %zp412 %a = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %z, <8 x bfloat> %x, <8 x bfloat> %y)413 %b = bitcast i8 %mask to <8 x i1>414 %c = select <8 x i1> %b, <8 x bfloat> %a, <8 x bfloat> zeroinitializer415 ret <8 x bfloat> %c416}417 418define <16 x bfloat> @fma_123_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z) {419; CHECK-LABEL: fma_123_v16bf16:420; CHECK: # %bb.0:421; CHECK-NEXT: vfmadd213bf16 %ymm2, %ymm1, %ymm0422; CHECK-NEXT: retq423 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z)424 ret <16 x bfloat> %a425}426 427define <16 x bfloat> @fma_213_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z) {428; CHECK-LABEL: fma_213_v16bf16:429; CHECK: # %bb.0:430; CHECK-NEXT: vfmadd213bf16 %ymm2, %ymm1, %ymm0431; CHECK-NEXT: retq432 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %x, <16 x bfloat> %z)433 ret <16 x bfloat> %a434}435 436define <16 x bfloat> @fma_231_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z) {437; CHECK-LABEL: fma_231_v16bf16:438; CHECK: # %bb.0:439; CHECK-NEXT: vfmadd231bf16 %ymm1, %ymm2, %ymm0440; CHECK-NEXT: retq441 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %z, <16 x bfloat> %x)442 ret <16 x bfloat> %a443}444 445define <16 x bfloat> @fma_321_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z) {446; CHECK-LABEL: fma_321_v16bf16:447; CHECK: # %bb.0:448; CHECK-NEXT: vfmadd231bf16 %ymm1, %ymm2, %ymm0449; CHECK-NEXT: retq450 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %y, <16 x bfloat> %x)451 ret <16 x bfloat> %a452}453 454define <16 x bfloat> @fma_132_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z) {455; CHECK-LABEL: fma_132_v16bf16:456; CHECK: # %bb.0:457; CHECK-NEXT: vfmadd213bf16 %ymm1, %ymm2, %ymm0458; CHECK-NEXT: retq459 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %z, <16 x bfloat> %y)460 ret <16 x bfloat> %a461}462 463define <16 x bfloat> @fma_312_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z) {464; CHECK-LABEL: fma_312_v16bf16:465; CHECK: # %bb.0:466; CHECK-NEXT: vfmadd213bf16 %ymm1, %ymm2, %ymm0467; CHECK-NEXT: retq468 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %x, <16 x bfloat> %y)469 ret <16 x bfloat> %a470}471 472define <16 x bfloat> @fma_load_123_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp) {473; CHECK-LABEL: fma_load_123_v16bf16:474; CHECK: # %bb.0:475; CHECK-NEXT: vfmadd213bf16 (%rdi), %ymm1, %ymm0476; CHECK-NEXT: retq477 %z = load <16 x bfloat>, ptr %zp478 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z)479 ret <16 x bfloat> %a480}481 482define <16 x bfloat> @fma_load_213_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp) {483; CHECK-LABEL: fma_load_213_v16bf16:484; CHECK: # %bb.0:485; CHECK-NEXT: vfmadd213bf16 (%rdi), %ymm1, %ymm0486; CHECK-NEXT: retq487 %z = load <16 x bfloat>, ptr %zp488 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %x, <16 x bfloat> %z)489 ret <16 x bfloat> %a490}491 492define <16 x bfloat> @fma_load_231_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp) {493; CHECK-LABEL: fma_load_231_v16bf16:494; CHECK: # %bb.0:495; CHECK-NEXT: vfmadd231bf16 (%rdi), %ymm1, %ymm0496; CHECK-NEXT: retq497 %z = load <16 x bfloat>, ptr %zp498 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %z, <16 x bfloat> %x)499 ret <16 x bfloat> %a500}501 502define <16 x bfloat> @fma_load_321_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp) {503; CHECK-LABEL: fma_load_321_v16bf16:504; CHECK: # %bb.0:505; CHECK-NEXT: vfmadd231bf16 (%rdi), %ymm1, %ymm0506; CHECK-NEXT: retq507 %z = load <16 x bfloat>, ptr %zp508 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %y, <16 x bfloat> %x)509 ret <16 x bfloat> %a510}511 512define <16 x bfloat> @fma_load_132_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp) {513; CHECK-LABEL: fma_load_132_v16bf16:514; CHECK: # %bb.0:515; CHECK-NEXT: vfmadd132bf16 (%rdi), %ymm1, %ymm0516; CHECK-NEXT: retq517 %z = load <16 x bfloat>, ptr %zp518 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %z, <16 x bfloat> %y)519 ret <16 x bfloat> %a520}521 522define <16 x bfloat> @fma_load_312_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp) {523; CHECK-LABEL: fma_load_312_v16bf16:524; CHECK: # %bb.0:525; CHECK-NEXT: vfmadd132bf16 (%rdi), %ymm1, %ymm0526; CHECK-NEXT: retq527 %z = load <16 x bfloat>, ptr %zp528 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %x, <16 x bfloat> %y)529 ret <16 x bfloat> %a530}531 532define <16 x bfloat> @fma_mask_123_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {533; CHECK-LABEL: fma_mask_123_v16bf16:534; CHECK: # %bb.0:535; CHECK-NEXT: kmovd %edi, %k1536; CHECK-NEXT: vfmadd132bf16 %ymm1, %ymm2, %ymm0 {%k1}537; CHECK-NEXT: retq538 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z)539 %b = bitcast i16 %mask to <16 x i1>540 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x541 ret <16 x bfloat> %c542}543 544define <16 x bfloat> @fma_mask_213_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {545; CHECK-LABEL: fma_mask_213_v16bf16:546; CHECK: # %bb.0:547; CHECK-NEXT: kmovd %edi, %k1548; CHECK-NEXT: vfmadd213bf16 %ymm2, %ymm1, %ymm0 {%k1}549; CHECK-NEXT: retq550 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %x, <16 x bfloat> %z)551 %b = bitcast i16 %mask to <16 x i1>552 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x553 ret <16 x bfloat> %c554}555 556define <16 x bfloat> @fma_mask_231_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {557; CHECK-LABEL: fma_mask_231_v16bf16:558; CHECK: # %bb.0:559; CHECK-NEXT: kmovd %edi, %k1560; CHECK-NEXT: vfmadd231bf16 %ymm2, %ymm1, %ymm0 {%k1}561; CHECK-NEXT: retq562 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %z, <16 x bfloat> %x)563 %b = bitcast i16 %mask to <16 x i1>564 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x565 ret <16 x bfloat> %c566}567 568define <16 x bfloat> @fma_mask_321_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {569; CHECK-LABEL: fma_mask_321_v16bf16:570; CHECK: # %bb.0:571; CHECK-NEXT: kmovd %edi, %k1572; CHECK-NEXT: vfmadd231bf16 %ymm1, %ymm2, %ymm0 {%k1}573; CHECK-NEXT: retq574 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %y, <16 x bfloat> %x)575 %b = bitcast i16 %mask to <16 x i1>576 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x577 ret <16 x bfloat> %c578}579 580define <16 x bfloat> @fma_mask_132_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {581; CHECK-LABEL: fma_mask_132_v16bf16:582; CHECK: # %bb.0:583; CHECK-NEXT: kmovd %edi, %k1584; CHECK-NEXT: vfmadd132bf16 %ymm2, %ymm1, %ymm0 {%k1}585; CHECK-NEXT: retq586 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %z, <16 x bfloat> %y)587 %b = bitcast i16 %mask to <16 x i1>588 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x589 ret <16 x bfloat> %c590}591 592define <16 x bfloat> @fma_mask_312_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {593; CHECK-LABEL: fma_mask_312_v16bf16:594; CHECK: # %bb.0:595; CHECK-NEXT: kmovd %edi, %k1596; CHECK-NEXT: vfmadd213bf16 %ymm1, %ymm2, %ymm0 {%k1}597; CHECK-NEXT: retq598 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %x, <16 x bfloat> %y)599 %b = bitcast i16 %mask to <16 x i1>600 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x601 ret <16 x bfloat> %c602}603 604define <16 x bfloat> @fma_maskz_123_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {605; CHECK-LABEL: fma_maskz_123_v16bf16:606; CHECK: # %bb.0:607; CHECK-NEXT: kmovd %edi, %k1608; CHECK-NEXT: vfmadd213bf16 %ymm2, %ymm1, %ymm0 {%k1} {z}609; CHECK-NEXT: retq610 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z)611 %b = bitcast i16 %mask to <16 x i1>612 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer613 ret <16 x bfloat> %c614}615 616define <16 x bfloat> @fma_maskz_213_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {617; CHECK-LABEL: fma_maskz_213_v16bf16:618; CHECK: # %bb.0:619; CHECK-NEXT: kmovd %edi, %k1620; CHECK-NEXT: vfmadd213bf16 %ymm2, %ymm1, %ymm0 {%k1} {z}621; CHECK-NEXT: retq622 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %x, <16 x bfloat> %z)623 %b = bitcast i16 %mask to <16 x i1>624 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer625 ret <16 x bfloat> %c626}627 628define <16 x bfloat> @fma_maskz_231_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {629; CHECK-LABEL: fma_maskz_231_v16bf16:630; CHECK: # %bb.0:631; CHECK-NEXT: kmovd %edi, %k1632; CHECK-NEXT: vfmadd231bf16 %ymm1, %ymm2, %ymm0 {%k1} {z}633; CHECK-NEXT: retq634 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %z, <16 x bfloat> %x)635 %b = bitcast i16 %mask to <16 x i1>636 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer637 ret <16 x bfloat> %c638}639 640define <16 x bfloat> @fma_maskz_321_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {641; CHECK-LABEL: fma_maskz_321_v16bf16:642; CHECK: # %bb.0:643; CHECK-NEXT: kmovd %edi, %k1644; CHECK-NEXT: vfmadd231bf16 %ymm1, %ymm2, %ymm0 {%k1} {z}645; CHECK-NEXT: retq646 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %y, <16 x bfloat> %x)647 %b = bitcast i16 %mask to <16 x i1>648 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer649 ret <16 x bfloat> %c650}651 652define <16 x bfloat> @fma_maskz_132_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {653; CHECK-LABEL: fma_maskz_132_v16bf16:654; CHECK: # %bb.0:655; CHECK-NEXT: kmovd %edi, %k1656; CHECK-NEXT: vfmadd213bf16 %ymm1, %ymm2, %ymm0 {%k1} {z}657; CHECK-NEXT: retq658 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %z, <16 x bfloat> %y)659 %b = bitcast i16 %mask to <16 x i1>660 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer661 ret <16 x bfloat> %c662}663 664define <16 x bfloat> @fma_maskz_312_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z, i16 %mask) {665; CHECK-LABEL: fma_maskz_312_v16bf16:666; CHECK: # %bb.0:667; CHECK-NEXT: kmovd %edi, %k1668; CHECK-NEXT: vfmadd213bf16 %ymm1, %ymm2, %ymm0 {%k1} {z}669; CHECK-NEXT: retq670 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %x, <16 x bfloat> %y)671 %b = bitcast i16 %mask to <16 x i1>672 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer673 ret <16 x bfloat> %c674}675 676define <16 x bfloat> @fma_mask_load_123_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {677; CHECK-LABEL: fma_mask_load_123_v16bf16:678; CHECK: # %bb.0:679; CHECK-NEXT: kmovd %esi, %k1680; CHECK-NEXT: vfmadd213bf16 (%rdi), %ymm1, %ymm0 {%k1}681; CHECK-NEXT: retq682 %z = load <16 x bfloat>, ptr %zp683 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z)684 %b = bitcast i16 %mask to <16 x i1>685 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x686 ret <16 x bfloat> %c687}688 689define <16 x bfloat> @fma_mask_load_213_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {690; CHECK-LABEL: fma_mask_load_213_v16bf16:691; CHECK: # %bb.0:692; CHECK-NEXT: kmovd %esi, %k1693; CHECK-NEXT: vfmadd213bf16 (%rdi), %ymm1, %ymm0 {%k1}694; CHECK-NEXT: retq695 %z = load <16 x bfloat>, ptr %zp696 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %x, <16 x bfloat> %z)697 %b = bitcast i16 %mask to <16 x i1>698 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x699 ret <16 x bfloat> %c700}701 702define <16 x bfloat> @fma_mask_load_231_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {703; CHECK-LABEL: fma_mask_load_231_v16bf16:704; CHECK: # %bb.0:705; CHECK-NEXT: kmovd %esi, %k1706; CHECK-NEXT: vfmadd231bf16 (%rdi), %ymm1, %ymm0 {%k1}707; CHECK-NEXT: retq708 %z = load <16 x bfloat>, ptr %zp709 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %z, <16 x bfloat> %x)710 %b = bitcast i16 %mask to <16 x i1>711 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x712 ret <16 x bfloat> %c713}714 715define <16 x bfloat> @fma_mask_load_321_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {716; CHECK-LABEL: fma_mask_load_321_v16bf16:717; CHECK: # %bb.0:718; CHECK-NEXT: kmovd %esi, %k1719; CHECK-NEXT: vfmadd231bf16 (%rdi), %ymm1, %ymm0 {%k1}720; CHECK-NEXT: retq721 %z = load <16 x bfloat>, ptr %zp722 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %y, <16 x bfloat> %x)723 %b = bitcast i16 %mask to <16 x i1>724 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x725 ret <16 x bfloat> %c726}727 728define <16 x bfloat> @fma_mask_load_132_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {729; CHECK-LABEL: fma_mask_load_132_v16bf16:730; CHECK: # %bb.0:731; CHECK-NEXT: kmovd %esi, %k1732; CHECK-NEXT: vfmadd132bf16 (%rdi), %ymm1, %ymm0 {%k1}733; CHECK-NEXT: retq734 %z = load <16 x bfloat>, ptr %zp735 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %z, <16 x bfloat> %y)736 %b = bitcast i16 %mask to <16 x i1>737 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x738 ret <16 x bfloat> %c739}740 741define <16 x bfloat> @fma_mask_load_312_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {742; CHECK-LABEL: fma_mask_load_312_v16bf16:743; CHECK: # %bb.0:744; CHECK-NEXT: kmovd %esi, %k1745; CHECK-NEXT: vfmadd132bf16 (%rdi), %ymm1, %ymm0 {%k1}746; CHECK-NEXT: retq747 %z = load <16 x bfloat>, ptr %zp748 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %x, <16 x bfloat> %y)749 %b = bitcast i16 %mask to <16 x i1>750 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> %x751 ret <16 x bfloat> %c752}753 754define <16 x bfloat> @fma_maskz_load_123_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {755; CHECK-LABEL: fma_maskz_load_123_v16bf16:756; CHECK: # %bb.0:757; CHECK-NEXT: kmovd %esi, %k1758; CHECK-NEXT: vfmadd213bf16 (%rdi), %ymm1, %ymm0 {%k1} {z}759; CHECK-NEXT: retq760 %z = load <16 x bfloat>, ptr %zp761 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, <16 x bfloat> %z)762 %b = bitcast i16 %mask to <16 x i1>763 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer764 ret <16 x bfloat> %c765}766 767define <16 x bfloat> @fma_maskz_load_213_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {768; CHECK-LABEL: fma_maskz_load_213_v16bf16:769; CHECK: # %bb.0:770; CHECK-NEXT: kmovd %esi, %k1771; CHECK-NEXT: vfmadd213bf16 (%rdi), %ymm1, %ymm0 {%k1} {z}772; CHECK-NEXT: retq773 %z = load <16 x bfloat>, ptr %zp774 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %x, <16 x bfloat> %z)775 %b = bitcast i16 %mask to <16 x i1>776 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer777 ret <16 x bfloat> %c778}779 780define <16 x bfloat> @fma_maskz_load_231_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {781; CHECK-LABEL: fma_maskz_load_231_v16bf16:782; CHECK: # %bb.0:783; CHECK-NEXT: kmovd %esi, %k1784; CHECK-NEXT: vfmadd231bf16 (%rdi), %ymm1, %ymm0 {%k1} {z}785; CHECK-NEXT: retq786 %z = load <16 x bfloat>, ptr %zp787 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %y, <16 x bfloat> %z, <16 x bfloat> %x)788 %b = bitcast i16 %mask to <16 x i1>789 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer790 ret <16 x bfloat> %c791}792 793define <16 x bfloat> @fma_maskz_load_321_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {794; CHECK-LABEL: fma_maskz_load_321_v16bf16:795; CHECK: # %bb.0:796; CHECK-NEXT: kmovd %esi, %k1797; CHECK-NEXT: vfmadd231bf16 (%rdi), %ymm1, %ymm0 {%k1} {z}798; CHECK-NEXT: retq799 %z = load <16 x bfloat>, ptr %zp800 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %y, <16 x bfloat> %x)801 %b = bitcast i16 %mask to <16 x i1>802 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer803 ret <16 x bfloat> %c804}805 806define <16 x bfloat> @fma_maskz_load_132_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {807; CHECK-LABEL: fma_maskz_load_132_v16bf16:808; CHECK: # %bb.0:809; CHECK-NEXT: kmovd %esi, %k1810; CHECK-NEXT: vfmadd132bf16 (%rdi), %ymm1, %ymm0 {%k1} {z}811; CHECK-NEXT: retq812 %z = load <16 x bfloat>, ptr %zp813 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %x, <16 x bfloat> %z, <16 x bfloat> %y)814 %b = bitcast i16 %mask to <16 x i1>815 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer816 ret <16 x bfloat> %c817}818 819define <16 x bfloat> @fma_maskz_load_312_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y, ptr %zp, i16 %mask) {820; CHECK-LABEL: fma_maskz_load_312_v16bf16:821; CHECK: # %bb.0:822; CHECK-NEXT: kmovd %esi, %k1823; CHECK-NEXT: vfmadd132bf16 (%rdi), %ymm1, %ymm0 {%k1} {z}824; CHECK-NEXT: retq825 %z = load <16 x bfloat>, ptr %zp826 %a = call <16 x bfloat> @llvm.fma.v16bf16(<16 x bfloat> %z, <16 x bfloat> %x, <16 x bfloat> %y)827 %b = bitcast i16 %mask to <16 x i1>828 %c = select <16 x i1> %b, <16 x bfloat> %a, <16 x bfloat> zeroinitializer829 ret <16 x bfloat> %c830}831 832define <32 x bfloat> @fma_123_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z) {833; CHECK-LABEL: fma_123_v32bf16:834; CHECK: # %bb.0:835; CHECK-NEXT: vfmadd213bf16 %zmm2, %zmm1, %zmm0836; CHECK-NEXT: retq837 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z)838 ret <32 x bfloat> %a839}840 841define <32 x bfloat> @fma_213_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z) {842; CHECK-LABEL: fma_213_v32bf16:843; CHECK: # %bb.0:844; CHECK-NEXT: vfmadd213bf16 %zmm2, %zmm1, %zmm0845; CHECK-NEXT: retq846 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %x, <32 x bfloat> %z)847 ret <32 x bfloat> %a848}849 850define <32 x bfloat> @fma_231_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z) {851; CHECK-LABEL: fma_231_v32bf16:852; CHECK: # %bb.0:853; CHECK-NEXT: vfmadd231bf16 %zmm1, %zmm2, %zmm0854; CHECK-NEXT: retq855 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %z, <32 x bfloat> %x)856 ret <32 x bfloat> %a857}858 859define <32 x bfloat> @fma_321_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z) {860; CHECK-LABEL: fma_321_v32bf16:861; CHECK: # %bb.0:862; CHECK-NEXT: vfmadd231bf16 %zmm1, %zmm2, %zmm0863; CHECK-NEXT: retq864 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %y, <32 x bfloat> %x)865 ret <32 x bfloat> %a866}867 868define <32 x bfloat> @fma_132_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z) {869; CHECK-LABEL: fma_132_v32bf16:870; CHECK: # %bb.0:871; CHECK-NEXT: vfmadd213bf16 %zmm1, %zmm2, %zmm0872; CHECK-NEXT: retq873 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %z, <32 x bfloat> %y)874 ret <32 x bfloat> %a875}876 877define <32 x bfloat> @fma_312_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z) {878; CHECK-LABEL: fma_312_v32bf16:879; CHECK: # %bb.0:880; CHECK-NEXT: vfmadd213bf16 %zmm1, %zmm2, %zmm0881; CHECK-NEXT: retq882 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %x, <32 x bfloat> %y)883 ret <32 x bfloat> %a884}885 886define <32 x bfloat> @fma_load_123_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp) {887; CHECK-LABEL: fma_load_123_v32bf16:888; CHECK: # %bb.0:889; CHECK-NEXT: vfmadd213bf16 (%rdi), %zmm1, %zmm0890; CHECK-NEXT: retq891 %z = load <32 x bfloat>, ptr %zp892 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z)893 ret <32 x bfloat> %a894}895 896define <32 x bfloat> @fma_load_213_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp) {897; CHECK-LABEL: fma_load_213_v32bf16:898; CHECK: # %bb.0:899; CHECK-NEXT: vfmadd213bf16 (%rdi), %zmm1, %zmm0900; CHECK-NEXT: retq901 %z = load <32 x bfloat>, ptr %zp902 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %x, <32 x bfloat> %z)903 ret <32 x bfloat> %a904}905 906define <32 x bfloat> @fma_load_231_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp) {907; CHECK-LABEL: fma_load_231_v32bf16:908; CHECK: # %bb.0:909; CHECK-NEXT: vfmadd231bf16 (%rdi), %zmm1, %zmm0910; CHECK-NEXT: retq911 %z = load <32 x bfloat>, ptr %zp912 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %z, <32 x bfloat> %x)913 ret <32 x bfloat> %a914}915 916define <32 x bfloat> @fma_load_321_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp) {917; CHECK-LABEL: fma_load_321_v32bf16:918; CHECK: # %bb.0:919; CHECK-NEXT: vfmadd231bf16 (%rdi), %zmm1, %zmm0920; CHECK-NEXT: retq921 %z = load <32 x bfloat>, ptr %zp922 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %y, <32 x bfloat> %x)923 ret <32 x bfloat> %a924}925 926define <32 x bfloat> @fma_load_132_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp) {927; CHECK-LABEL: fma_load_132_v32bf16:928; CHECK: # %bb.0:929; CHECK-NEXT: vfmadd132bf16 (%rdi), %zmm1, %zmm0930; CHECK-NEXT: retq931 %z = load <32 x bfloat>, ptr %zp932 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %z, <32 x bfloat> %y)933 ret <32 x bfloat> %a934}935 936define <32 x bfloat> @fma_load_312_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp) {937; CHECK-LABEL: fma_load_312_v32bf16:938; CHECK: # %bb.0:939; CHECK-NEXT: vfmadd132bf16 (%rdi), %zmm1, %zmm0940; CHECK-NEXT: retq941 %z = load <32 x bfloat>, ptr %zp942 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %x, <32 x bfloat> %y)943 ret <32 x bfloat> %a944}945 946define <32 x bfloat> @fma_mask_123_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {947; CHECK-LABEL: fma_mask_123_v32bf16:948; CHECK: # %bb.0:949; CHECK-NEXT: kmovd %edi, %k1950; CHECK-NEXT: vfmadd132bf16 %zmm1, %zmm2, %zmm0 {%k1}951; CHECK-NEXT: retq952 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z)953 %b = bitcast i32 %mask to <32 x i1>954 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x955 ret <32 x bfloat> %c956}957 958define <32 x bfloat> @fma_mask_213_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {959; CHECK-LABEL: fma_mask_213_v32bf16:960; CHECK: # %bb.0:961; CHECK-NEXT: kmovd %edi, %k1962; CHECK-NEXT: vfmadd213bf16 %zmm2, %zmm1, %zmm0 {%k1}963; CHECK-NEXT: retq964 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %x, <32 x bfloat> %z)965 %b = bitcast i32 %mask to <32 x i1>966 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x967 ret <32 x bfloat> %c968}969 970define <32 x bfloat> @fma_mask_231_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {971; CHECK-LABEL: fma_mask_231_v32bf16:972; CHECK: # %bb.0:973; CHECK-NEXT: kmovd %edi, %k1974; CHECK-NEXT: vfmadd231bf16 %zmm2, %zmm1, %zmm0 {%k1}975; CHECK-NEXT: retq976 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %z, <32 x bfloat> %x)977 %b = bitcast i32 %mask to <32 x i1>978 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x979 ret <32 x bfloat> %c980}981 982define <32 x bfloat> @fma_mask_321_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {983; CHECK-LABEL: fma_mask_321_v32bf16:984; CHECK: # %bb.0:985; CHECK-NEXT: kmovd %edi, %k1986; CHECK-NEXT: vfmadd231bf16 %zmm1, %zmm2, %zmm0 {%k1}987; CHECK-NEXT: retq988 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %y, <32 x bfloat> %x)989 %b = bitcast i32 %mask to <32 x i1>990 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x991 ret <32 x bfloat> %c992}993 994define <32 x bfloat> @fma_mask_132_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {995; CHECK-LABEL: fma_mask_132_v32bf16:996; CHECK: # %bb.0:997; CHECK-NEXT: kmovd %edi, %k1998; CHECK-NEXT: vfmadd132bf16 %zmm2, %zmm1, %zmm0 {%k1}999; CHECK-NEXT: retq1000 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %z, <32 x bfloat> %y)1001 %b = bitcast i32 %mask to <32 x i1>1002 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x1003 ret <32 x bfloat> %c1004}1005 1006define <32 x bfloat> @fma_mask_312_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {1007; CHECK-LABEL: fma_mask_312_v32bf16:1008; CHECK: # %bb.0:1009; CHECK-NEXT: kmovd %edi, %k11010; CHECK-NEXT: vfmadd213bf16 %zmm1, %zmm2, %zmm0 {%k1}1011; CHECK-NEXT: retq1012 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %x, <32 x bfloat> %y)1013 %b = bitcast i32 %mask to <32 x i1>1014 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x1015 ret <32 x bfloat> %c1016}1017 1018define <32 x bfloat> @fma_maskz_123_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {1019; CHECK-LABEL: fma_maskz_123_v32bf16:1020; CHECK: # %bb.0:1021; CHECK-NEXT: kmovd %edi, %k11022; CHECK-NEXT: vfmadd213bf16 %zmm2, %zmm1, %zmm0 {%k1} {z}1023; CHECK-NEXT: retq1024 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z)1025 %b = bitcast i32 %mask to <32 x i1>1026 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer1027 ret <32 x bfloat> %c1028}1029 1030define <32 x bfloat> @fma_maskz_213_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {1031; CHECK-LABEL: fma_maskz_213_v32bf16:1032; CHECK: # %bb.0:1033; CHECK-NEXT: kmovd %edi, %k11034; CHECK-NEXT: vfmadd213bf16 %zmm2, %zmm1, %zmm0 {%k1} {z}1035; CHECK-NEXT: retq1036 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %x, <32 x bfloat> %z)1037 %b = bitcast i32 %mask to <32 x i1>1038 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer1039 ret <32 x bfloat> %c1040}1041 1042define <32 x bfloat> @fma_maskz_231_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {1043; CHECK-LABEL: fma_maskz_231_v32bf16:1044; CHECK: # %bb.0:1045; CHECK-NEXT: kmovd %edi, %k11046; CHECK-NEXT: vfmadd231bf16 %zmm1, %zmm2, %zmm0 {%k1} {z}1047; CHECK-NEXT: retq1048 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %z, <32 x bfloat> %x)1049 %b = bitcast i32 %mask to <32 x i1>1050 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer1051 ret <32 x bfloat> %c1052}1053 1054define <32 x bfloat> @fma_maskz_321_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {1055; CHECK-LABEL: fma_maskz_321_v32bf16:1056; CHECK: # %bb.0:1057; CHECK-NEXT: kmovd %edi, %k11058; CHECK-NEXT: vfmadd231bf16 %zmm1, %zmm2, %zmm0 {%k1} {z}1059; CHECK-NEXT: retq1060 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %y, <32 x bfloat> %x)1061 %b = bitcast i32 %mask to <32 x i1>1062 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer1063 ret <32 x bfloat> %c1064}1065 1066define <32 x bfloat> @fma_maskz_132_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {1067; CHECK-LABEL: fma_maskz_132_v32bf16:1068; CHECK: # %bb.0:1069; CHECK-NEXT: kmovd %edi, %k11070; CHECK-NEXT: vfmadd213bf16 %zmm1, %zmm2, %zmm0 {%k1} {z}1071; CHECK-NEXT: retq1072 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %z, <32 x bfloat> %y)1073 %b = bitcast i32 %mask to <32 x i1>1074 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer1075 ret <32 x bfloat> %c1076}1077 1078define <32 x bfloat> @fma_maskz_312_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z, i32 %mask) {1079; CHECK-LABEL: fma_maskz_312_v32bf16:1080; CHECK: # %bb.0:1081; CHECK-NEXT: kmovd %edi, %k11082; CHECK-NEXT: vfmadd213bf16 %zmm1, %zmm2, %zmm0 {%k1} {z}1083; CHECK-NEXT: retq1084 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %x, <32 x bfloat> %y)1085 %b = bitcast i32 %mask to <32 x i1>1086 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer1087 ret <32 x bfloat> %c1088}1089 1090define <32 x bfloat> @fma_mask_load_123_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {1091; CHECK-LABEL: fma_mask_load_123_v32bf16:1092; CHECK: # %bb.0:1093; CHECK-NEXT: kmovd %esi, %k11094; CHECK-NEXT: vfmadd213bf16 (%rdi), %zmm1, %zmm0 {%k1}1095; CHECK-NEXT: retq1096 %z = load <32 x bfloat>, ptr %zp1097 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z)1098 %b = bitcast i32 %mask to <32 x i1>1099 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x1100 ret <32 x bfloat> %c1101}1102 1103define <32 x bfloat> @fma_mask_load_213_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {1104; CHECK-LABEL: fma_mask_load_213_v32bf16:1105; CHECK: # %bb.0:1106; CHECK-NEXT: kmovd %esi, %k11107; CHECK-NEXT: vfmadd213bf16 (%rdi), %zmm1, %zmm0 {%k1}1108; CHECK-NEXT: retq1109 %z = load <32 x bfloat>, ptr %zp1110 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %x, <32 x bfloat> %z)1111 %b = bitcast i32 %mask to <32 x i1>1112 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x1113 ret <32 x bfloat> %c1114}1115 1116define <32 x bfloat> @fma_mask_load_231_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {1117; CHECK-LABEL: fma_mask_load_231_v32bf16:1118; CHECK: # %bb.0:1119; CHECK-NEXT: kmovd %esi, %k11120; CHECK-NEXT: vfmadd231bf16 (%rdi), %zmm1, %zmm0 {%k1}1121; CHECK-NEXT: retq1122 %z = load <32 x bfloat>, ptr %zp1123 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %z, <32 x bfloat> %x)1124 %b = bitcast i32 %mask to <32 x i1>1125 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x1126 ret <32 x bfloat> %c1127}1128 1129define <32 x bfloat> @fma_mask_load_321_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {1130; CHECK-LABEL: fma_mask_load_321_v32bf16:1131; CHECK: # %bb.0:1132; CHECK-NEXT: kmovd %esi, %k11133; CHECK-NEXT: vfmadd231bf16 (%rdi), %zmm1, %zmm0 {%k1}1134; CHECK-NEXT: retq1135 %z = load <32 x bfloat>, ptr %zp1136 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %y, <32 x bfloat> %x)1137 %b = bitcast i32 %mask to <32 x i1>1138 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x1139 ret <32 x bfloat> %c1140}1141 1142define <32 x bfloat> @fma_mask_load_132_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {1143; CHECK-LABEL: fma_mask_load_132_v32bf16:1144; CHECK: # %bb.0:1145; CHECK-NEXT: kmovd %esi, %k11146; CHECK-NEXT: vfmadd132bf16 (%rdi), %zmm1, %zmm0 {%k1}1147; CHECK-NEXT: retq1148 %z = load <32 x bfloat>, ptr %zp1149 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %z, <32 x bfloat> %y)1150 %b = bitcast i32 %mask to <32 x i1>1151 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x1152 ret <32 x bfloat> %c1153}1154 1155define <32 x bfloat> @fma_mask_load_312_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {1156; CHECK-LABEL: fma_mask_load_312_v32bf16:1157; CHECK: # %bb.0:1158; CHECK-NEXT: kmovd %esi, %k11159; CHECK-NEXT: vfmadd132bf16 (%rdi), %zmm1, %zmm0 {%k1}1160; CHECK-NEXT: retq1161 %z = load <32 x bfloat>, ptr %zp1162 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %x, <32 x bfloat> %y)1163 %b = bitcast i32 %mask to <32 x i1>1164 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> %x1165 ret <32 x bfloat> %c1166}1167 1168define <32 x bfloat> @fma_maskz_load_123_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {1169; CHECK-LABEL: fma_maskz_load_123_v32bf16:1170; CHECK: # %bb.0:1171; CHECK-NEXT: kmovd %esi, %k11172; CHECK-NEXT: vfmadd213bf16 (%rdi), %zmm1, %zmm0 {%k1} {z}1173; CHECK-NEXT: retq1174 %z = load <32 x bfloat>, ptr %zp1175 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, <32 x bfloat> %z)1176 %b = bitcast i32 %mask to <32 x i1>1177 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer1178 ret <32 x bfloat> %c1179}1180 1181define <32 x bfloat> @fma_maskz_load_213_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {1182; CHECK-LABEL: fma_maskz_load_213_v32bf16:1183; CHECK: # %bb.0:1184; CHECK-NEXT: kmovd %esi, %k11185; CHECK-NEXT: vfmadd213bf16 (%rdi), %zmm1, %zmm0 {%k1} {z}1186; CHECK-NEXT: retq1187 %z = load <32 x bfloat>, ptr %zp1188 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %x, <32 x bfloat> %z)1189 %b = bitcast i32 %mask to <32 x i1>1190 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer1191 ret <32 x bfloat> %c1192}1193 1194define <32 x bfloat> @fma_maskz_load_231_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {1195; CHECK-LABEL: fma_maskz_load_231_v32bf16:1196; CHECK: # %bb.0:1197; CHECK-NEXT: kmovd %esi, %k11198; CHECK-NEXT: vfmadd231bf16 (%rdi), %zmm1, %zmm0 {%k1} {z}1199; CHECK-NEXT: retq1200 %z = load <32 x bfloat>, ptr %zp1201 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %y, <32 x bfloat> %z, <32 x bfloat> %x)1202 %b = bitcast i32 %mask to <32 x i1>1203 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer1204 ret <32 x bfloat> %c1205}1206 1207define <32 x bfloat> @fma_maskz_load_321_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {1208; CHECK-LABEL: fma_maskz_load_321_v32bf16:1209; CHECK: # %bb.0:1210; CHECK-NEXT: kmovd %esi, %k11211; CHECK-NEXT: vfmadd231bf16 (%rdi), %zmm1, %zmm0 {%k1} {z}1212; CHECK-NEXT: retq1213 %z = load <32 x bfloat>, ptr %zp1214 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %y, <32 x bfloat> %x)1215 %b = bitcast i32 %mask to <32 x i1>1216 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer1217 ret <32 x bfloat> %c1218}1219 1220define <32 x bfloat> @fma_maskz_load_132_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {1221; CHECK-LABEL: fma_maskz_load_132_v32bf16:1222; CHECK: # %bb.0:1223; CHECK-NEXT: kmovd %esi, %k11224; CHECK-NEXT: vfmadd132bf16 (%rdi), %zmm1, %zmm0 {%k1} {z}1225; CHECK-NEXT: retq1226 %z = load <32 x bfloat>, ptr %zp1227 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %x, <32 x bfloat> %z, <32 x bfloat> %y)1228 %b = bitcast i32 %mask to <32 x i1>1229 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer1230 ret <32 x bfloat> %c1231}1232 1233define <32 x bfloat> @fma_maskz_load_312_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y, ptr %zp, i32 %mask) {1234; CHECK-LABEL: fma_maskz_load_312_v32bf16:1235; CHECK: # %bb.0:1236; CHECK-NEXT: kmovd %esi, %k11237; CHECK-NEXT: vfmadd132bf16 (%rdi), %zmm1, %zmm0 {%k1} {z}1238; CHECK-NEXT: retq1239 %z = load <32 x bfloat>, ptr %zp1240 %a = call <32 x bfloat> @llvm.fma.v32bf16(<32 x bfloat> %z, <32 x bfloat> %x, <32 x bfloat> %y)1241 %b = bitcast i32 %mask to <32 x i1>1242 %c = select <32 x i1> %b, <32 x bfloat> %a, <32 x bfloat> zeroinitializer1243 ret <32 x bfloat> %c1244}1245