711 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=SVE3; RUN: llc -mattr=+sve,+bf16,+sve-b16b16 < %s | FileCheck %s --check-prefixes=SVE-B16B164 5target triple = "aarch64-unknown-linux-gnu"6 7define <vscale x 8 x bfloat> @fmla_nxv8bf16(<vscale x 8 x bfloat> %acc, <vscale x 8 x bfloat> %m1, <vscale x 8 x bfloat> %m2) {8; SVE-LABEL: fmla_nxv8bf16:9; SVE: // %bb.0:10; SVE-NEXT: uunpkhi z3.s, z0.h11; SVE-NEXT: uunpklo z0.s, z0.h12; SVE-NEXT: uunpkhi z4.s, z2.h13; SVE-NEXT: uunpkhi z5.s, z1.h14; SVE-NEXT: uunpklo z2.s, z2.h15; SVE-NEXT: uunpklo z1.s, z1.h16; SVE-NEXT: ptrue p0.s17; SVE-NEXT: lsl z3.s, z3.s, #1618; SVE-NEXT: lsl z0.s, z0.s, #1619; SVE-NEXT: bfmlalb z3.s, z5.h, z4.h20; SVE-NEXT: bfmlalb z0.s, z1.h, z2.h21; SVE-NEXT: bfcvt z1.h, p0/m, z3.s22; SVE-NEXT: bfcvt z0.h, p0/m, z0.s23; SVE-NEXT: uzp1 z0.h, z0.h, z1.h24; SVE-NEXT: ret25;26; SVE-B16B16-LABEL: fmla_nxv8bf16:27; SVE-B16B16: // %bb.0:28; SVE-B16B16-NEXT: ptrue p0.h29; SVE-B16B16-NEXT: bfmla z0.h, p0/m, z1.h, z2.h30; SVE-B16B16-NEXT: ret31 %mul = fmul contract <vscale x 8 x bfloat> %m1, %m232 %res = fadd contract <vscale x 8 x bfloat> %acc, %mul33 ret <vscale x 8 x bfloat> %res34}35 36define <vscale x 4 x bfloat> @fmla_nxv4bf16(<vscale x 4 x bfloat> %acc, <vscale x 4 x bfloat> %m1, <vscale x 4 x bfloat> %m2) {37; SVE-LABEL: fmla_nxv4bf16:38; SVE: // %bb.0:39; SVE-NEXT: lsl z0.s, z0.s, #1640; SVE-NEXT: ptrue p0.s41; SVE-NEXT: bfmlalb z0.s, z1.h, z2.h42; SVE-NEXT: bfcvt z0.h, p0/m, z0.s43; SVE-NEXT: ret44;45; SVE-B16B16-LABEL: fmla_nxv4bf16:46; SVE-B16B16: // %bb.0:47; SVE-B16B16-NEXT: ptrue p0.s48; SVE-B16B16-NEXT: bfmla z0.h, p0/m, z1.h, z2.h49; SVE-B16B16-NEXT: ret50 %mul = fmul contract <vscale x 4 x bfloat> %m1, %m251 %res = fadd contract <vscale x 4 x bfloat> %acc, %mul52 ret <vscale x 4 x bfloat> %res53}54 55define <vscale x 2 x bfloat> @fmla_nxv2bf16(<vscale x 2 x bfloat> %acc, <vscale x 2 x bfloat> %m1, <vscale x 2 x bfloat> %m2) {56; SVE-LABEL: fmla_nxv2bf16:57; SVE: // %bb.0:58; SVE-NEXT: lsl z2.s, z2.s, #1659; SVE-NEXT: lsl z1.s, z1.s, #1660; SVE-NEXT: lsl z0.s, z0.s, #1661; SVE-NEXT: ptrue p0.d62; SVE-NEXT: fmla z0.s, p0/m, z1.s, z2.s63; SVE-NEXT: bfcvt z0.h, p0/m, z0.s64; SVE-NEXT: ret65;66; SVE-B16B16-LABEL: fmla_nxv2bf16:67; SVE-B16B16: // %bb.0:68; SVE-B16B16-NEXT: ptrue p0.d69; SVE-B16B16-NEXT: bfmla z0.h, p0/m, z1.h, z2.h70; SVE-B16B16-NEXT: ret71 %mul = fmul contract <vscale x 2 x bfloat> %m1, %m272 %res = fadd contract <vscale x 2 x bfloat> %acc, %mul73 ret <vscale x 2 x bfloat> %res74}75 76define <vscale x 8 x bfloat> @fmls_nxv8bf16(<vscale x 8 x bfloat> %acc, <vscale x 8 x bfloat> %m1, <vscale x 8 x bfloat> %m2) {77; SVE-LABEL: fmls_nxv8bf16:78; SVE: // %bb.0:79; SVE-NEXT: ptrue p0.h80; SVE-NEXT: uunpkhi z3.s, z0.h81; SVE-NEXT: uunpklo z0.s, z0.h82; SVE-NEXT: uunpkhi z5.s, z2.h83; SVE-NEXT: uunpklo z2.s, z2.h84; SVE-NEXT: fneg z1.h, p0/m, z1.h85; SVE-NEXT: ptrue p0.s86; SVE-NEXT: lsl z3.s, z3.s, #1687; SVE-NEXT: lsl z0.s, z0.s, #1688; SVE-NEXT: uunpkhi z4.s, z1.h89; SVE-NEXT: uunpklo z1.s, z1.h90; SVE-NEXT: bfmlalb z3.s, z4.h, z5.h91; SVE-NEXT: bfmlalb z0.s, z1.h, z2.h92; SVE-NEXT: bfcvt z1.h, p0/m, z3.s93; SVE-NEXT: bfcvt z0.h, p0/m, z0.s94; SVE-NEXT: uzp1 z0.h, z0.h, z1.h95; SVE-NEXT: ret96;97; SVE-B16B16-LABEL: fmls_nxv8bf16:98; SVE-B16B16: // %bb.0:99; SVE-B16B16-NEXT: ptrue p0.h100; SVE-B16B16-NEXT: bfmls z0.h, p0/m, z1.h, z2.h101; SVE-B16B16-NEXT: ret102 %mul = fmul contract <vscale x 8 x bfloat> %m1, %m2103 %res = fsub contract <vscale x 8 x bfloat> %acc, %mul104 ret <vscale x 8 x bfloat> %res105}106 107define <vscale x 4 x bfloat> @fmls_nxv4bf16(<vscale x 4 x bfloat> %acc, <vscale x 4 x bfloat> %m1, <vscale x 4 x bfloat> %m2) {108; SVE-LABEL: fmls_nxv4bf16:109; SVE: // %bb.0:110; SVE-NEXT: ptrue p0.s111; SVE-NEXT: lsl z0.s, z0.s, #16112; SVE-NEXT: fneg z1.h, p0/m, z1.h113; SVE-NEXT: bfmlalb z0.s, z1.h, z2.h114; SVE-NEXT: bfcvt z0.h, p0/m, z0.s115; SVE-NEXT: ret116;117; SVE-B16B16-LABEL: fmls_nxv4bf16:118; SVE-B16B16: // %bb.0:119; SVE-B16B16-NEXT: ptrue p0.s120; SVE-B16B16-NEXT: bfmls z0.h, p0/m, z1.h, z2.h121; SVE-B16B16-NEXT: ret122 %mul = fmul contract <vscale x 4 x bfloat> %m1, %m2123 %res = fsub contract <vscale x 4 x bfloat> %acc, %mul124 ret <vscale x 4 x bfloat> %res125}126 127define <vscale x 2 x bfloat> @fmls_nxv2bf16(<vscale x 2 x bfloat> %acc, <vscale x 2 x bfloat> %m1, <vscale x 2 x bfloat> %m2) {128; SVE-LABEL: fmls_nxv2bf16:129; SVE: // %bb.0:130; SVE-NEXT: ptrue p0.d131; SVE-NEXT: lsl z2.s, z2.s, #16132; SVE-NEXT: lsl z0.s, z0.s, #16133; SVE-NEXT: fneg z1.h, p0/m, z1.h134; SVE-NEXT: lsl z1.s, z1.s, #16135; SVE-NEXT: fmla z0.s, p0/m, z1.s, z2.s136; SVE-NEXT: bfcvt z0.h, p0/m, z0.s137; SVE-NEXT: ret138;139; SVE-B16B16-LABEL: fmls_nxv2bf16:140; SVE-B16B16: // %bb.0:141; SVE-B16B16-NEXT: ptrue p0.d142; SVE-B16B16-NEXT: bfmls z0.h, p0/m, z1.h, z2.h143; SVE-B16B16-NEXT: ret144 %mul = fmul contract <vscale x 2 x bfloat> %m1, %m2145 %res = fsub contract <vscale x 2 x bfloat> %acc, %mul146 ret <vscale x 2 x bfloat> %res147}148 149define <vscale x 8 x bfloat> @fmla_sel_nxv8bf16(<vscale x 8 x i1> %pred, <vscale x 8 x bfloat> %acc, <vscale x 8 x bfloat> %m1, <vscale x 8 x bfloat> %m2) {150; SVE-LABEL: fmla_sel_nxv8bf16:151; SVE: // %bb.0:152; SVE-NEXT: uunpkhi z3.s, z0.h153; SVE-NEXT: uunpklo z4.s, z0.h154; SVE-NEXT: uunpkhi z5.s, z2.h155; SVE-NEXT: uunpkhi z6.s, z1.h156; SVE-NEXT: uunpklo z2.s, z2.h157; SVE-NEXT: uunpklo z1.s, z1.h158; SVE-NEXT: ptrue p1.s159; SVE-NEXT: lsl z3.s, z3.s, #16160; SVE-NEXT: lsl z4.s, z4.s, #16161; SVE-NEXT: bfmlalb z3.s, z6.h, z5.h162; SVE-NEXT: bfmlalb z4.s, z1.h, z2.h163; SVE-NEXT: bfcvt z1.h, p1/m, z3.s164; SVE-NEXT: bfcvt z2.h, p1/m, z4.s165; SVE-NEXT: uzp1 z1.h, z2.h, z1.h166; SVE-NEXT: mov z0.h, p0/m, z1.h167; SVE-NEXT: ret168;169; SVE-B16B16-LABEL: fmla_sel_nxv8bf16:170; SVE-B16B16: // %bb.0:171; SVE-B16B16-NEXT: bfmla z0.h, p0/m, z1.h, z2.h172; SVE-B16B16-NEXT: ret173 %mul = fmul contract <vscale x 8 x bfloat> %m1, %m2174 %add = fadd contract <vscale x 8 x bfloat> %acc, %mul175 %res = select <vscale x 8 x i1> %pred, <vscale x 8 x bfloat> %add, <vscale x 8 x bfloat> %acc176 ret <vscale x 8 x bfloat> %res177}178 179define <vscale x 4 x bfloat> @fmla_sel_nxv4bf16(<vscale x 4 x i1> %pred, <vscale x 4 x bfloat> %acc, <vscale x 4 x bfloat> %m1, <vscale x 4 x bfloat> %m2) {180; SVE-LABEL: fmla_sel_nxv4bf16:181; SVE: // %bb.0:182; SVE-NEXT: lsl z3.s, z0.s, #16183; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h184; SVE-NEXT: bfcvt z0.h, p0/m, z3.s185; SVE-NEXT: ret186;187; SVE-B16B16-LABEL: fmla_sel_nxv4bf16:188; SVE-B16B16: // %bb.0:189; SVE-B16B16-NEXT: bfmla z0.h, p0/m, z1.h, z2.h190; SVE-B16B16-NEXT: ret191 %mul = fmul contract <vscale x 4 x bfloat> %m1, %m2192 %add = fadd contract <vscale x 4 x bfloat> %acc, %mul193 %res = select <vscale x 4 x i1> %pred, <vscale x 4 x bfloat> %add, <vscale x 4 x bfloat> %acc194 ret <vscale x 4 x bfloat> %res195}196 197define <vscale x 2 x bfloat> @fmla_sel_nxv2bf16(<vscale x 2 x i1> %pred, <vscale x 2 x bfloat> %acc, <vscale x 2 x bfloat> %m1, <vscale x 2 x bfloat> %m2) {198; SVE-LABEL: fmla_sel_nxv2bf16:199; SVE: // %bb.0:200; SVE-NEXT: lsl z2.s, z2.s, #16201; SVE-NEXT: lsl z1.s, z1.s, #16202; SVE-NEXT: lsl z3.s, z0.s, #16203; SVE-NEXT: ptrue p1.d204; SVE-NEXT: fmad z1.s, p1/m, z2.s, z3.s205; SVE-NEXT: bfcvt z0.h, p0/m, z1.s206; SVE-NEXT: ret207;208; SVE-B16B16-LABEL: fmla_sel_nxv2bf16:209; SVE-B16B16: // %bb.0:210; SVE-B16B16-NEXT: bfmla z0.h, p0/m, z1.h, z2.h211; SVE-B16B16-NEXT: ret212 %mul = fmul contract <vscale x 2 x bfloat> %m1, %m2213 %add = fadd contract <vscale x 2 x bfloat> %acc, %mul214 %res = select <vscale x 2 x i1> %pred, <vscale x 2 x bfloat> %add, <vscale x 2 x bfloat> %acc215 ret <vscale x 2 x bfloat> %res216}217 218define <vscale x 8 x bfloat> @fmls_sel_nxv8bf16(<vscale x 8 x i1> %pred, <vscale x 8 x bfloat> %acc, <vscale x 8 x bfloat> %m1, <vscale x 8 x bfloat> %m2) {219; SVE-LABEL: fmls_sel_nxv8bf16:220; SVE: // %bb.0:221; SVE-NEXT: ptrue p1.h222; SVE-NEXT: uunpkhi z3.s, z0.h223; SVE-NEXT: uunpklo z4.s, z0.h224; SVE-NEXT: uunpkhi z6.s, z2.h225; SVE-NEXT: uunpklo z2.s, z2.h226; SVE-NEXT: fneg z1.h, p1/m, z1.h227; SVE-NEXT: ptrue p1.s228; SVE-NEXT: lsl z3.s, z3.s, #16229; SVE-NEXT: lsl z4.s, z4.s, #16230; SVE-NEXT: uunpkhi z5.s, z1.h231; SVE-NEXT: uunpklo z1.s, z1.h232; SVE-NEXT: bfmlalb z3.s, z5.h, z6.h233; SVE-NEXT: bfmlalb z4.s, z1.h, z2.h234; SVE-NEXT: bfcvt z1.h, p1/m, z3.s235; SVE-NEXT: bfcvt z2.h, p1/m, z4.s236; SVE-NEXT: uzp1 z1.h, z2.h, z1.h237; SVE-NEXT: mov z0.h, p0/m, z1.h238; SVE-NEXT: ret239;240; SVE-B16B16-LABEL: fmls_sel_nxv8bf16:241; SVE-B16B16: // %bb.0:242; SVE-B16B16-NEXT: bfmls z0.h, p0/m, z1.h, z2.h243; SVE-B16B16-NEXT: ret244 %mul = fmul contract <vscale x 8 x bfloat> %m1, %m2245 %sub = fsub contract <vscale x 8 x bfloat> %acc, %mul246 %res = select <vscale x 8 x i1> %pred, <vscale x 8 x bfloat> %sub, <vscale x 8 x bfloat> %acc247 ret <vscale x 8 x bfloat> %res248}249 250define <vscale x 4 x bfloat> @fmls_sel_nxv4bf16(<vscale x 4 x i1> %pred, <vscale x 4 x bfloat> %acc, <vscale x 4 x bfloat> %m1, <vscale x 4 x bfloat> %m2) {251; SVE-LABEL: fmls_sel_nxv4bf16:252; SVE: // %bb.0:253; SVE-NEXT: ptrue p1.s254; SVE-NEXT: lsl z3.s, z0.s, #16255; SVE-NEXT: fneg z1.h, p1/m, z1.h256; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h257; SVE-NEXT: bfcvt z0.h, p0/m, z3.s258; SVE-NEXT: ret259;260; SVE-B16B16-LABEL: fmls_sel_nxv4bf16:261; SVE-B16B16: // %bb.0:262; SVE-B16B16-NEXT: bfmls z0.h, p0/m, z1.h, z2.h263; SVE-B16B16-NEXT: ret264 %mul = fmul contract <vscale x 4 x bfloat> %m1, %m2265 %sub = fsub contract <vscale x 4 x bfloat> %acc, %mul266 %res = select <vscale x 4 x i1> %pred, <vscale x 4 x bfloat> %sub, <vscale x 4 x bfloat> %acc267 ret <vscale x 4 x bfloat> %res268}269 270define <vscale x 2 x bfloat> @fmls_sel_nxv2bf16(<vscale x 2 x i1> %pred, <vscale x 2 x bfloat> %acc, <vscale x 2 x bfloat> %m1, <vscale x 2 x bfloat> %m2) {271; SVE-LABEL: fmls_sel_nxv2bf16:272; SVE: // %bb.0:273; SVE-NEXT: ptrue p1.d274; SVE-NEXT: lsl z2.s, z2.s, #16275; SVE-NEXT: lsl z3.s, z0.s, #16276; SVE-NEXT: fneg z1.h, p1/m, z1.h277; SVE-NEXT: lsl z1.s, z1.s, #16278; SVE-NEXT: fmad z1.s, p1/m, z2.s, z3.s279; SVE-NEXT: bfcvt z0.h, p0/m, z1.s280; SVE-NEXT: ret281;282; SVE-B16B16-LABEL: fmls_sel_nxv2bf16:283; SVE-B16B16: // %bb.0:284; SVE-B16B16-NEXT: bfmls z0.h, p0/m, z1.h, z2.h285; SVE-B16B16-NEXT: ret286 %mul = fmul contract <vscale x 2 x bfloat> %m1, %m2287 %sub = fsub contract <vscale x 2 x bfloat> %acc, %mul288 %res = select <vscale x 2 x i1> %pred, <vscale x 2 x bfloat> %sub, <vscale x 2 x bfloat> %acc289 ret <vscale x 2 x bfloat> %res290}291 292define <vscale x 8 x bfloat> @fadd_sel_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> %mask) {293; SVE-LABEL: fadd_sel_nxv8bf16:294; SVE: // %bb.0:295; SVE-NEXT: uunpkhi z2.s, z1.h296; SVE-NEXT: uunpkhi z3.s, z0.h297; SVE-NEXT: uunpklo z1.s, z1.h298; SVE-NEXT: uunpklo z4.s, z0.h299; SVE-NEXT: ptrue p1.s300; SVE-NEXT: lsl z2.s, z2.s, #16301; SVE-NEXT: lsl z3.s, z3.s, #16302; SVE-NEXT: lsl z1.s, z1.s, #16303; SVE-NEXT: lsl z4.s, z4.s, #16304; SVE-NEXT: fadd z2.s, z3.s, z2.s305; SVE-NEXT: fadd z1.s, z4.s, z1.s306; SVE-NEXT: bfcvt z2.h, p1/m, z2.s307; SVE-NEXT: bfcvt z1.h, p1/m, z1.s308; SVE-NEXT: uzp1 z1.h, z1.h, z2.h309; SVE-NEXT: mov z0.h, p0/m, z1.h310; SVE-NEXT: ret311;312; SVE-B16B16-LABEL: fadd_sel_nxv8bf16:313; SVE-B16B16: // %bb.0:314; SVE-B16B16-NEXT: bfadd z0.h, p0/m, z0.h, z1.h315; SVE-B16B16-NEXT: ret316 %sel = select <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> zeroinitializer317 %fadd = fadd nsz <vscale x 8 x bfloat> %a, %sel318 ret <vscale x 8 x bfloat> %fadd319}320 321define <vscale x 8 x bfloat> @fsub_sel_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> %mask) {322; SVE-LABEL: fsub_sel_nxv8bf16:323; SVE: // %bb.0:324; SVE-NEXT: uunpkhi z2.s, z1.h325; SVE-NEXT: uunpkhi z3.s, z0.h326; SVE-NEXT: uunpklo z1.s, z1.h327; SVE-NEXT: uunpklo z4.s, z0.h328; SVE-NEXT: ptrue p1.s329; SVE-NEXT: lsl z2.s, z2.s, #16330; SVE-NEXT: lsl z3.s, z3.s, #16331; SVE-NEXT: lsl z1.s, z1.s, #16332; SVE-NEXT: lsl z4.s, z4.s, #16333; SVE-NEXT: fsub z2.s, z3.s, z2.s334; SVE-NEXT: fsub z1.s, z4.s, z1.s335; SVE-NEXT: bfcvt z2.h, p1/m, z2.s336; SVE-NEXT: bfcvt z1.h, p1/m, z1.s337; SVE-NEXT: uzp1 z1.h, z1.h, z2.h338; SVE-NEXT: mov z0.h, p0/m, z1.h339; SVE-NEXT: ret340;341; SVE-B16B16-LABEL: fsub_sel_nxv8bf16:342; SVE-B16B16: // %bb.0:343; SVE-B16B16-NEXT: bfsub z0.h, p0/m, z0.h, z1.h344; SVE-B16B16-NEXT: ret345 %sel = select <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> zeroinitializer346 %fsub = fsub <vscale x 8 x bfloat> %a, %sel347 ret <vscale x 8 x bfloat> %fsub348}349 350define <vscale x 8 x bfloat> @fadd_sel_negzero_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> %mask) {351; SVE-LABEL: fadd_sel_negzero_nxv8bf16:352; SVE: // %bb.0:353; SVE-NEXT: uunpkhi z2.s, z1.h354; SVE-NEXT: uunpkhi z3.s, z0.h355; SVE-NEXT: uunpklo z1.s, z1.h356; SVE-NEXT: uunpklo z4.s, z0.h357; SVE-NEXT: ptrue p1.s358; SVE-NEXT: lsl z2.s, z2.s, #16359; SVE-NEXT: lsl z3.s, z3.s, #16360; SVE-NEXT: lsl z1.s, z1.s, #16361; SVE-NEXT: lsl z4.s, z4.s, #16362; SVE-NEXT: fadd z2.s, z3.s, z2.s363; SVE-NEXT: fadd z1.s, z4.s, z1.s364; SVE-NEXT: bfcvt z2.h, p1/m, z2.s365; SVE-NEXT: bfcvt z1.h, p1/m, z1.s366; SVE-NEXT: uzp1 z1.h, z1.h, z2.h367; SVE-NEXT: mov z0.h, p0/m, z1.h368; SVE-NEXT: ret369;370; SVE-B16B16-LABEL: fadd_sel_negzero_nxv8bf16:371; SVE-B16B16: // %bb.0:372; SVE-B16B16-NEXT: bfadd z0.h, p0/m, z0.h, z1.h373; SVE-B16B16-NEXT: ret374 %nz = fneg <vscale x 8 x bfloat> zeroinitializer375 %sel = select <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %nz376 %fadd = fadd <vscale x 8 x bfloat> %a, %sel377 ret <vscale x 8 x bfloat> %fadd378}379 380define <vscale x 8 x bfloat> @fsub_sel_negzero_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> %mask) {381; SVE-LABEL: fsub_sel_negzero_nxv8bf16:382; SVE: // %bb.0:383; SVE-NEXT: uunpkhi z2.s, z1.h384; SVE-NEXT: uunpkhi z3.s, z0.h385; SVE-NEXT: uunpklo z1.s, z1.h386; SVE-NEXT: uunpklo z4.s, z0.h387; SVE-NEXT: ptrue p1.s388; SVE-NEXT: lsl z2.s, z2.s, #16389; SVE-NEXT: lsl z3.s, z3.s, #16390; SVE-NEXT: lsl z1.s, z1.s, #16391; SVE-NEXT: lsl z4.s, z4.s, #16392; SVE-NEXT: fsub z2.s, z3.s, z2.s393; SVE-NEXT: fsub z1.s, z4.s, z1.s394; SVE-NEXT: bfcvt z2.h, p1/m, z2.s395; SVE-NEXT: bfcvt z1.h, p1/m, z1.s396; SVE-NEXT: uzp1 z1.h, z1.h, z2.h397; SVE-NEXT: mov z0.h, p0/m, z1.h398; SVE-NEXT: ret399;400; SVE-B16B16-LABEL: fsub_sel_negzero_nxv8bf16:401; SVE-B16B16: // %bb.0:402; SVE-B16B16-NEXT: bfsub z0.h, p0/m, z0.h, z1.h403; SVE-B16B16-NEXT: ret404 %nz = fneg <vscale x 8 x bfloat> zeroinitializer405 %sel = select <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %nz406 %fsub = fsub nsz <vscale x 8 x bfloat> %a, %sel407 ret <vscale x 8 x bfloat> %fsub408}409 410define <vscale x 8 x bfloat> @fadd_sel_fmul_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, <vscale x 8 x i1> %mask) {411; SVE-LABEL: fadd_sel_fmul_nxv8bf16:412; SVE: // %bb.0:413; SVE-NEXT: mov z3.s, #0x80000000414; SVE-NEXT: mov z4.s, #0x80000000415; SVE-NEXT: ptrue p1.s416; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h417; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h418; SVE-NEXT: movi v2.2d, #0000000000000000419; SVE-NEXT: bfcvt z1.h, p1/m, z3.s420; SVE-NEXT: uunpkhi z3.s, z0.h421; SVE-NEXT: uunpklo z0.s, z0.h422; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s423; SVE-NEXT: lsl z3.s, z3.s, #16424; SVE-NEXT: lsl z0.s, z0.s, #16425; SVE-NEXT: sel z1.h, p0, z1.h, z2.h426; SVE-NEXT: uunpkhi z2.s, z1.h427; SVE-NEXT: uunpklo z1.s, z1.h428; SVE-NEXT: lsl z2.s, z2.s, #16429; SVE-NEXT: lsl z1.s, z1.s, #16430; SVE-NEXT: fadd z2.s, z3.s, z2.s431; SVE-NEXT: fadd z0.s, z0.s, z1.s432; SVE-NEXT: bfcvt z1.h, p1/m, z2.s433; SVE-NEXT: bfcvt z0.h, p1/m, z0.s434; SVE-NEXT: uzp1 z0.h, z0.h, z1.h435; SVE-NEXT: ret436;437; SVE-B16B16-LABEL: fadd_sel_fmul_nxv8bf16:438; SVE-B16B16: // %bb.0:439; SVE-B16B16-NEXT: movi v3.2d, #0000000000000000440; SVE-B16B16-NEXT: bfmul z1.h, z1.h, z2.h441; SVE-B16B16-NEXT: sel z1.h, p0, z1.h, z3.h442; SVE-B16B16-NEXT: bfadd z0.h, z0.h, z1.h443; SVE-B16B16-NEXT: ret444 %fmul = fmul <vscale x 8 x bfloat> %b, %c445 %sel = select <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> %fmul, <vscale x 8 x bfloat> zeroinitializer446 %fadd = fadd contract <vscale x 8 x bfloat> %a, %sel447 ret <vscale x 8 x bfloat> %fadd448}449 450define <vscale x 8 x bfloat> @fsub_sel_fmul_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, <vscale x 8 x i1> %mask) {451; SVE-LABEL: fsub_sel_fmul_nxv8bf16:452; SVE: // %bb.0:453; SVE-NEXT: mov z3.s, #0x80000000454; SVE-NEXT: mov z4.s, #0x80000000455; SVE-NEXT: ptrue p1.s456; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h457; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h458; SVE-NEXT: bfcvt z1.h, p1/m, z3.s459; SVE-NEXT: uunpkhi z3.s, z0.h460; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s461; SVE-NEXT: uunpklo z4.s, z0.h462; SVE-NEXT: lsl z3.s, z3.s, #16463; SVE-NEXT: uunpkhi z2.s, z1.h464; SVE-NEXT: uunpklo z1.s, z1.h465; SVE-NEXT: lsl z4.s, z4.s, #16466; SVE-NEXT: lsl z2.s, z2.s, #16467; SVE-NEXT: lsl z1.s, z1.s, #16468; SVE-NEXT: fsub z2.s, z3.s, z2.s469; SVE-NEXT: fsub z1.s, z4.s, z1.s470; SVE-NEXT: bfcvt z2.h, p1/m, z2.s471; SVE-NEXT: bfcvt z1.h, p1/m, z1.s472; SVE-NEXT: uzp1 z1.h, z1.h, z2.h473; SVE-NEXT: mov z0.h, p0/m, z1.h474; SVE-NEXT: ret475;476; SVE-B16B16-LABEL: fsub_sel_fmul_nxv8bf16:477; SVE-B16B16: // %bb.0:478; SVE-B16B16-NEXT: bfmls z0.h, p0/m, z1.h, z2.h479; SVE-B16B16-NEXT: ret480 %fmul = fmul <vscale x 8 x bfloat> %b, %c481 %sel = select <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> %fmul, <vscale x 8 x bfloat> zeroinitializer482 %fsub = fsub contract <vscale x 8 x bfloat> %a, %sel483 ret <vscale x 8 x bfloat> %fsub484}485 486define <vscale x 8 x bfloat> @fadd_sel_fmul_nsz_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, <vscale x 8 x i1> %mask) {487; SVE-LABEL: fadd_sel_fmul_nsz_nxv8bf16:488; SVE: // %bb.0:489; SVE-NEXT: mov z3.s, #0x80000000490; SVE-NEXT: mov z4.s, #0x80000000491; SVE-NEXT: ptrue p1.s492; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h493; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h494; SVE-NEXT: bfcvt z1.h, p1/m, z3.s495; SVE-NEXT: uunpkhi z3.s, z0.h496; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s497; SVE-NEXT: uunpklo z4.s, z0.h498; SVE-NEXT: lsl z3.s, z3.s, #16499; SVE-NEXT: uunpkhi z2.s, z1.h500; SVE-NEXT: uunpklo z1.s, z1.h501; SVE-NEXT: lsl z4.s, z4.s, #16502; SVE-NEXT: lsl z2.s, z2.s, #16503; SVE-NEXT: lsl z1.s, z1.s, #16504; SVE-NEXT: fadd z2.s, z3.s, z2.s505; SVE-NEXT: fadd z1.s, z4.s, z1.s506; SVE-NEXT: bfcvt z2.h, p1/m, z2.s507; SVE-NEXT: bfcvt z1.h, p1/m, z1.s508; SVE-NEXT: uzp1 z1.h, z1.h, z2.h509; SVE-NEXT: mov z0.h, p0/m, z1.h510; SVE-NEXT: ret511;512; SVE-B16B16-LABEL: fadd_sel_fmul_nsz_nxv8bf16:513; SVE-B16B16: // %bb.0:514; SVE-B16B16-NEXT: bfmla z0.h, p0/m, z1.h, z2.h515; SVE-B16B16-NEXT: ret516 %fmul = fmul <vscale x 8 x bfloat> %b, %c517 %sel = select <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> %fmul, <vscale x 8 x bfloat> zeroinitializer518 %fadd = fadd nsz contract <vscale x 8 x bfloat> %a, %sel519 ret <vscale x 8 x bfloat> %fadd520}521 522define <vscale x 8 x bfloat> @fsub_sel_fmul_nsz_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, <vscale x 8 x i1> %mask) {523; SVE-LABEL: fsub_sel_fmul_nsz_nxv8bf16:524; SVE: // %bb.0:525; SVE-NEXT: mov z3.s, #0x80000000526; SVE-NEXT: mov z4.s, #0x80000000527; SVE-NEXT: ptrue p1.s528; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h529; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h530; SVE-NEXT: bfcvt z1.h, p1/m, z3.s531; SVE-NEXT: uunpkhi z3.s, z0.h532; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s533; SVE-NEXT: uunpklo z4.s, z0.h534; SVE-NEXT: lsl z3.s, z3.s, #16535; SVE-NEXT: uunpkhi z2.s, z1.h536; SVE-NEXT: uunpklo z1.s, z1.h537; SVE-NEXT: lsl z4.s, z4.s, #16538; SVE-NEXT: lsl z2.s, z2.s, #16539; SVE-NEXT: lsl z1.s, z1.s, #16540; SVE-NEXT: fsub z2.s, z3.s, z2.s541; SVE-NEXT: fsub z1.s, z4.s, z1.s542; SVE-NEXT: bfcvt z2.h, p1/m, z2.s543; SVE-NEXT: bfcvt z1.h, p1/m, z1.s544; SVE-NEXT: uzp1 z1.h, z1.h, z2.h545; SVE-NEXT: mov z0.h, p0/m, z1.h546; SVE-NEXT: ret547;548; SVE-B16B16-LABEL: fsub_sel_fmul_nsz_nxv8bf16:549; SVE-B16B16: // %bb.0:550; SVE-B16B16-NEXT: bfmls z0.h, p0/m, z1.h, z2.h551; SVE-B16B16-NEXT: ret552 %fmul = fmul <vscale x 8 x bfloat> %b, %c553 %sel = select <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> %fmul, <vscale x 8 x bfloat> zeroinitializer554 %fsub = fsub nsz contract <vscale x 8 x bfloat> %a, %sel555 ret <vscale x 8 x bfloat> %fsub556}557 558define <vscale x 8 x bfloat> @fadd_sel_fmul_negzero_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, <vscale x 8 x i1> %mask) {559; SVE-LABEL: fadd_sel_fmul_negzero_nxv8bf16:560; SVE: // %bb.0:561; SVE-NEXT: mov z3.s, #0x80000000562; SVE-NEXT: mov z4.s, #0x80000000563; SVE-NEXT: ptrue p1.s564; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h565; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h566; SVE-NEXT: bfcvt z1.h, p1/m, z3.s567; SVE-NEXT: uunpkhi z3.s, z0.h568; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s569; SVE-NEXT: uunpklo z4.s, z0.h570; SVE-NEXT: lsl z3.s, z3.s, #16571; SVE-NEXT: uunpkhi z2.s, z1.h572; SVE-NEXT: uunpklo z1.s, z1.h573; SVE-NEXT: lsl z4.s, z4.s, #16574; SVE-NEXT: lsl z2.s, z2.s, #16575; SVE-NEXT: lsl z1.s, z1.s, #16576; SVE-NEXT: fadd z2.s, z3.s, z2.s577; SVE-NEXT: fadd z1.s, z4.s, z1.s578; SVE-NEXT: bfcvt z2.h, p1/m, z2.s579; SVE-NEXT: bfcvt z1.h, p1/m, z1.s580; SVE-NEXT: uzp1 z1.h, z1.h, z2.h581; SVE-NEXT: mov z0.h, p0/m, z1.h582; SVE-NEXT: ret583;584; SVE-B16B16-LABEL: fadd_sel_fmul_negzero_nxv8bf16:585; SVE-B16B16: // %bb.0:586; SVE-B16B16-NEXT: bfmla z0.h, p0/m, z1.h, z2.h587; SVE-B16B16-NEXT: ret588 %fmul = fmul <vscale x 8 x bfloat> %b, %c589 %nz = fneg <vscale x 8 x bfloat> zeroinitializer590 %sel = select <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> %fmul, <vscale x 8 x bfloat> %nz591 %fadd = fadd contract <vscale x 8 x bfloat> %a, %sel592 ret <vscale x 8 x bfloat> %fadd593}594 595define <vscale x 8 x bfloat> @fsub_sel_fmul_negzero_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, <vscale x 8 x i1> %mask) {596; SVE-LABEL: fsub_sel_fmul_negzero_nxv8bf16:597; SVE: // %bb.0:598; SVE-NEXT: mov z3.s, #0x80000000599; SVE-NEXT: mov z4.s, #0x80000000600; SVE-NEXT: ptrue p1.s601; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h602; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h603; SVE-NEXT: dupm z2.h, #0x8000604; SVE-NEXT: bfcvt z1.h, p1/m, z3.s605; SVE-NEXT: uunpkhi z3.s, z0.h606; SVE-NEXT: uunpklo z0.s, z0.h607; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s608; SVE-NEXT: lsl z3.s, z3.s, #16609; SVE-NEXT: lsl z0.s, z0.s, #16610; SVE-NEXT: sel z1.h, p0, z1.h, z2.h611; SVE-NEXT: uunpkhi z2.s, z1.h612; SVE-NEXT: uunpklo z1.s, z1.h613; SVE-NEXT: lsl z2.s, z2.s, #16614; SVE-NEXT: lsl z1.s, z1.s, #16615; SVE-NEXT: fsub z2.s, z3.s, z2.s616; SVE-NEXT: fsub z0.s, z0.s, z1.s617; SVE-NEXT: bfcvt z1.h, p1/m, z2.s618; SVE-NEXT: bfcvt z0.h, p1/m, z0.s619; SVE-NEXT: uzp1 z0.h, z0.h, z1.h620; SVE-NEXT: ret621;622; SVE-B16B16-LABEL: fsub_sel_fmul_negzero_nxv8bf16:623; SVE-B16B16: // %bb.0:624; SVE-B16B16-NEXT: dupm z3.h, #0x8000625; SVE-B16B16-NEXT: bfmul z1.h, z1.h, z2.h626; SVE-B16B16-NEXT: sel z1.h, p0, z1.h, z3.h627; SVE-B16B16-NEXT: bfsub z0.h, z0.h, z1.h628; SVE-B16B16-NEXT: ret629 %fmul = fmul <vscale x 8 x bfloat> %b, %c630 %nz = fneg <vscale x 8 x bfloat> zeroinitializer631 %sel = select <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> %fmul, <vscale x 8 x bfloat> %nz632 %fsub = fsub contract <vscale x 8 x bfloat> %a, %sel633 ret <vscale x 8 x bfloat> %fsub634}635 636define <vscale x 8 x bfloat> @fadd_sel_fmul_negzero_nsz_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, <vscale x 8 x i1> %mask) {637; SVE-LABEL: fadd_sel_fmul_negzero_nsz_nxv8bf16:638; SVE: // %bb.0:639; SVE-NEXT: mov z3.s, #0x80000000640; SVE-NEXT: mov z4.s, #0x80000000641; SVE-NEXT: ptrue p1.s642; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h643; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h644; SVE-NEXT: bfcvt z1.h, p1/m, z3.s645; SVE-NEXT: uunpkhi z3.s, z0.h646; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s647; SVE-NEXT: uunpklo z4.s, z0.h648; SVE-NEXT: lsl z3.s, z3.s, #16649; SVE-NEXT: uunpkhi z2.s, z1.h650; SVE-NEXT: uunpklo z1.s, z1.h651; SVE-NEXT: lsl z4.s, z4.s, #16652; SVE-NEXT: lsl z2.s, z2.s, #16653; SVE-NEXT: lsl z1.s, z1.s, #16654; SVE-NEXT: fadd z2.s, z3.s, z2.s655; SVE-NEXT: fadd z1.s, z4.s, z1.s656; SVE-NEXT: bfcvt z2.h, p1/m, z2.s657; SVE-NEXT: bfcvt z1.h, p1/m, z1.s658; SVE-NEXT: uzp1 z1.h, z1.h, z2.h659; SVE-NEXT: mov z0.h, p0/m, z1.h660; SVE-NEXT: ret661;662; SVE-B16B16-LABEL: fadd_sel_fmul_negzero_nsz_nxv8bf16:663; SVE-B16B16: // %bb.0:664; SVE-B16B16-NEXT: bfmla z0.h, p0/m, z1.h, z2.h665; SVE-B16B16-NEXT: ret666 %fmul = fmul <vscale x 8 x bfloat> %b, %c667 %nz = fneg <vscale x 8 x bfloat> zeroinitializer668 %sel = select <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> %fmul, <vscale x 8 x bfloat> %nz669 %fadd = fadd nsz contract <vscale x 8 x bfloat> %a, %sel670 ret <vscale x 8 x bfloat> %fadd671}672 673define <vscale x 8 x bfloat> @fsub_sel_fmul_negzero_nsz_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c, <vscale x 8 x i1> %mask) {674; SVE-LABEL: fsub_sel_fmul_negzero_nsz_nxv8bf16:675; SVE: // %bb.0:676; SVE-NEXT: mov z3.s, #0x80000000677; SVE-NEXT: mov z4.s, #0x80000000678; SVE-NEXT: ptrue p1.s679; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h680; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h681; SVE-NEXT: bfcvt z1.h, p1/m, z3.s682; SVE-NEXT: uunpkhi z3.s, z0.h683; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s684; SVE-NEXT: uunpklo z4.s, z0.h685; SVE-NEXT: lsl z3.s, z3.s, #16686; SVE-NEXT: uunpkhi z2.s, z1.h687; SVE-NEXT: uunpklo z1.s, z1.h688; SVE-NEXT: lsl z4.s, z4.s, #16689; SVE-NEXT: lsl z2.s, z2.s, #16690; SVE-NEXT: lsl z1.s, z1.s, #16691; SVE-NEXT: fsub z2.s, z3.s, z2.s692; SVE-NEXT: fsub z1.s, z4.s, z1.s693; SVE-NEXT: bfcvt z2.h, p1/m, z2.s694; SVE-NEXT: bfcvt z1.h, p1/m, z1.s695; SVE-NEXT: uzp1 z1.h, z1.h, z2.h696; SVE-NEXT: mov z0.h, p0/m, z1.h697; SVE-NEXT: ret698;699; SVE-B16B16-LABEL: fsub_sel_fmul_negzero_nsz_nxv8bf16:700; SVE-B16B16: // %bb.0:701; SVE-B16B16-NEXT: bfmls z0.h, p0/m, z1.h, z2.h702; SVE-B16B16-NEXT: ret703 %fmul = fmul <vscale x 8 x bfloat> %b, %c704 %nz = fneg <vscale x 8 x bfloat> zeroinitializer705 %sel = select <vscale x 8 x i1> %mask, <vscale x 8 x bfloat> %fmul, <vscale x 8 x bfloat> %nz706 %fsub = fsub nsz contract <vscale x 8 x bfloat> %a, %sel707 ret <vscale x 8 x bfloat> %fsub708}709 710declare <vscale x 8 x bfloat> @llvm.fma.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)711