790 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16,NOB16B16-NONSTREAMING3; RUN: llc -mattr=+sve,+bf16,+sve-b16b16 < %s | FileCheck %s --check-prefixes=CHECK,B16B164; RUN: llc -mattr=+sme,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16,NOB16B16-STREAMING5; RUN: llc -mattr=+sme2,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,B16B166 7target triple = "aarch64-unknown-linux-gnu"8 9;10; FABS11;12 13define <vscale x 2 x bfloat> @fabs_nxv2bf16(<vscale x 2 x bfloat> %a) {14; CHECK-LABEL: fabs_nxv2bf16:15; CHECK: // %bb.0:16; CHECK-NEXT: ptrue p0.d17; CHECK-NEXT: fabs z0.h, p0/m, z0.h18; CHECK-NEXT: ret19 %res = call <vscale x 2 x bfloat> @llvm.fabs.nxv2bf16(<vscale x 2 x bfloat> %a)20 ret <vscale x 2 x bfloat> %res21}22 23define <vscale x 4 x bfloat> @fabs_nxv4bf16(<vscale x 4 x bfloat> %a) {24; CHECK-LABEL: fabs_nxv4bf16:25; CHECK: // %bb.0:26; CHECK-NEXT: ptrue p0.s27; CHECK-NEXT: fabs z0.h, p0/m, z0.h28; CHECK-NEXT: ret29 %res = call <vscale x 4 x bfloat> @llvm.fabs.nxv4bf16(<vscale x 4 x bfloat> %a)30 ret <vscale x 4 x bfloat> %res31}32 33define <vscale x 8 x bfloat> @fabs_nxv8bf16(<vscale x 8 x bfloat> %a) {34; CHECK-LABEL: fabs_nxv8bf16:35; CHECK: // %bb.0:36; CHECK-NEXT: ptrue p0.h37; CHECK-NEXT: fabs z0.h, p0/m, z0.h38; CHECK-NEXT: ret39 %res = call <vscale x 8 x bfloat> @llvm.fabs.nxv8bf16(<vscale x 8 x bfloat> %a)40 ret <vscale x 8 x bfloat> %res41}42 43;44; FADD45;46 47define <vscale x 2 x bfloat> @fadd_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {48; NOB16B16-LABEL: fadd_nxv2bf16:49; NOB16B16: // %bb.0:50; NOB16B16-NEXT: lsl z1.s, z1.s, #1651; NOB16B16-NEXT: lsl z0.s, z0.s, #1652; NOB16B16-NEXT: ptrue p0.d53; NOB16B16-NEXT: fadd z0.s, p0/m, z0.s, z1.s54; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s55; NOB16B16-NEXT: ret56;57; B16B16-LABEL: fadd_nxv2bf16:58; B16B16: // %bb.0:59; B16B16-NEXT: ptrue p0.d60; B16B16-NEXT: bfadd z0.h, p0/m, z0.h, z1.h61; B16B16-NEXT: ret62 %res = fadd <vscale x 2 x bfloat> %a, %b63 ret <vscale x 2 x bfloat> %res64}65 66define <vscale x 4 x bfloat> @fadd_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {67; NOB16B16-LABEL: fadd_nxv4bf16:68; NOB16B16: // %bb.0:69; NOB16B16-NEXT: lsl z1.s, z1.s, #1670; NOB16B16-NEXT: lsl z0.s, z0.s, #1671; NOB16B16-NEXT: ptrue p0.s72; NOB16B16-NEXT: fadd z0.s, z0.s, z1.s73; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s74; NOB16B16-NEXT: ret75;76; B16B16-LABEL: fadd_nxv4bf16:77; B16B16: // %bb.0:78; B16B16-NEXT: ptrue p0.s79; B16B16-NEXT: bfadd z0.h, p0/m, z0.h, z1.h80; B16B16-NEXT: ret81 %res = fadd <vscale x 4 x bfloat> %a, %b82 ret <vscale x 4 x bfloat> %res83}84 85define <vscale x 8 x bfloat> @fadd_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {86; NOB16B16-LABEL: fadd_nxv8bf16:87; NOB16B16: // %bb.0:88; NOB16B16-NEXT: uunpkhi z2.s, z1.h89; NOB16B16-NEXT: uunpkhi z3.s, z0.h90; NOB16B16-NEXT: uunpklo z1.s, z1.h91; NOB16B16-NEXT: uunpklo z0.s, z0.h92; NOB16B16-NEXT: ptrue p0.s93; NOB16B16-NEXT: lsl z2.s, z2.s, #1694; NOB16B16-NEXT: lsl z3.s, z3.s, #1695; NOB16B16-NEXT: lsl z1.s, z1.s, #1696; NOB16B16-NEXT: lsl z0.s, z0.s, #1697; NOB16B16-NEXT: fadd z2.s, z3.s, z2.s98; NOB16B16-NEXT: fadd z0.s, z0.s, z1.s99; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s100; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s101; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h102; NOB16B16-NEXT: ret103;104; B16B16-LABEL: fadd_nxv8bf16:105; B16B16: // %bb.0:106; B16B16-NEXT: bfadd z0.h, z0.h, z1.h107; B16B16-NEXT: ret108 %res = fadd <vscale x 8 x bfloat> %a, %b109 ret <vscale x 8 x bfloat> %res110}111 112;113; FDIV114;115 116define <vscale x 2 x bfloat> @fdiv_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {117; CHECK-LABEL: fdiv_nxv2bf16:118; CHECK: // %bb.0:119; CHECK-NEXT: lsl z1.s, z1.s, #16120; CHECK-NEXT: lsl z0.s, z0.s, #16121; CHECK-NEXT: ptrue p0.d122; CHECK-NEXT: fdiv z0.s, p0/m, z0.s, z1.s123; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s124; CHECK-NEXT: ret125 %res = fdiv <vscale x 2 x bfloat> %a, %b126 ret <vscale x 2 x bfloat> %res127}128 129define <vscale x 4 x bfloat> @fdiv_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {130; CHECK-LABEL: fdiv_nxv4bf16:131; CHECK: // %bb.0:132; CHECK-NEXT: lsl z1.s, z1.s, #16133; CHECK-NEXT: lsl z0.s, z0.s, #16134; CHECK-NEXT: ptrue p0.s135; CHECK-NEXT: fdiv z0.s, p0/m, z0.s, z1.s136; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s137; CHECK-NEXT: ret138 %res = fdiv <vscale x 4 x bfloat> %a, %b139 ret <vscale x 4 x bfloat> %res140}141 142define <vscale x 8 x bfloat> @fdiv_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {143; CHECK-LABEL: fdiv_nxv8bf16:144; CHECK: // %bb.0:145; CHECK-NEXT: uunpkhi z2.s, z1.h146; CHECK-NEXT: uunpkhi z3.s, z0.h147; CHECK-NEXT: uunpklo z1.s, z1.h148; CHECK-NEXT: uunpklo z0.s, z0.h149; CHECK-NEXT: ptrue p0.s150; CHECK-NEXT: lsl z2.s, z2.s, #16151; CHECK-NEXT: lsl z3.s, z3.s, #16152; CHECK-NEXT: lsl z1.s, z1.s, #16153; CHECK-NEXT: lsl z0.s, z0.s, #16154; CHECK-NEXT: fdivr z2.s, p0/m, z2.s, z3.s155; CHECK-NEXT: fdiv z0.s, p0/m, z0.s, z1.s156; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s157; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s158; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h159; CHECK-NEXT: ret160 %res = fdiv <vscale x 8 x bfloat> %a, %b161 ret <vscale x 8 x bfloat> %res162}163 164;165; FMAX166;167 168define <vscale x 2 x bfloat> @fmax_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {169; NOB16B16-LABEL: fmax_nxv2bf16:170; NOB16B16: // %bb.0:171; NOB16B16-NEXT: lsl z1.s, z1.s, #16172; NOB16B16-NEXT: lsl z0.s, z0.s, #16173; NOB16B16-NEXT: ptrue p0.d174; NOB16B16-NEXT: fmax z0.s, p0/m, z0.s, z1.s175; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s176; NOB16B16-NEXT: ret177;178; B16B16-LABEL: fmax_nxv2bf16:179; B16B16: // %bb.0:180; B16B16-NEXT: ptrue p0.d181; B16B16-NEXT: bfmax z0.h, p0/m, z0.h, z1.h182; B16B16-NEXT: ret183 %res = call <vscale x 2 x bfloat> @llvm.maximum.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b)184 ret <vscale x 2 x bfloat> %res185}186 187define <vscale x 4 x bfloat> @fmax_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {188; NOB16B16-LABEL: fmax_nxv4bf16:189; NOB16B16: // %bb.0:190; NOB16B16-NEXT: lsl z1.s, z1.s, #16191; NOB16B16-NEXT: lsl z0.s, z0.s, #16192; NOB16B16-NEXT: ptrue p0.s193; NOB16B16-NEXT: fmax z0.s, p0/m, z0.s, z1.s194; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s195; NOB16B16-NEXT: ret196;197; B16B16-LABEL: fmax_nxv4bf16:198; B16B16: // %bb.0:199; B16B16-NEXT: ptrue p0.s200; B16B16-NEXT: bfmax z0.h, p0/m, z0.h, z1.h201; B16B16-NEXT: ret202 %res = call <vscale x 4 x bfloat> @llvm.maximum.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b)203 ret <vscale x 4 x bfloat> %res204}205 206define <vscale x 8 x bfloat> @fmax_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {207; NOB16B16-LABEL: fmax_nxv8bf16:208; NOB16B16: // %bb.0:209; NOB16B16-NEXT: uunpkhi z2.s, z1.h210; NOB16B16-NEXT: uunpkhi z3.s, z0.h211; NOB16B16-NEXT: uunpklo z1.s, z1.h212; NOB16B16-NEXT: uunpklo z0.s, z0.h213; NOB16B16-NEXT: ptrue p0.s214; NOB16B16-NEXT: lsl z2.s, z2.s, #16215; NOB16B16-NEXT: lsl z3.s, z3.s, #16216; NOB16B16-NEXT: lsl z1.s, z1.s, #16217; NOB16B16-NEXT: lsl z0.s, z0.s, #16218; NOB16B16-NEXT: fmax z2.s, p0/m, z2.s, z3.s219; NOB16B16-NEXT: fmax z0.s, p0/m, z0.s, z1.s220; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s221; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s222; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h223; NOB16B16-NEXT: ret224;225; B16B16-LABEL: fmax_nxv8bf16:226; B16B16: // %bb.0:227; B16B16-NEXT: ptrue p0.h228; B16B16-NEXT: bfmax z0.h, p0/m, z0.h, z1.h229; B16B16-NEXT: ret230 %res = call <vscale x 8 x bfloat> @llvm.maximum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b)231 ret <vscale x 8 x bfloat> %res232}233 234;235; FMAXNM236;237 238define <vscale x 2 x bfloat> @fmaxnm_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {239; NOB16B16-LABEL: fmaxnm_nxv2bf16:240; NOB16B16: // %bb.0:241; NOB16B16-NEXT: lsl z1.s, z1.s, #16242; NOB16B16-NEXT: lsl z0.s, z0.s, #16243; NOB16B16-NEXT: ptrue p0.d244; NOB16B16-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s245; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s246; NOB16B16-NEXT: ret247;248; B16B16-LABEL: fmaxnm_nxv2bf16:249; B16B16: // %bb.0:250; B16B16-NEXT: ptrue p0.d251; B16B16-NEXT: bfmaxnm z0.h, p0/m, z0.h, z1.h252; B16B16-NEXT: ret253 %res = call <vscale x 2 x bfloat> @llvm.maxnum.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b)254 ret <vscale x 2 x bfloat> %res255}256 257define <vscale x 4 x bfloat> @fmaxnm_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {258; NOB16B16-LABEL: fmaxnm_nxv4bf16:259; NOB16B16: // %bb.0:260; NOB16B16-NEXT: lsl z1.s, z1.s, #16261; NOB16B16-NEXT: lsl z0.s, z0.s, #16262; NOB16B16-NEXT: ptrue p0.s263; NOB16B16-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s264; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s265; NOB16B16-NEXT: ret266;267; B16B16-LABEL: fmaxnm_nxv4bf16:268; B16B16: // %bb.0:269; B16B16-NEXT: ptrue p0.s270; B16B16-NEXT: bfmaxnm z0.h, p0/m, z0.h, z1.h271; B16B16-NEXT: ret272 %res = call <vscale x 4 x bfloat> @llvm.maxnum.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b)273 ret <vscale x 4 x bfloat> %res274}275 276define <vscale x 8 x bfloat> @fmaxnm_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {277; NOB16B16-LABEL: fmaxnm_nxv8bf16:278; NOB16B16: // %bb.0:279; NOB16B16-NEXT: uunpkhi z2.s, z1.h280; NOB16B16-NEXT: uunpkhi z3.s, z0.h281; NOB16B16-NEXT: uunpklo z1.s, z1.h282; NOB16B16-NEXT: uunpklo z0.s, z0.h283; NOB16B16-NEXT: ptrue p0.s284; NOB16B16-NEXT: lsl z2.s, z2.s, #16285; NOB16B16-NEXT: lsl z3.s, z3.s, #16286; NOB16B16-NEXT: lsl z1.s, z1.s, #16287; NOB16B16-NEXT: lsl z0.s, z0.s, #16288; NOB16B16-NEXT: fmaxnm z2.s, p0/m, z2.s, z3.s289; NOB16B16-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s290; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s291; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s292; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h293; NOB16B16-NEXT: ret294;295; B16B16-LABEL: fmaxnm_nxv8bf16:296; B16B16: // %bb.0:297; B16B16-NEXT: ptrue p0.h298; B16B16-NEXT: bfmaxnm z0.h, p0/m, z0.h, z1.h299; B16B16-NEXT: ret300 %res = call <vscale x 8 x bfloat> @llvm.maxnum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b)301 ret <vscale x 8 x bfloat> %res302}303 304;305; FMIN306;307 308define <vscale x 2 x bfloat> @fmin_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {309; NOB16B16-LABEL: fmin_nxv2bf16:310; NOB16B16: // %bb.0:311; NOB16B16-NEXT: lsl z1.s, z1.s, #16312; NOB16B16-NEXT: lsl z0.s, z0.s, #16313; NOB16B16-NEXT: ptrue p0.d314; NOB16B16-NEXT: fmin z0.s, p0/m, z0.s, z1.s315; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s316; NOB16B16-NEXT: ret317;318; B16B16-LABEL: fmin_nxv2bf16:319; B16B16: // %bb.0:320; B16B16-NEXT: ptrue p0.d321; B16B16-NEXT: bfmin z0.h, p0/m, z0.h, z1.h322; B16B16-NEXT: ret323 %res = call <vscale x 2 x bfloat> @llvm.minimum.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b)324 ret <vscale x 2 x bfloat> %res325}326 327define <vscale x 4 x bfloat> @fmin_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {328; NOB16B16-LABEL: fmin_nxv4bf16:329; NOB16B16: // %bb.0:330; NOB16B16-NEXT: lsl z1.s, z1.s, #16331; NOB16B16-NEXT: lsl z0.s, z0.s, #16332; NOB16B16-NEXT: ptrue p0.s333; NOB16B16-NEXT: fmin z0.s, p0/m, z0.s, z1.s334; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s335; NOB16B16-NEXT: ret336;337; B16B16-LABEL: fmin_nxv4bf16:338; B16B16: // %bb.0:339; B16B16-NEXT: ptrue p0.s340; B16B16-NEXT: bfmin z0.h, p0/m, z0.h, z1.h341; B16B16-NEXT: ret342 %res = call <vscale x 4 x bfloat> @llvm.minimum.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b)343 ret <vscale x 4 x bfloat> %res344}345 346define <vscale x 8 x bfloat> @fmin_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {347; NOB16B16-LABEL: fmin_nxv8bf16:348; NOB16B16: // %bb.0:349; NOB16B16-NEXT: uunpkhi z2.s, z1.h350; NOB16B16-NEXT: uunpkhi z3.s, z0.h351; NOB16B16-NEXT: uunpklo z1.s, z1.h352; NOB16B16-NEXT: uunpklo z0.s, z0.h353; NOB16B16-NEXT: ptrue p0.s354; NOB16B16-NEXT: lsl z2.s, z2.s, #16355; NOB16B16-NEXT: lsl z3.s, z3.s, #16356; NOB16B16-NEXT: lsl z1.s, z1.s, #16357; NOB16B16-NEXT: lsl z0.s, z0.s, #16358; NOB16B16-NEXT: fmin z2.s, p0/m, z2.s, z3.s359; NOB16B16-NEXT: fmin z0.s, p0/m, z0.s, z1.s360; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s361; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s362; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h363; NOB16B16-NEXT: ret364;365; B16B16-LABEL: fmin_nxv8bf16:366; B16B16: // %bb.0:367; B16B16-NEXT: ptrue p0.h368; B16B16-NEXT: bfmin z0.h, p0/m, z0.h, z1.h369; B16B16-NEXT: ret370 %res = call <vscale x 8 x bfloat> @llvm.minimum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b)371 ret <vscale x 8 x bfloat> %res372}373 374;375; FMINNM376;377 378define <vscale x 2 x bfloat> @fminnm_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {379; NOB16B16-LABEL: fminnm_nxv2bf16:380; NOB16B16: // %bb.0:381; NOB16B16-NEXT: lsl z1.s, z1.s, #16382; NOB16B16-NEXT: lsl z0.s, z0.s, #16383; NOB16B16-NEXT: ptrue p0.d384; NOB16B16-NEXT: fminnm z0.s, p0/m, z0.s, z1.s385; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s386; NOB16B16-NEXT: ret387;388; B16B16-LABEL: fminnm_nxv2bf16:389; B16B16: // %bb.0:390; B16B16-NEXT: ptrue p0.d391; B16B16-NEXT: bfminnm z0.h, p0/m, z0.h, z1.h392; B16B16-NEXT: ret393 %res = call <vscale x 2 x bfloat> @llvm.minnum.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b)394 ret <vscale x 2 x bfloat> %res395}396 397define <vscale x 4 x bfloat> @fminnm_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {398; NOB16B16-LABEL: fminnm_nxv4bf16:399; NOB16B16: // %bb.0:400; NOB16B16-NEXT: lsl z1.s, z1.s, #16401; NOB16B16-NEXT: lsl z0.s, z0.s, #16402; NOB16B16-NEXT: ptrue p0.s403; NOB16B16-NEXT: fminnm z0.s, p0/m, z0.s, z1.s404; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s405; NOB16B16-NEXT: ret406;407; B16B16-LABEL: fminnm_nxv4bf16:408; B16B16: // %bb.0:409; B16B16-NEXT: ptrue p0.s410; B16B16-NEXT: bfminnm z0.h, p0/m, z0.h, z1.h411; B16B16-NEXT: ret412 %res = call <vscale x 4 x bfloat> @llvm.minnum.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b)413 ret <vscale x 4 x bfloat> %res414}415 416define <vscale x 8 x bfloat> @fminnm_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {417; NOB16B16-LABEL: fminnm_nxv8bf16:418; NOB16B16: // %bb.0:419; NOB16B16-NEXT: uunpkhi z2.s, z1.h420; NOB16B16-NEXT: uunpkhi z3.s, z0.h421; NOB16B16-NEXT: uunpklo z1.s, z1.h422; NOB16B16-NEXT: uunpklo z0.s, z0.h423; NOB16B16-NEXT: ptrue p0.s424; NOB16B16-NEXT: lsl z2.s, z2.s, #16425; NOB16B16-NEXT: lsl z3.s, z3.s, #16426; NOB16B16-NEXT: lsl z1.s, z1.s, #16427; NOB16B16-NEXT: lsl z0.s, z0.s, #16428; NOB16B16-NEXT: fminnm z2.s, p0/m, z2.s, z3.s429; NOB16B16-NEXT: fminnm z0.s, p0/m, z0.s, z1.s430; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s431; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s432; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h433; NOB16B16-NEXT: ret434;435; B16B16-LABEL: fminnm_nxv8bf16:436; B16B16: // %bb.0:437; B16B16-NEXT: ptrue p0.h438; B16B16-NEXT: bfminnm z0.h, p0/m, z0.h, z1.h439; B16B16-NEXT: ret440 %res = call <vscale x 8 x bfloat> @llvm.minnum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b)441 ret <vscale x 8 x bfloat> %res442}443 444;445; FMLA446;447 448define <vscale x 2 x bfloat> @fmla_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b, <vscale x 2 x bfloat> %c) {449; NOB16B16-LABEL: fmla_nxv2bf16:450; NOB16B16: // %bb.0:451; NOB16B16-NEXT: lsl z1.s, z1.s, #16452; NOB16B16-NEXT: lsl z0.s, z0.s, #16453; NOB16B16-NEXT: lsl z2.s, z2.s, #16454; NOB16B16-NEXT: ptrue p0.d455; NOB16B16-NEXT: fmad z0.s, p0/m, z1.s, z2.s456; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s457; NOB16B16-NEXT: ret458;459; B16B16-LABEL: fmla_nxv2bf16:460; B16B16: // %bb.0:461; B16B16-NEXT: ptrue p0.d462; B16B16-NEXT: bfmla z0.h, p0/m, z1.h, z2.h463; B16B16-NEXT: ret464 %res = call <vscale x 2 x bfloat> @llvm.fma.nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b, <vscale x 2 x bfloat> %c)465 ret <vscale x 2 x bfloat> %res466}467 468define <vscale x 4 x bfloat> @fmla_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b, <vscale x 4 x bfloat> %c) {469; NOB16B16-LABEL: fmla_nxv4bf16:470; NOB16B16: // %bb.0:471; NOB16B16-NEXT: lsl z2.s, z2.s, #16472; NOB16B16-NEXT: ptrue p0.s473; NOB16B16-NEXT: bfmlalb z2.s, z0.h, z1.h474; NOB16B16-NEXT: bfcvt z0.h, p0/m, z2.s475; NOB16B16-NEXT: ret476;477; B16B16-LABEL: fmla_nxv4bf16:478; B16B16: // %bb.0:479; B16B16-NEXT: ptrue p0.s480; B16B16-NEXT: bfmla z0.h, p0/m, z1.h, z2.h481; B16B16-NEXT: ret482 %res = call <vscale x 4 x bfloat> @llvm.fma.nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b, <vscale x 4 x bfloat> %c)483 ret <vscale x 4 x bfloat> %res484}485 486define <vscale x 8 x bfloat> @fmla_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) {487; NOB16B16-LABEL: fmla_nxv8bf16:488; NOB16B16: // %bb.0:489; NOB16B16-NEXT: uunpkhi z3.s, z2.h490; NOB16B16-NEXT: uunpklo z2.s, z2.h491; NOB16B16-NEXT: uunpkhi z4.s, z1.h492; NOB16B16-NEXT: uunpkhi z5.s, z0.h493; NOB16B16-NEXT: uunpklo z1.s, z1.h494; NOB16B16-NEXT: uunpklo z0.s, z0.h495; NOB16B16-NEXT: ptrue p0.s496; NOB16B16-NEXT: lsl z3.s, z3.s, #16497; NOB16B16-NEXT: lsl z2.s, z2.s, #16498; NOB16B16-NEXT: bfmlalb z3.s, z5.h, z4.h499; NOB16B16-NEXT: bfmlalb z2.s, z0.h, z1.h500; NOB16B16-NEXT: bfcvt z0.h, p0/m, z3.s501; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s502; NOB16B16-NEXT: uzp1 z0.h, z1.h, z0.h503; NOB16B16-NEXT: ret504;505; B16B16-LABEL: fmla_nxv8bf16:506; B16B16: // %bb.0:507; B16B16-NEXT: ptrue p0.h508; B16B16-NEXT: bfmla z0.h, p0/m, z1.h, z2.h509; B16B16-NEXT: ret510 %res = call <vscale x 8 x bfloat> @llvm.fma.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c)511 ret <vscale x 8 x bfloat> %res512}513 514;515; FMUL516;517 518define <vscale x 2 x bfloat> @fmul_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {519; NOB16B16-LABEL: fmul_nxv2bf16:520; NOB16B16: // %bb.0:521; NOB16B16-NEXT: lsl z1.s, z1.s, #16522; NOB16B16-NEXT: lsl z0.s, z0.s, #16523; NOB16B16-NEXT: ptrue p0.d524; NOB16B16-NEXT: fmul z0.s, p0/m, z0.s, z1.s525; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s526; NOB16B16-NEXT: ret527;528; B16B16-LABEL: fmul_nxv2bf16:529; B16B16: // %bb.0:530; B16B16-NEXT: ptrue p0.d531; B16B16-NEXT: bfmul z0.h, p0/m, z0.h, z1.h532; B16B16-NEXT: ret533 %res = fmul nsz <vscale x 2 x bfloat> %a, %b534 ret <vscale x 2 x bfloat> %res535}536 537define <vscale x 4 x bfloat> @fmul_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {538; NOB16B16-NONSTREAMING-LABEL: fmul_nxv4bf16:539; NOB16B16-NONSTREAMING: // %bb.0:540; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000541; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s542; NOB16B16-NONSTREAMING-NEXT: bfmlalb z2.s, z0.h, z1.h543; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z2.s544; NOB16B16-NONSTREAMING-NEXT: ret545;546; B16B16-LABEL: fmul_nxv4bf16:547; B16B16: // %bb.0:548; B16B16-NEXT: ptrue p0.s549; B16B16-NEXT: bfmul z0.h, p0/m, z0.h, z1.h550; B16B16-NEXT: ret551;552; NOB16B16-STREAMING-LABEL: fmul_nxv4bf16:553; NOB16B16-STREAMING: // %bb.0:554; NOB16B16-STREAMING-NEXT: mov z2.s, #0 // =0x0555; NOB16B16-STREAMING-NEXT: ptrue p0.s556; NOB16B16-STREAMING-NEXT: bfmlalb z2.s, z0.h, z1.h557; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z2.s558; NOB16B16-STREAMING-NEXT: ret559 %res = fmul nsz <vscale x 4 x bfloat> %a, %b560 ret <vscale x 4 x bfloat> %res561}562 563define <vscale x 8 x bfloat> @fmul_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {564; NOB16B16-NONSTREAMING-LABEL: fmul_nxv8bf16:565; NOB16B16-NONSTREAMING: // %bb.0:566; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000567; NOB16B16-NONSTREAMING-NEXT: movi v3.2d, #0000000000000000568; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s569; NOB16B16-NONSTREAMING-NEXT: bfmlalb z2.s, z0.h, z1.h570; NOB16B16-NONSTREAMING-NEXT: bfmlalt z3.s, z0.h, z1.h571; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z2.s572; NOB16B16-NONSTREAMING-NEXT: bfcvtnt z0.h, p0/m, z3.s573; NOB16B16-NONSTREAMING-NEXT: ret574;575; B16B16-LABEL: fmul_nxv8bf16:576; B16B16: // %bb.0:577; B16B16-NEXT: bfmul z0.h, z0.h, z1.h578; B16B16-NEXT: ret579;580; NOB16B16-STREAMING-LABEL: fmul_nxv8bf16:581; NOB16B16-STREAMING: // %bb.0:582; NOB16B16-STREAMING-NEXT: mov z2.s, #0 // =0x0583; NOB16B16-STREAMING-NEXT: mov z3.s, #0 // =0x0584; NOB16B16-STREAMING-NEXT: ptrue p0.s585; NOB16B16-STREAMING-NEXT: bfmlalb z2.s, z0.h, z1.h586; NOB16B16-STREAMING-NEXT: bfmlalt z3.s, z0.h, z1.h587; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z2.s588; NOB16B16-STREAMING-NEXT: bfcvtnt z0.h, p0/m, z3.s589; NOB16B16-STREAMING-NEXT: ret590 %res = fmul nsz <vscale x 8 x bfloat> %a, %b591 ret <vscale x 8 x bfloat> %res592}593 594define <vscale x 8 x bfloat> @fmul_nxv8bf16_no_nsz(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {595; NOB16B16-LABEL: fmul_nxv8bf16_no_nsz:596; NOB16B16: // %bb.0:597; NOB16B16-NEXT: mov z2.s, #0x80000000598; NOB16B16-NEXT: mov z3.s, #0x80000000599; NOB16B16-NEXT: ptrue p0.s600; NOB16B16-NEXT: bfmlalb z2.s, z0.h, z1.h601; NOB16B16-NEXT: bfmlalt z3.s, z0.h, z1.h602; NOB16B16-NEXT: bfcvt z0.h, p0/m, z2.s603; NOB16B16-NEXT: bfcvtnt z0.h, p0/m, z3.s604; NOB16B16-NEXT: ret605;606; B16B16-LABEL: fmul_nxv8bf16_no_nsz:607; B16B16: // %bb.0:608; B16B16-NEXT: bfmul z0.h, z0.h, z1.h609; B16B16-NEXT: ret610 %res = fmul <vscale x 8 x bfloat> %a, %b611 ret <vscale x 8 x bfloat> %res612}613 614;615; FNEG616;617 618define <vscale x 2 x bfloat> @fneg_nxv2bf16(<vscale x 2 x bfloat> %a) {619; CHECK-LABEL: fneg_nxv2bf16:620; CHECK: // %bb.0:621; CHECK-NEXT: ptrue p0.d622; CHECK-NEXT: fneg z0.h, p0/m, z0.h623; CHECK-NEXT: ret624 %res = fneg <vscale x 2 x bfloat> %a625 ret <vscale x 2 x bfloat> %res626}627 628define <vscale x 4 x bfloat> @fneg_nxv4bf16(<vscale x 4 x bfloat> %a) {629; CHECK-LABEL: fneg_nxv4bf16:630; CHECK: // %bb.0:631; CHECK-NEXT: ptrue p0.s632; CHECK-NEXT: fneg z0.h, p0/m, z0.h633; CHECK-NEXT: ret634 %res = fneg <vscale x 4 x bfloat> %a635 ret <vscale x 4 x bfloat> %res636}637 638define <vscale x 8 x bfloat> @fneg_nxv8bf16(<vscale x 8 x bfloat> %a) {639; CHECK-LABEL: fneg_nxv8bf16:640; CHECK: // %bb.0:641; CHECK-NEXT: ptrue p0.h642; CHECK-NEXT: fneg z0.h, p0/m, z0.h643; CHECK-NEXT: ret644 %res = fneg <vscale x 8 x bfloat> %a645 ret <vscale x 8 x bfloat> %res646}647 648;649; FSQRT650;651 652define <vscale x 2 x bfloat> @fsqrt_nxv2bf16(<vscale x 2 x bfloat> %a) {653; CHECK-LABEL: fsqrt_nxv2bf16:654; CHECK: // %bb.0:655; CHECK-NEXT: lsl z0.s, z0.s, #16656; CHECK-NEXT: ptrue p0.d657; CHECK-NEXT: fsqrt z0.s, p0/m, z0.s658; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s659; CHECK-NEXT: ret660 %res = call <vscale x 2 x bfloat> @llvm.sqrt.nxv2bf16(<vscale x 2 x bfloat> %a)661 ret <vscale x 2 x bfloat> %res662}663 664define <vscale x 4 x bfloat> @fsqrt_nxv4bf16(<vscale x 4 x bfloat> %a) {665; CHECK-LABEL: fsqrt_nxv4bf16:666; CHECK: // %bb.0:667; CHECK-NEXT: lsl z0.s, z0.s, #16668; CHECK-NEXT: ptrue p0.s669; CHECK-NEXT: fsqrt z0.s, p0/m, z0.s670; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s671; CHECK-NEXT: ret672 %res = call <vscale x 4 x bfloat> @llvm.sqrt.nxv4bf16(<vscale x 4 x bfloat> %a)673 ret <vscale x 4 x bfloat> %res674}675 676define <vscale x 8 x bfloat> @fsqrt_nxv8bf16(<vscale x 8 x bfloat> %a) {677; CHECK-LABEL: fsqrt_nxv8bf16:678; CHECK: // %bb.0:679; CHECK-NEXT: uunpkhi z1.s, z0.h680; CHECK-NEXT: uunpklo z0.s, z0.h681; CHECK-NEXT: ptrue p0.s682; CHECK-NEXT: lsl z1.s, z1.s, #16683; CHECK-NEXT: lsl z0.s, z0.s, #16684; CHECK-NEXT: fsqrt z1.s, p0/m, z1.s685; CHECK-NEXT: fsqrt z0.s, p0/m, z0.s686; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s687; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s688; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h689; CHECK-NEXT: ret690 %res = call <vscale x 8 x bfloat> @llvm.sqrt.nxv8bf16(<vscale x 8 x bfloat> %a)691 ret <vscale x 8 x bfloat> %res692}693 694;695; FSUB696;697 698define <vscale x 2 x bfloat> @fsub_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {699; NOB16B16-LABEL: fsub_nxv2bf16:700; NOB16B16: // %bb.0:701; NOB16B16-NEXT: lsl z1.s, z1.s, #16702; NOB16B16-NEXT: lsl z0.s, z0.s, #16703; NOB16B16-NEXT: ptrue p0.d704; NOB16B16-NEXT: fsub z0.s, p0/m, z0.s, z1.s705; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s706; NOB16B16-NEXT: ret707;708; B16B16-LABEL: fsub_nxv2bf16:709; B16B16: // %bb.0:710; B16B16-NEXT: ptrue p0.d711; B16B16-NEXT: bfsub z0.h, p0/m, z0.h, z1.h712; B16B16-NEXT: ret713 %res = fsub <vscale x 2 x bfloat> %a, %b714 ret <vscale x 2 x bfloat> %res715}716 717define <vscale x 4 x bfloat> @fsub_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {718; NOB16B16-LABEL: fsub_nxv4bf16:719; NOB16B16: // %bb.0:720; NOB16B16-NEXT: lsl z1.s, z1.s, #16721; NOB16B16-NEXT: lsl z0.s, z0.s, #16722; NOB16B16-NEXT: ptrue p0.s723; NOB16B16-NEXT: fsub z0.s, z0.s, z1.s724; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s725; NOB16B16-NEXT: ret726;727; B16B16-LABEL: fsub_nxv4bf16:728; B16B16: // %bb.0:729; B16B16-NEXT: ptrue p0.s730; B16B16-NEXT: bfsub z0.h, p0/m, z0.h, z1.h731; B16B16-NEXT: ret732 %res = fsub <vscale x 4 x bfloat> %a, %b733 ret <vscale x 4 x bfloat> %res734}735 736define <vscale x 8 x bfloat> @fsub_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {737; NOB16B16-LABEL: fsub_nxv8bf16:738; NOB16B16: // %bb.0:739; NOB16B16-NEXT: uunpkhi z2.s, z1.h740; NOB16B16-NEXT: uunpkhi z3.s, z0.h741; NOB16B16-NEXT: uunpklo z1.s, z1.h742; NOB16B16-NEXT: uunpklo z0.s, z0.h743; NOB16B16-NEXT: ptrue p0.s744; NOB16B16-NEXT: lsl z2.s, z2.s, #16745; NOB16B16-NEXT: lsl z3.s, z3.s, #16746; NOB16B16-NEXT: lsl z1.s, z1.s, #16747; NOB16B16-NEXT: lsl z0.s, z0.s, #16748; NOB16B16-NEXT: fsub z2.s, z3.s, z2.s749; NOB16B16-NEXT: fsub z0.s, z0.s, z1.s750; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s751; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s752; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h753; NOB16B16-NEXT: ret754;755; B16B16-LABEL: fsub_nxv8bf16:756; B16B16: // %bb.0:757; B16B16-NEXT: bfsub z0.h, z0.h, z1.h758; B16B16-NEXT: ret759 %res = fsub <vscale x 8 x bfloat> %a, %b760 ret <vscale x 8 x bfloat> %res761}762 763declare <vscale x 2 x bfloat> @llvm.fabs.nxv2bf16(<vscale x 2 x bfloat>)764declare <vscale x 4 x bfloat> @llvm.fabs.nxv4bf16(<vscale x 4 x bfloat>)765declare <vscale x 8 x bfloat> @llvm.fabs.nxv8bf16(<vscale x 8 x bfloat>)766 767declare <vscale x 2 x bfloat> @llvm.fma.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x bfloat>, <vscale x 2 x bfloat>)768declare <vscale x 4 x bfloat> @llvm.fma.nxv4bf16(<vscale x 4 x bfloat>, <vscale x 4 x bfloat>, <vscale x 4 x bfloat>)769declare <vscale x 8 x bfloat> @llvm.fma.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)770 771declare <vscale x 2 x bfloat> @llvm.maximum.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x bfloat>)772declare <vscale x 4 x bfloat> @llvm.maximum.nxv4bf16(<vscale x 4 x bfloat>, <vscale x 4 x bfloat>)773declare <vscale x 8 x bfloat> @llvm.maximum.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)774 775declare <vscale x 2 x bfloat> @llvm.maxnum.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x bfloat>)776declare <vscale x 4 x bfloat> @llvm.maxnum.nxv4bf16(<vscale x 4 x bfloat>, <vscale x 4 x bfloat>)777declare <vscale x 8 x bfloat> @llvm.maxnum.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)778 779declare <vscale x 2 x bfloat> @llvm.minimum.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x bfloat>)780declare <vscale x 4 x bfloat> @llvm.minimum.nxv4bf16(<vscale x 4 x bfloat>, <vscale x 4 x bfloat>)781declare <vscale x 8 x bfloat> @llvm.minimum.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)782 783declare <vscale x 2 x bfloat> @llvm.minnum.nxv2bf16(<vscale x 2 x bfloat>, <vscale x 2 x bfloat>)784declare <vscale x 4 x bfloat> @llvm.minnum.nxv4bf16(<vscale x 4 x bfloat>, <vscale x 4 x bfloat>)785declare <vscale x 8 x bfloat> @llvm.minnum.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>)786 787declare <vscale x 2 x bfloat> @llvm.sqrt.nxv2bf16(<vscale x 2 x bfloat>)788declare <vscale x 4 x bfloat> @llvm.sqrt.nxv4bf16(<vscale x 4 x bfloat>)789declare <vscale x 8 x bfloat> @llvm.sqrt.nxv8bf16(<vscale x 8 x bfloat>)790