280 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s -check-prefixes=CHECK,SVE3; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s -check-prefixes=CHECK,SME4 5target triple = "aarch64-unknown-linux-gnu"6 7; FADDV8 9define bfloat @faddv_nxv2bf16(<vscale x 2 x bfloat> %a) {10; CHECK-LABEL: faddv_nxv2bf16:11; CHECK: // %bb.0:12; CHECK-NEXT: lsl z0.s, z0.s, #1613; CHECK-NEXT: ptrue p0.d14; CHECK-NEXT: faddv s0, p0, z0.s15; CHECK-NEXT: bfcvt h0, s016; CHECK-NEXT: ret17 %res = call fast bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat zeroinitializer, <vscale x 2 x bfloat> %a)18 ret bfloat %res19}20 21define bfloat @faddv_nxv4bf16(<vscale x 4 x bfloat> %a) {22; CHECK-LABEL: faddv_nxv4bf16:23; CHECK: // %bb.0:24; CHECK-NEXT: lsl z0.s, z0.s, #1625; CHECK-NEXT: ptrue p0.s26; CHECK-NEXT: faddv s0, p0, z0.s27; CHECK-NEXT: bfcvt h0, s028; CHECK-NEXT: ret29 %res = call fast bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat zeroinitializer, <vscale x 4 x bfloat> %a)30 ret bfloat %res31}32 33define bfloat @faddv_nxv8bf16(<vscale x 8 x bfloat> %a) {34; CHECK-LABEL: faddv_nxv8bf16:35; CHECK: // %bb.0:36; CHECK-NEXT: uunpkhi z1.s, z0.h37; CHECK-NEXT: uunpklo z0.s, z0.h38; CHECK-NEXT: ptrue p0.s39; CHECK-NEXT: lsl z1.s, z1.s, #1640; CHECK-NEXT: lsl z0.s, z0.s, #1641; CHECK-NEXT: fadd z0.s, z0.s, z1.s42; CHECK-NEXT: faddv s0, p0, z0.s43; CHECK-NEXT: bfcvt h0, s044; CHECK-NEXT: ret45 %res = call fast bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat zeroinitializer, <vscale x 8 x bfloat> %a)46 ret bfloat %res47}48 49; FMAXNMV50 51define bfloat @fmaxv_nxv2bf16(<vscale x 2 x bfloat> %a) {52; CHECK-LABEL: fmaxv_nxv2bf16:53; CHECK: // %bb.0:54; CHECK-NEXT: lsl z0.s, z0.s, #1655; CHECK-NEXT: ptrue p0.d56; CHECK-NEXT: fmaxnmv s0, p0, z0.s57; CHECK-NEXT: bfcvt h0, s058; CHECK-NEXT: ret59 %res = call bfloat @llvm.vector.reduce.fmax.nxv2bf16(<vscale x 2 x bfloat> %a)60 ret bfloat %res61}62 63define bfloat @fmaxv_nxv4bf16(<vscale x 4 x bfloat> %a) {64; CHECK-LABEL: fmaxv_nxv4bf16:65; CHECK: // %bb.0:66; CHECK-NEXT: lsl z0.s, z0.s, #1667; CHECK-NEXT: ptrue p0.s68; CHECK-NEXT: fmaxnmv s0, p0, z0.s69; CHECK-NEXT: bfcvt h0, s070; CHECK-NEXT: ret71 %res = call bfloat @llvm.vector.reduce.fmax.nxv4bf16(<vscale x 4 x bfloat> %a)72 ret bfloat %res73}74 75define bfloat @fmaxv_nxv8bf16(<vscale x 8 x bfloat> %a) {76; CHECK-LABEL: fmaxv_nxv8bf16:77; CHECK: // %bb.0:78; CHECK-NEXT: uunpkhi z1.s, z0.h79; CHECK-NEXT: uunpklo z0.s, z0.h80; CHECK-NEXT: ptrue p0.s81; CHECK-NEXT: lsl z1.s, z1.s, #1682; CHECK-NEXT: lsl z0.s, z0.s, #1683; CHECK-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s84; CHECK-NEXT: fmaxnmv s0, p0, z0.s85; CHECK-NEXT: bfcvt h0, s086; CHECK-NEXT: ret87 %res = call bfloat @llvm.vector.reduce.fmax.nxv8bf16(<vscale x 8 x bfloat> %a)88 ret bfloat %res89}90 91; FMINNMV92 93define bfloat @fminv_nxv2bf16(<vscale x 2 x bfloat> %a) {94; CHECK-LABEL: fminv_nxv2bf16:95; CHECK: // %bb.0:96; CHECK-NEXT: lsl z0.s, z0.s, #1697; CHECK-NEXT: ptrue p0.d98; CHECK-NEXT: fminnmv s0, p0, z0.s99; CHECK-NEXT: bfcvt h0, s0100; CHECK-NEXT: ret101 %res = call bfloat @llvm.vector.reduce.fmin.nxv2bf16(<vscale x 2 x bfloat> %a)102 ret bfloat %res103}104 105define bfloat @fminv_nxv4bf16(<vscale x 4 x bfloat> %a) {106; CHECK-LABEL: fminv_nxv4bf16:107; CHECK: // %bb.0:108; CHECK-NEXT: lsl z0.s, z0.s, #16109; CHECK-NEXT: ptrue p0.s110; CHECK-NEXT: fminnmv s0, p0, z0.s111; CHECK-NEXT: bfcvt h0, s0112; CHECK-NEXT: ret113 %res = call bfloat @llvm.vector.reduce.fmin.nxv4bf16(<vscale x 4 x bfloat> %a)114 ret bfloat %res115}116 117define bfloat @fminv_nxv8bf16(<vscale x 8 x bfloat> %a) {118; CHECK-LABEL: fminv_nxv8bf16:119; CHECK: // %bb.0:120; CHECK-NEXT: uunpkhi z1.s, z0.h121; CHECK-NEXT: uunpklo z0.s, z0.h122; CHECK-NEXT: ptrue p0.s123; CHECK-NEXT: lsl z1.s, z1.s, #16124; CHECK-NEXT: lsl z0.s, z0.s, #16125; CHECK-NEXT: fminnm z0.s, p0/m, z0.s, z1.s126; CHECK-NEXT: fminnmv s0, p0, z0.s127; CHECK-NEXT: bfcvt h0, s0128; CHECK-NEXT: ret129 %res = call bfloat @llvm.vector.reduce.fmin.nxv8bf16(<vscale x 8 x bfloat> %a)130 ret bfloat %res131}132 133; FMAXV134 135define bfloat @fmaximumv_nxv2bf16(<vscale x 2 x bfloat> %a) {136; CHECK-LABEL: fmaximumv_nxv2bf16:137; CHECK: // %bb.0:138; CHECK-NEXT: lsl z0.s, z0.s, #16139; CHECK-NEXT: ptrue p0.d140; CHECK-NEXT: fmaxv s0, p0, z0.s141; CHECK-NEXT: bfcvt h0, s0142; CHECK-NEXT: ret143 %res = call bfloat @llvm.vector.reduce.fmaximum.nxv2bf16(<vscale x 2 x bfloat> %a)144 ret bfloat %res145}146 147define bfloat @fmaximumv_nxv4bf16(<vscale x 4 x bfloat> %a) {148; CHECK-LABEL: fmaximumv_nxv4bf16:149; CHECK: // %bb.0:150; CHECK-NEXT: lsl z0.s, z0.s, #16151; CHECK-NEXT: ptrue p0.s152; CHECK-NEXT: fmaxv s0, p0, z0.s153; CHECK-NEXT: bfcvt h0, s0154; CHECK-NEXT: ret155 %res = call bfloat @llvm.vector.reduce.fmaximum.nxv4bf16(<vscale x 4 x bfloat> %a)156 ret bfloat %res157}158 159define bfloat @fmaximumv_nxv8bf16(<vscale x 8 x bfloat> %a) {160; CHECK-LABEL: fmaximumv_nxv8bf16:161; CHECK: // %bb.0:162; CHECK-NEXT: uunpkhi z1.s, z0.h163; CHECK-NEXT: uunpklo z0.s, z0.h164; CHECK-NEXT: ptrue p0.s165; CHECK-NEXT: lsl z1.s, z1.s, #16166; CHECK-NEXT: lsl z0.s, z0.s, #16167; CHECK-NEXT: fmax z0.s, p0/m, z0.s, z1.s168; CHECK-NEXT: fmaxv s0, p0, z0.s169; CHECK-NEXT: bfcvt h0, s0170; CHECK-NEXT: ret171 %res = call bfloat @llvm.vector.reduce.fmaximum.nxv8bf16(<vscale x 8 x bfloat> %a)172 ret bfloat %res173}174 175; FMINV176 177define bfloat @fminimumv_nxv2bf16(<vscale x 2 x bfloat> %a) {178; CHECK-LABEL: fminimumv_nxv2bf16:179; CHECK: // %bb.0:180; CHECK-NEXT: lsl z0.s, z0.s, #16181; CHECK-NEXT: ptrue p0.d182; CHECK-NEXT: fminv s0, p0, z0.s183; CHECK-NEXT: bfcvt h0, s0184; CHECK-NEXT: ret185 %res = call bfloat @llvm.vector.reduce.fminimum.nxv2bf16(<vscale x 2 x bfloat> %a)186 ret bfloat %res187}188 189define bfloat @fminimumv_nxv4bf16(<vscale x 4 x bfloat> %a) {190; CHECK-LABEL: fminimumv_nxv4bf16:191; CHECK: // %bb.0:192; CHECK-NEXT: lsl z0.s, z0.s, #16193; CHECK-NEXT: ptrue p0.s194; CHECK-NEXT: fminv s0, p0, z0.s195; CHECK-NEXT: bfcvt h0, s0196; CHECK-NEXT: ret197 %res = call bfloat @llvm.vector.reduce.fminimum.nxv4bf16(<vscale x 4 x bfloat> %a)198 ret bfloat %res199}200 201define bfloat @fminimumv_nxv8bf16(<vscale x 8 x bfloat> %a) {202; CHECK-LABEL: fminimumv_nxv8bf16:203; CHECK: // %bb.0:204; CHECK-NEXT: uunpkhi z1.s, z0.h205; CHECK-NEXT: uunpklo z0.s, z0.h206; CHECK-NEXT: ptrue p0.s207; CHECK-NEXT: lsl z1.s, z1.s, #16208; CHECK-NEXT: lsl z0.s, z0.s, #16209; CHECK-NEXT: fmin z0.s, p0/m, z0.s, z1.s210; CHECK-NEXT: fminv s0, p0, z0.s211; CHECK-NEXT: bfcvt h0, s0212; CHECK-NEXT: ret213 %res = call bfloat @llvm.vector.reduce.fminimum.nxv8bf16(<vscale x 8 x bfloat> %a)214 ret bfloat %res215}216 217; The reduction is performed at a higher precision. Because add operations218; can utilise that precision, its result must be rounded even if it's then219; promoted.220define float @promoted_fadd(<vscale x 4 x bfloat> %a) {221; SVE-LABEL: promoted_fadd:222; SVE: // %bb.0:223; SVE-NEXT: lsl z0.s, z0.s, #16224; SVE-NEXT: ptrue p0.s225; SVE-NEXT: faddv s0, p0, z0.s226; SVE-NEXT: bfcvt h0, s0227; SVE-NEXT: shll v0.4s, v0.4h, #16228; SVE-NEXT: // kill: def $s0 killed $s0 killed $q0229; SVE-NEXT: ret230;231; SME-LABEL: promoted_fadd:232; SME: // %bb.0:233; SME-NEXT: lsl z0.s, z0.s, #16234; SME-NEXT: ptrue p0.s235; SME-NEXT: faddv s0, p0, z0.s236; SME-NEXT: bfcvt h0, s0237; SME-NEXT: fmov w8, s0238; SME-NEXT: lsl w8, w8, #16239; SME-NEXT: fmov s0, w8240; SME-NEXT: ret241 %rdx = call fast bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat zeroinitializer, <vscale x 4 x bfloat> %a)242 %res = fpext bfloat %rdx to float243 ret float %res244}245 246; The reduction is performed at a higher precision. Because min/max operations247; don't utilise that precision, its result can be used directly.248define float @promoted_fmax(<vscale x 4 x bfloat> %a) {249; CHECK-LABEL: promoted_fmax:250; CHECK: // %bb.0:251; CHECK-NEXT: lsl z0.s, z0.s, #16252; CHECK-NEXT: ptrue p0.s253; CHECK-NEXT: fmaxnmv s0, p0, z0.s254; CHECK-NEXT: // kill: def $s0 killed $s0 killed $z0255; CHECK-NEXT: ret256 %rdx = call bfloat @llvm.vector.reduce.fmax.nxv4bf16(<vscale x 4 x bfloat> %a)257 %res = fpext bfloat %rdx to float258 ret float %res259}260 261declare bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat, <vscale x 2 x bfloat>)262declare bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat, <vscale x 4 x bfloat>)263declare bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat, <vscale x 8 x bfloat>)264 265declare bfloat @llvm.vector.reduce.fmax.nxv2bf16(<vscale x 2 x bfloat>)266declare bfloat @llvm.vector.reduce.fmax.nxv4bf16(<vscale x 4 x bfloat>)267declare bfloat @llvm.vector.reduce.fmax.nxv8bf16(<vscale x 8 x bfloat>)268 269declare bfloat @llvm.vector.reduce.fmin.nxv2bf16(<vscale x 2 x bfloat>)270declare bfloat @llvm.vector.reduce.fmin.nxv4bf16(<vscale x 4 x bfloat>)271declare bfloat @llvm.vector.reduce.fmin.nxv8bf16(<vscale x 8 x bfloat>)272 273declare bfloat @llvm.vector.reduce.fmaximum.nxv2bf16(<vscale x 2 x bfloat>)274declare bfloat @llvm.vector.reduce.fmaximum.nxv4bf16(<vscale x 4 x bfloat>)275declare bfloat @llvm.vector.reduce.fmaximum.nxv8bf16(<vscale x 8 x bfloat>)276 277declare bfloat @llvm.vector.reduce.fminimum.nxv2bf16(<vscale x 2 x bfloat>)278declare bfloat @llvm.vector.reduce.fminimum.nxv4bf16(<vscale x 4 x bfloat>)279declare bfloat @llvm.vector.reduce.fminimum.nxv8bf16(<vscale x 8 x bfloat>)280