277 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 62; RUN: llc -mattr=+sve2 < %s | FileCheck %s --check-prefixes=CHECK,SVE23; RUN: llc -mattr=+sve2p1 < %s | FileCheck %s --check-prefixes=CHECK,SVE2P14 5target triple = "aarch64-linux-gnu"6 7define void @fdot_v4f32(ptr %accptr, ptr %aptr, ptr %bptr) {8; SVE2-LABEL: fdot_v4f32:9; SVE2: // %bb.0: // %entry10; SVE2-NEXT: ldr q0, [x1]11; SVE2-NEXT: ldr q1, [x2]12; SVE2-NEXT: fcvtl v2.4s, v0.4h13; SVE2-NEXT: fcvtl v3.4s, v1.4h14; SVE2-NEXT: fcvtl2 v0.4s, v0.8h15; SVE2-NEXT: fcvtl2 v1.4s, v1.8h16; SVE2-NEXT: fmul v2.4s, v2.4s, v3.4s17; SVE2-NEXT: ldr q3, [x0]18; SVE2-NEXT: fmul v0.4s, v0.4s, v1.4s19; SVE2-NEXT: fadd v1.4s, v3.4s, v2.4s20; SVE2-NEXT: fadd v0.4s, v1.4s, v0.4s21; SVE2-NEXT: str q0, [x0]22; SVE2-NEXT: ret23;24; SVE2P1-LABEL: fdot_v4f32:25; SVE2P1: // %bb.0: // %entry26; SVE2P1-NEXT: ldr q0, [x0]27; SVE2P1-NEXT: ldr q1, [x1]28; SVE2P1-NEXT: ldr q2, [x2]29; SVE2P1-NEXT: fdot z0.s, z1.h, z2.h30; SVE2P1-NEXT: str q0, [x0]31; SVE2P1-NEXT: ret32entry:33 %acc = load <4 x float>, ptr %accptr34 %a = load <8 x half>, ptr %aptr35 %b = load <8 x half>, ptr %bptr36 %a.wide = fpext <8 x half> %a to <8 x float>37 %b.wide = fpext <8 x half> %b to <8 x float>38 %mult = fmul <8 x float> %a.wide, %b.wide39 %partial.reduce = call <4 x float> @llvm.vector.partial.reduce.fadd(<4 x float> %acc, <8 x float> %mult)40 store <4 x float> %partial.reduce, ptr %accptr41 ret void42}43 44define void @fdot_wide_v8f32(ptr %accptr, ptr %aptr, ptr %bptr) vscale_range(2,0) {45; SVE2-LABEL: fdot_wide_v8f32:46; SVE2: // %bb.0: // %entry47; SVE2-NEXT: ptrue p0.s, vl848; SVE2-NEXT: mov x8, #8 // =0x849; SVE2-NEXT: ld1h { z0.s }, p0/z, [x1]50; SVE2-NEXT: ld1h { z1.s }, p0/z, [x2]51; SVE2-NEXT: ld1h { z2.s }, p0/z, [x1, x8, lsl #1]52; SVE2-NEXT: ld1h { z3.s }, p0/z, [x2, x8, lsl #1]53; SVE2-NEXT: fcvt z0.s, p0/m, z0.h54; SVE2-NEXT: fcvt z1.s, p0/m, z1.h55; SVE2-NEXT: fcvt z2.s, p0/m, z2.h56; SVE2-NEXT: fcvt z3.s, p0/m, z3.h57; SVE2-NEXT: fmul z0.s, p0/m, z0.s, z1.s58; SVE2-NEXT: ld1w { z1.s }, p0/z, [x0]59; SVE2-NEXT: fmul z2.s, p0/m, z2.s, z3.s60; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z1.s61; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z2.s62; SVE2-NEXT: st1w { z0.s }, p0, [x0]63; SVE2-NEXT: ret64;65; SVE2P1-LABEL: fdot_wide_v8f32:66; SVE2P1: // %bb.0: // %entry67; SVE2P1-NEXT: ptrue p0.s, vl868; SVE2P1-NEXT: ptrue p1.h, vl1669; SVE2P1-NEXT: ld1w { z0.s }, p0/z, [x0]70; SVE2P1-NEXT: ld1h { z1.h }, p1/z, [x1]71; SVE2P1-NEXT: ld1h { z2.h }, p1/z, [x2]72; SVE2P1-NEXT: fdot z0.s, z1.h, z2.h73; SVE2P1-NEXT: st1w { z0.s }, p0, [x0]74; SVE2P1-NEXT: ret75entry:76 %acc = load <8 x float>, ptr %accptr77 %a = load <16 x half>, ptr %aptr78 %b = load <16 x half>, ptr %bptr79 %a.wide = fpext <16 x half> %a to <16 x float>80 %b.wide = fpext <16 x half> %b to <16 x float>81 %mult = fmul <16 x float> %a.wide, %b.wide82 %partial.reduce = call <8 x float> @llvm.vector.partial.reduce.fadd(<8 x float> %acc, <16 x float> %mult)83 store <8 x float> %partial.reduce, ptr %accptr84 ret void85}86 87define void @fdot_wide_v16f32(ptr %accptr, ptr %aptr, ptr %bptr) vscale_range(4,0) {88; SVE2-LABEL: fdot_wide_v16f32:89; SVE2: // %bb.0: // %entry90; SVE2-NEXT: ptrue p0.s, vl1691; SVE2-NEXT: mov x8, #16 // =0x1092; SVE2-NEXT: ld1h { z0.s }, p0/z, [x1]93; SVE2-NEXT: ld1h { z1.s }, p0/z, [x2]94; SVE2-NEXT: ld1h { z2.s }, p0/z, [x1, x8, lsl #1]95; SVE2-NEXT: ld1h { z3.s }, p0/z, [x2, x8, lsl #1]96; SVE2-NEXT: fcvt z0.s, p0/m, z0.h97; SVE2-NEXT: fcvt z1.s, p0/m, z1.h98; SVE2-NEXT: fcvt z2.s, p0/m, z2.h99; SVE2-NEXT: fcvt z3.s, p0/m, z3.h100; SVE2-NEXT: fmul z0.s, p0/m, z0.s, z1.s101; SVE2-NEXT: ld1w { z1.s }, p0/z, [x0]102; SVE2-NEXT: fmul z2.s, p0/m, z2.s, z3.s103; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z1.s104; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z2.s105; SVE2-NEXT: st1w { z0.s }, p0, [x0]106; SVE2-NEXT: ret107;108; SVE2P1-LABEL: fdot_wide_v16f32:109; SVE2P1: // %bb.0: // %entry110; SVE2P1-NEXT: ptrue p0.s, vl16111; SVE2P1-NEXT: ptrue p1.h, vl32112; SVE2P1-NEXT: ld1w { z0.s }, p0/z, [x0]113; SVE2P1-NEXT: ld1h { z1.h }, p1/z, [x1]114; SVE2P1-NEXT: ld1h { z2.h }, p1/z, [x2]115; SVE2P1-NEXT: fdot z0.s, z1.h, z2.h116; SVE2P1-NEXT: st1w { z0.s }, p0, [x0]117; SVE2P1-NEXT: ret118entry:119 %acc = load <16 x float>, ptr %accptr120 %a = load <32 x half>, ptr %aptr121 %b = load <32 x half>, ptr %bptr122 %a.wide = fpext <32 x half> %a to <32 x float>123 %b.wide = fpext <32 x half> %b to <32 x float>124 %mult = fmul <32 x float> %a.wide, %b.wide125 %partial.reduce = call <16 x float> @llvm.vector.partial.reduce.fadd(<16 x float> %acc, <32 x float> %mult)126 store <16 x float> %partial.reduce, ptr %accptr127 ret void128}129 130define void @fdot_wide_v32f32(ptr %accptr, ptr %aptr, ptr %bptr) vscale_range(8,0) {131; SVE2-LABEL: fdot_wide_v32f32:132; SVE2: // %bb.0: // %entry133; SVE2-NEXT: ptrue p0.s, vl32134; SVE2-NEXT: mov x8, #32 // =0x20135; SVE2-NEXT: ld1h { z0.s }, p0/z, [x1]136; SVE2-NEXT: ld1h { z1.s }, p0/z, [x2]137; SVE2-NEXT: ld1h { z2.s }, p0/z, [x1, x8, lsl #1]138; SVE2-NEXT: ld1h { z3.s }, p0/z, [x2, x8, lsl #1]139; SVE2-NEXT: fcvt z0.s, p0/m, z0.h140; SVE2-NEXT: fcvt z1.s, p0/m, z1.h141; SVE2-NEXT: fcvt z2.s, p0/m, z2.h142; SVE2-NEXT: fcvt z3.s, p0/m, z3.h143; SVE2-NEXT: fmul z0.s, p0/m, z0.s, z1.s144; SVE2-NEXT: ld1w { z1.s }, p0/z, [x0]145; SVE2-NEXT: fmul z2.s, p0/m, z2.s, z3.s146; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z1.s147; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z2.s148; SVE2-NEXT: st1w { z0.s }, p0, [x0]149; SVE2-NEXT: ret150;151; SVE2P1-LABEL: fdot_wide_v32f32:152; SVE2P1: // %bb.0: // %entry153; SVE2P1-NEXT: ptrue p0.s, vl32154; SVE2P1-NEXT: ptrue p1.h, vl64155; SVE2P1-NEXT: ld1w { z0.s }, p0/z, [x0]156; SVE2P1-NEXT: ld1h { z1.h }, p1/z, [x1]157; SVE2P1-NEXT: ld1h { z2.h }, p1/z, [x2]158; SVE2P1-NEXT: fdot z0.s, z1.h, z2.h159; SVE2P1-NEXT: st1w { z0.s }, p0, [x0]160; SVE2P1-NEXT: ret161entry:162 %acc = load <32 x float>, ptr %accptr163 %a = load <64 x half>, ptr %aptr164 %b = load <64 x half>, ptr %bptr165 %a.wide = fpext <64 x half> %a to <64 x float>166 %b.wide = fpext <64 x half> %b to <64 x float>167 %mult = fmul <64 x float> %a.wide, %b.wide168 %partial.reduce = call <32 x float> @llvm.vector.partial.reduce.fadd(<32 x float> %acc, <64 x float> %mult)169 store <32 x float> %partial.reduce, ptr %accptr170 ret void171}172 173define void @fdot_wide_v64f32(ptr %accptr, ptr %aptr, ptr %bptr) vscale_range(16,0) {174; SVE2-LABEL: fdot_wide_v64f32:175; SVE2: // %bb.0: // %entry176; SVE2-NEXT: ptrue p0.s, vl64177; SVE2-NEXT: mov x8, #64 // =0x40178; SVE2-NEXT: ld1h { z0.s }, p0/z, [x1]179; SVE2-NEXT: ld1h { z1.s }, p0/z, [x2]180; SVE2-NEXT: ld1h { z2.s }, p0/z, [x1, x8, lsl #1]181; SVE2-NEXT: ld1h { z3.s }, p0/z, [x2, x8, lsl #1]182; SVE2-NEXT: fcvt z0.s, p0/m, z0.h183; SVE2-NEXT: fcvt z1.s, p0/m, z1.h184; SVE2-NEXT: fcvt z2.s, p0/m, z2.h185; SVE2-NEXT: fcvt z3.s, p0/m, z3.h186; SVE2-NEXT: fmul z0.s, p0/m, z0.s, z1.s187; SVE2-NEXT: ld1w { z1.s }, p0/z, [x0]188; SVE2-NEXT: fmul z2.s, p0/m, z2.s, z3.s189; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z1.s190; SVE2-NEXT: fadd z0.s, p0/m, z0.s, z2.s191; SVE2-NEXT: st1w { z0.s }, p0, [x0]192; SVE2-NEXT: ret193;194; SVE2P1-LABEL: fdot_wide_v64f32:195; SVE2P1: // %bb.0: // %entry196; SVE2P1-NEXT: ptrue p0.s, vl64197; SVE2P1-NEXT: ptrue p1.h, vl128198; SVE2P1-NEXT: ld1w { z0.s }, p0/z, [x0]199; SVE2P1-NEXT: ld1h { z1.h }, p1/z, [x1]200; SVE2P1-NEXT: ld1h { z2.h }, p1/z, [x2]201; SVE2P1-NEXT: fdot z0.s, z1.h, z2.h202; SVE2P1-NEXT: st1w { z0.s }, p0, [x0]203; SVE2P1-NEXT: ret204entry:205 %acc = load <64 x float>, ptr %accptr206 %a = load <128 x half>, ptr %aptr207 %b = load <128 x half>, ptr %bptr208 %a.wide = fpext <128 x half> %a to <128 x float>209 %b.wide = fpext <128 x half> %b to <128 x float>210 %mult = fmul <128 x float> %a.wide, %b.wide211 %partial.reduce = call <64 x float> @llvm.vector.partial.reduce.fadd(<64 x float> %acc, <128 x float> %mult)212 store <64 x float> %partial.reduce, ptr %accptr213 ret void214}215 216define <4 x float> @fixed_fdot_wide(<4 x float> %acc, <8 x half> %a, <8 x half> %b) {217; SVE2-LABEL: fixed_fdot_wide:218; SVE2: // %bb.0: // %entry219; SVE2-NEXT: fcvtl v3.4s, v1.4h220; SVE2-NEXT: fcvtl v4.4s, v2.4h221; SVE2-NEXT: fcvtl2 v1.4s, v1.8h222; SVE2-NEXT: fcvtl2 v2.4s, v2.8h223; SVE2-NEXT: fmul v3.4s, v3.4s, v4.4s224; SVE2-NEXT: fmul v1.4s, v1.4s, v2.4s225; SVE2-NEXT: fadd v0.4s, v0.4s, v3.4s226; SVE2-NEXT: fadd v0.4s, v0.4s, v1.4s227; SVE2-NEXT: ret228;229; SVE2P1-LABEL: fixed_fdot_wide:230; SVE2P1: // %bb.0: // %entry231; SVE2P1-NEXT: // kill: def $q0 killed $q0 def $z0232; SVE2P1-NEXT: // kill: def $q2 killed $q2 def $z2233; SVE2P1-NEXT: // kill: def $q1 killed $q1 def $z1234; SVE2P1-NEXT: fdot z0.s, z1.h, z2.h235; SVE2P1-NEXT: // kill: def $q0 killed $q0 killed $z0236; SVE2P1-NEXT: ret237entry:238 %a.wide = fpext <8 x half> %a to <8 x float>239 %b.wide = fpext <8 x half> %b to <8 x float>240 %mult = fmul <8 x float> %a.wide, %b.wide241 %partial.reduce = call <4 x float> @llvm.vector.partial.reduce.fadd(<4 x float> %acc, <8 x float> %mult)242 ret <4 x float> %partial.reduce243}244 245define <8 x half> @partial_reduce_half(<8 x half> %acc, <16 x half> %a) {246; CHECK-LABEL: partial_reduce_half:247; CHECK: // %bb.0: // %entry248; CHECK-NEXT: fadd v0.8h, v0.8h, v1.8h249; CHECK-NEXT: fadd v0.8h, v0.8h, v2.8h250; CHECK-NEXT: ret251entry:252 %partial.reduce = call <8 x half> @llvm.vector.partial.reduce.fadd(<8 x half> %acc, <16 x half> %a)253 ret <8 x half> %partial.reduce254}255 256define <4 x float> @partial_reduce_float(<4 x float> %acc, <8 x float> %a) {257; CHECK-LABEL: partial_reduce_float:258; CHECK: // %bb.0: // %entry259; CHECK-NEXT: fadd v0.4s, v0.4s, v1.4s260; CHECK-NEXT: fadd v0.4s, v0.4s, v2.4s261; CHECK-NEXT: ret262entry:263 %partial.reduce = call <4 x float> @llvm.vector.partial.reduce.fadd(<4 x float> %acc, <8 x float> %a)264 ret <4 x float> %partial.reduce265}266 267define <2 x double> @partial_reduce_double(<2 x double> %acc, <4 x double> %a) {268; CHECK-LABEL: partial_reduce_double:269; CHECK: // %bb.0: // %entry270; CHECK-NEXT: fadd v0.2d, v0.2d, v1.2d271; CHECK-NEXT: fadd v0.2d, v0.2d, v2.2d272; CHECK-NEXT: ret273entry:274 %partial.reduce = call <2 x double> @llvm.vector.partial.reduce.fadd(<2 x double> %acc, <4 x double> %a)275 ret <2 x double> %partial.reduce276}277