360 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -O3 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2563; RUN: llc -O3 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5124; RUN: llc -O3 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5125 6target triple = "aarch64-unknown-linux-gnu"7 8;9; FMA10;11 12; Don't use SVE for 64-bit vectors.13define <4 x half> @fma_v4f16(<4 x half> %op1, <4 x half> %op2, <4 x half> %op3) vscale_range(2,0) #0 {14; CHECK-LABEL: fma_v4f16:15; CHECK: // %bb.0:16; CHECK-NEXT: fmla v2.4h, v0.4h, v1.4h17; CHECK-NEXT: fmov d0, d218; CHECK-NEXT: ret19 %mul = fmul contract <4 x half> %op1, %op220 %res = fadd contract <4 x half> %mul, %op321 ret <4 x half> %res22}23 24; Don't use SVE for 128-bit vectors.25define <8 x half> @fma_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3) vscale_range(2,0) #0 {26; CHECK-LABEL: fma_v8f16:27; CHECK: // %bb.0:28; CHECK-NEXT: fmla v2.8h, v0.8h, v1.8h29; CHECK-NEXT: mov v0.16b, v2.16b30; CHECK-NEXT: ret31 %mul = fmul contract <8 x half> %op1, %op232 %res = fadd contract <8 x half> %mul, %op333 ret <8 x half> %res34}35 36define void @fma_v16f16(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 {37; CHECK-LABEL: fma_v16f16:38; CHECK: // %bb.0:39; CHECK-NEXT: ptrue p0.h, vl1640; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]41; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]42; CHECK-NEXT: ld1h { z2.h }, p0/z, [x2]43; CHECK-NEXT: fmad z0.h, p0/m, z1.h, z2.h44; CHECK-NEXT: st1h { z0.h }, p0, [x0]45; CHECK-NEXT: ret46 %op1 = load <16 x half>, ptr %a47 %op2 = load <16 x half>, ptr %b48 %op3 = load <16 x half>, ptr %c49 %mul = fmul contract <16 x half> %op1, %op250 %res = fadd contract <16 x half> %mul, %op351 store <16 x half> %res, ptr %a52 ret void53}54 55define void @fma_v32f16(ptr %a, ptr %b, ptr %c) #0 {56; VBITS_GE_256-LABEL: fma_v32f16:57; VBITS_GE_256: // %bb.0:58; VBITS_GE_256-NEXT: ptrue p0.h, vl1659; VBITS_GE_256-NEXT: mov x8, #16 // =0x1060; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]61; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]62; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x2, x8, lsl #1]63; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x0]64; VBITS_GE_256-NEXT: ld1h { z4.h }, p0/z, [x1]65; VBITS_GE_256-NEXT: ld1h { z5.h }, p0/z, [x2]66; VBITS_GE_256-NEXT: fmad z0.h, p0/m, z1.h, z2.h67; VBITS_GE_256-NEXT: fmad z3.h, p0/m, z4.h, z5.h68; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]69; VBITS_GE_256-NEXT: st1h { z3.h }, p0, [x0]70; VBITS_GE_256-NEXT: ret71;72; VBITS_GE_512-LABEL: fma_v32f16:73; VBITS_GE_512: // %bb.0:74; VBITS_GE_512-NEXT: ptrue p0.h, vl3275; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]76; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]77; VBITS_GE_512-NEXT: ld1h { z2.h }, p0/z, [x2]78; VBITS_GE_512-NEXT: fmad z0.h, p0/m, z1.h, z2.h79; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]80; VBITS_GE_512-NEXT: ret81 %op1 = load <32 x half>, ptr %a82 %op2 = load <32 x half>, ptr %b83 %op3 = load <32 x half>, ptr %c84 %mul = fmul contract <32 x half> %op1, %op285 %res = fadd contract <32 x half> %mul, %op386 store <32 x half> %res, ptr %a87 ret void88}89 90define void @fma_v64f16(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 {91; CHECK-LABEL: fma_v64f16:92; CHECK: // %bb.0:93; CHECK-NEXT: ptrue p0.h, vl6494; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]95; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]96; CHECK-NEXT: ld1h { z2.h }, p0/z, [x2]97; CHECK-NEXT: fmad z0.h, p0/m, z1.h, z2.h98; CHECK-NEXT: st1h { z0.h }, p0, [x0]99; CHECK-NEXT: ret100 %op1 = load <64 x half>, ptr %a101 %op2 = load <64 x half>, ptr %b102 %op3 = load <64 x half>, ptr %c103 %mul = fmul contract <64 x half> %op1, %op2104 %res = fadd contract <64 x half> %mul, %op3105 store <64 x half> %res, ptr %a106 ret void107}108 109define void @fma_v128f16(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 {110; CHECK-LABEL: fma_v128f16:111; CHECK: // %bb.0:112; CHECK-NEXT: ptrue p0.h, vl128113; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]114; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]115; CHECK-NEXT: ld1h { z2.h }, p0/z, [x2]116; CHECK-NEXT: fmad z0.h, p0/m, z1.h, z2.h117; CHECK-NEXT: st1h { z0.h }, p0, [x0]118; CHECK-NEXT: ret119 %op1 = load <128 x half>, ptr %a120 %op2 = load <128 x half>, ptr %b121 %op3 = load <128 x half>, ptr %c122 %mul = fmul contract <128 x half> %op1, %op2123 %res = fadd contract <128 x half> %mul, %op3124 store <128 x half> %res, ptr %a125 ret void126}127 128; Don't use SVE for 64-bit vectors.129define <2 x float> @fma_v2f32(<2 x float> %op1, <2 x float> %op2, <2 x float> %op3) vscale_range(2,0) #0 {130; CHECK-LABEL: fma_v2f32:131; CHECK: // %bb.0:132; CHECK-NEXT: fmla v2.2s, v0.2s, v1.2s133; CHECK-NEXT: fmov d0, d2134; CHECK-NEXT: ret135 %mul = fmul contract <2 x float> %op1, %op2136 %res = fadd contract <2 x float> %mul, %op3137 ret <2 x float> %res138}139 140; Don't use SVE for 128-bit vectors.141define <4 x float> @fma_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %op3) vscale_range(2,0) #0 {142; CHECK-LABEL: fma_v4f32:143; CHECK: // %bb.0:144; CHECK-NEXT: fmla v2.4s, v0.4s, v1.4s145; CHECK-NEXT: mov v0.16b, v2.16b146; CHECK-NEXT: ret147 %mul = fmul contract <4 x float> %op1, %op2148 %res = fadd contract <4 x float> %mul, %op3149 ret <4 x float> %res150}151 152define void @fma_v8f32(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 {153; CHECK-LABEL: fma_v8f32:154; CHECK: // %bb.0:155; CHECK-NEXT: ptrue p0.s, vl8156; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]157; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]158; CHECK-NEXT: ld1w { z2.s }, p0/z, [x2]159; CHECK-NEXT: fmad z0.s, p0/m, z1.s, z2.s160; CHECK-NEXT: st1w { z0.s }, p0, [x0]161; CHECK-NEXT: ret162 %op1 = load <8 x float>, ptr %a163 %op2 = load <8 x float>, ptr %b164 %op3 = load <8 x float>, ptr %c165 %mul = fmul contract <8 x float> %op1, %op2166 %res = fadd contract <8 x float> %mul, %op3167 store <8 x float> %res, ptr %a168 ret void169}170 171define void @fma_v16f32(ptr %a, ptr %b, ptr %c) #0 {172; VBITS_GE_256-LABEL: fma_v16f32:173; VBITS_GE_256: // %bb.0:174; VBITS_GE_256-NEXT: ptrue p0.s, vl8175; VBITS_GE_256-NEXT: mov x8, #8 // =0x8176; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]177; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]178; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x2, x8, lsl #2]179; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x0]180; VBITS_GE_256-NEXT: ld1w { z4.s }, p0/z, [x1]181; VBITS_GE_256-NEXT: ld1w { z5.s }, p0/z, [x2]182; VBITS_GE_256-NEXT: fmad z0.s, p0/m, z1.s, z2.s183; VBITS_GE_256-NEXT: fmad z3.s, p0/m, z4.s, z5.s184; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]185; VBITS_GE_256-NEXT: st1w { z3.s }, p0, [x0]186; VBITS_GE_256-NEXT: ret187;188; VBITS_GE_512-LABEL: fma_v16f32:189; VBITS_GE_512: // %bb.0:190; VBITS_GE_512-NEXT: ptrue p0.s, vl16191; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]192; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]193; VBITS_GE_512-NEXT: ld1w { z2.s }, p0/z, [x2]194; VBITS_GE_512-NEXT: fmad z0.s, p0/m, z1.s, z2.s195; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]196; VBITS_GE_512-NEXT: ret197 %op1 = load <16 x float>, ptr %a198 %op2 = load <16 x float>, ptr %b199 %op3 = load <16 x float>, ptr %c200 %mul = fmul contract <16 x float> %op1, %op2201 %res = fadd contract <16 x float> %mul, %op3202 store <16 x float> %res, ptr %a203 ret void204}205 206define void @fma_v32f32(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 {207; CHECK-LABEL: fma_v32f32:208; CHECK: // %bb.0:209; CHECK-NEXT: ptrue p0.s, vl32210; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]211; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]212; CHECK-NEXT: ld1w { z2.s }, p0/z, [x2]213; CHECK-NEXT: fmad z0.s, p0/m, z1.s, z2.s214; CHECK-NEXT: st1w { z0.s }, p0, [x0]215; CHECK-NEXT: ret216 %op1 = load <32 x float>, ptr %a217 %op2 = load <32 x float>, ptr %b218 %op3 = load <32 x float>, ptr %c219 %mul = fmul contract <32 x float> %op1, %op2220 %res = fadd contract <32 x float> %mul, %op3221 store <32 x float> %res, ptr %a222 ret void223}224 225define void @fma_v64f32(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 {226; CHECK-LABEL: fma_v64f32:227; CHECK: // %bb.0:228; CHECK-NEXT: ptrue p0.s, vl64229; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]230; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]231; CHECK-NEXT: ld1w { z2.s }, p0/z, [x2]232; CHECK-NEXT: fmad z0.s, p0/m, z1.s, z2.s233; CHECK-NEXT: st1w { z0.s }, p0, [x0]234; CHECK-NEXT: ret235 %op1 = load <64 x float>, ptr %a236 %op2 = load <64 x float>, ptr %b237 %op3 = load <64 x float>, ptr %c238 %mul = fmul contract <64 x float> %op1, %op2239 %res = fadd contract <64 x float> %mul, %op3240 store <64 x float> %res, ptr %a241 ret void242}243 244; Don't use SVE for 64-bit vectors.245define <1 x double> @fma_v1f64(<1 x double> %op1, <1 x double> %op2, <1 x double> %op3) vscale_range(2,0) #0 {246; CHECK-LABEL: fma_v1f64:247; CHECK: // %bb.0:248; CHECK-NEXT: fmadd d0, d0, d1, d2249; CHECK-NEXT: ret250 %mul = fmul contract <1 x double> %op1, %op2251 %res = fadd contract <1 x double> %mul, %op3252 ret <1 x double> %res253}254 255; Don't use SVE for 128-bit vectors.256define <2 x double> @fma_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double> %op3) vscale_range(2,0) #0 {257; CHECK-LABEL: fma_v2f64:258; CHECK: // %bb.0:259; CHECK-NEXT: fmla v2.2d, v0.2d, v1.2d260; CHECK-NEXT: mov v0.16b, v2.16b261; CHECK-NEXT: ret262 %mul = fmul contract <2 x double> %op1, %op2263 %res = fadd contract <2 x double> %mul, %op3264 ret <2 x double> %res265}266 267define void @fma_v4f64(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 {268; CHECK-LABEL: fma_v4f64:269; CHECK: // %bb.0:270; CHECK-NEXT: ptrue p0.d, vl4271; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]272; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]273; CHECK-NEXT: ld1d { z2.d }, p0/z, [x2]274; CHECK-NEXT: fmad z0.d, p0/m, z1.d, z2.d275; CHECK-NEXT: st1d { z0.d }, p0, [x0]276; CHECK-NEXT: ret277 %op1 = load <4 x double>, ptr %a278 %op2 = load <4 x double>, ptr %b279 %op3 = load <4 x double>, ptr %c280 %mul = fmul contract <4 x double> %op1, %op2281 %res = fadd contract <4 x double> %mul, %op3282 store <4 x double> %res, ptr %a283 ret void284}285 286define void @fma_v8f64(ptr %a, ptr %b, ptr %c) #0 {287; VBITS_GE_256-LABEL: fma_v8f64:288; VBITS_GE_256: // %bb.0:289; VBITS_GE_256-NEXT: ptrue p0.d, vl4290; VBITS_GE_256-NEXT: mov x8, #4 // =0x4291; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]292; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]293; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x2, x8, lsl #3]294; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x0]295; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1]296; VBITS_GE_256-NEXT: ld1d { z5.d }, p0/z, [x2]297; VBITS_GE_256-NEXT: fmad z0.d, p0/m, z1.d, z2.d298; VBITS_GE_256-NEXT: fmad z3.d, p0/m, z4.d, z5.d299; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]300; VBITS_GE_256-NEXT: st1d { z3.d }, p0, [x0]301; VBITS_GE_256-NEXT: ret302;303; VBITS_GE_512-LABEL: fma_v8f64:304; VBITS_GE_512: // %bb.0:305; VBITS_GE_512-NEXT: ptrue p0.d, vl8306; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]307; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]308; VBITS_GE_512-NEXT: ld1d { z2.d }, p0/z, [x2]309; VBITS_GE_512-NEXT: fmad z0.d, p0/m, z1.d, z2.d310; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]311; VBITS_GE_512-NEXT: ret312 %op1 = load <8 x double>, ptr %a313 %op2 = load <8 x double>, ptr %b314 %op3 = load <8 x double>, ptr %c315 %mul = fmul contract <8 x double> %op1, %op2316 %res = fadd contract <8 x double> %mul, %op3317 store <8 x double> %res, ptr %a318 ret void319}320 321define void @fma_v16f64(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 {322; CHECK-LABEL: fma_v16f64:323; CHECK: // %bb.0:324; CHECK-NEXT: ptrue p0.d, vl16325; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]326; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]327; CHECK-NEXT: ld1d { z2.d }, p0/z, [x2]328; CHECK-NEXT: fmad z0.d, p0/m, z1.d, z2.d329; CHECK-NEXT: st1d { z0.d }, p0, [x0]330; CHECK-NEXT: ret331 %op1 = load <16 x double>, ptr %a332 %op2 = load <16 x double>, ptr %b333 %op3 = load <16 x double>, ptr %c334 %mul = fmul contract <16 x double> %op1, %op2335 %res = fadd contract <16 x double> %mul, %op3336 store <16 x double> %res, ptr %a337 ret void338}339 340define void @fma_v32f64(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 {341; CHECK-LABEL: fma_v32f64:342; CHECK: // %bb.0:343; CHECK-NEXT: ptrue p0.d, vl32344; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]345; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]346; CHECK-NEXT: ld1d { z2.d }, p0/z, [x2]347; CHECK-NEXT: fmad z0.d, p0/m, z1.d, z2.d348; CHECK-NEXT: st1d { z0.d }, p0, [x0]349; CHECK-NEXT: ret350 %op1 = load <32 x double>, ptr %a351 %op2 = load <32 x double>, ptr %b352 %op3 = load <32 x double>, ptr %c353 %mul = fmul contract <32 x double> %op1, %op2354 %res = fadd contract <32 x double> %mul, %op3355 store <32 x double> %res, ptr %a356 ret void357}358 359attributes #0 = { "target-features"="+sve" }360