1607 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 22; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP163; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP164; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP165; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP166 7define double @fma_f64(double %a, double %b, double %c) {8; CHECK-LABEL: fma_f64:9; CHECK: // %bb.0: // %entry10; CHECK-NEXT: fmadd d0, d0, d1, d211; CHECK-NEXT: ret12entry:13 %d = call double @llvm.fma.f64(double %a, double %b, double %c)14 ret double %d15}16 17define float @fma_f32(float %a, float %b, float %c) {18; CHECK-LABEL: fma_f32:19; CHECK: // %bb.0: // %entry20; CHECK-NEXT: fmadd s0, s0, s1, s221; CHECK-NEXT: ret22entry:23 %d = call float @llvm.fma.f32(float %a, float %b, float %c)24 ret float %d25}26 27define half @fma_f16(half %a, half %b, half %c) {28; CHECK-SD-NOFP16-LABEL: fma_f16:29; CHECK-SD-NOFP16: // %bb.0: // %entry30; CHECK-SD-NOFP16-NEXT: fcvt s2, h231; CHECK-SD-NOFP16-NEXT: fcvt s1, h132; CHECK-SD-NOFP16-NEXT: fcvt s0, h033; CHECK-SD-NOFP16-NEXT: fmadd s0, s0, s1, s234; CHECK-SD-NOFP16-NEXT: fcvt h0, s035; CHECK-SD-NOFP16-NEXT: ret36;37; CHECK-SD-FP16-LABEL: fma_f16:38; CHECK-SD-FP16: // %bb.0: // %entry39; CHECK-SD-FP16-NEXT: fmadd h0, h0, h1, h240; CHECK-SD-FP16-NEXT: ret41;42; CHECK-GI-NOFP16-LABEL: fma_f16:43; CHECK-GI-NOFP16: // %bb.0: // %entry44; CHECK-GI-NOFP16-NEXT: fcvt s0, h045; CHECK-GI-NOFP16-NEXT: fcvt s1, h146; CHECK-GI-NOFP16-NEXT: fcvt s2, h247; CHECK-GI-NOFP16-NEXT: fmadd s0, s0, s1, s248; CHECK-GI-NOFP16-NEXT: fcvt h0, s049; CHECK-GI-NOFP16-NEXT: ret50;51; CHECK-GI-FP16-LABEL: fma_f16:52; CHECK-GI-FP16: // %bb.0: // %entry53; CHECK-GI-FP16-NEXT: fmadd h0, h0, h1, h254; CHECK-GI-FP16-NEXT: ret55entry:56 %d = call half @llvm.fma.f16(half %a, half %b, half %c)57 ret half %d58}59 60define <2 x double> @fma_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c) {61; CHECK-LABEL: fma_v2f64:62; CHECK: // %bb.0: // %entry63; CHECK-NEXT: fmla v2.2d, v1.2d, v0.2d64; CHECK-NEXT: mov v0.16b, v2.16b65; CHECK-NEXT: ret66entry:67 %d = call <2 x double> @llvm.fma.v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c)68 ret <2 x double> %d69}70 71define <3 x double> @fma_v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c) {72; CHECK-SD-LABEL: fma_v3f64:73; CHECK-SD: // %bb.0: // %entry74; CHECK-SD-NEXT: // kill: def $d6 killed $d6 def $q675; CHECK-SD-NEXT: // kill: def $d3 killed $d3 def $q376; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q077; CHECK-SD-NEXT: // kill: def $d7 killed $d7 def $q778; CHECK-SD-NEXT: // kill: def $d4 killed $d4 def $q479; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q180; CHECK-SD-NEXT: // kill: def $d5 killed $d5 def $q581; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q282; CHECK-SD-NEXT: mov v3.d[1], v4.d[0]83; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]84; CHECK-SD-NEXT: mov v6.d[1], v7.d[0]85; CHECK-SD-NEXT: fmla v6.2d, v3.2d, v0.2d86; CHECK-SD-NEXT: ldr d3, [sp]87; CHECK-SD-NEXT: fmla v3.2d, v5.2d, v2.2d88; CHECK-SD-NEXT: fmov d0, d689; CHECK-SD-NEXT: ext v1.16b, v6.16b, v6.16b, #890; CHECK-SD-NEXT: // kill: def $d1 killed $d1 killed $q191; CHECK-SD-NEXT: fmov d2, d392; CHECK-SD-NEXT: ret93;94; CHECK-GI-LABEL: fma_v3f64:95; CHECK-GI: // %bb.0: // %entry96; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q097; CHECK-GI-NEXT: // kill: def $d3 killed $d3 def $q398; CHECK-GI-NEXT: // kill: def $d6 killed $d6 def $q699; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1100; CHECK-GI-NEXT: // kill: def $d4 killed $d4 def $q4101; CHECK-GI-NEXT: // kill: def $d7 killed $d7 def $q7102; CHECK-GI-NEXT: mov v0.d[1], v1.d[0]103; CHECK-GI-NEXT: mov v3.d[1], v4.d[0]104; CHECK-GI-NEXT: mov v6.d[1], v7.d[0]105; CHECK-GI-NEXT: fmla v6.2d, v3.2d, v0.2d106; CHECK-GI-NEXT: ldr d0, [sp]107; CHECK-GI-NEXT: fmadd d2, d2, d5, d0108; CHECK-GI-NEXT: mov d1, v6.d[1]109; CHECK-GI-NEXT: fmov d0, d6110; CHECK-GI-NEXT: ret111entry:112 %d = call <3 x double> @llvm.fma.v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c)113 ret <3 x double> %d114}115 116define <4 x double> @fma_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c) {117; CHECK-LABEL: fma_v4f64:118; CHECK: // %bb.0: // %entry119; CHECK-NEXT: fmla v4.2d, v2.2d, v0.2d120; CHECK-NEXT: fmla v5.2d, v3.2d, v1.2d121; CHECK-NEXT: mov v0.16b, v4.16b122; CHECK-NEXT: mov v1.16b, v5.16b123; CHECK-NEXT: ret124entry:125 %d = call <4 x double> @llvm.fma.v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c)126 ret <4 x double> %d127}128 129define <2 x float> @fma_v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c) {130; CHECK-LABEL: fma_v2f32:131; CHECK: // %bb.0: // %entry132; CHECK-NEXT: fmla v2.2s, v1.2s, v0.2s133; CHECK-NEXT: fmov d0, d2134; CHECK-NEXT: ret135entry:136 %d = call <2 x float> @llvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c)137 ret <2 x float> %d138}139 140define <3 x float> @fma_v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c) {141; CHECK-LABEL: fma_v3f32:142; CHECK: // %bb.0: // %entry143; CHECK-NEXT: fmla v2.4s, v1.4s, v0.4s144; CHECK-NEXT: mov v0.16b, v2.16b145; CHECK-NEXT: ret146entry:147 %d = call <3 x float> @llvm.fma.v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c)148 ret <3 x float> %d149}150 151define <4 x float> @fma_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {152; CHECK-LABEL: fma_v4f32:153; CHECK: // %bb.0: // %entry154; CHECK-NEXT: fmla v2.4s, v1.4s, v0.4s155; CHECK-NEXT: mov v0.16b, v2.16b156; CHECK-NEXT: ret157entry:158 %d = call <4 x float> @llvm.fma.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c)159 ret <4 x float> %d160}161 162define <8 x float> @fma_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) {163; CHECK-LABEL: fma_v8f32:164; CHECK: // %bb.0: // %entry165; CHECK-NEXT: fmla v4.4s, v2.4s, v0.4s166; CHECK-NEXT: fmla v5.4s, v3.4s, v1.4s167; CHECK-NEXT: mov v0.16b, v4.16b168; CHECK-NEXT: mov v1.16b, v5.16b169; CHECK-NEXT: ret170entry:171 %d = call <8 x float> @llvm.fma.v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c)172 ret <8 x float> %d173}174 175define <7 x half> @fma_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {176; CHECK-SD-NOFP16-LABEL: fma_v7f16:177; CHECK-SD-NOFP16: // %bb.0: // %entry178; CHECK-SD-NOFP16-NEXT: mov h3, v2.h[1]179; CHECK-SD-NOFP16-NEXT: mov h4, v1.h[1]180; CHECK-SD-NOFP16-NEXT: mov h5, v0.h[1]181; CHECK-SD-NOFP16-NEXT: fcvt s6, h2182; CHECK-SD-NOFP16-NEXT: fcvt s7, h1183; CHECK-SD-NOFP16-NEXT: fcvt s16, h0184; CHECK-SD-NOFP16-NEXT: mov h17, v2.h[2]185; CHECK-SD-NOFP16-NEXT: mov h18, v1.h[2]186; CHECK-SD-NOFP16-NEXT: mov h19, v0.h[2]187; CHECK-SD-NOFP16-NEXT: fcvt s3, h3188; CHECK-SD-NOFP16-NEXT: fcvt s4, h4189; CHECK-SD-NOFP16-NEXT: fcvt s5, h5190; CHECK-SD-NOFP16-NEXT: fmadd s6, s16, s7, s6191; CHECK-SD-NOFP16-NEXT: fcvt s7, h17192; CHECK-SD-NOFP16-NEXT: fcvt s16, h18193; CHECK-SD-NOFP16-NEXT: fcvt s17, h19194; CHECK-SD-NOFP16-NEXT: mov h18, v1.h[3]195; CHECK-SD-NOFP16-NEXT: mov h19, v0.h[3]196; CHECK-SD-NOFP16-NEXT: fmadd s4, s5, s4, s3197; CHECK-SD-NOFP16-NEXT: mov h5, v2.h[3]198; CHECK-SD-NOFP16-NEXT: fcvt h3, s6199; CHECK-SD-NOFP16-NEXT: fmadd s6, s17, s16, s7200; CHECK-SD-NOFP16-NEXT: mov h17, v2.h[4]201; CHECK-SD-NOFP16-NEXT: fcvt s7, h18202; CHECK-SD-NOFP16-NEXT: fcvt s16, h19203; CHECK-SD-NOFP16-NEXT: mov h18, v1.h[4]204; CHECK-SD-NOFP16-NEXT: fcvt h4, s4205; CHECK-SD-NOFP16-NEXT: fcvt s5, h5206; CHECK-SD-NOFP16-NEXT: mov h19, v0.h[4]207; CHECK-SD-NOFP16-NEXT: fcvt h6, s6208; CHECK-SD-NOFP16-NEXT: fcvt s17, h17209; CHECK-SD-NOFP16-NEXT: fcvt s18, h18210; CHECK-SD-NOFP16-NEXT: mov v3.h[1], v4.h[0]211; CHECK-SD-NOFP16-NEXT: mov h4, v2.h[5]212; CHECK-SD-NOFP16-NEXT: fmadd s5, s16, s7, s5213; CHECK-SD-NOFP16-NEXT: mov h7, v1.h[5]214; CHECK-SD-NOFP16-NEXT: mov h16, v0.h[5]215; CHECK-SD-NOFP16-NEXT: fcvt s19, h19216; CHECK-SD-NOFP16-NEXT: mov v3.h[2], v6.h[0]217; CHECK-SD-NOFP16-NEXT: mov h6, v2.h[6]218; CHECK-SD-NOFP16-NEXT: fcvt s4, h4219; CHECK-SD-NOFP16-NEXT: fcvt s7, h7220; CHECK-SD-NOFP16-NEXT: fcvt s16, h16221; CHECK-SD-NOFP16-NEXT: fcvt h5, s5222; CHECK-SD-NOFP16-NEXT: fmadd s17, s19, s18, s17223; CHECK-SD-NOFP16-NEXT: mov h18, v1.h[6]224; CHECK-SD-NOFP16-NEXT: mov h19, v0.h[6]225; CHECK-SD-NOFP16-NEXT: mov h2, v2.h[7]226; CHECK-SD-NOFP16-NEXT: mov h1, v1.h[7]227; CHECK-SD-NOFP16-NEXT: mov h0, v0.h[7]228; CHECK-SD-NOFP16-NEXT: fmadd s4, s16, s7, s4229; CHECK-SD-NOFP16-NEXT: mov v3.h[3], v5.h[0]230; CHECK-SD-NOFP16-NEXT: fcvt s5, h6231; CHECK-SD-NOFP16-NEXT: fcvt s6, h18232; CHECK-SD-NOFP16-NEXT: fcvt s7, h19233; CHECK-SD-NOFP16-NEXT: fcvt h16, s17234; CHECK-SD-NOFP16-NEXT: fcvt s2, h2235; CHECK-SD-NOFP16-NEXT: fcvt s1, h1236; CHECK-SD-NOFP16-NEXT: fcvt s0, h0237; CHECK-SD-NOFP16-NEXT: fcvt h4, s4238; CHECK-SD-NOFP16-NEXT: fmadd s5, s7, s6, s5239; CHECK-SD-NOFP16-NEXT: mov v3.h[4], v16.h[0]240; CHECK-SD-NOFP16-NEXT: fmadd s0, s0, s1, s2241; CHECK-SD-NOFP16-NEXT: mov v3.h[5], v4.h[0]242; CHECK-SD-NOFP16-NEXT: fcvt h4, s5243; CHECK-SD-NOFP16-NEXT: fcvt h0, s0244; CHECK-SD-NOFP16-NEXT: mov v3.h[6], v4.h[0]245; CHECK-SD-NOFP16-NEXT: mov v3.h[7], v0.h[0]246; CHECK-SD-NOFP16-NEXT: mov v0.16b, v3.16b247; CHECK-SD-NOFP16-NEXT: ret248;249; CHECK-SD-FP16-LABEL: fma_v7f16:250; CHECK-SD-FP16: // %bb.0: // %entry251; CHECK-SD-FP16-NEXT: fmla v2.8h, v1.8h, v0.8h252; CHECK-SD-FP16-NEXT: mov v0.16b, v2.16b253; CHECK-SD-FP16-NEXT: ret254;255; CHECK-GI-NOFP16-LABEL: fma_v7f16:256; CHECK-GI-NOFP16: // %bb.0: // %entry257; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v0.4h258; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v1.4h259; CHECK-GI-NOFP16-NEXT: fcvtl v5.4s, v2.4h260; CHECK-GI-NOFP16-NEXT: mov v6.h[0], v0.h[4]261; CHECK-GI-NOFP16-NEXT: fmla v5.4s, v4.4s, v3.4s262; CHECK-GI-NOFP16-NEXT: mov v3.h[0], v1.h[4]263; CHECK-GI-NOFP16-NEXT: mov v4.h[0], v2.h[4]264; CHECK-GI-NOFP16-NEXT: mov v6.h[1], v0.h[5]265; CHECK-GI-NOFP16-NEXT: mov v3.h[1], v1.h[5]266; CHECK-GI-NOFP16-NEXT: mov v4.h[1], v2.h[5]267; CHECK-GI-NOFP16-NEXT: fcvtn v5.4h, v5.4s268; CHECK-GI-NOFP16-NEXT: mov v6.h[2], v0.h[6]269; CHECK-GI-NOFP16-NEXT: mov v3.h[2], v1.h[6]270; CHECK-GI-NOFP16-NEXT: mov v4.h[2], v2.h[6]271; CHECK-GI-NOFP16-NEXT: mov v0.h[0], v5.h[0]272; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v6.4h273; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v3.4h274; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v4.4h275; CHECK-GI-NOFP16-NEXT: mov v0.h[1], v5.h[1]276; CHECK-GI-NOFP16-NEXT: fmla v3.4s, v2.4s, v1.4s277; CHECK-GI-NOFP16-NEXT: mov v0.h[2], v5.h[2]278; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v3.4s279; CHECK-GI-NOFP16-NEXT: mov v0.h[3], v5.h[3]280; CHECK-GI-NOFP16-NEXT: mov v0.h[4], v1.h[0]281; CHECK-GI-NOFP16-NEXT: mov v0.h[5], v1.h[1]282; CHECK-GI-NOFP16-NEXT: mov v0.h[6], v1.h[2]283; CHECK-GI-NOFP16-NEXT: ret284;285; CHECK-GI-FP16-LABEL: fma_v7f16:286; CHECK-GI-FP16: // %bb.0: // %entry287; CHECK-GI-FP16-NEXT: fmla v2.8h, v1.8h, v0.8h288; CHECK-GI-FP16-NEXT: mov v0.16b, v2.16b289; CHECK-GI-FP16-NEXT: ret290entry:291 %d = call <7 x half> @llvm.fma.v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c)292 ret <7 x half> %d293}294 295define <4 x half> @fma_v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c) {296; CHECK-SD-NOFP16-LABEL: fma_v4f16:297; CHECK-SD-NOFP16: // %bb.0: // %entry298; CHECK-SD-NOFP16-NEXT: // kill: def $d2 killed $d2 def $q2299; CHECK-SD-NOFP16-NEXT: // kill: def $d1 killed $d1 def $q1300; CHECK-SD-NOFP16-NEXT: // kill: def $d0 killed $d0 def $q0301; CHECK-SD-NOFP16-NEXT: mov h3, v2.h[1]302; CHECK-SD-NOFP16-NEXT: mov h4, v1.h[1]303; CHECK-SD-NOFP16-NEXT: mov h5, v0.h[1]304; CHECK-SD-NOFP16-NEXT: fcvt s6, h2305; CHECK-SD-NOFP16-NEXT: fcvt s7, h1306; CHECK-SD-NOFP16-NEXT: fcvt s16, h0307; CHECK-SD-NOFP16-NEXT: mov h17, v2.h[2]308; CHECK-SD-NOFP16-NEXT: mov h18, v1.h[2]309; CHECK-SD-NOFP16-NEXT: mov h19, v0.h[2]310; CHECK-SD-NOFP16-NEXT: mov h2, v2.h[3]311; CHECK-SD-NOFP16-NEXT: mov h1, v1.h[3]312; CHECK-SD-NOFP16-NEXT: fcvt s3, h3313; CHECK-SD-NOFP16-NEXT: fcvt s4, h4314; CHECK-SD-NOFP16-NEXT: fcvt s5, h5315; CHECK-SD-NOFP16-NEXT: fmadd s6, s16, s7, s6316; CHECK-SD-NOFP16-NEXT: mov h16, v0.h[3]317; CHECK-SD-NOFP16-NEXT: fcvt s7, h19318; CHECK-SD-NOFP16-NEXT: fcvt s2, h2319; CHECK-SD-NOFP16-NEXT: fcvt s1, h1320; CHECK-SD-NOFP16-NEXT: fmadd s3, s5, s4, s3321; CHECK-SD-NOFP16-NEXT: fcvt s4, h17322; CHECK-SD-NOFP16-NEXT: fcvt s5, h18323; CHECK-SD-NOFP16-NEXT: fcvt h0, s6324; CHECK-SD-NOFP16-NEXT: fmadd s4, s7, s5, s4325; CHECK-SD-NOFP16-NEXT: fcvt h3, s3326; CHECK-SD-NOFP16-NEXT: fcvt s5, h16327; CHECK-SD-NOFP16-NEXT: mov v0.h[1], v3.h[0]328; CHECK-SD-NOFP16-NEXT: fcvt h3, s4329; CHECK-SD-NOFP16-NEXT: fmadd s1, s5, s1, s2330; CHECK-SD-NOFP16-NEXT: mov v0.h[2], v3.h[0]331; CHECK-SD-NOFP16-NEXT: fcvt h1, s1332; CHECK-SD-NOFP16-NEXT: mov v0.h[3], v1.h[0]333; CHECK-SD-NOFP16-NEXT: // kill: def $d0 killed $d0 killed $q0334; CHECK-SD-NOFP16-NEXT: ret335;336; CHECK-SD-FP16-LABEL: fma_v4f16:337; CHECK-SD-FP16: // %bb.0: // %entry338; CHECK-SD-FP16-NEXT: fmla v2.4h, v1.4h, v0.4h339; CHECK-SD-FP16-NEXT: fmov d0, d2340; CHECK-SD-FP16-NEXT: ret341;342; CHECK-GI-NOFP16-LABEL: fma_v4f16:343; CHECK-GI-NOFP16: // %bb.0: // %entry344; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h345; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h346; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v2.4h347; CHECK-GI-NOFP16-NEXT: fmla v2.4s, v1.4s, v0.4s348; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v2.4s349; CHECK-GI-NOFP16-NEXT: ret350;351; CHECK-GI-FP16-LABEL: fma_v4f16:352; CHECK-GI-FP16: // %bb.0: // %entry353; CHECK-GI-FP16-NEXT: fmla v2.4h, v1.4h, v0.4h354; CHECK-GI-FP16-NEXT: fmov d0, d2355; CHECK-GI-FP16-NEXT: ret356entry:357 %d = call <4 x half> @llvm.fma.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c)358 ret <4 x half> %d359}360 361define <8 x half> @fma_v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {362; CHECK-SD-NOFP16-LABEL: fma_v8f16:363; CHECK-SD-NOFP16: // %bb.0: // %entry364; CHECK-SD-NOFP16-NEXT: mov h3, v2.h[1]365; CHECK-SD-NOFP16-NEXT: mov h4, v1.h[1]366; CHECK-SD-NOFP16-NEXT: mov h5, v0.h[1]367; CHECK-SD-NOFP16-NEXT: fcvt s6, h2368; CHECK-SD-NOFP16-NEXT: fcvt s7, h1369; CHECK-SD-NOFP16-NEXT: fcvt s16, h0370; CHECK-SD-NOFP16-NEXT: mov h17, v2.h[2]371; CHECK-SD-NOFP16-NEXT: mov h18, v1.h[2]372; CHECK-SD-NOFP16-NEXT: mov h19, v0.h[2]373; CHECK-SD-NOFP16-NEXT: fcvt s3, h3374; CHECK-SD-NOFP16-NEXT: fcvt s4, h4375; CHECK-SD-NOFP16-NEXT: fcvt s5, h5376; CHECK-SD-NOFP16-NEXT: fmadd s6, s16, s7, s6377; CHECK-SD-NOFP16-NEXT: fcvt s7, h17378; CHECK-SD-NOFP16-NEXT: fcvt s16, h18379; CHECK-SD-NOFP16-NEXT: fcvt s17, h19380; CHECK-SD-NOFP16-NEXT: mov h18, v1.h[3]381; CHECK-SD-NOFP16-NEXT: mov h19, v0.h[3]382; CHECK-SD-NOFP16-NEXT: fmadd s4, s5, s4, s3383; CHECK-SD-NOFP16-NEXT: mov h5, v2.h[3]384; CHECK-SD-NOFP16-NEXT: fcvt h3, s6385; CHECK-SD-NOFP16-NEXT: fmadd s6, s17, s16, s7386; CHECK-SD-NOFP16-NEXT: mov h17, v2.h[4]387; CHECK-SD-NOFP16-NEXT: fcvt s7, h18388; CHECK-SD-NOFP16-NEXT: fcvt s16, h19389; CHECK-SD-NOFP16-NEXT: mov h18, v1.h[4]390; CHECK-SD-NOFP16-NEXT: fcvt h4, s4391; CHECK-SD-NOFP16-NEXT: fcvt s5, h5392; CHECK-SD-NOFP16-NEXT: mov h19, v0.h[4]393; CHECK-SD-NOFP16-NEXT: fcvt h6, s6394; CHECK-SD-NOFP16-NEXT: fcvt s17, h17395; CHECK-SD-NOFP16-NEXT: fcvt s18, h18396; CHECK-SD-NOFP16-NEXT: mov v3.h[1], v4.h[0]397; CHECK-SD-NOFP16-NEXT: mov h4, v2.h[5]398; CHECK-SD-NOFP16-NEXT: fmadd s5, s16, s7, s5399; CHECK-SD-NOFP16-NEXT: mov h7, v1.h[5]400; CHECK-SD-NOFP16-NEXT: mov h16, v0.h[5]401; CHECK-SD-NOFP16-NEXT: fcvt s19, h19402; CHECK-SD-NOFP16-NEXT: mov v3.h[2], v6.h[0]403; CHECK-SD-NOFP16-NEXT: mov h6, v2.h[6]404; CHECK-SD-NOFP16-NEXT: fcvt s4, h4405; CHECK-SD-NOFP16-NEXT: fcvt s7, h7406; CHECK-SD-NOFP16-NEXT: fcvt s16, h16407; CHECK-SD-NOFP16-NEXT: fcvt h5, s5408; CHECK-SD-NOFP16-NEXT: fmadd s17, s19, s18, s17409; CHECK-SD-NOFP16-NEXT: mov h18, v1.h[6]410; CHECK-SD-NOFP16-NEXT: mov h19, v0.h[6]411; CHECK-SD-NOFP16-NEXT: mov h2, v2.h[7]412; CHECK-SD-NOFP16-NEXT: mov h1, v1.h[7]413; CHECK-SD-NOFP16-NEXT: mov h0, v0.h[7]414; CHECK-SD-NOFP16-NEXT: fmadd s4, s16, s7, s4415; CHECK-SD-NOFP16-NEXT: mov v3.h[3], v5.h[0]416; CHECK-SD-NOFP16-NEXT: fcvt s5, h6417; CHECK-SD-NOFP16-NEXT: fcvt s6, h18418; CHECK-SD-NOFP16-NEXT: fcvt s7, h19419; CHECK-SD-NOFP16-NEXT: fcvt h16, s17420; CHECK-SD-NOFP16-NEXT: fcvt s2, h2421; CHECK-SD-NOFP16-NEXT: fcvt s1, h1422; CHECK-SD-NOFP16-NEXT: fcvt s0, h0423; CHECK-SD-NOFP16-NEXT: fcvt h4, s4424; CHECK-SD-NOFP16-NEXT: fmadd s5, s7, s6, s5425; CHECK-SD-NOFP16-NEXT: mov v3.h[4], v16.h[0]426; CHECK-SD-NOFP16-NEXT: fmadd s0, s0, s1, s2427; CHECK-SD-NOFP16-NEXT: mov v3.h[5], v4.h[0]428; CHECK-SD-NOFP16-NEXT: fcvt h4, s5429; CHECK-SD-NOFP16-NEXT: fcvt h0, s0430; CHECK-SD-NOFP16-NEXT: mov v3.h[6], v4.h[0]431; CHECK-SD-NOFP16-NEXT: mov v3.h[7], v0.h[0]432; CHECK-SD-NOFP16-NEXT: mov v0.16b, v3.16b433; CHECK-SD-NOFP16-NEXT: ret434;435; CHECK-SD-FP16-LABEL: fma_v8f16:436; CHECK-SD-FP16: // %bb.0: // %entry437; CHECK-SD-FP16-NEXT: fmla v2.8h, v1.8h, v0.8h438; CHECK-SD-FP16-NEXT: mov v0.16b, v2.16b439; CHECK-SD-FP16-NEXT: ret440;441; CHECK-GI-NOFP16-LABEL: fma_v8f16:442; CHECK-GI-NOFP16: // %bb.0: // %entry443; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v0.4h444; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v1.4h445; CHECK-GI-NOFP16-NEXT: fcvtl v5.4s, v2.4h446; CHECK-GI-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h447; CHECK-GI-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h448; CHECK-GI-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h449; CHECK-GI-NOFP16-NEXT: fmla v5.4s, v4.4s, v3.4s450; CHECK-GI-NOFP16-NEXT: fmla v2.4s, v1.4s, v0.4s451; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v5.4s452; CHECK-GI-NOFP16-NEXT: fcvtn2 v0.8h, v2.4s453; CHECK-GI-NOFP16-NEXT: ret454;455; CHECK-GI-FP16-LABEL: fma_v8f16:456; CHECK-GI-FP16: // %bb.0: // %entry457; CHECK-GI-FP16-NEXT: fmla v2.8h, v1.8h, v0.8h458; CHECK-GI-FP16-NEXT: mov v0.16b, v2.16b459; CHECK-GI-FP16-NEXT: ret460entry:461 %d = call <8 x half> @llvm.fma.v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c)462 ret <8 x half> %d463}464 465define <16 x half> @fma_v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c) {466; CHECK-SD-NOFP16-LABEL: fma_v16f16:467; CHECK-SD-NOFP16: // %bb.0: // %entry468; CHECK-SD-NOFP16-NEXT: mov h6, v4.h[1]469; CHECK-SD-NOFP16-NEXT: mov h7, v2.h[1]470; CHECK-SD-NOFP16-NEXT: mov h16, v0.h[1]471; CHECK-SD-NOFP16-NEXT: fcvt s17, h4472; CHECK-SD-NOFP16-NEXT: fcvt s18, h2473; CHECK-SD-NOFP16-NEXT: fcvt s19, h0474; CHECK-SD-NOFP16-NEXT: mov h20, v4.h[2]475; CHECK-SD-NOFP16-NEXT: mov h21, v2.h[2]476; CHECK-SD-NOFP16-NEXT: mov h22, v0.h[2]477; CHECK-SD-NOFP16-NEXT: mov h23, v4.h[3]478; CHECK-SD-NOFP16-NEXT: mov h24, v2.h[3]479; CHECK-SD-NOFP16-NEXT: mov h25, v0.h[3]480; CHECK-SD-NOFP16-NEXT: fcvt s6, h6481; CHECK-SD-NOFP16-NEXT: fcvt s7, h7482; CHECK-SD-NOFP16-NEXT: fcvt s16, h16483; CHECK-SD-NOFP16-NEXT: fmadd s17, s19, s18, s17484; CHECK-SD-NOFP16-NEXT: mov h26, v1.h[1]485; CHECK-SD-NOFP16-NEXT: fcvt s27, h5486; CHECK-SD-NOFP16-NEXT: fcvt s18, h20487; CHECK-SD-NOFP16-NEXT: fcvt s19, h21488; CHECK-SD-NOFP16-NEXT: fcvt s20, h22489; CHECK-SD-NOFP16-NEXT: fcvt s21, h23490; CHECK-SD-NOFP16-NEXT: fcvt s22, h24491; CHECK-SD-NOFP16-NEXT: fcvt s23, h25492; CHECK-SD-NOFP16-NEXT: fmadd s7, s16, s7, s6493; CHECK-SD-NOFP16-NEXT: mov h24, v5.h[1]494; CHECK-SD-NOFP16-NEXT: mov h25, v3.h[1]495; CHECK-SD-NOFP16-NEXT: fcvt h6, s17496; CHECK-SD-NOFP16-NEXT: fcvt s28, h3497; CHECK-SD-NOFP16-NEXT: fcvt s29, h1498; CHECK-SD-NOFP16-NEXT: fmadd s19, s20, s19, s18499; CHECK-SD-NOFP16-NEXT: fcvt s26, h26500; CHECK-SD-NOFP16-NEXT: mov h16, v4.h[4]501; CHECK-SD-NOFP16-NEXT: fmadd s21, s23, s22, s21502; CHECK-SD-NOFP16-NEXT: mov h22, v3.h[2]503; CHECK-SD-NOFP16-NEXT: mov h23, v1.h[2]504; CHECK-SD-NOFP16-NEXT: fcvt h20, s7505; CHECK-SD-NOFP16-NEXT: fcvt s24, h24506; CHECK-SD-NOFP16-NEXT: fcvt s25, h25507; CHECK-SD-NOFP16-NEXT: mov h17, v2.h[4]508; CHECK-SD-NOFP16-NEXT: mov h18, v0.h[4]509; CHECK-SD-NOFP16-NEXT: mov h7, v4.h[5]510; CHECK-SD-NOFP16-NEXT: fcvt h19, s19511; CHECK-SD-NOFP16-NEXT: mov h30, v2.h[5]512; CHECK-SD-NOFP16-NEXT: fcvt s16, h16513; CHECK-SD-NOFP16-NEXT: fcvt h21, s21514; CHECK-SD-NOFP16-NEXT: mov h31, v1.h[4]515; CHECK-SD-NOFP16-NEXT: fmadd s24, s26, s25, s24516; CHECK-SD-NOFP16-NEXT: fmadd s25, s29, s28, s27517; CHECK-SD-NOFP16-NEXT: mov v6.h[1], v20.h[0]518; CHECK-SD-NOFP16-NEXT: mov h20, v5.h[2]519; CHECK-SD-NOFP16-NEXT: mov h26, v5.h[3]520; CHECK-SD-NOFP16-NEXT: mov h27, v3.h[3]521; CHECK-SD-NOFP16-NEXT: mov h28, v1.h[3]522; CHECK-SD-NOFP16-NEXT: fcvt s17, h17523; CHECK-SD-NOFP16-NEXT: fcvt s18, h18524; CHECK-SD-NOFP16-NEXT: fcvt s29, h7525; CHECK-SD-NOFP16-NEXT: fcvt s30, h30526; CHECK-SD-NOFP16-NEXT: mov v6.h[2], v19.h[0]527; CHECK-SD-NOFP16-NEXT: fcvt h24, s24528; CHECK-SD-NOFP16-NEXT: fcvt h7, s25529; CHECK-SD-NOFP16-NEXT: fcvt s19, h20530; CHECK-SD-NOFP16-NEXT: fcvt s20, h22531; CHECK-SD-NOFP16-NEXT: fcvt s22, h23532; CHECK-SD-NOFP16-NEXT: fmadd s16, s18, s17, s16533; CHECK-SD-NOFP16-NEXT: mov h23, v0.h[5]534; CHECK-SD-NOFP16-NEXT: fcvt s25, h26535; CHECK-SD-NOFP16-NEXT: fcvt s26, h27536; CHECK-SD-NOFP16-NEXT: fcvt s27, h28537; CHECK-SD-NOFP16-NEXT: mov h18, v4.h[6]538; CHECK-SD-NOFP16-NEXT: mov v6.h[3], v21.h[0]539; CHECK-SD-NOFP16-NEXT: mov v7.h[1], v24.h[0]540; CHECK-SD-NOFP16-NEXT: mov h24, v5.h[5]541; CHECK-SD-NOFP16-NEXT: fmadd s19, s22, s20, s19542; CHECK-SD-NOFP16-NEXT: mov h20, v5.h[4]543; CHECK-SD-NOFP16-NEXT: mov h22, v3.h[4]544; CHECK-SD-NOFP16-NEXT: fcvt s23, h23545; CHECK-SD-NOFP16-NEXT: mov h28, v0.h[6]546; CHECK-SD-NOFP16-NEXT: fcvt h16, s16547; CHECK-SD-NOFP16-NEXT: fcvt s18, h18548; CHECK-SD-NOFP16-NEXT: mov h4, v4.h[7]549; CHECK-SD-NOFP16-NEXT: mov h0, v0.h[7]550; CHECK-SD-NOFP16-NEXT: fcvt s20, h20551; CHECK-SD-NOFP16-NEXT: fcvt s21, h22552; CHECK-SD-NOFP16-NEXT: fcvt s22, h31553; CHECK-SD-NOFP16-NEXT: fmadd s17, s23, s30, s29554; CHECK-SD-NOFP16-NEXT: fmadd s23, s27, s26, s25555; CHECK-SD-NOFP16-NEXT: fcvt h19, s19556; CHECK-SD-NOFP16-NEXT: mov h25, v3.h[5]557; CHECK-SD-NOFP16-NEXT: mov h26, v1.h[5]558; CHECK-SD-NOFP16-NEXT: mov h27, v2.h[6]559; CHECK-SD-NOFP16-NEXT: mov h29, v1.h[6]560; CHECK-SD-NOFP16-NEXT: mov h2, v2.h[7]561; CHECK-SD-NOFP16-NEXT: mov h1, v1.h[7]562; CHECK-SD-NOFP16-NEXT: fmadd s20, s22, s21, s20563; CHECK-SD-NOFP16-NEXT: mov h21, v5.h[6]564; CHECK-SD-NOFP16-NEXT: mov h22, v3.h[6]565; CHECK-SD-NOFP16-NEXT: mov v7.h[2], v19.h[0]566; CHECK-SD-NOFP16-NEXT: fcvt h19, s23567; CHECK-SD-NOFP16-NEXT: fcvt s23, h24568; CHECK-SD-NOFP16-NEXT: fcvt s24, h25569; CHECK-SD-NOFP16-NEXT: fcvt s25, h26570; CHECK-SD-NOFP16-NEXT: fcvt s26, h27571; CHECK-SD-NOFP16-NEXT: fcvt s27, h28572; CHECK-SD-NOFP16-NEXT: fcvt s28, h29573; CHECK-SD-NOFP16-NEXT: mov h5, v5.h[7]574; CHECK-SD-NOFP16-NEXT: fcvt s21, h21575; CHECK-SD-NOFP16-NEXT: fcvt s22, h22576; CHECK-SD-NOFP16-NEXT: mov h3, v3.h[7]577; CHECK-SD-NOFP16-NEXT: mov v7.h[3], v19.h[0]578; CHECK-SD-NOFP16-NEXT: fcvt h19, s20579; CHECK-SD-NOFP16-NEXT: mov v6.h[4], v16.h[0]580; CHECK-SD-NOFP16-NEXT: fmadd s20, s25, s24, s23581; CHECK-SD-NOFP16-NEXT: fcvt h16, s17582; CHECK-SD-NOFP16-NEXT: fcvt s4, h4583; CHECK-SD-NOFP16-NEXT: fmadd s18, s27, s26, s18584; CHECK-SD-NOFP16-NEXT: fcvt s2, h2585; CHECK-SD-NOFP16-NEXT: fcvt s0, h0586; CHECK-SD-NOFP16-NEXT: fmadd s21, s28, s22, s21587; CHECK-SD-NOFP16-NEXT: fcvt s5, h5588; CHECK-SD-NOFP16-NEXT: fcvt s3, h3589; CHECK-SD-NOFP16-NEXT: mov v7.h[4], v19.h[0]590; CHECK-SD-NOFP16-NEXT: fcvt s1, h1591; CHECK-SD-NOFP16-NEXT: fcvt h17, s20592; CHECK-SD-NOFP16-NEXT: mov v6.h[5], v16.h[0]593; CHECK-SD-NOFP16-NEXT: fmadd s0, s0, s2, s4594; CHECK-SD-NOFP16-NEXT: fcvt h2, s18595; CHECK-SD-NOFP16-NEXT: fcvt h4, s21596; CHECK-SD-NOFP16-NEXT: fmadd s1, s1, s3, s5597; CHECK-SD-NOFP16-NEXT: mov v7.h[5], v17.h[0]598; CHECK-SD-NOFP16-NEXT: mov v6.h[6], v2.h[0]599; CHECK-SD-NOFP16-NEXT: fcvt h0, s0600; CHECK-SD-NOFP16-NEXT: fcvt h1, s1601; CHECK-SD-NOFP16-NEXT: mov v7.h[6], v4.h[0]602; CHECK-SD-NOFP16-NEXT: mov v6.h[7], v0.h[0]603; CHECK-SD-NOFP16-NEXT: mov v7.h[7], v1.h[0]604; CHECK-SD-NOFP16-NEXT: mov v0.16b, v6.16b605; CHECK-SD-NOFP16-NEXT: mov v1.16b, v7.16b606; CHECK-SD-NOFP16-NEXT: ret607;608; CHECK-SD-FP16-LABEL: fma_v16f16:609; CHECK-SD-FP16: // %bb.0: // %entry610; CHECK-SD-FP16-NEXT: fmla v4.8h, v2.8h, v0.8h611; CHECK-SD-FP16-NEXT: fmla v5.8h, v3.8h, v1.8h612; CHECK-SD-FP16-NEXT: mov v0.16b, v4.16b613; CHECK-SD-FP16-NEXT: mov v1.16b, v5.16b614; CHECK-SD-FP16-NEXT: ret615;616; CHECK-GI-NOFP16-LABEL: fma_v16f16:617; CHECK-GI-NOFP16: // %bb.0: // %entry618; CHECK-GI-NOFP16-NEXT: fcvtl v6.4s, v0.4h619; CHECK-GI-NOFP16-NEXT: fcvtl v7.4s, v1.4h620; CHECK-GI-NOFP16-NEXT: fcvtl v16.4s, v2.4h621; CHECK-GI-NOFP16-NEXT: fcvtl v17.4s, v3.4h622; CHECK-GI-NOFP16-NEXT: fcvtl v18.4s, v4.4h623; CHECK-GI-NOFP16-NEXT: fcvtl v19.4s, v5.4h624; CHECK-GI-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h625; CHECK-GI-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h626; CHECK-GI-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h627; CHECK-GI-NOFP16-NEXT: fcvtl2 v3.4s, v3.8h628; CHECK-GI-NOFP16-NEXT: fcvtl2 v4.4s, v4.8h629; CHECK-GI-NOFP16-NEXT: fcvtl2 v5.4s, v5.8h630; CHECK-GI-NOFP16-NEXT: fmla v18.4s, v16.4s, v6.4s631; CHECK-GI-NOFP16-NEXT: fmla v19.4s, v17.4s, v7.4s632; CHECK-GI-NOFP16-NEXT: fmla v4.4s, v2.4s, v0.4s633; CHECK-GI-NOFP16-NEXT: fmla v5.4s, v3.4s, v1.4s634; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v18.4s635; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v19.4s636; CHECK-GI-NOFP16-NEXT: fcvtn2 v0.8h, v4.4s637; CHECK-GI-NOFP16-NEXT: fcvtn2 v1.8h, v5.4s638; CHECK-GI-NOFP16-NEXT: ret639;640; CHECK-GI-FP16-LABEL: fma_v16f16:641; CHECK-GI-FP16: // %bb.0: // %entry642; CHECK-GI-FP16-NEXT: fmla v4.8h, v2.8h, v0.8h643; CHECK-GI-FP16-NEXT: fmla v5.8h, v3.8h, v1.8h644; CHECK-GI-FP16-NEXT: mov v0.16b, v4.16b645; CHECK-GI-FP16-NEXT: mov v1.16b, v5.16b646; CHECK-GI-FP16-NEXT: ret647entry:648 %d = call <16 x half> @llvm.fma.v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c)649 ret <16 x half> %d650}651 652define double @fmuladd_f64(double %a, double %b, double %c) {653; CHECK-LABEL: fmuladd_f64:654; CHECK: // %bb.0: // %entry655; CHECK-NEXT: fmadd d0, d0, d1, d2656; CHECK-NEXT: ret657entry:658 %d = call double @llvm.fmuladd.f64(double %a, double %b, double %c)659 ret double %d660}661 662define float @fmuladd_f32(float %a, float %b, float %c) {663; CHECK-LABEL: fmuladd_f32:664; CHECK: // %bb.0: // %entry665; CHECK-NEXT: fmadd s0, s0, s1, s2666; CHECK-NEXT: ret667entry:668 %d = call float @llvm.fmuladd.f32(float %a, float %b, float %c)669 ret float %d670}671 672define half @fmuladd_f16(half %a, half %b, half %c) {673; CHECK-SD-NOFP16-LABEL: fmuladd_f16:674; CHECK-SD-NOFP16: // %bb.0: // %entry675; CHECK-SD-NOFP16-NEXT: fcvt s1, h1676; CHECK-SD-NOFP16-NEXT: fcvt s0, h0677; CHECK-SD-NOFP16-NEXT: fmul s0, s0, s1678; CHECK-SD-NOFP16-NEXT: fcvt s1, h2679; CHECK-SD-NOFP16-NEXT: fcvt h0, s0680; CHECK-SD-NOFP16-NEXT: fcvt s0, h0681; CHECK-SD-NOFP16-NEXT: fadd s0, s0, s1682; CHECK-SD-NOFP16-NEXT: fcvt h0, s0683; CHECK-SD-NOFP16-NEXT: ret684;685; CHECK-SD-FP16-LABEL: fmuladd_f16:686; CHECK-SD-FP16: // %bb.0: // %entry687; CHECK-SD-FP16-NEXT: fmadd h0, h0, h1, h2688; CHECK-SD-FP16-NEXT: ret689;690; CHECK-GI-NOFP16-LABEL: fmuladd_f16:691; CHECK-GI-NOFP16: // %bb.0: // %entry692; CHECK-GI-NOFP16-NEXT: fcvt s0, h0693; CHECK-GI-NOFP16-NEXT: fcvt s1, h1694; CHECK-GI-NOFP16-NEXT: fmul s0, s0, s1695; CHECK-GI-NOFP16-NEXT: fcvt s1, h2696; CHECK-GI-NOFP16-NEXT: fcvt h0, s0697; CHECK-GI-NOFP16-NEXT: fcvt s0, h0698; CHECK-GI-NOFP16-NEXT: fadd s0, s0, s1699; CHECK-GI-NOFP16-NEXT: fcvt h0, s0700; CHECK-GI-NOFP16-NEXT: ret701;702; CHECK-GI-FP16-LABEL: fmuladd_f16:703; CHECK-GI-FP16: // %bb.0: // %entry704; CHECK-GI-FP16-NEXT: fmadd h0, h0, h1, h2705; CHECK-GI-FP16-NEXT: ret706entry:707 %d = call half @llvm.fmuladd.f16(half %a, half %b, half %c)708 ret half %d709}710 711define <2 x double> @fmuladd_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c) {712; CHECK-LABEL: fmuladd_v2f64:713; CHECK: // %bb.0: // %entry714; CHECK-NEXT: fmla v2.2d, v1.2d, v0.2d715; CHECK-NEXT: mov v0.16b, v2.16b716; CHECK-NEXT: ret717entry:718 %d = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c)719 ret <2 x double> %d720}721 722define <3 x double> @fmuladd_v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c) {723; CHECK-SD-LABEL: fmuladd_v3f64:724; CHECK-SD: // %bb.0: // %entry725; CHECK-SD-NEXT: // kill: def $d6 killed $d6 def $q6726; CHECK-SD-NEXT: // kill: def $d3 killed $d3 def $q3727; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0728; CHECK-SD-NEXT: // kill: def $d7 killed $d7 def $q7729; CHECK-SD-NEXT: // kill: def $d4 killed $d4 def $q4730; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1731; CHECK-SD-NEXT: // kill: def $d5 killed $d5 def $q5732; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q2733; CHECK-SD-NEXT: mov v3.d[1], v4.d[0]734; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]735; CHECK-SD-NEXT: mov v6.d[1], v7.d[0]736; CHECK-SD-NEXT: fmla v6.2d, v3.2d, v0.2d737; CHECK-SD-NEXT: ldr d3, [sp]738; CHECK-SD-NEXT: fmla v3.2d, v5.2d, v2.2d739; CHECK-SD-NEXT: fmov d0, d6740; CHECK-SD-NEXT: ext v1.16b, v6.16b, v6.16b, #8741; CHECK-SD-NEXT: // kill: def $d1 killed $d1 killed $q1742; CHECK-SD-NEXT: fmov d2, d3743; CHECK-SD-NEXT: ret744;745; CHECK-GI-LABEL: fmuladd_v3f64:746; CHECK-GI: // %bb.0: // %entry747; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0748; CHECK-GI-NEXT: // kill: def $d3 killed $d3 def $q3749; CHECK-GI-NEXT: // kill: def $d6 killed $d6 def $q6750; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1751; CHECK-GI-NEXT: // kill: def $d4 killed $d4 def $q4752; CHECK-GI-NEXT: // kill: def $d7 killed $d7 def $q7753; CHECK-GI-NEXT: mov v0.d[1], v1.d[0]754; CHECK-GI-NEXT: mov v3.d[1], v4.d[0]755; CHECK-GI-NEXT: mov v6.d[1], v7.d[0]756; CHECK-GI-NEXT: fmla v6.2d, v3.2d, v0.2d757; CHECK-GI-NEXT: ldr d0, [sp]758; CHECK-GI-NEXT: fmadd d2, d2, d5, d0759; CHECK-GI-NEXT: mov d1, v6.d[1]760; CHECK-GI-NEXT: fmov d0, d6761; CHECK-GI-NEXT: ret762entry:763 %d = call <3 x double> @llvm.fmuladd.v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c)764 ret <3 x double> %d765}766 767define <4 x double> @fmuladd_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c) {768; CHECK-LABEL: fmuladd_v4f64:769; CHECK: // %bb.0: // %entry770; CHECK-NEXT: fmla v4.2d, v2.2d, v0.2d771; CHECK-NEXT: fmla v5.2d, v3.2d, v1.2d772; CHECK-NEXT: mov v0.16b, v4.16b773; CHECK-NEXT: mov v1.16b, v5.16b774; CHECK-NEXT: ret775entry:776 %d = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c)777 ret <4 x double> %d778}779 780define <2 x float> @fmuladd_v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c) {781; CHECK-LABEL: fmuladd_v2f32:782; CHECK: // %bb.0: // %entry783; CHECK-NEXT: fmla v2.2s, v1.2s, v0.2s784; CHECK-NEXT: fmov d0, d2785; CHECK-NEXT: ret786entry:787 %d = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c)788 ret <2 x float> %d789}790 791define <3 x float> @fmuladd_v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c) {792; CHECK-LABEL: fmuladd_v3f32:793; CHECK: // %bb.0: // %entry794; CHECK-NEXT: fmla v2.4s, v1.4s, v0.4s795; CHECK-NEXT: mov v0.16b, v2.16b796; CHECK-NEXT: ret797entry:798 %d = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c)799 ret <3 x float> %d800}801 802define <4 x float> @fmuladd_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {803; CHECK-LABEL: fmuladd_v4f32:804; CHECK: // %bb.0: // %entry805; CHECK-NEXT: fmla v2.4s, v1.4s, v0.4s806; CHECK-NEXT: mov v0.16b, v2.16b807; CHECK-NEXT: ret808entry:809 %d = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c)810 ret <4 x float> %d811}812 813define <8 x float> @fmuladd_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) {814; CHECK-LABEL: fmuladd_v8f32:815; CHECK: // %bb.0: // %entry816; CHECK-NEXT: fmla v4.4s, v2.4s, v0.4s817; CHECK-NEXT: fmla v5.4s, v3.4s, v1.4s818; CHECK-NEXT: mov v0.16b, v4.16b819; CHECK-NEXT: mov v1.16b, v5.16b820; CHECK-NEXT: ret821entry:822 %d = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c)823 ret <8 x float> %d824}825 826define <7 x half> @fmuladd_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {827; CHECK-SD-NOFP16-LABEL: fmuladd_v7f16:828; CHECK-SD-NOFP16: // %bb.0: // %entry829; CHECK-SD-NOFP16-NEXT: fcvtl v3.4s, v1.4h830; CHECK-SD-NOFP16-NEXT: fcvtl v4.4s, v0.4h831; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h832; CHECK-SD-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h833; CHECK-SD-NOFP16-NEXT: fmul v3.4s, v4.4s, v3.4s834; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s835; CHECK-SD-NOFP16-NEXT: fcvtn v1.4h, v3.4s836; CHECK-SD-NOFP16-NEXT: fcvtl v3.4s, v2.4h837; CHECK-SD-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h838; CHECK-SD-NOFP16-NEXT: fcvtn2 v1.8h, v0.4s839; CHECK-SD-NOFP16-NEXT: fcvtl v0.4s, v1.4h840; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h841; CHECK-SD-NOFP16-NEXT: fadd v0.4s, v0.4s, v3.4s842; CHECK-SD-NOFP16-NEXT: fadd v1.4s, v1.4s, v2.4s843; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v0.4s844; CHECK-SD-NOFP16-NEXT: fcvtn2 v0.8h, v1.4s845; CHECK-SD-NOFP16-NEXT: ret846;847; CHECK-SD-FP16-LABEL: fmuladd_v7f16:848; CHECK-SD-FP16: // %bb.0: // %entry849; CHECK-SD-FP16-NEXT: fmla v2.8h, v1.8h, v0.8h850; CHECK-SD-FP16-NEXT: mov v0.16b, v2.16b851; CHECK-SD-FP16-NEXT: ret852;853; CHECK-GI-NOFP16-LABEL: fmuladd_v7f16:854; CHECK-GI-NOFP16: // %bb.0: // %entry855; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v0.4h856; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v1.4h857; CHECK-GI-NOFP16-NEXT: mov v5.h[0], v0.h[4]858; CHECK-GI-NOFP16-NEXT: fmul v3.4s, v3.4s, v4.4s859; CHECK-GI-NOFP16-NEXT: mov v4.h[0], v1.h[4]860; CHECK-GI-NOFP16-NEXT: mov v5.h[1], v0.h[5]861; CHECK-GI-NOFP16-NEXT: mov v4.h[1], v1.h[5]862; CHECK-GI-NOFP16-NEXT: fcvtn v3.4h, v3.4s863; CHECK-GI-NOFP16-NEXT: mov v5.h[2], v0.h[6]864; CHECK-GI-NOFP16-NEXT: mov v4.h[2], v1.h[6]865; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v3.4h866; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v2.4h867; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v5.4h868; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v4.4h869; CHECK-GI-NOFP16-NEXT: fadd v0.4s, v0.4s, v1.4s870; CHECK-GI-NOFP16-NEXT: mov v1.h[0], v2.h[4]871; CHECK-GI-NOFP16-NEXT: fmul v3.4s, v3.4s, v4.4s872; CHECK-GI-NOFP16-NEXT: mov v1.h[1], v2.h[5]873; CHECK-GI-NOFP16-NEXT: fcvtn v4.4h, v0.4s874; CHECK-GI-NOFP16-NEXT: fcvtn v3.4h, v3.4s875; CHECK-GI-NOFP16-NEXT: mov v1.h[2], v2.h[6]876; CHECK-GI-NOFP16-NEXT: mov v0.h[0], v4.h[0]877; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v3.4h878; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h879; CHECK-GI-NOFP16-NEXT: mov v0.h[1], v4.h[1]880; CHECK-GI-NOFP16-NEXT: fadd v1.4s, v2.4s, v1.4s881; CHECK-GI-NOFP16-NEXT: mov v0.h[2], v4.h[2]882; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v1.4s883; CHECK-GI-NOFP16-NEXT: mov v0.h[3], v4.h[3]884; CHECK-GI-NOFP16-NEXT: mov v0.h[4], v1.h[0]885; CHECK-GI-NOFP16-NEXT: mov v0.h[5], v1.h[1]886; CHECK-GI-NOFP16-NEXT: mov v0.h[6], v1.h[2]887; CHECK-GI-NOFP16-NEXT: ret888;889; CHECK-GI-FP16-LABEL: fmuladd_v7f16:890; CHECK-GI-FP16: // %bb.0: // %entry891; CHECK-GI-FP16-NEXT: fmla v2.8h, v1.8h, v0.8h892; CHECK-GI-FP16-NEXT: mov v0.16b, v2.16b893; CHECK-GI-FP16-NEXT: ret894entry:895 %d = call <7 x half> @llvm.fmuladd.v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c)896 ret <7 x half> %d897}898 899define <4 x half> @fmuladd_v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c) {900; CHECK-SD-NOFP16-LABEL: fmuladd_v4f16:901; CHECK-SD-NOFP16: // %bb.0: // %entry902; CHECK-SD-NOFP16-NEXT: fcvtl v1.4s, v1.4h903; CHECK-SD-NOFP16-NEXT: fcvtl v0.4s, v0.4h904; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s905; CHECK-SD-NOFP16-NEXT: fcvtl v1.4s, v2.4h906; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v0.4s907; CHECK-SD-NOFP16-NEXT: fcvtl v0.4s, v0.4h908; CHECK-SD-NOFP16-NEXT: fadd v0.4s, v0.4s, v1.4s909; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v0.4s910; CHECK-SD-NOFP16-NEXT: ret911;912; CHECK-SD-FP16-LABEL: fmuladd_v4f16:913; CHECK-SD-FP16: // %bb.0: // %entry914; CHECK-SD-FP16-NEXT: fmla v2.4h, v1.4h, v0.4h915; CHECK-SD-FP16-NEXT: fmov d0, d2916; CHECK-SD-FP16-NEXT: ret917;918; CHECK-GI-NOFP16-LABEL: fmuladd_v4f16:919; CHECK-GI-NOFP16: // %bb.0: // %entry920; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h921; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h922; CHECK-GI-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s923; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v2.4h924; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v0.4s925; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h926; CHECK-GI-NOFP16-NEXT: fadd v0.4s, v0.4s, v1.4s927; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v0.4s928; CHECK-GI-NOFP16-NEXT: ret929;930; CHECK-GI-FP16-LABEL: fmuladd_v4f16:931; CHECK-GI-FP16: // %bb.0: // %entry932; CHECK-GI-FP16-NEXT: fmla v2.4h, v1.4h, v0.4h933; CHECK-GI-FP16-NEXT: fmov d0, d2934; CHECK-GI-FP16-NEXT: ret935entry:936 %d = call <4 x half> @llvm.fmuladd.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c)937 ret <4 x half> %d938}939 940define <8 x half> @fmuladd_v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {941; CHECK-SD-NOFP16-LABEL: fmuladd_v8f16:942; CHECK-SD-NOFP16: // %bb.0: // %entry943; CHECK-SD-NOFP16-NEXT: fcvtl v3.4s, v1.4h944; CHECK-SD-NOFP16-NEXT: fcvtl v4.4s, v0.4h945; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h946; CHECK-SD-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h947; CHECK-SD-NOFP16-NEXT: fmul v3.4s, v4.4s, v3.4s948; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s949; CHECK-SD-NOFP16-NEXT: fcvtn v1.4h, v3.4s950; CHECK-SD-NOFP16-NEXT: fcvtl v3.4s, v2.4h951; CHECK-SD-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h952; CHECK-SD-NOFP16-NEXT: fcvtn2 v1.8h, v0.4s953; CHECK-SD-NOFP16-NEXT: fcvtl v0.4s, v1.4h954; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h955; CHECK-SD-NOFP16-NEXT: fadd v0.4s, v0.4s, v3.4s956; CHECK-SD-NOFP16-NEXT: fadd v1.4s, v1.4s, v2.4s957; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v0.4s958; CHECK-SD-NOFP16-NEXT: fcvtn2 v0.8h, v1.4s959; CHECK-SD-NOFP16-NEXT: ret960;961; CHECK-SD-FP16-LABEL: fmuladd_v8f16:962; CHECK-SD-FP16: // %bb.0: // %entry963; CHECK-SD-FP16-NEXT: fmla v2.8h, v1.8h, v0.8h964; CHECK-SD-FP16-NEXT: mov v0.16b, v2.16b965; CHECK-SD-FP16-NEXT: ret966;967; CHECK-GI-NOFP16-LABEL: fmuladd_v8f16:968; CHECK-GI-NOFP16: // %bb.0: // %entry969; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v0.4h970; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v1.4h971; CHECK-GI-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h972; CHECK-GI-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h973; CHECK-GI-NOFP16-NEXT: fmul v3.4s, v3.4s, v4.4s974; CHECK-GI-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s975; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v3.4s976; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v2.4h977; CHECK-GI-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h978; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v0.4s979; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h980; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h981; CHECK-GI-NOFP16-NEXT: fadd v1.4s, v1.4s, v3.4s982; CHECK-GI-NOFP16-NEXT: fadd v2.4s, v0.4s, v2.4s983; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v1.4s984; CHECK-GI-NOFP16-NEXT: fcvtn2 v0.8h, v2.4s985; CHECK-GI-NOFP16-NEXT: ret986;987; CHECK-GI-FP16-LABEL: fmuladd_v8f16:988; CHECK-GI-FP16: // %bb.0: // %entry989; CHECK-GI-FP16-NEXT: fmla v2.8h, v1.8h, v0.8h990; CHECK-GI-FP16-NEXT: mov v0.16b, v2.16b991; CHECK-GI-FP16-NEXT: ret992entry:993 %d = call <8 x half> @llvm.fmuladd.v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c)994 ret <8 x half> %d995}996 997define <16 x half> @fmuladd_v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c) {998; CHECK-SD-NOFP16-LABEL: fmuladd_v16f16:999; CHECK-SD-NOFP16: // %bb.0: // %entry1000; CHECK-SD-NOFP16-NEXT: fcvtl v6.4s, v2.4h1001; CHECK-SD-NOFP16-NEXT: fcvtl v7.4s, v0.4h1002; CHECK-SD-NOFP16-NEXT: fcvtl v16.4s, v3.4h1003; CHECK-SD-NOFP16-NEXT: fcvtl v17.4s, v1.4h1004; CHECK-SD-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h1005; CHECK-SD-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h1006; CHECK-SD-NOFP16-NEXT: fcvtl2 v3.4s, v3.8h1007; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h1008; CHECK-SD-NOFP16-NEXT: fmul v6.4s, v7.4s, v6.4s1009; CHECK-SD-NOFP16-NEXT: fmul v7.4s, v17.4s, v16.4s1010; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v2.4s1011; CHECK-SD-NOFP16-NEXT: fmul v1.4s, v1.4s, v3.4s1012; CHECK-SD-NOFP16-NEXT: fcvtn v2.4h, v6.4s1013; CHECK-SD-NOFP16-NEXT: fcvtl v6.4s, v5.4h1014; CHECK-SD-NOFP16-NEXT: fcvtn v3.4h, v7.4s1015; CHECK-SD-NOFP16-NEXT: fcvtn2 v2.8h, v0.4s1016; CHECK-SD-NOFP16-NEXT: fcvtl v0.4s, v4.4h1017; CHECK-SD-NOFP16-NEXT: fcvtl2 v4.4s, v4.8h1018; CHECK-SD-NOFP16-NEXT: fcvtn2 v3.8h, v1.4s1019; CHECK-SD-NOFP16-NEXT: fcvtl v1.4s, v2.4h1020; CHECK-SD-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h1021; CHECK-SD-NOFP16-NEXT: fcvtl v7.4s, v3.4h1022; CHECK-SD-NOFP16-NEXT: fadd v0.4s, v1.4s, v0.4s1023; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v3.8h1024; CHECK-SD-NOFP16-NEXT: fcvtl2 v3.4s, v5.8h1025; CHECK-SD-NOFP16-NEXT: fadd v5.4s, v7.4s, v6.4s1026; CHECK-SD-NOFP16-NEXT: fadd v2.4s, v2.4s, v4.4s1027; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v0.4s1028; CHECK-SD-NOFP16-NEXT: fadd v3.4s, v1.4s, v3.4s1029; CHECK-SD-NOFP16-NEXT: fcvtn v1.4h, v5.4s1030; CHECK-SD-NOFP16-NEXT: fcvtn2 v0.8h, v2.4s1031; CHECK-SD-NOFP16-NEXT: fcvtn2 v1.8h, v3.4s1032; CHECK-SD-NOFP16-NEXT: ret1033;1034; CHECK-SD-FP16-LABEL: fmuladd_v16f16:1035; CHECK-SD-FP16: // %bb.0: // %entry1036; CHECK-SD-FP16-NEXT: fmla v4.8h, v2.8h, v0.8h1037; CHECK-SD-FP16-NEXT: fmla v5.8h, v3.8h, v1.8h1038; CHECK-SD-FP16-NEXT: mov v0.16b, v4.16b1039; CHECK-SD-FP16-NEXT: mov v1.16b, v5.16b1040; CHECK-SD-FP16-NEXT: ret1041;1042; CHECK-GI-NOFP16-LABEL: fmuladd_v16f16:1043; CHECK-GI-NOFP16: // %bb.0: // %entry1044; CHECK-GI-NOFP16-NEXT: fcvtl v6.4s, v0.4h1045; CHECK-GI-NOFP16-NEXT: fcvtl v7.4s, v2.4h1046; CHECK-GI-NOFP16-NEXT: fcvtl v16.4s, v1.4h1047; CHECK-GI-NOFP16-NEXT: fcvtl v17.4s, v3.4h1048; CHECK-GI-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h1049; CHECK-GI-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h1050; CHECK-GI-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h1051; CHECK-GI-NOFP16-NEXT: fcvtl2 v3.4s, v3.8h1052; CHECK-GI-NOFP16-NEXT: fmul v6.4s, v6.4s, v7.4s1053; CHECK-GI-NOFP16-NEXT: fmul v7.4s, v16.4s, v17.4s1054; CHECK-GI-NOFP16-NEXT: fmul v0.4s, v0.4s, v2.4s1055; CHECK-GI-NOFP16-NEXT: fmul v1.4s, v1.4s, v3.4s1056; CHECK-GI-NOFP16-NEXT: fcvtn v2.4h, v6.4s1057; CHECK-GI-NOFP16-NEXT: fcvtl v6.4s, v4.4h1058; CHECK-GI-NOFP16-NEXT: fcvtl2 v4.4s, v4.8h1059; CHECK-GI-NOFP16-NEXT: fcvtn v3.4h, v7.4s1060; CHECK-GI-NOFP16-NEXT: fcvtl v7.4s, v5.4h1061; CHECK-GI-NOFP16-NEXT: fcvtl2 v5.4s, v5.8h1062; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v0.4s1063; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v1.4s1064; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v2.4h1065; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v3.4h1066; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h1067; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h1068; CHECK-GI-NOFP16-NEXT: fadd v2.4s, v2.4s, v6.4s1069; CHECK-GI-NOFP16-NEXT: fadd v3.4s, v3.4s, v7.4s1070; CHECK-GI-NOFP16-NEXT: fadd v4.4s, v0.4s, v4.4s1071; CHECK-GI-NOFP16-NEXT: fadd v5.4s, v1.4s, v5.4s1072; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v2.4s1073; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v3.4s1074; CHECK-GI-NOFP16-NEXT: fcvtn2 v0.8h, v4.4s1075; CHECK-GI-NOFP16-NEXT: fcvtn2 v1.8h, v5.4s1076; CHECK-GI-NOFP16-NEXT: ret1077;1078; CHECK-GI-FP16-LABEL: fmuladd_v16f16:1079; CHECK-GI-FP16: // %bb.0: // %entry1080; CHECK-GI-FP16-NEXT: fmla v4.8h, v2.8h, v0.8h1081; CHECK-GI-FP16-NEXT: fmla v5.8h, v3.8h, v1.8h1082; CHECK-GI-FP16-NEXT: mov v0.16b, v4.16b1083; CHECK-GI-FP16-NEXT: mov v1.16b, v5.16b1084; CHECK-GI-FP16-NEXT: ret1085entry:1086 %d = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c)1087 ret <16 x half> %d1088}1089 1090define double @fmul_f64(double %a, double %b, double %c) {1091; CHECK-LABEL: fmul_f64:1092; CHECK: // %bb.0: // %entry1093; CHECK-NEXT: fmadd d0, d0, d1, d21094; CHECK-NEXT: ret1095entry:1096 %d = fmul fast double %a, %b1097 %e = fadd fast double %d, %c1098 ret double %e1099}1100 1101define float @fmul_f32(float %a, float %b, float %c) {1102; CHECK-LABEL: fmul_f32:1103; CHECK: // %bb.0: // %entry1104; CHECK-NEXT: fmadd s0, s0, s1, s21105; CHECK-NEXT: ret1106entry:1107 %d = fmul fast float %a, %b1108 %e = fadd fast float %d, %c1109 ret float %e1110}1111 1112define half @fmul_f16(half %a, half %b, half %c) {1113; CHECK-SD-NOFP16-LABEL: fmul_f16:1114; CHECK-SD-NOFP16: // %bb.0: // %entry1115; CHECK-SD-NOFP16-NEXT: fcvt s1, h11116; CHECK-SD-NOFP16-NEXT: fcvt s0, h01117; CHECK-SD-NOFP16-NEXT: fcvt s2, h21118; CHECK-SD-NOFP16-NEXT: fmadd s0, s0, s1, s21119; CHECK-SD-NOFP16-NEXT: fcvt h0, s01120; CHECK-SD-NOFP16-NEXT: ret1121;1122; CHECK-SD-FP16-LABEL: fmul_f16:1123; CHECK-SD-FP16: // %bb.0: // %entry1124; CHECK-SD-FP16-NEXT: fmadd h0, h0, h1, h21125; CHECK-SD-FP16-NEXT: ret1126;1127; CHECK-GI-NOFP16-LABEL: fmul_f16:1128; CHECK-GI-NOFP16: // %bb.0: // %entry1129; CHECK-GI-NOFP16-NEXT: fcvt s0, h01130; CHECK-GI-NOFP16-NEXT: fcvt s1, h11131; CHECK-GI-NOFP16-NEXT: fmul s0, s0, s11132; CHECK-GI-NOFP16-NEXT: fcvt s1, h21133; CHECK-GI-NOFP16-NEXT: fcvt h0, s01134; CHECK-GI-NOFP16-NEXT: fcvt s0, h01135; CHECK-GI-NOFP16-NEXT: fadd s0, s0, s11136; CHECK-GI-NOFP16-NEXT: fcvt h0, s01137; CHECK-GI-NOFP16-NEXT: ret1138;1139; CHECK-GI-FP16-LABEL: fmul_f16:1140; CHECK-GI-FP16: // %bb.0: // %entry1141; CHECK-GI-FP16-NEXT: fmadd h0, h0, h1, h21142; CHECK-GI-FP16-NEXT: ret1143entry:1144 %d = fmul fast half %a, %b1145 %e = fadd fast half %d, %c1146 ret half %e1147}1148 1149define <2 x double> @fmul_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c) {1150; CHECK-SD-LABEL: fmul_v2f64:1151; CHECK-SD: // %bb.0: // %entry1152; CHECK-SD-NEXT: fmla v2.2d, v1.2d, v0.2d1153; CHECK-SD-NEXT: mov v0.16b, v2.16b1154; CHECK-SD-NEXT: ret1155;1156; CHECK-GI-LABEL: fmul_v2f64:1157; CHECK-GI: // %bb.0: // %entry1158; CHECK-GI-NEXT: fmla v2.2d, v0.2d, v1.2d1159; CHECK-GI-NEXT: mov v0.16b, v2.16b1160; CHECK-GI-NEXT: ret1161entry:1162 %d = fmul fast <2 x double> %a, %b1163 %e = fadd fast <2 x double> %d, %c1164 ret <2 x double> %e1165}1166 1167define <3 x double> @fmul_v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c) {1168; CHECK-SD-LABEL: fmul_v3f64:1169; CHECK-SD: // %bb.0: // %entry1170; CHECK-SD-NEXT: // kill: def $d6 killed $d6 def $q61171; CHECK-SD-NEXT: // kill: def $d3 killed $d3 def $q31172; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q01173; CHECK-SD-NEXT: // kill: def $d7 killed $d7 def $q71174; CHECK-SD-NEXT: // kill: def $d4 killed $d4 def $q41175; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q11176; CHECK-SD-NEXT: // kill: def $d5 killed $d5 def $q51177; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q21178; CHECK-SD-NEXT: mov v3.d[1], v4.d[0]1179; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]1180; CHECK-SD-NEXT: mov v6.d[1], v7.d[0]1181; CHECK-SD-NEXT: fmla v6.2d, v3.2d, v0.2d1182; CHECK-SD-NEXT: ldr d3, [sp]1183; CHECK-SD-NEXT: fmla v3.2d, v5.2d, v2.2d1184; CHECK-SD-NEXT: fmov d0, d61185; CHECK-SD-NEXT: ext v1.16b, v6.16b, v6.16b, #81186; CHECK-SD-NEXT: // kill: def $d1 killed $d1 killed $q11187; CHECK-SD-NEXT: fmov d2, d31188; CHECK-SD-NEXT: ret1189;1190; CHECK-GI-LABEL: fmul_v3f64:1191; CHECK-GI: // %bb.0: // %entry1192; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q01193; CHECK-GI-NEXT: // kill: def $d3 killed $d3 def $q31194; CHECK-GI-NEXT: // kill: def $d6 killed $d6 def $q61195; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q11196; CHECK-GI-NEXT: // kill: def $d4 killed $d4 def $q41197; CHECK-GI-NEXT: // kill: def $d7 killed $d7 def $q71198; CHECK-GI-NEXT: mov v0.d[1], v1.d[0]1199; CHECK-GI-NEXT: mov v3.d[1], v4.d[0]1200; CHECK-GI-NEXT: mov v6.d[1], v7.d[0]1201; CHECK-GI-NEXT: fmla v6.2d, v0.2d, v3.2d1202; CHECK-GI-NEXT: ldr d0, [sp]1203; CHECK-GI-NEXT: fmadd d2, d2, d5, d01204; CHECK-GI-NEXT: mov d1, v6.d[1]1205; CHECK-GI-NEXT: fmov d0, d61206; CHECK-GI-NEXT: ret1207entry:1208 %d = fmul fast <3 x double> %a, %b1209 %e = fadd fast <3 x double> %d, %c1210 ret <3 x double> %e1211}1212 1213define <4 x double> @fmul_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c) {1214; CHECK-SD-LABEL: fmul_v4f64:1215; CHECK-SD: // %bb.0: // %entry1216; CHECK-SD-NEXT: fmla v4.2d, v2.2d, v0.2d1217; CHECK-SD-NEXT: fmla v5.2d, v3.2d, v1.2d1218; CHECK-SD-NEXT: mov v0.16b, v4.16b1219; CHECK-SD-NEXT: mov v1.16b, v5.16b1220; CHECK-SD-NEXT: ret1221;1222; CHECK-GI-LABEL: fmul_v4f64:1223; CHECK-GI: // %bb.0: // %entry1224; CHECK-GI-NEXT: fmla v4.2d, v0.2d, v2.2d1225; CHECK-GI-NEXT: fmla v5.2d, v1.2d, v3.2d1226; CHECK-GI-NEXT: mov v0.16b, v4.16b1227; CHECK-GI-NEXT: mov v1.16b, v5.16b1228; CHECK-GI-NEXT: ret1229entry:1230 %d = fmul fast <4 x double> %a, %b1231 %e = fadd fast <4 x double> %d, %c1232 ret <4 x double> %e1233}1234 1235define <2 x float> @fmul_v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c) {1236; CHECK-SD-LABEL: fmul_v2f32:1237; CHECK-SD: // %bb.0: // %entry1238; CHECK-SD-NEXT: fmla v2.2s, v1.2s, v0.2s1239; CHECK-SD-NEXT: fmov d0, d21240; CHECK-SD-NEXT: ret1241;1242; CHECK-GI-LABEL: fmul_v2f32:1243; CHECK-GI: // %bb.0: // %entry1244; CHECK-GI-NEXT: fmla v2.2s, v0.2s, v1.2s1245; CHECK-GI-NEXT: fmov d0, d21246; CHECK-GI-NEXT: ret1247entry:1248 %d = fmul fast <2 x float> %a, %b1249 %e = fadd fast <2 x float> %d, %c1250 ret <2 x float> %e1251}1252 1253define <3 x float> @fmul_v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c) {1254; CHECK-SD-LABEL: fmul_v3f32:1255; CHECK-SD: // %bb.0: // %entry1256; CHECK-SD-NEXT: fmla v2.4s, v1.4s, v0.4s1257; CHECK-SD-NEXT: mov v0.16b, v2.16b1258; CHECK-SD-NEXT: ret1259;1260; CHECK-GI-LABEL: fmul_v3f32:1261; CHECK-GI: // %bb.0: // %entry1262; CHECK-GI-NEXT: fmla v2.4s, v0.4s, v1.4s1263; CHECK-GI-NEXT: mov v0.16b, v2.16b1264; CHECK-GI-NEXT: ret1265entry:1266 %d = fmul fast <3 x float> %a, %b1267 %e = fadd fast <3 x float> %d, %c1268 ret <3 x float> %e1269}1270 1271define <4 x float> @fmul_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {1272; CHECK-SD-LABEL: fmul_v4f32:1273; CHECK-SD: // %bb.0: // %entry1274; CHECK-SD-NEXT: fmla v2.4s, v1.4s, v0.4s1275; CHECK-SD-NEXT: mov v0.16b, v2.16b1276; CHECK-SD-NEXT: ret1277;1278; CHECK-GI-LABEL: fmul_v4f32:1279; CHECK-GI: // %bb.0: // %entry1280; CHECK-GI-NEXT: fmla v2.4s, v0.4s, v1.4s1281; CHECK-GI-NEXT: mov v0.16b, v2.16b1282; CHECK-GI-NEXT: ret1283entry:1284 %d = fmul fast <4 x float> %a, %b1285 %e = fadd fast <4 x float> %d, %c1286 ret <4 x float> %e1287}1288 1289define <8 x float> @fmul_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) {1290; CHECK-SD-LABEL: fmul_v8f32:1291; CHECK-SD: // %bb.0: // %entry1292; CHECK-SD-NEXT: fmla v4.4s, v2.4s, v0.4s1293; CHECK-SD-NEXT: fmla v5.4s, v3.4s, v1.4s1294; CHECK-SD-NEXT: mov v0.16b, v4.16b1295; CHECK-SD-NEXT: mov v1.16b, v5.16b1296; CHECK-SD-NEXT: ret1297;1298; CHECK-GI-LABEL: fmul_v8f32:1299; CHECK-GI: // %bb.0: // %entry1300; CHECK-GI-NEXT: fmla v4.4s, v0.4s, v2.4s1301; CHECK-GI-NEXT: fmla v5.4s, v1.4s, v3.4s1302; CHECK-GI-NEXT: mov v0.16b, v4.16b1303; CHECK-GI-NEXT: mov v1.16b, v5.16b1304; CHECK-GI-NEXT: ret1305entry:1306 %d = fmul fast <8 x float> %a, %b1307 %e = fadd fast <8 x float> %d, %c1308 ret <8 x float> %e1309}1310 1311define <7 x half> @fmul_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {1312; CHECK-SD-NOFP16-LABEL: fmul_v7f16:1313; CHECK-SD-NOFP16: // %bb.0: // %entry1314; CHECK-SD-NOFP16-NEXT: fcvtl v3.4s, v1.4h1315; CHECK-SD-NOFP16-NEXT: fcvtl v4.4s, v0.4h1316; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h1317; CHECK-SD-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h1318; CHECK-SD-NOFP16-NEXT: fmul v3.4s, v4.4s, v3.4s1319; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s1320; CHECK-SD-NOFP16-NEXT: fcvtn v1.4h, v3.4s1321; CHECK-SD-NOFP16-NEXT: fcvtl v3.4s, v2.4h1322; CHECK-SD-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h1323; CHECK-SD-NOFP16-NEXT: fcvtn2 v1.8h, v0.4s1324; CHECK-SD-NOFP16-NEXT: fcvtl v0.4s, v1.4h1325; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h1326; CHECK-SD-NOFP16-NEXT: fadd v0.4s, v0.4s, v3.4s1327; CHECK-SD-NOFP16-NEXT: fadd v1.4s, v1.4s, v2.4s1328; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v0.4s1329; CHECK-SD-NOFP16-NEXT: fcvtn2 v0.8h, v1.4s1330; CHECK-SD-NOFP16-NEXT: ret1331;1332; CHECK-SD-FP16-LABEL: fmul_v7f16:1333; CHECK-SD-FP16: // %bb.0: // %entry1334; CHECK-SD-FP16-NEXT: fmla v2.8h, v1.8h, v0.8h1335; CHECK-SD-FP16-NEXT: mov v0.16b, v2.16b1336; CHECK-SD-FP16-NEXT: ret1337;1338; CHECK-GI-NOFP16-LABEL: fmul_v7f16:1339; CHECK-GI-NOFP16: // %bb.0: // %entry1340; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v0.4h1341; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v1.4h1342; CHECK-GI-NOFP16-NEXT: mov v5.h[0], v0.h[4]1343; CHECK-GI-NOFP16-NEXT: fmul v3.4s, v3.4s, v4.4s1344; CHECK-GI-NOFP16-NEXT: mov v4.h[0], v1.h[4]1345; CHECK-GI-NOFP16-NEXT: mov v5.h[1], v0.h[5]1346; CHECK-GI-NOFP16-NEXT: mov v4.h[1], v1.h[5]1347; CHECK-GI-NOFP16-NEXT: fcvtn v3.4h, v3.4s1348; CHECK-GI-NOFP16-NEXT: mov v5.h[2], v0.h[6]1349; CHECK-GI-NOFP16-NEXT: mov v4.h[2], v1.h[6]1350; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v3.4h1351; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v2.4h1352; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v5.4h1353; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v4.4h1354; CHECK-GI-NOFP16-NEXT: fadd v0.4s, v0.4s, v1.4s1355; CHECK-GI-NOFP16-NEXT: mov v1.h[0], v2.h[4]1356; CHECK-GI-NOFP16-NEXT: fmul v3.4s, v3.4s, v4.4s1357; CHECK-GI-NOFP16-NEXT: mov v1.h[1], v2.h[5]1358; CHECK-GI-NOFP16-NEXT: fcvtn v4.4h, v0.4s1359; CHECK-GI-NOFP16-NEXT: fcvtn v3.4h, v3.4s1360; CHECK-GI-NOFP16-NEXT: mov v1.h[2], v2.h[6]1361; CHECK-GI-NOFP16-NEXT: mov v0.h[0], v4.h[0]1362; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v3.4h1363; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h1364; CHECK-GI-NOFP16-NEXT: mov v0.h[1], v4.h[1]1365; CHECK-GI-NOFP16-NEXT: fadd v1.4s, v2.4s, v1.4s1366; CHECK-GI-NOFP16-NEXT: mov v0.h[2], v4.h[2]1367; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v1.4s1368; CHECK-GI-NOFP16-NEXT: mov v0.h[3], v4.h[3]1369; CHECK-GI-NOFP16-NEXT: mov v0.h[4], v1.h[0]1370; CHECK-GI-NOFP16-NEXT: mov v0.h[5], v1.h[1]1371; CHECK-GI-NOFP16-NEXT: mov v0.h[6], v1.h[2]1372; CHECK-GI-NOFP16-NEXT: ret1373;1374; CHECK-GI-FP16-LABEL: fmul_v7f16:1375; CHECK-GI-FP16: // %bb.0: // %entry1376; CHECK-GI-FP16-NEXT: fmla v2.8h, v0.8h, v1.8h1377; CHECK-GI-FP16-NEXT: mov v0.16b, v2.16b1378; CHECK-GI-FP16-NEXT: ret1379entry:1380 %d = fmul fast <7 x half> %a, %b1381 %e = fadd fast <7 x half> %d, %c1382 ret <7 x half> %e1383}1384 1385define <4 x half> @fmul_v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c) {1386; CHECK-SD-NOFP16-LABEL: fmul_v4f16:1387; CHECK-SD-NOFP16: // %bb.0: // %entry1388; CHECK-SD-NOFP16-NEXT: fcvtl v1.4s, v1.4h1389; CHECK-SD-NOFP16-NEXT: fcvtl v0.4s, v0.4h1390; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s1391; CHECK-SD-NOFP16-NEXT: fcvtl v1.4s, v2.4h1392; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v0.4s1393; CHECK-SD-NOFP16-NEXT: fcvtl v0.4s, v0.4h1394; CHECK-SD-NOFP16-NEXT: fadd v0.4s, v0.4s, v1.4s1395; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v0.4s1396; CHECK-SD-NOFP16-NEXT: ret1397;1398; CHECK-SD-FP16-LABEL: fmul_v4f16:1399; CHECK-SD-FP16: // %bb.0: // %entry1400; CHECK-SD-FP16-NEXT: fmla v2.4h, v1.4h, v0.4h1401; CHECK-SD-FP16-NEXT: fmov d0, d21402; CHECK-SD-FP16-NEXT: ret1403;1404; CHECK-GI-NOFP16-LABEL: fmul_v4f16:1405; CHECK-GI-NOFP16: // %bb.0: // %entry1406; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h1407; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h1408; CHECK-GI-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s1409; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v2.4h1410; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v0.4s1411; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h1412; CHECK-GI-NOFP16-NEXT: fadd v0.4s, v0.4s, v1.4s1413; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v0.4s1414; CHECK-GI-NOFP16-NEXT: ret1415;1416; CHECK-GI-FP16-LABEL: fmul_v4f16:1417; CHECK-GI-FP16: // %bb.0: // %entry1418; CHECK-GI-FP16-NEXT: fmla v2.4h, v0.4h, v1.4h1419; CHECK-GI-FP16-NEXT: fmov d0, d21420; CHECK-GI-FP16-NEXT: ret1421entry:1422 %d = fmul fast <4 x half> %a, %b1423 %e = fadd fast <4 x half> %d, %c1424 ret <4 x half> %e1425}1426 1427define <8 x half> @fmul_v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {1428; CHECK-SD-NOFP16-LABEL: fmul_v8f16:1429; CHECK-SD-NOFP16: // %bb.0: // %entry1430; CHECK-SD-NOFP16-NEXT: fcvtl v3.4s, v1.4h1431; CHECK-SD-NOFP16-NEXT: fcvtl v4.4s, v0.4h1432; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h1433; CHECK-SD-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h1434; CHECK-SD-NOFP16-NEXT: fmul v3.4s, v4.4s, v3.4s1435; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s1436; CHECK-SD-NOFP16-NEXT: fcvtn v1.4h, v3.4s1437; CHECK-SD-NOFP16-NEXT: fcvtl v3.4s, v2.4h1438; CHECK-SD-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h1439; CHECK-SD-NOFP16-NEXT: fcvtn2 v1.8h, v0.4s1440; CHECK-SD-NOFP16-NEXT: fcvtl v0.4s, v1.4h1441; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h1442; CHECK-SD-NOFP16-NEXT: fadd v0.4s, v0.4s, v3.4s1443; CHECK-SD-NOFP16-NEXT: fadd v1.4s, v1.4s, v2.4s1444; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v0.4s1445; CHECK-SD-NOFP16-NEXT: fcvtn2 v0.8h, v1.4s1446; CHECK-SD-NOFP16-NEXT: ret1447;1448; CHECK-SD-FP16-LABEL: fmul_v8f16:1449; CHECK-SD-FP16: // %bb.0: // %entry1450; CHECK-SD-FP16-NEXT: fmla v2.8h, v1.8h, v0.8h1451; CHECK-SD-FP16-NEXT: mov v0.16b, v2.16b1452; CHECK-SD-FP16-NEXT: ret1453;1454; CHECK-GI-NOFP16-LABEL: fmul_v8f16:1455; CHECK-GI-NOFP16: // %bb.0: // %entry1456; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v0.4h1457; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v1.4h1458; CHECK-GI-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h1459; CHECK-GI-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h1460; CHECK-GI-NOFP16-NEXT: fmul v3.4s, v3.4s, v4.4s1461; CHECK-GI-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s1462; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v3.4s1463; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v2.4h1464; CHECK-GI-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h1465; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v0.4s1466; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h1467; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h1468; CHECK-GI-NOFP16-NEXT: fadd v1.4s, v1.4s, v3.4s1469; CHECK-GI-NOFP16-NEXT: fadd v2.4s, v0.4s, v2.4s1470; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v1.4s1471; CHECK-GI-NOFP16-NEXT: fcvtn2 v0.8h, v2.4s1472; CHECK-GI-NOFP16-NEXT: ret1473;1474; CHECK-GI-FP16-LABEL: fmul_v8f16:1475; CHECK-GI-FP16: // %bb.0: // %entry1476; CHECK-GI-FP16-NEXT: fmla v2.8h, v0.8h, v1.8h1477; CHECK-GI-FP16-NEXT: mov v0.16b, v2.16b1478; CHECK-GI-FP16-NEXT: ret1479entry:1480 %d = fmul fast <8 x half> %a, %b1481 %e = fadd fast <8 x half> %d, %c1482 ret <8 x half> %e1483}1484 1485define <16 x half> @fmul_v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c) {1486; CHECK-SD-NOFP16-LABEL: fmul_v16f16:1487; CHECK-SD-NOFP16: // %bb.0: // %entry1488; CHECK-SD-NOFP16-NEXT: fcvtl v6.4s, v2.4h1489; CHECK-SD-NOFP16-NEXT: fcvtl v7.4s, v0.4h1490; CHECK-SD-NOFP16-NEXT: fcvtl v16.4s, v3.4h1491; CHECK-SD-NOFP16-NEXT: fcvtl v17.4s, v1.4h1492; CHECK-SD-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h1493; CHECK-SD-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h1494; CHECK-SD-NOFP16-NEXT: fcvtl2 v3.4s, v3.8h1495; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h1496; CHECK-SD-NOFP16-NEXT: fmul v6.4s, v7.4s, v6.4s1497; CHECK-SD-NOFP16-NEXT: fmul v7.4s, v17.4s, v16.4s1498; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v2.4s1499; CHECK-SD-NOFP16-NEXT: fmul v1.4s, v1.4s, v3.4s1500; CHECK-SD-NOFP16-NEXT: fcvtn v2.4h, v6.4s1501; CHECK-SD-NOFP16-NEXT: fcvtl v6.4s, v5.4h1502; CHECK-SD-NOFP16-NEXT: fcvtn v3.4h, v7.4s1503; CHECK-SD-NOFP16-NEXT: fcvtn2 v2.8h, v0.4s1504; CHECK-SD-NOFP16-NEXT: fcvtl v0.4s, v4.4h1505; CHECK-SD-NOFP16-NEXT: fcvtl2 v4.4s, v4.8h1506; CHECK-SD-NOFP16-NEXT: fcvtn2 v3.8h, v1.4s1507; CHECK-SD-NOFP16-NEXT: fcvtl v1.4s, v2.4h1508; CHECK-SD-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h1509; CHECK-SD-NOFP16-NEXT: fcvtl v7.4s, v3.4h1510; CHECK-SD-NOFP16-NEXT: fadd v0.4s, v1.4s, v0.4s1511; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v3.8h1512; CHECK-SD-NOFP16-NEXT: fcvtl2 v3.4s, v5.8h1513; CHECK-SD-NOFP16-NEXT: fadd v5.4s, v7.4s, v6.4s1514; CHECK-SD-NOFP16-NEXT: fadd v2.4s, v2.4s, v4.4s1515; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v0.4s1516; CHECK-SD-NOFP16-NEXT: fadd v3.4s, v1.4s, v3.4s1517; CHECK-SD-NOFP16-NEXT: fcvtn v1.4h, v5.4s1518; CHECK-SD-NOFP16-NEXT: fcvtn2 v0.8h, v2.4s1519; CHECK-SD-NOFP16-NEXT: fcvtn2 v1.8h, v3.4s1520; CHECK-SD-NOFP16-NEXT: ret1521;1522; CHECK-SD-FP16-LABEL: fmul_v16f16:1523; CHECK-SD-FP16: // %bb.0: // %entry1524; CHECK-SD-FP16-NEXT: fmla v4.8h, v2.8h, v0.8h1525; CHECK-SD-FP16-NEXT: fmla v5.8h, v3.8h, v1.8h1526; CHECK-SD-FP16-NEXT: mov v0.16b, v4.16b1527; CHECK-SD-FP16-NEXT: mov v1.16b, v5.16b1528; CHECK-SD-FP16-NEXT: ret1529;1530; CHECK-GI-NOFP16-LABEL: fmul_v16f16:1531; CHECK-GI-NOFP16: // %bb.0: // %entry1532; CHECK-GI-NOFP16-NEXT: fcvtl v6.4s, v0.4h1533; CHECK-GI-NOFP16-NEXT: fcvtl v7.4s, v2.4h1534; CHECK-GI-NOFP16-NEXT: fcvtl v16.4s, v1.4h1535; CHECK-GI-NOFP16-NEXT: fcvtl v17.4s, v3.4h1536; CHECK-GI-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h1537; CHECK-GI-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h1538; CHECK-GI-NOFP16-NEXT: fcvtl2 v2.4s, v2.8h1539; CHECK-GI-NOFP16-NEXT: fcvtl2 v3.4s, v3.8h1540; CHECK-GI-NOFP16-NEXT: fmul v6.4s, v6.4s, v7.4s1541; CHECK-GI-NOFP16-NEXT: fmul v7.4s, v16.4s, v17.4s1542; CHECK-GI-NOFP16-NEXT: fmul v0.4s, v0.4s, v2.4s1543; CHECK-GI-NOFP16-NEXT: fmul v1.4s, v1.4s, v3.4s1544; CHECK-GI-NOFP16-NEXT: fcvtn v2.4h, v6.4s1545; CHECK-GI-NOFP16-NEXT: fcvtl v6.4s, v4.4h1546; CHECK-GI-NOFP16-NEXT: fcvtl2 v4.4s, v4.8h1547; CHECK-GI-NOFP16-NEXT: fcvtn v3.4h, v7.4s1548; CHECK-GI-NOFP16-NEXT: fcvtl v7.4s, v5.4h1549; CHECK-GI-NOFP16-NEXT: fcvtl2 v5.4s, v5.8h1550; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v0.4s1551; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v1.4s1552; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v2.4h1553; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v3.4h1554; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h1555; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h1556; CHECK-GI-NOFP16-NEXT: fadd v2.4s, v2.4s, v6.4s1557; CHECK-GI-NOFP16-NEXT: fadd v3.4s, v3.4s, v7.4s1558; CHECK-GI-NOFP16-NEXT: fadd v4.4s, v0.4s, v4.4s1559; CHECK-GI-NOFP16-NEXT: fadd v5.4s, v1.4s, v5.4s1560; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v2.4s1561; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v3.4s1562; CHECK-GI-NOFP16-NEXT: fcvtn2 v0.8h, v4.4s1563; CHECK-GI-NOFP16-NEXT: fcvtn2 v1.8h, v5.4s1564; CHECK-GI-NOFP16-NEXT: ret1565;1566; CHECK-GI-FP16-LABEL: fmul_v16f16:1567; CHECK-GI-FP16: // %bb.0: // %entry1568; CHECK-GI-FP16-NEXT: fmla v4.8h, v0.8h, v2.8h1569; CHECK-GI-FP16-NEXT: fmla v5.8h, v1.8h, v3.8h1570; CHECK-GI-FP16-NEXT: mov v0.16b, v4.16b1571; CHECK-GI-FP16-NEXT: mov v1.16b, v5.16b1572; CHECK-GI-FP16-NEXT: ret1573entry:1574 %d = fmul fast <16 x half> %a, %b1575 %e = fadd fast <16 x half> %d, %c1576 ret <16 x half> %e1577}1578 1579declare <16 x half> @llvm.fma.v16f16(<16 x half>, <16 x half>, <16 x half>)1580declare <16 x half> @llvm.fmuladd.v16f16(<16 x half>, <16 x half>, <16 x half>)1581declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)1582declare <2 x double> @llvm.fmuladd.v2f64(<2 x double>, <2 x double>, <2 x double>)1583declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)1584declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>)1585declare <3 x double> @llvm.fma.v3f64(<3 x double>, <3 x double>, <3 x double>)1586declare <3 x double> @llvm.fmuladd.v3f64(<3 x double>, <3 x double>, <3 x double>)1587declare <3 x float> @llvm.fma.v3f32(<3 x float>, <3 x float>, <3 x float>)1588declare <3 x float> @llvm.fmuladd.v3f32(<3 x float>, <3 x float>, <3 x float>)1589declare <4 x double> @llvm.fma.v4f64(<4 x double>, <4 x double>, <4 x double>)1590declare <4 x double> @llvm.fmuladd.v4f64(<4 x double>, <4 x double>, <4 x double>)1591declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)1592declare <4 x float> @llvm.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>)1593declare <4 x half> @llvm.fma.v4f16(<4 x half>, <4 x half>, <4 x half>)1594declare <4 x half> @llvm.fmuladd.v4f16(<4 x half>, <4 x half>, <4 x half>)1595declare <7 x half> @llvm.fma.v7f16(<7 x half>, <7 x half>, <7 x half>)1596declare <7 x half> @llvm.fmuladd.v7f16(<7 x half>, <7 x half>, <7 x half>)1597declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>)1598declare <8 x float> @llvm.fmuladd.v8f32(<8 x float>, <8 x float>, <8 x float>)1599declare <8 x half> @llvm.fma.v8f16(<8 x half>, <8 x half>, <8 x half>)1600declare <8 x half> @llvm.fmuladd.v8f16(<8 x half>, <8 x half>, <8 x half>)1601declare double @llvm.fma.f64(double, double, double)1602declare double @llvm.fmuladd.f64(double, double, double)1603declare float @llvm.fma.f32(float, float, float)1604declare float @llvm.fmuladd.f32(float, float, float)1605declare half @llvm.fma.f16(half, half, half)1606declare half @llvm.fmuladd.f16(half, half, half)1607