683 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mattr=+sve -force-streaming-compatible < %s | FileCheck %s3; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE4 5target triple = "aarch64-unknown-linux-gnu"6 7;8; FMA9;10 11define <4 x half> @fma_v4f16(<4 x half> %op1, <4 x half> %op2, <4 x half> %op3) {12; CHECK-LABEL: fma_v4f16:13; CHECK: // %bb.0:14; CHECK-NEXT: ptrue p0.h, vl415; CHECK-NEXT: // kill: def $d0 killed $d0 def $z016; CHECK-NEXT: // kill: def $d2 killed $d2 def $z217; CHECK-NEXT: // kill: def $d1 killed $d1 def $z118; CHECK-NEXT: fmad z0.h, p0/m, z1.h, z2.h19; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z020; CHECK-NEXT: ret21;22; NONEON-NOSVE-LABEL: fma_v4f16:23; NONEON-NOSVE: // %bb.0:24; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!25; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 3226; NONEON-NOSVE-NEXT: ldr h0, [sp, #14]27; NONEON-NOSVE-NEXT: ldr h1, [sp, #6]28; NONEON-NOSVE-NEXT: str d2, [sp, #16]29; NONEON-NOSVE-NEXT: ldr h2, [sp, #22]30; NONEON-NOSVE-NEXT: ldr h6, [sp, #12]31; NONEON-NOSVE-NEXT: ldr h7, [sp, #4]32; NONEON-NOSVE-NEXT: fcvt s0, h033; NONEON-NOSVE-NEXT: fcvt s1, h134; NONEON-NOSVE-NEXT: ldr h4, [sp, #10]35; NONEON-NOSVE-NEXT: fcvt s2, h236; NONEON-NOSVE-NEXT: fcvt s6, h637; NONEON-NOSVE-NEXT: fcvt s7, h738; NONEON-NOSVE-NEXT: ldr h5, [sp, #2]39; NONEON-NOSVE-NEXT: fcvt s4, h440; NONEON-NOSVE-NEXT: ldr h3, [sp]41; NONEON-NOSVE-NEXT: fmul s0, s1, s042; NONEON-NOSVE-NEXT: fcvt s5, h543; NONEON-NOSVE-NEXT: ldr h1, [sp, #8]44; NONEON-NOSVE-NEXT: fcvt s3, h345; NONEON-NOSVE-NEXT: fcvt s1, h146; NONEON-NOSVE-NEXT: fcvt h0, s047; NONEON-NOSVE-NEXT: fmul s1, s3, s148; NONEON-NOSVE-NEXT: fcvt s0, h049; NONEON-NOSVE-NEXT: fcvt h1, s150; NONEON-NOSVE-NEXT: fadd s0, s0, s251; NONEON-NOSVE-NEXT: fmul s2, s7, s652; NONEON-NOSVE-NEXT: fcvt s1, h153; NONEON-NOSVE-NEXT: fcvt h0, s054; NONEON-NOSVE-NEXT: fcvt h2, s255; NONEON-NOSVE-NEXT: str h0, [sp, #30]56; NONEON-NOSVE-NEXT: ldr h0, [sp, #20]57; NONEON-NOSVE-NEXT: fcvt s2, h258; NONEON-NOSVE-NEXT: fcvt s0, h059; NONEON-NOSVE-NEXT: fadd s0, s2, s060; NONEON-NOSVE-NEXT: fmul s2, s5, s461; NONEON-NOSVE-NEXT: fcvt h0, s062; NONEON-NOSVE-NEXT: fcvt h2, s263; NONEON-NOSVE-NEXT: str h0, [sp, #28]64; NONEON-NOSVE-NEXT: ldr h0, [sp, #18]65; NONEON-NOSVE-NEXT: fcvt s2, h266; NONEON-NOSVE-NEXT: fcvt s0, h067; NONEON-NOSVE-NEXT: fadd s0, s2, s068; NONEON-NOSVE-NEXT: fcvt h0, s069; NONEON-NOSVE-NEXT: str h0, [sp, #26]70; NONEON-NOSVE-NEXT: ldr h0, [sp, #16]71; NONEON-NOSVE-NEXT: fcvt s0, h072; NONEON-NOSVE-NEXT: fadd s0, s1, s073; NONEON-NOSVE-NEXT: fcvt h0, s074; NONEON-NOSVE-NEXT: str h0, [sp, #24]75; NONEON-NOSVE-NEXT: ldr d0, [sp, #24]76; NONEON-NOSVE-NEXT: add sp, sp, #3277; NONEON-NOSVE-NEXT: ret78 %mul = fmul contract <4 x half> %op1, %op279 %res = fadd contract <4 x half> %mul, %op380 ret <4 x half> %res81}82 83define <8 x half> @fma_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3) {84; CHECK-LABEL: fma_v8f16:85; CHECK: // %bb.0:86; CHECK-NEXT: ptrue p0.h, vl887; CHECK-NEXT: // kill: def $q0 killed $q0 def $z088; CHECK-NEXT: // kill: def $q2 killed $q2 def $z289; CHECK-NEXT: // kill: def $q1 killed $q1 def $z190; CHECK-NEXT: fmad z0.h, p0/m, z1.h, z2.h91; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z092; CHECK-NEXT: ret93;94; NONEON-NOSVE-LABEL: fma_v8f16:95; NONEON-NOSVE: // %bb.0:96; NONEON-NOSVE-NEXT: stp q0, q1, [sp, #-64]!97; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 6498; NONEON-NOSVE-NEXT: ldr h0, [sp, #30]99; NONEON-NOSVE-NEXT: ldr h1, [sp, #14]100; NONEON-NOSVE-NEXT: str q2, [sp, #32]101; NONEON-NOSVE-NEXT: ldr h2, [sp, #46]102; NONEON-NOSVE-NEXT: ldr h22, [sp, #28]103; NONEON-NOSVE-NEXT: ldr h23, [sp, #12]104; NONEON-NOSVE-NEXT: fcvt s3, h0105; NONEON-NOSVE-NEXT: fcvt s1, h1106; NONEON-NOSVE-NEXT: ldr h20, [sp, #26]107; NONEON-NOSVE-NEXT: fcvt s2, h2108; NONEON-NOSVE-NEXT: fcvt s22, h22109; NONEON-NOSVE-NEXT: fcvt s23, h23110; NONEON-NOSVE-NEXT: ldr h21, [sp, #10]111; NONEON-NOSVE-NEXT: fcvt s20, h20112; NONEON-NOSVE-NEXT: ldr h18, [sp, #24]113; NONEON-NOSVE-NEXT: ldr h19, [sp, #8]114; NONEON-NOSVE-NEXT: ldr h16, [sp, #22]115; NONEON-NOSVE-NEXT: ldr h17, [sp, #6]116; NONEON-NOSVE-NEXT: fmul s5, s1, s3117; NONEON-NOSVE-NEXT: fcvt s21, h21118; NONEON-NOSVE-NEXT: fcvt s18, h18119; NONEON-NOSVE-NEXT: fcvt s19, h19120; NONEON-NOSVE-NEXT: fcvt s16, h16121; NONEON-NOSVE-NEXT: fcvt s17, h17122; NONEON-NOSVE-NEXT: ldr h6, [sp, #20]123; NONEON-NOSVE-NEXT: ldr h7, [sp, #4]124; NONEON-NOSVE-NEXT: ldr h3, [sp, #18]125; NONEON-NOSVE-NEXT: ldr h4, [sp, #2]126; NONEON-NOSVE-NEXT: ldr h0, [sp, #16]127; NONEON-NOSVE-NEXT: ldr h1, [sp]128; NONEON-NOSVE-NEXT: fcvt h5, s5129; NONEON-NOSVE-NEXT: fcvt s6, h6130; NONEON-NOSVE-NEXT: fcvt s7, h7131; NONEON-NOSVE-NEXT: fcvt s3, h3132; NONEON-NOSVE-NEXT: fcvt s4, h4133; NONEON-NOSVE-NEXT: fcvt s0, h0134; NONEON-NOSVE-NEXT: fcvt s1, h1135; NONEON-NOSVE-NEXT: fcvt s5, h5136; NONEON-NOSVE-NEXT: fmul s3, s4, s3137; NONEON-NOSVE-NEXT: fmul s0, s1, s0138; NONEON-NOSVE-NEXT: fadd s2, s5, s2139; NONEON-NOSVE-NEXT: fmul s5, s23, s22140; NONEON-NOSVE-NEXT: fcvt h3, s3141; NONEON-NOSVE-NEXT: fcvt h0, s0142; NONEON-NOSVE-NEXT: fcvt h2, s2143; NONEON-NOSVE-NEXT: fcvt h5, s5144; NONEON-NOSVE-NEXT: fcvt s3, h3145; NONEON-NOSVE-NEXT: fcvt s0, h0146; NONEON-NOSVE-NEXT: str h2, [sp, #62]147; NONEON-NOSVE-NEXT: ldr h2, [sp, #44]148; NONEON-NOSVE-NEXT: fcvt s5, h5149; NONEON-NOSVE-NEXT: fcvt s2, h2150; NONEON-NOSVE-NEXT: fadd s2, s5, s2151; NONEON-NOSVE-NEXT: fmul s5, s21, s20152; NONEON-NOSVE-NEXT: fcvt h2, s2153; NONEON-NOSVE-NEXT: fcvt h5, s5154; NONEON-NOSVE-NEXT: str h2, [sp, #60]155; NONEON-NOSVE-NEXT: ldr h2, [sp, #42]156; NONEON-NOSVE-NEXT: fcvt s5, h5157; NONEON-NOSVE-NEXT: fcvt s2, h2158; NONEON-NOSVE-NEXT: fadd s2, s5, s2159; NONEON-NOSVE-NEXT: fmul s5, s19, s18160; NONEON-NOSVE-NEXT: fcvt h2, s2161; NONEON-NOSVE-NEXT: fcvt h5, s5162; NONEON-NOSVE-NEXT: str h2, [sp, #58]163; NONEON-NOSVE-NEXT: ldr h2, [sp, #40]164; NONEON-NOSVE-NEXT: fcvt s5, h5165; NONEON-NOSVE-NEXT: fcvt s2, h2166; NONEON-NOSVE-NEXT: fadd s2, s5, s2167; NONEON-NOSVE-NEXT: fmul s5, s17, s16168; NONEON-NOSVE-NEXT: fcvt h2, s2169; NONEON-NOSVE-NEXT: fcvt h5, s5170; NONEON-NOSVE-NEXT: str h2, [sp, #56]171; NONEON-NOSVE-NEXT: ldr h2, [sp, #38]172; NONEON-NOSVE-NEXT: fcvt s5, h5173; NONEON-NOSVE-NEXT: fcvt s2, h2174; NONEON-NOSVE-NEXT: fadd s2, s5, s2175; NONEON-NOSVE-NEXT: fmul s5, s7, s6176; NONEON-NOSVE-NEXT: fcvt h2, s2177; NONEON-NOSVE-NEXT: fcvt h5, s5178; NONEON-NOSVE-NEXT: str h2, [sp, #54]179; NONEON-NOSVE-NEXT: ldr h2, [sp, #36]180; NONEON-NOSVE-NEXT: fcvt s5, h5181; NONEON-NOSVE-NEXT: fcvt s2, h2182; NONEON-NOSVE-NEXT: fadd s2, s5, s2183; NONEON-NOSVE-NEXT: fcvt h2, s2184; NONEON-NOSVE-NEXT: str h2, [sp, #52]185; NONEON-NOSVE-NEXT: ldr h2, [sp, #34]186; NONEON-NOSVE-NEXT: fcvt s2, h2187; NONEON-NOSVE-NEXT: fadd s2, s3, s2188; NONEON-NOSVE-NEXT: fcvt h1, s2189; NONEON-NOSVE-NEXT: str h1, [sp, #50]190; NONEON-NOSVE-NEXT: ldr h1, [sp, #32]191; NONEON-NOSVE-NEXT: fcvt s1, h1192; NONEON-NOSVE-NEXT: fadd s0, s0, s1193; NONEON-NOSVE-NEXT: fcvt h0, s0194; NONEON-NOSVE-NEXT: str h0, [sp, #48]195; NONEON-NOSVE-NEXT: ldr q0, [sp, #48]196; NONEON-NOSVE-NEXT: add sp, sp, #64197; NONEON-NOSVE-NEXT: ret198 %mul = fmul contract <8 x half> %op1, %op2199 %res = fadd contract <8 x half> %mul, %op3200 ret <8 x half> %res201}202 203define void @fma_v16f16(ptr %a, ptr %b, ptr %c) {204; CHECK-LABEL: fma_v16f16:205; CHECK: // %bb.0:206; CHECK-NEXT: ldp q0, q4, [x1]207; CHECK-NEXT: ptrue p0.h, vl8208; CHECK-NEXT: ldp q1, q5, [x2]209; CHECK-NEXT: ldp q2, q3, [x0]210; CHECK-NEXT: fmad z0.h, p0/m, z2.h, z1.h211; CHECK-NEXT: fmad z3.h, p0/m, z4.h, z5.h212; CHECK-NEXT: stp q0, q3, [x0]213; CHECK-NEXT: ret214;215; NONEON-NOSVE-LABEL: fma_v16f16:216; NONEON-NOSVE: // %bb.0:217; NONEON-NOSVE-NEXT: sub sp, sp, #208218; NONEON-NOSVE-NEXT: stp d15, d14, [sp, #144] // 16-byte Folded Spill219; NONEON-NOSVE-NEXT: stp d13, d12, [sp, #160] // 16-byte Folded Spill220; NONEON-NOSVE-NEXT: stp d11, d10, [sp, #176] // 16-byte Folded Spill221; NONEON-NOSVE-NEXT: stp d9, d8, [sp, #192] // 16-byte Folded Spill222; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 208223; NONEON-NOSVE-NEXT: .cfi_offset b8, -8224; NONEON-NOSVE-NEXT: .cfi_offset b9, -16225; NONEON-NOSVE-NEXT: .cfi_offset b10, -24226; NONEON-NOSVE-NEXT: .cfi_offset b11, -32227; NONEON-NOSVE-NEXT: .cfi_offset b12, -40228; NONEON-NOSVE-NEXT: .cfi_offset b13, -48229; NONEON-NOSVE-NEXT: .cfi_offset b14, -56230; NONEON-NOSVE-NEXT: .cfi_offset b15, -64231; NONEON-NOSVE-NEXT: ldp q0, q1, [x1]232; NONEON-NOSVE-NEXT: ldp q3, q2, [x0]233; NONEON-NOSVE-NEXT: ldp q18, q19, [x2]234; NONEON-NOSVE-NEXT: stp q3, q0, [sp, #16]235; NONEON-NOSVE-NEXT: stp q2, q1, [sp, #64]236; NONEON-NOSVE-NEXT: ldr h24, [sp, #46]237; NONEON-NOSVE-NEXT: ldr h25, [sp, #30]238; NONEON-NOSVE-NEXT: ldr h0, [sp, #94]239; NONEON-NOSVE-NEXT: ldr h1, [sp, #78]240; NONEON-NOSVE-NEXT: str q19, [sp, #96]241; NONEON-NOSVE-NEXT: str q18, [sp, #48]242; NONEON-NOSVE-NEXT: ldr h18, [sp, #110]243; NONEON-NOSVE-NEXT: ldr h15, [sp, #92]244; NONEON-NOSVE-NEXT: fcvt s20, h0245; NONEON-NOSVE-NEXT: fcvt s21, h1246; NONEON-NOSVE-NEXT: ldr h0, [sp, #32]247; NONEON-NOSVE-NEXT: fcvt s18, h18248; NONEON-NOSVE-NEXT: ldr h13, [sp, #90]249; NONEON-NOSVE-NEXT: ldr h14, [sp, #74]250; NONEON-NOSVE-NEXT: str h0, [sp, #14] // 2-byte Spill251; NONEON-NOSVE-NEXT: ldr h0, [sp, #76]252; NONEON-NOSVE-NEXT: ldr h11, [sp, #88]253; NONEON-NOSVE-NEXT: ldr h12, [sp, #72]254; NONEON-NOSVE-NEXT: ldr h9, [sp, #86]255; NONEON-NOSVE-NEXT: ldr h10, [sp, #70]256; NONEON-NOSVE-NEXT: fmul s30, s21, s20257; NONEON-NOSVE-NEXT: fcvt s0, h0258; NONEON-NOSVE-NEXT: ldr h31, [sp, #84]259; NONEON-NOSVE-NEXT: ldr h8, [sp, #68]260; NONEON-NOSVE-NEXT: ldr h28, [sp, #82]261; NONEON-NOSVE-NEXT: ldr h29, [sp, #66]262; NONEON-NOSVE-NEXT: ldr h26, [sp, #80]263; NONEON-NOSVE-NEXT: ldr h27, [sp, #64]264; NONEON-NOSVE-NEXT: ldr h22, [sp, #44]265; NONEON-NOSVE-NEXT: ldr h23, [sp, #28]266; NONEON-NOSVE-NEXT: ldr h20, [sp, #42]267; NONEON-NOSVE-NEXT: ldr h21, [sp, #26]268; NONEON-NOSVE-NEXT: fcvt h19, s30269; NONEON-NOSVE-NEXT: fcvt s30, h15270; NONEON-NOSVE-NEXT: ldr h16, [sp, #40]271; NONEON-NOSVE-NEXT: ldr h17, [sp, #24]272; NONEON-NOSVE-NEXT: ldr h6, [sp, #38]273; NONEON-NOSVE-NEXT: ldr h7, [sp, #22]274; NONEON-NOSVE-NEXT: fcvt s16, h16275; NONEON-NOSVE-NEXT: ldr h4, [sp, #36]276; NONEON-NOSVE-NEXT: ldr h5, [sp, #20]277; NONEON-NOSVE-NEXT: fcvt s17, h17278; NONEON-NOSVE-NEXT: fcvt s6, h6279; NONEON-NOSVE-NEXT: fcvt s7, h7280; NONEON-NOSVE-NEXT: fcvt s19, h19281; NONEON-NOSVE-NEXT: fmul s0, s0, s30282; NONEON-NOSVE-NEXT: fcvt s30, h14283; NONEON-NOSVE-NEXT: fcvt s4, h4284; NONEON-NOSVE-NEXT: fcvt s5, h5285; NONEON-NOSVE-NEXT: ldr h2, [sp, #34]286; NONEON-NOSVE-NEXT: ldr h3, [sp, #18]287; NONEON-NOSVE-NEXT: ldr h1, [sp, #16]288; NONEON-NOSVE-NEXT: fmul s16, s17, s16289; NONEON-NOSVE-NEXT: fmul s6, s7, s6290; NONEON-NOSVE-NEXT: fcvt s2, h2291; NONEON-NOSVE-NEXT: fadd s18, s19, s18292; NONEON-NOSVE-NEXT: fcvt h0, s0293; NONEON-NOSVE-NEXT: fcvt s19, h13294; NONEON-NOSVE-NEXT: fmul s4, s5, s4295; NONEON-NOSVE-NEXT: fcvt s3, h3296; NONEON-NOSVE-NEXT: fcvt s1, h1297; NONEON-NOSVE-NEXT: ldp d15, d14, [sp, #144] // 16-byte Folded Reload298; NONEON-NOSVE-NEXT: fcvt h16, s16299; NONEON-NOSVE-NEXT: fcvt h6, s6300; NONEON-NOSVE-NEXT: fcvt h18, s18301; NONEON-NOSVE-NEXT: fcvt s0, h0302; NONEON-NOSVE-NEXT: fcvt h4, s4303; NONEON-NOSVE-NEXT: fmul s2, s3, s2304; NONEON-NOSVE-NEXT: ldr h3, [sp, #14] // 2-byte Reload305; NONEON-NOSVE-NEXT: fcvt s16, h16306; NONEON-NOSVE-NEXT: fcvt s6, h6307; NONEON-NOSVE-NEXT: fcvt s3, h3308; NONEON-NOSVE-NEXT: str h18, [sp, #142]309; NONEON-NOSVE-NEXT: ldr h18, [sp, #108]310; NONEON-NOSVE-NEXT: fcvt s4, h4311; NONEON-NOSVE-NEXT: fcvt h2, s2312; NONEON-NOSVE-NEXT: fcvt s18, h18313; NONEON-NOSVE-NEXT: fmul s1, s1, s3314; NONEON-NOSVE-NEXT: fcvt s2, h2315; NONEON-NOSVE-NEXT: fadd s0, s0, s18316; NONEON-NOSVE-NEXT: fmul s18, s30, s19317; NONEON-NOSVE-NEXT: fcvt s19, h11318; NONEON-NOSVE-NEXT: fcvt s30, h12319; NONEON-NOSVE-NEXT: fcvt h1, s1320; NONEON-NOSVE-NEXT: ldp d13, d12, [sp, #160] // 16-byte Folded Reload321; NONEON-NOSVE-NEXT: fcvt h0, s0322; NONEON-NOSVE-NEXT: fcvt h18, s18323; NONEON-NOSVE-NEXT: fcvt s1, h1324; NONEON-NOSVE-NEXT: str h0, [sp, #140]325; NONEON-NOSVE-NEXT: ldr h0, [sp, #106]326; NONEON-NOSVE-NEXT: fcvt s18, h18327; NONEON-NOSVE-NEXT: fcvt s0, h0328; NONEON-NOSVE-NEXT: fadd s0, s18, s0329; NONEON-NOSVE-NEXT: fmul s18, s30, s19330; NONEON-NOSVE-NEXT: fcvt s19, h9331; NONEON-NOSVE-NEXT: fcvt s30, h10332; NONEON-NOSVE-NEXT: ldp d11, d10, [sp, #176] // 16-byte Folded Reload333; NONEON-NOSVE-NEXT: fcvt h0, s0334; NONEON-NOSVE-NEXT: fcvt h18, s18335; NONEON-NOSVE-NEXT: str h0, [sp, #138]336; NONEON-NOSVE-NEXT: ldr h0, [sp, #104]337; NONEON-NOSVE-NEXT: fcvt s18, h18338; NONEON-NOSVE-NEXT: fcvt s0, h0339; NONEON-NOSVE-NEXT: fadd s0, s18, s0340; NONEON-NOSVE-NEXT: fmul s18, s30, s19341; NONEON-NOSVE-NEXT: fcvt s19, h31342; NONEON-NOSVE-NEXT: fcvt s30, h8343; NONEON-NOSVE-NEXT: ldp d9, d8, [sp, #192] // 16-byte Folded Reload344; NONEON-NOSVE-NEXT: fcvt h0, s0345; NONEON-NOSVE-NEXT: fcvt h18, s18346; NONEON-NOSVE-NEXT: str h0, [sp, #136]347; NONEON-NOSVE-NEXT: ldr h0, [sp, #102]348; NONEON-NOSVE-NEXT: fcvt s18, h18349; NONEON-NOSVE-NEXT: fcvt s0, h0350; NONEON-NOSVE-NEXT: fadd s0, s18, s0351; NONEON-NOSVE-NEXT: fmul s18, s30, s19352; NONEON-NOSVE-NEXT: fcvt s19, h28353; NONEON-NOSVE-NEXT: fcvt s28, h29354; NONEON-NOSVE-NEXT: fcvt h0, s0355; NONEON-NOSVE-NEXT: fcvt h18, s18356; NONEON-NOSVE-NEXT: str h0, [sp, #134]357; NONEON-NOSVE-NEXT: ldr h0, [sp, #100]358; NONEON-NOSVE-NEXT: fcvt s18, h18359; NONEON-NOSVE-NEXT: fcvt s0, h0360; NONEON-NOSVE-NEXT: fadd s0, s18, s0361; NONEON-NOSVE-NEXT: fmul s18, s28, s19362; NONEON-NOSVE-NEXT: fcvt s19, h26363; NONEON-NOSVE-NEXT: fcvt s26, h27364; NONEON-NOSVE-NEXT: fcvt h0, s0365; NONEON-NOSVE-NEXT: fcvt h18, s18366; NONEON-NOSVE-NEXT: str h0, [sp, #132]367; NONEON-NOSVE-NEXT: ldr h0, [sp, #98]368; NONEON-NOSVE-NEXT: fcvt s18, h18369; NONEON-NOSVE-NEXT: fcvt s0, h0370; NONEON-NOSVE-NEXT: fadd s0, s18, s0371; NONEON-NOSVE-NEXT: fmul s18, s26, s19372; NONEON-NOSVE-NEXT: fcvt s19, h24373; NONEON-NOSVE-NEXT: fcvt s24, h25374; NONEON-NOSVE-NEXT: fcvt h0, s0375; NONEON-NOSVE-NEXT: fcvt h18, s18376; NONEON-NOSVE-NEXT: str h0, [sp, #130]377; NONEON-NOSVE-NEXT: ldr h0, [sp, #96]378; NONEON-NOSVE-NEXT: fcvt s18, h18379; NONEON-NOSVE-NEXT: fcvt s0, h0380; NONEON-NOSVE-NEXT: fadd s0, s18, s0381; NONEON-NOSVE-NEXT: fmul s18, s24, s19382; NONEON-NOSVE-NEXT: fcvt s19, h22383; NONEON-NOSVE-NEXT: fcvt s22, h23384; NONEON-NOSVE-NEXT: fcvt h0, s0385; NONEON-NOSVE-NEXT: fcvt h18, s18386; NONEON-NOSVE-NEXT: str h0, [sp, #128]387; NONEON-NOSVE-NEXT: ldr h0, [sp, #62]388; NONEON-NOSVE-NEXT: fcvt s18, h18389; NONEON-NOSVE-NEXT: fcvt s0, h0390; NONEON-NOSVE-NEXT: fadd s0, s18, s0391; NONEON-NOSVE-NEXT: fmul s18, s22, s19392; NONEON-NOSVE-NEXT: fcvt s19, h20393; NONEON-NOSVE-NEXT: fcvt s20, h21394; NONEON-NOSVE-NEXT: fcvt h0, s0395; NONEON-NOSVE-NEXT: fcvt h18, s18396; NONEON-NOSVE-NEXT: str h0, [sp, #126]397; NONEON-NOSVE-NEXT: ldr h0, [sp, #60]398; NONEON-NOSVE-NEXT: fcvt s18, h18399; NONEON-NOSVE-NEXT: fcvt s0, h0400; NONEON-NOSVE-NEXT: fadd s0, s18, s0401; NONEON-NOSVE-NEXT: fmul s18, s20, s19402; NONEON-NOSVE-NEXT: fcvt h0, s0403; NONEON-NOSVE-NEXT: fcvt h18, s18404; NONEON-NOSVE-NEXT: str h0, [sp, #124]405; NONEON-NOSVE-NEXT: ldr h0, [sp, #58]406; NONEON-NOSVE-NEXT: fcvt s18, h18407; NONEON-NOSVE-NEXT: fcvt s0, h0408; NONEON-NOSVE-NEXT: fadd s0, s18, s0409; NONEON-NOSVE-NEXT: fcvt h0, s0410; NONEON-NOSVE-NEXT: str h0, [sp, #122]411; NONEON-NOSVE-NEXT: ldr h0, [sp, #56]412; NONEON-NOSVE-NEXT: fcvt s0, h0413; NONEON-NOSVE-NEXT: fadd s0, s16, s0414; NONEON-NOSVE-NEXT: fcvt h0, s0415; NONEON-NOSVE-NEXT: str h0, [sp, #120]416; NONEON-NOSVE-NEXT: ldr h0, [sp, #54]417; NONEON-NOSVE-NEXT: fcvt s0, h0418; NONEON-NOSVE-NEXT: fadd s0, s6, s0419; NONEON-NOSVE-NEXT: fcvt h0, s0420; NONEON-NOSVE-NEXT: str h0, [sp, #118]421; NONEON-NOSVE-NEXT: ldr h0, [sp, #52]422; NONEON-NOSVE-NEXT: fcvt s0, h0423; NONEON-NOSVE-NEXT: fadd s0, s4, s0424; NONEON-NOSVE-NEXT: fcvt h0, s0425; NONEON-NOSVE-NEXT: str h0, [sp, #116]426; NONEON-NOSVE-NEXT: ldr h0, [sp, #50]427; NONEON-NOSVE-NEXT: fcvt s0, h0428; NONEON-NOSVE-NEXT: fadd s0, s2, s0429; NONEON-NOSVE-NEXT: fcvt h0, s0430; NONEON-NOSVE-NEXT: str h0, [sp, #114]431; NONEON-NOSVE-NEXT: ldr h0, [sp, #48]432; NONEON-NOSVE-NEXT: fcvt s0, h0433; NONEON-NOSVE-NEXT: fadd s0, s1, s0434; NONEON-NOSVE-NEXT: fcvt h0, s0435; NONEON-NOSVE-NEXT: str h0, [sp, #112]436; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #112]437; NONEON-NOSVE-NEXT: stp q0, q1, [x0]438; NONEON-NOSVE-NEXT: add sp, sp, #208439; NONEON-NOSVE-NEXT: ret440 %op1 = load <16 x half>, ptr %a441 %op2 = load <16 x half>, ptr %b442 %op3 = load <16 x half>, ptr %c443 %mul = fmul contract <16 x half> %op1, %op2444 %res = fadd contract <16 x half> %mul, %op3445 store <16 x half> %res, ptr %a446 ret void447}448 449define <2 x float> @fma_v2f32(<2 x float> %op1, <2 x float> %op2, <2 x float> %op3) {450; CHECK-LABEL: fma_v2f32:451; CHECK: // %bb.0:452; CHECK-NEXT: ptrue p0.s, vl2453; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0454; CHECK-NEXT: // kill: def $d2 killed $d2 def $z2455; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1456; CHECK-NEXT: fmad z0.s, p0/m, z1.s, z2.s457; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0458; CHECK-NEXT: ret459;460; NONEON-NOSVE-LABEL: fma_v2f32:461; NONEON-NOSVE: // %bb.0:462; NONEON-NOSVE-NEXT: sub sp, sp, #32463; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32464; NONEON-NOSVE-NEXT: stp d1, d2, [sp, #8]465; NONEON-NOSVE-NEXT: str d0, [sp]466; NONEON-NOSVE-NEXT: ldp s1, s3, [sp, #8]467; NONEON-NOSVE-NEXT: ldp s2, s4, [sp]468; NONEON-NOSVE-NEXT: ldr s0, [sp, #20]469; NONEON-NOSVE-NEXT: fmadd s5, s4, s3, s0470; NONEON-NOSVE-NEXT: ldr s0, [sp, #16]471; NONEON-NOSVE-NEXT: fmadd s0, s2, s1, s0472; NONEON-NOSVE-NEXT: stp s0, s5, [sp, #24]473; NONEON-NOSVE-NEXT: ldr d0, [sp, #24]474; NONEON-NOSVE-NEXT: add sp, sp, #32475; NONEON-NOSVE-NEXT: ret476 %mul = fmul contract <2 x float> %op1, %op2477 %res = fadd contract <2 x float> %mul, %op3478 ret <2 x float> %res479}480 481define <4 x float> @fma_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %op3) {482; CHECK-LABEL: fma_v4f32:483; CHECK: // %bb.0:484; CHECK-NEXT: ptrue p0.s, vl4485; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0486; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2487; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1488; CHECK-NEXT: fmad z0.s, p0/m, z1.s, z2.s489; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0490; CHECK-NEXT: ret491;492; NONEON-NOSVE-LABEL: fma_v4f32:493; NONEON-NOSVE: // %bb.0:494; NONEON-NOSVE-NEXT: sub sp, sp, #64495; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64496; NONEON-NOSVE-NEXT: stp q1, q2, [sp, #16]497; NONEON-NOSVE-NEXT: str q0, [sp]498; NONEON-NOSVE-NEXT: ldp s1, s3, [sp, #24]499; NONEON-NOSVE-NEXT: ldp s2, s4, [sp, #8]500; NONEON-NOSVE-NEXT: ldr s0, [sp, #44]501; NONEON-NOSVE-NEXT: fmadd s5, s4, s3, s0502; NONEON-NOSVE-NEXT: ldr s0, [sp, #40]503; NONEON-NOSVE-NEXT: fmadd s0, s2, s1, s0504; NONEON-NOSVE-NEXT: ldp s1, s3, [sp, #16]505; NONEON-NOSVE-NEXT: ldp s2, s4, [sp]506; NONEON-NOSVE-NEXT: stp s0, s5, [sp, #56]507; NONEON-NOSVE-NEXT: ldr s0, [sp, #36]508; NONEON-NOSVE-NEXT: fmadd s5, s4, s3, s0509; NONEON-NOSVE-NEXT: ldr s0, [sp, #32]510; NONEON-NOSVE-NEXT: fmadd s0, s2, s1, s0511; NONEON-NOSVE-NEXT: stp s0, s5, [sp, #48]512; NONEON-NOSVE-NEXT: ldr q0, [sp, #48]513; NONEON-NOSVE-NEXT: add sp, sp, #64514; NONEON-NOSVE-NEXT: ret515 %mul = fmul contract <4 x float> %op1, %op2516 %res = fadd contract <4 x float> %mul, %op3517 ret <4 x float> %res518}519 520define void @fma_v8f32(ptr %a, ptr %b, ptr %c) {521; CHECK-LABEL: fma_v8f32:522; CHECK: // %bb.0:523; CHECK-NEXT: ldp q0, q4, [x1]524; CHECK-NEXT: ptrue p0.s, vl4525; CHECK-NEXT: ldp q1, q5, [x2]526; CHECK-NEXT: ldp q2, q3, [x0]527; CHECK-NEXT: fmad z0.s, p0/m, z2.s, z1.s528; CHECK-NEXT: fmad z3.s, p0/m, z4.s, z5.s529; CHECK-NEXT: stp q0, q3, [x0]530; CHECK-NEXT: ret531;532; NONEON-NOSVE-LABEL: fma_v8f32:533; NONEON-NOSVE: // %bb.0:534; NONEON-NOSVE-NEXT: sub sp, sp, #128535; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 128536; NONEON-NOSVE-NEXT: ldp q1, q0, [x2]537; NONEON-NOSVE-NEXT: ldp q2, q3, [x1]538; NONEON-NOSVE-NEXT: ldp q4, q5, [x0]539; NONEON-NOSVE-NEXT: stp q3, q0, [sp, #64]540; NONEON-NOSVE-NEXT: stp q4, q2, [sp]541; NONEON-NOSVE-NEXT: ldr s0, [sp, #92]542; NONEON-NOSVE-NEXT: stp q1, q5, [sp, #32]543; NONEON-NOSVE-NEXT: ldp s1, s3, [sp, #72]544; NONEON-NOSVE-NEXT: ldp s2, s4, [sp, #56]545; NONEON-NOSVE-NEXT: fmadd s5, s4, s3, s0546; NONEON-NOSVE-NEXT: ldr s0, [sp, #88]547; NONEON-NOSVE-NEXT: fmadd s0, s2, s1, s0548; NONEON-NOSVE-NEXT: ldp s1, s3, [sp, #64]549; NONEON-NOSVE-NEXT: ldp s2, s4, [sp, #48]550; NONEON-NOSVE-NEXT: stp s0, s5, [sp, #120]551; NONEON-NOSVE-NEXT: ldr s0, [sp, #84]552; NONEON-NOSVE-NEXT: fmadd s5, s4, s3, s0553; NONEON-NOSVE-NEXT: ldr s0, [sp, #80]554; NONEON-NOSVE-NEXT: fmadd s0, s2, s1, s0555; NONEON-NOSVE-NEXT: ldp s1, s3, [sp, #24]556; NONEON-NOSVE-NEXT: ldp s2, s4, [sp, #8]557; NONEON-NOSVE-NEXT: stp s0, s5, [sp, #112]558; NONEON-NOSVE-NEXT: ldr s0, [sp, #44]559; NONEON-NOSVE-NEXT: fmadd s5, s4, s3, s0560; NONEON-NOSVE-NEXT: ldr s0, [sp, #40]561; NONEON-NOSVE-NEXT: fmadd s0, s2, s1, s0562; NONEON-NOSVE-NEXT: ldp s1, s3, [sp, #16]563; NONEON-NOSVE-NEXT: ldp s2, s4, [sp]564; NONEON-NOSVE-NEXT: stp s0, s5, [sp, #104]565; NONEON-NOSVE-NEXT: ldr s0, [sp, #36]566; NONEON-NOSVE-NEXT: fmadd s5, s4, s3, s0567; NONEON-NOSVE-NEXT: ldr s0, [sp, #32]568; NONEON-NOSVE-NEXT: fmadd s0, s2, s1, s0569; NONEON-NOSVE-NEXT: stp s0, s5, [sp, #96]570; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #96]571; NONEON-NOSVE-NEXT: stp q0, q1, [x0]572; NONEON-NOSVE-NEXT: add sp, sp, #128573; NONEON-NOSVE-NEXT: ret574 %op1 = load <8 x float>, ptr %a575 %op2 = load <8 x float>, ptr %b576 %op3 = load <8 x float>, ptr %c577 %mul = fmul contract <8 x float> %op1, %op2578 %res = fadd contract <8 x float> %mul, %op3579 store <8 x float> %res, ptr %a580 ret void581}582 583define <1 x double> @fma_v1f64(<1 x double> %op1, <1 x double> %op2, <1 x double> %op3) {584; CHECK-LABEL: fma_v1f64:585; CHECK: // %bb.0:586; CHECK-NEXT: fmadd d0, d0, d1, d2587; CHECK-NEXT: ret588;589; NONEON-NOSVE-LABEL: fma_v1f64:590; NONEON-NOSVE: // %bb.0:591; NONEON-NOSVE-NEXT: sub sp, sp, #16592; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16593; NONEON-NOSVE-NEXT: fmadd d0, d0, d1, d2594; NONEON-NOSVE-NEXT: str d0, [sp, #8]595; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]596; NONEON-NOSVE-NEXT: add sp, sp, #16597; NONEON-NOSVE-NEXT: ret598 %mul = fmul contract <1 x double> %op1, %op2599 %res = fadd contract <1 x double> %mul, %op3600 ret <1 x double> %res601}602 603define <2 x double> @fma_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double> %op3) {604; CHECK-LABEL: fma_v2f64:605; CHECK: // %bb.0:606; CHECK-NEXT: ptrue p0.d, vl2607; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0608; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2609; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1610; CHECK-NEXT: fmad z0.d, p0/m, z1.d, z2.d611; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0612; CHECK-NEXT: ret613;614; NONEON-NOSVE-LABEL: fma_v2f64:615; NONEON-NOSVE: // %bb.0:616; NONEON-NOSVE-NEXT: sub sp, sp, #64617; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64618; NONEON-NOSVE-NEXT: stp q1, q2, [sp, #16]619; NONEON-NOSVE-NEXT: str q0, [sp]620; NONEON-NOSVE-NEXT: ldp d1, d3, [sp, #16]621; NONEON-NOSVE-NEXT: ldp d2, d4, [sp]622; NONEON-NOSVE-NEXT: ldr d0, [sp, #40]623; NONEON-NOSVE-NEXT: fmadd d5, d4, d3, d0624; NONEON-NOSVE-NEXT: ldr d0, [sp, #32]625; NONEON-NOSVE-NEXT: fmadd d0, d2, d1, d0626; NONEON-NOSVE-NEXT: stp d0, d5, [sp, #48]627; NONEON-NOSVE-NEXT: ldr q0, [sp, #48]628; NONEON-NOSVE-NEXT: add sp, sp, #64629; NONEON-NOSVE-NEXT: ret630 %mul = fmul contract <2 x double> %op1, %op2631 %res = fadd contract <2 x double> %mul, %op3632 ret <2 x double> %res633}634 635define void @fma_v4f64(ptr %a, ptr %b, ptr %c) {636; CHECK-LABEL: fma_v4f64:637; CHECK: // %bb.0:638; CHECK-NEXT: ldp q0, q4, [x1]639; CHECK-NEXT: ptrue p0.d, vl2640; CHECK-NEXT: ldp q1, q5, [x2]641; CHECK-NEXT: ldp q2, q3, [x0]642; CHECK-NEXT: fmad z0.d, p0/m, z2.d, z1.d643; CHECK-NEXT: fmad z3.d, p0/m, z4.d, z5.d644; CHECK-NEXT: stp q0, q3, [x0]645; CHECK-NEXT: ret646;647; NONEON-NOSVE-LABEL: fma_v4f64:648; NONEON-NOSVE: // %bb.0:649; NONEON-NOSVE-NEXT: sub sp, sp, #128650; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 128651; NONEON-NOSVE-NEXT: ldp q1, q0, [x2]652; NONEON-NOSVE-NEXT: ldp q2, q3, [x1]653; NONEON-NOSVE-NEXT: ldp q4, q5, [x0]654; NONEON-NOSVE-NEXT: stp q3, q0, [sp, #64]655; NONEON-NOSVE-NEXT: stp q4, q2, [sp]656; NONEON-NOSVE-NEXT: ldr d0, [sp, #88]657; NONEON-NOSVE-NEXT: stp q1, q5, [sp, #32]658; NONEON-NOSVE-NEXT: ldp d1, d3, [sp, #64]659; NONEON-NOSVE-NEXT: ldp d2, d4, [sp, #48]660; NONEON-NOSVE-NEXT: fmadd d5, d4, d3, d0661; NONEON-NOSVE-NEXT: ldr d0, [sp, #80]662; NONEON-NOSVE-NEXT: fmadd d0, d2, d1, d0663; NONEON-NOSVE-NEXT: ldp d1, d3, [sp, #16]664; NONEON-NOSVE-NEXT: ldp d2, d4, [sp]665; NONEON-NOSVE-NEXT: stp d0, d5, [sp, #112]666; NONEON-NOSVE-NEXT: ldr d0, [sp, #40]667; NONEON-NOSVE-NEXT: fmadd d5, d4, d3, d0668; NONEON-NOSVE-NEXT: ldr d0, [sp, #32]669; NONEON-NOSVE-NEXT: fmadd d0, d2, d1, d0670; NONEON-NOSVE-NEXT: stp d0, d5, [sp, #96]671; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #96]672; NONEON-NOSVE-NEXT: stp q0, q1, [x0]673; NONEON-NOSVE-NEXT: add sp, sp, #128674; NONEON-NOSVE-NEXT: ret675 %op1 = load <4 x double>, ptr %a676 %op2 = load <4 x double>, ptr %b677 %op3 = load <4 x double>, ptr %c678 %mul = fmul contract <4 x double> %op1, %op2679 %res = fadd contract <4 x double> %mul, %op3680 store <4 x double> %res, ptr %a681 ret void682}683