495 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s --mattr=+sve -o - | FileCheck %s3 4target triple = "aarch64"5 6%"class.std::complex" = type { { double, double } }7 8; Zero initialized reduction9;10; complex<double> x = 0.0 + 0.0i;11; for (int i = 0; i < 100; ++i)12; x += a[i] * b[i];13;14define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {15; CHECK-LABEL: complex_mul_v2f64:16; CHECK: // %bb.0: // %entry17; CHECK-NEXT: movi v0.2d, #000000000000000018; CHECK-NEXT: movi v1.2d, #000000000000000019; CHECK-NEXT: cntd x820; CHECK-NEXT: neg x9, x821; CHECK-NEXT: mov w10, #100 // =0x6422; CHECK-NEXT: ptrue p0.d23; CHECK-NEXT: and x9, x9, x1024; CHECK-NEXT: rdvl x10, #225; CHECK-NEXT: .LBB0_1: // %vector.body26; CHECK-NEXT: // =>This Inner Loop Header: Depth=127; CHECK-NEXT: ldr z2, [x0, #1, mul vl]28; CHECK-NEXT: ldr z3, [x0]29; CHECK-NEXT: subs x9, x9, x830; CHECK-NEXT: ldr z4, [x1, #1, mul vl]31; CHECK-NEXT: ldr z5, [x1]32; CHECK-NEXT: add x1, x1, x1033; CHECK-NEXT: add x0, x0, x1034; CHECK-NEXT: fcmla z1.d, p0/m, z5.d, z3.d, #035; CHECK-NEXT: fcmla z0.d, p0/m, z4.d, z2.d, #036; CHECK-NEXT: fcmla z1.d, p0/m, z5.d, z3.d, #9037; CHECK-NEXT: fcmla z0.d, p0/m, z4.d, z2.d, #9038; CHECK-NEXT: b.ne .LBB0_139; CHECK-NEXT: // %bb.2: // %exit.block40; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d41; CHECK-NEXT: uzp2 z1.d, z1.d, z0.d42; CHECK-NEXT: faddv d0, p0, z2.d43; CHECK-NEXT: faddv d1, p0, z1.d44; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z045; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z146; CHECK-NEXT: ret47entry:48 %0 = tail call i64 @llvm.vscale.i64()49 %1 = shl nuw nsw i64 %0, 150 %n.mod.vf = urem i64 100, %151 %n.vec = sub nuw nsw i64 100, %n.mod.vf52 %2 = shl nuw nsw i64 %0, 553 br label %vector.body54 55vector.body: ; preds = %vector.body, %entry56 %lsr.iv31 = phi i64 [ %lsr.iv.next32, %vector.body ], [ %n.vec, %entry ]57 %lsr.iv27 = phi i64 [ %lsr.iv.next28, %vector.body ], [ 0, %entry ]58 %vec.phi = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %16, %vector.body ]59 %vec.phi12 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %14, %vector.body ]60 %scevgep46 = getelementptr i8, ptr %a, i64 %lsr.iv2761 %scevgep47 = getelementptr i8, ptr %b, i64 %lsr.iv2762 %wide.vec = load <vscale x 4 x double>, ptr %scevgep46, align 863 %3 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec)64 %4 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %3, 065 %5 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %3, 166 %wide.vec30 = load <vscale x 4 x double>, ptr %scevgep47, align 867 %6 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec30)68 %7 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %6, 069 %8 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %6, 170 %9 = fmul fast <vscale x 2 x double> %8, %471 %10 = fmul fast <vscale x 2 x double> %7, %572 %11 = fmul fast <vscale x 2 x double> %7, %473 %12 = fadd fast <vscale x 2 x double> %11, %vec.phi1274 %13 = fmul fast <vscale x 2 x double> %8, %575 %14 = fsub fast <vscale x 2 x double> %12, %1376 %15 = fadd fast <vscale x 2 x double> %10, %vec.phi77 %16 = fadd fast <vscale x 2 x double> %15, %978 %lsr.iv.next28 = add i64 %lsr.iv27, %279 %lsr.iv.next32 = sub i64 %lsr.iv31, %180 %17 = icmp eq i64 %lsr.iv.next32, 081 br i1 %17, label %exit.block, label %vector.body82 83exit.block: ; preds = %vector.body84 %18 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %14)85 %19 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %16)86 %.fca.0.0.insert = insertvalue %"class.std::complex" poison, double %18, 0, 087 %.fca.0.1.insert = insertvalue %"class.std::complex" %.fca.0.0.insert, double %19, 0, 188 ret %"class.std::complex" %.fca.0.1.insert89}90 91; Fixed value initialized reduction92;93; complex<double> x = 2.0 + 1.0i;94; for (int i = 0; i < 100; ++i)95; x += a[i] * b[i];96;97define %"class.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) {98; CHECK-LABEL: complex_mul_nonzero_init_v2f64:99; CHECK: // %bb.0: // %entry100; CHECK-NEXT: movi v0.2d, #0000000000000000101; CHECK-NEXT: fmov d1, #1.00000000102; CHECK-NEXT: cntd x8103; CHECK-NEXT: fmov d2, #2.00000000104; CHECK-NEXT: ptrue p0.d, vl1105; CHECK-NEXT: neg x9, x8106; CHECK-NEXT: mov w10, #100 // =0x64107; CHECK-NEXT: and x9, x9, x10108; CHECK-NEXT: rdvl x10, #2109; CHECK-NEXT: sel z1.d, p0, z1.d, z0.d110; CHECK-NEXT: sel z2.d, p0, z2.d, z0.d111; CHECK-NEXT: ptrue p0.d112; CHECK-NEXT: zip2 z0.d, z2.d, z1.d113; CHECK-NEXT: zip1 z1.d, z2.d, z1.d114; CHECK-NEXT: .LBB1_1: // %vector.body115; CHECK-NEXT: // =>This Inner Loop Header: Depth=1116; CHECK-NEXT: ldr z2, [x0, #1, mul vl]117; CHECK-NEXT: ldr z3, [x0]118; CHECK-NEXT: subs x9, x9, x8119; CHECK-NEXT: ldr z4, [x1, #1, mul vl]120; CHECK-NEXT: ldr z5, [x1]121; CHECK-NEXT: add x1, x1, x10122; CHECK-NEXT: add x0, x0, x10123; CHECK-NEXT: fcmla z1.d, p0/m, z5.d, z3.d, #0124; CHECK-NEXT: fcmla z0.d, p0/m, z4.d, z2.d, #0125; CHECK-NEXT: fcmla z1.d, p0/m, z5.d, z3.d, #90126; CHECK-NEXT: fcmla z0.d, p0/m, z4.d, z2.d, #90127; CHECK-NEXT: b.ne .LBB1_1128; CHECK-NEXT: // %bb.2: // %exit.block129; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d130; CHECK-NEXT: uzp2 z1.d, z1.d, z0.d131; CHECK-NEXT: faddv d0, p0, z2.d132; CHECK-NEXT: faddv d1, p0, z1.d133; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0134; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1135; CHECK-NEXT: ret136entry:137 %0 = tail call i64 @llvm.vscale.i64()138 %1 = shl nuw nsw i64 %0, 1139 %n.mod.vf = urem i64 100, %1140 %n.vec = sub nuw nsw i64 100, %n.mod.vf141 %2 = shl nuw nsw i64 %0, 5142 br label %vector.body143 144vector.body: ; preds = %vector.body, %entry145 %lsr.iv31 = phi i64 [ %lsr.iv.next32, %vector.body ], [ %n.vec, %entry ]146 %lsr.iv27 = phi i64 [ %lsr.iv.next28, %vector.body ], [ 0, %entry ]147 %vec.phi = phi <vscale x 2 x double> [ insertelement (<vscale x 2 x double> zeroinitializer, double 1.000000e+00, i32 0), %entry ], [ %16, %vector.body ]148 %vec.phi12 = phi <vscale x 2 x double> [ insertelement (<vscale x 2 x double> zeroinitializer, double 2.000000e+0, i32 0), %entry ], [ %14, %vector.body ]149 %scevgep46 = getelementptr i8, ptr %a, i64 %lsr.iv27150 %scevgep47 = getelementptr i8, ptr %b, i64 %lsr.iv27151 %wide.vec = load <vscale x 4 x double>, ptr %scevgep46, align 8152 %3 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec)153 %4 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %3, 0154 %5 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %3, 1155 %wide.vec30 = load <vscale x 4 x double>, ptr %scevgep47, align 8156 %6 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec30)157 %7 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %6, 0158 %8 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %6, 1159 %9 = fmul fast <vscale x 2 x double> %8, %4160 %10 = fmul fast <vscale x 2 x double> %7, %5161 %11 = fmul fast <vscale x 2 x double> %7, %4162 %12 = fadd fast <vscale x 2 x double> %11, %vec.phi12163 %13 = fmul fast <vscale x 2 x double> %8, %5164 %14 = fsub fast <vscale x 2 x double> %12, %13165 %15 = fadd fast <vscale x 2 x double> %10, %vec.phi166 %16 = fadd fast <vscale x 2 x double> %15, %9167 %lsr.iv.next28 = add i64 %lsr.iv27, %2168 %lsr.iv.next32 = sub i64 %lsr.iv31, %1169 %17 = icmp eq i64 %lsr.iv.next32, 0170 br i1 %17, label %exit.block, label %vector.body171 172exit.block: ; preds = %vector.body173 %18 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %14)174 %19 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %16)175 %.fca.0.0.insert = insertvalue %"class.std::complex" poison, double %18, 0, 0176 %.fca.0.1.insert = insertvalue %"class.std::complex" %.fca.0.0.insert, double %19, 0, 1177 ret %"class.std::complex" %.fca.0.1.insert178}179 180; Loop unrolled with factor 2181;182define %"class.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {183; CHECK-LABEL: complex_mul_v2f64_unrolled:184; CHECK: // %bb.0: // %entry185; CHECK-NEXT: movi v0.2d, #0000000000000000186; CHECK-NEXT: movi v1.2d, #0000000000000000187; CHECK-NEXT: cntw x8188; CHECK-NEXT: movi v2.2d, #0000000000000000189; CHECK-NEXT: movi v3.2d, #0000000000000000190; CHECK-NEXT: neg x9, x8191; CHECK-NEXT: mov w10, #1000 // =0x3e8192; CHECK-NEXT: ptrue p0.d193; CHECK-NEXT: and x9, x9, x10194; CHECK-NEXT: rdvl x10, #4195; CHECK-NEXT: .LBB2_1: // %vector.body196; CHECK-NEXT: // =>This Inner Loop Header: Depth=1197; CHECK-NEXT: ldr z4, [x0, #1, mul vl]198; CHECK-NEXT: ldr z5, [x0]199; CHECK-NEXT: subs x9, x9, x8200; CHECK-NEXT: ldr z6, [x0, #3, mul vl]201; CHECK-NEXT: ldr z7, [x1, #1, mul vl]202; CHECK-NEXT: ldr z16, [x1]203; CHECK-NEXT: ldr z17, [x0, #2, mul vl]204; CHECK-NEXT: add x0, x0, x10205; CHECK-NEXT: ldr z18, [x1, #3, mul vl]206; CHECK-NEXT: ldr z19, [x1, #2, mul vl]207; CHECK-NEXT: add x1, x1, x10208; CHECK-NEXT: fcmla z1.d, p0/m, z16.d, z5.d, #0209; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z4.d, #0210; CHECK-NEXT: fcmla z3.d, p0/m, z18.d, z6.d, #0211; CHECK-NEXT: fcmla z2.d, p0/m, z19.d, z17.d, #0212; CHECK-NEXT: fcmla z1.d, p0/m, z16.d, z5.d, #90213; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z4.d, #90214; CHECK-NEXT: fcmla z3.d, p0/m, z18.d, z6.d, #90215; CHECK-NEXT: fcmla z2.d, p0/m, z19.d, z17.d, #90216; CHECK-NEXT: b.ne .LBB2_1217; CHECK-NEXT: // %bb.2: // %exit.block218; CHECK-NEXT: uzp1 z4.d, z2.d, z3.d219; CHECK-NEXT: uzp1 z5.d, z1.d, z0.d220; CHECK-NEXT: uzp2 z2.d, z2.d, z3.d221; CHECK-NEXT: uzp2 z0.d, z1.d, z0.d222; CHECK-NEXT: fadd z1.d, z4.d, z5.d223; CHECK-NEXT: fadd z2.d, z2.d, z0.d224; CHECK-NEXT: faddv d0, p0, z1.d225; CHECK-NEXT: faddv d1, p0, z2.d226; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0227; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1228; CHECK-NEXT: ret229entry:230 %0 = tail call i64 @llvm.vscale.i64()231 %1 = shl nuw nsw i64 %0, 2232 %n.mod.vf = urem i64 1000, %1233 %n.vec = sub i64 1000, %n.mod.vf234 %2 = shl nuw nsw i64 %0, 6235 %3 = shl nuw nsw i64 %0, 5236 %scevgep61 = getelementptr i8, ptr %b, i64 %3237 %scevgep63 = getelementptr i8, ptr %a, i64 %3238 br label %vector.body239 240vector.body: ; preds = %vector.body, %entry241 %lsr.iv38 = phi i64 [ %lsr.iv.next39, %vector.body ], [ %n.vec, %entry ]242 %lsr.iv34 = phi i64 [ %lsr.iv.next35, %vector.body ], [ 0, %entry ]243 %vec.phi = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %30, %vector.body ]244 %vec.phi12 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %31, %vector.body ]245 %vec.phi13 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %26, %vector.body ]246 %vec.phi14 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %27, %vector.body ]247 %scevgep57 = getelementptr i8, ptr %a, i64 %lsr.iv34248 %scevgep64 = getelementptr i8, ptr %scevgep63, i64 %lsr.iv34249 %scevgep58 = getelementptr i8, ptr %b, i64 %lsr.iv34250 %scevgep62 = getelementptr i8, ptr %scevgep61, i64 %lsr.iv34251 %wide.vec = load <vscale x 4 x double>, ptr %scevgep57, align 8252 %wide.vec32 = load <vscale x 4 x double>, ptr %scevgep64, align 8253 %4 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec)254 %5 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec32)255 %6 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %4, 0256 %7 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %5, 0257 %8 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %4, 1258 %9 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %5, 1259 %wide.vec34 = load <vscale x 4 x double>, ptr %scevgep58, align 8260 %wide.vec35 = load <vscale x 4 x double>, ptr %scevgep62, align 8261 %10 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec34)262 %11 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec35)263 %12 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %10, 0264 %13 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %11, 0265 %14 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %10, 1266 %15 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %11, 1267 %16 = fmul fast <vscale x 2 x double> %14, %6268 %17 = fmul fast <vscale x 2 x double> %15, %7269 %18 = fmul fast <vscale x 2 x double> %12, %8270 %19 = fmul fast <vscale x 2 x double> %13, %9271 %20 = fmul fast <vscale x 2 x double> %12, %6272 %21 = fmul fast <vscale x 2 x double> %13, %7273 %22 = fadd fast <vscale x 2 x double> %20, %vec.phi13274 %23 = fadd fast <vscale x 2 x double> %21, %vec.phi14275 %24 = fmul fast <vscale x 2 x double> %14, %8276 %25 = fmul fast <vscale x 2 x double> %15, %9277 %26 = fsub fast <vscale x 2 x double> %22, %24278 %27 = fsub fast <vscale x 2 x double> %23, %25279 %28 = fadd fast <vscale x 2 x double> %18, %vec.phi280 %29 = fadd fast <vscale x 2 x double> %19, %vec.phi12281 %30 = fadd fast <vscale x 2 x double> %28, %16282 %31 = fadd fast <vscale x 2 x double> %29, %17283 %lsr.iv.next35 = add i64 %lsr.iv34, %2284 %lsr.iv.next39 = sub i64 %lsr.iv38, %1285 %32 = icmp eq i64 %lsr.iv.next39, 0286 br i1 %32, label %exit.block, label %vector.body287 288exit.block: ; preds = %vector.body289 %bin.rdx15 = fadd fast <vscale x 2 x double> %27, %26290 %33 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %bin.rdx15)291 %bin.rdx = fadd fast <vscale x 2 x double> %31, %30292 %34 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %bin.rdx)293 %.fca.0.0.insert = insertvalue %"class.std::complex" poison, double %33, 0, 0294 %.fca.0.1.insert = insertvalue %"class.std::complex" %.fca.0.0.insert, double %34, 0, 1295 ret %"class.std::complex" %.fca.0.1.insert296}297 298; Integer and floating point complex number reduction in the same loop:299; complex<double> *s = ...;300; int *a = ...;301;302; for (int i = 0; i < N; ++i) {303; sum += s[i];304; int_sum += a[i];305; }306;307define dso_local %"class.std::complex" @reduction_mix(ptr %a, ptr %b, ptr noalias nocapture noundef readnone %c, [2 x double] %d.coerce, ptr nocapture noundef readonly %s, ptr nocapture noundef writeonly %outs) local_unnamed_addr #0 {308; CHECK-LABEL: reduction_mix:309; CHECK: // %bb.0: // %entry310; CHECK-NEXT: movi v2.2d, #0000000000000000311; CHECK-NEXT: movi v0.2d, #0000000000000000312; CHECK-NEXT: cntd x9313; CHECK-NEXT: movi v1.2d, #0000000000000000314; CHECK-NEXT: neg x10, x9315; CHECK-NEXT: mov w11, #100 // =0x64316; CHECK-NEXT: ptrue p0.d317; CHECK-NEXT: mov x8, xzr318; CHECK-NEXT: and x10, x10, x11319; CHECK-NEXT: rdvl x11, #2320; CHECK-NEXT: .LBB3_1: // %vector.body321; CHECK-NEXT: // =>This Inner Loop Header: Depth=1322; CHECK-NEXT: ldr z3, [x0]323; CHECK-NEXT: ldr z4, [x0, #1, mul vl]324; CHECK-NEXT: add x0, x0, x11325; CHECK-NEXT: ld1w { z5.d }, p0/z, [x3, x8, lsl #2]326; CHECK-NEXT: add x8, x8, x9327; CHECK-NEXT: cmp x10, x8328; CHECK-NEXT: fadd z1.d, z4.d, z1.d329; CHECK-NEXT: fadd z0.d, z3.d, z0.d330; CHECK-NEXT: add z2.d, z5.d, z2.d331; CHECK-NEXT: b.ne .LBB3_1332; CHECK-NEXT: // %bb.2: // %middle.block333; CHECK-NEXT: uzp2 z3.d, z0.d, z1.d334; CHECK-NEXT: uzp1 z1.d, z0.d, z1.d335; CHECK-NEXT: uaddv d2, p0, z2.d336; CHECK-NEXT: faddv d0, p0, z3.d337; CHECK-NEXT: faddv d1, p0, z1.d338; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0339; CHECK-NEXT: str s2, [x4]340; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1341; CHECK-NEXT: ret342entry:343 %0 = tail call i64 @llvm.vscale.i64()344 %1 = shl nuw nsw i64 %0, 1345 %n.mod.vf = urem i64 100, %1346 %n.vec = sub nuw nsw i64 100, %n.mod.vf347 %2 = tail call i64 @llvm.vscale.i64()348 %3 = shl nuw nsw i64 %2, 1349 br label %vector.body350 351vector.body: ; preds = %vector.body, %entry352 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]353 %vec.phi = phi <vscale x 2 x i32> [ zeroinitializer, %entry ], [ %5, %vector.body ]354 %vec.phi13 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %9, %vector.body ]355 %vec.phi14 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %10, %vector.body ]356 %4 = getelementptr inbounds i32, ptr %s, i64 %index357 %wide.load = load <vscale x 2 x i32>, ptr %4, align 4358 %5 = add <vscale x 2 x i32> %wide.load, %vec.phi359 %6 = getelementptr inbounds %"class.std::complex", ptr %a, i64 %index360 %wide.vec = load <vscale x 4 x double>, ptr %6, align 8361 %strided.vec = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %wide.vec)362 %7 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 0363 %8 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 1364 %9 = fadd fast <vscale x 2 x double> %7, %vec.phi13365 %10 = fadd fast <vscale x 2 x double> %8, %vec.phi14366 %index.next = add nuw i64 %index, %3367 %11 = icmp eq i64 %index.next, %n.vec368 br i1 %11, label %middle.block, label %vector.body369 370middle.block: ; preds = %vector.body371 %12 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %10)372 %13 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double -0.000000e+00, <vscale x 2 x double> %9)373 %14 = tail call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> %5)374 store i32 %14, ptr %outs, align 4375 %.fca.0.0.insert = insertvalue %"class.std::complex" poison, double %12, 0, 0376 %.fca.0.1.insert = insertvalue %"class.std::complex" %.fca.0.0.insert, double %13, 0, 1377 ret %"class.std::complex" %.fca.0.1.insert378}379 380 381; Zero initialized double reduction382; struct foo {383; complex<double> v1, v2;384; };385;386;complex<double> foo(struct foo *a, struct foo *b, int n) {387; complex<double> x = 0.0 + 0.0i;388; complex<double> y = 0.0 + 0.0i;389; for (int i = 0; i < n; i++) {390; struct foo t1 = a[i];391; struct foo t2 = b[i];392; x += t1.v1 * t2.v1;393; y += t1.v2 * t2.v2;394; }395; return x + y;396;}397%struct.foo2 = type { %"class.std::complex", %"class.std::complex" }398 399define %"class.std::complex" @double_complex_mul_v2f64(ptr noundef readonly captures(none) %src1, ptr noundef readonly captures(none) %src2, i64 noundef %nvec) {400; CHECK-LABEL: double_complex_mul_v2f64:401; CHECK: // %bb.0: // %entry402; CHECK-NEXT: movi v0.2d, #0000000000000000403; CHECK-NEXT: movi v2.2d, #0000000000000000404; CHECK-NEXT: cntd x8405; CHECK-NEXT: movi v1.2d, #0000000000000000406; CHECK-NEXT: movi v3.2d, #0000000000000000407; CHECK-NEXT: rdvl x9, #4408; CHECK-NEXT: ptrue p0.d409; CHECK-NEXT: .LBB4_1: // %vector.body410; CHECK-NEXT: // =>This Inner Loop Header: Depth=1411; CHECK-NEXT: ld4d { z4.d - z7.d }, p0/z, [x0]412; CHECK-NEXT: subs x2, x2, x8413; CHECK-NEXT: add x0, x0, x9414; CHECK-NEXT: ld4d { z16.d - z19.d }, p0/z, [x1]415; CHECK-NEXT: add x1, x1, x9416; CHECK-NEXT: fmla z2.d, p0/m, z16.d, z4.d417; CHECK-NEXT: fmla z0.d, p0/m, z16.d, z5.d418; CHECK-NEXT: fmla z3.d, p0/m, z18.d, z6.d419; CHECK-NEXT: fmla z1.d, p0/m, z18.d, z7.d420; CHECK-NEXT: fmls z2.d, p0/m, z5.d, z17.d421; CHECK-NEXT: fmla z0.d, p0/m, z17.d, z4.d422; CHECK-NEXT: fmls z3.d, p0/m, z7.d, z19.d423; CHECK-NEXT: fmla z1.d, p0/m, z19.d, z6.d424; CHECK-NEXT: b.ne .LBB4_1425; CHECK-NEXT: // %bb.2: // %middle.block426; CHECK-NEXT: fadd z2.d, z2.d, z3.d427; CHECK-NEXT: fadd z1.d, z0.d, z1.d428; CHECK-NEXT: faddv d0, p0, z2.d429; CHECK-NEXT: faddv d1, p0, z1.d430; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0431; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1432; CHECK-NEXT: ret433entry:434 %vscale = tail call i64 @llvm.vscale.i64()435 %inc = shl nuw nsw i64 %vscale, 1436 br label %vector.body437 438vector.body:439 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]440 %vec.phi1 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %vec.phi1.next, %vector.body ]441 %vec.phi2 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %vec.phi2.next, %vector.body ]442 %vec.phi3 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %vec.phi3.next, %vector.body ]443 %vec.phi4 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %vec.phi4.next, %vector.body ]444 %gep1 = getelementptr inbounds nuw %struct.foo2, ptr %src1, i64 %index445 %wide.vec = load <vscale x 8 x double>, ptr %gep1, align 8446 %strided.vec = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave4.nxv8f64(<vscale x 8 x double> %wide.vec)447 %ext00 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 0448 %ext01 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 1449 %ext02 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 2450 %ext03 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 3451 %gep2 = getelementptr inbounds nuw %struct.foo2, ptr %src2, i64 %index452 %wide.vec73 = load <vscale x 8 x double>, ptr %gep2, align 8453 %strided.vec74 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave4.nxv8f64(<vscale x 8 x double> %wide.vec73)454 %ext10 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec74, 0455 %ext11 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec74, 1456 %ext12 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec74, 2457 %ext13 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec74, 3458 %fmul1 = fmul fast <vscale x 2 x double> %ext10, %ext00459 %fmul2 = fmul fast <vscale x 2 x double> %ext11, %ext00460 %fmul3 = fmul fast <vscale x 2 x double> %ext10, %ext01461 %fadd1 = fadd fast <vscale x 2 x double> %fmul1, %vec.phi2462 %fmul4 = fmul fast <vscale x 2 x double> %ext01, %ext11463 %vec.phi2.next = fsub fast <vscale x 2 x double> %fadd1, %fmul4464 %fadd2 = fadd fast <vscale x 2 x double> %fmul3, %vec.phi1465 %vec.phi1.next = fadd fast <vscale x 2 x double> %fadd2, %fmul2466 %fmul5 = fmul fast <vscale x 2 x double> %ext12, %ext02467 %fmul6 = fmul fast <vscale x 2 x double> %ext13, %ext02468 %fmul7 = fmul fast <vscale x 2 x double> %ext12, %ext03469 %fadd3 = fadd fast <vscale x 2 x double> %fmul5, %vec.phi4470 %fmul8 = fmul fast <vscale x 2 x double> %ext03, %ext13471 %vec.phi4.next = fsub fast <vscale x 2 x double> %fadd3, %fmul8472 %fadd4 = fadd fast <vscale x 2 x double> %fmul7, %vec.phi3473 %vec.phi3.next = fadd fast <vscale x 2 x double> %fadd4, %fmul6474 %index.next = add nuw i64 %index, %inc475 %cmp = icmp eq i64 %index.next, %nvec476 br i1 %cmp, label %middle.block, label %vector.body477 478middle.block:479 %final1 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %vec.phi1.next)480 %final2 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %vec.phi2.next)481 %final3 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %vec.phi3.next)482 %final4 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %vec.phi4.next)483 %last_fadd1 = fadd fast double %final2, %final4484 %last_fadd2 = fadd fast double %final1, %final3485 %.fca.0.0.insert = insertvalue %"class.std::complex" poison, double %last_fadd1, 0, 0486 %.fca.0.1.insert = insertvalue %"class.std::complex" %.fca.0.0.insert, double %last_fadd2, 0, 1487 ret %"class.std::complex" %.fca.0.1.insert488}489 490 491declare i64 @llvm.vscale.i64()492declare { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double>)493declare double @llvm.vector.reduce.fadd.nxv2f64(double, <vscale x 2 x double>)494declare i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32>)495