266 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s --mattr=+complxnum,+neon -o - | FileCheck %s3 4target triple = "aarch64"5 6%"struct.std::complex" = type { { double, double } }7 8; Zero initialized reduction9;10; complex<double> x = 0.0 + 0.0i;11; for (int i = 0; i < 100; ++i)12; x += a[i] * b[i];13;14define dso_local %"struct.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {15; CHECK-LABEL: complex_mul_v2f64:16; CHECK: // %bb.0: // %entry17; CHECK-NEXT: movi v0.2d, #000000000000000018; CHECK-NEXT: movi v1.2d, #000000000000000019; CHECK-NEXT: mov x8, xzr20; CHECK-NEXT: .LBB0_1: // %vector.body21; CHECK-NEXT: // =>This Inner Loop Header: Depth=122; CHECK-NEXT: add x9, x0, x823; CHECK-NEXT: add x10, x1, x824; CHECK-NEXT: add x8, x8, #3225; CHECK-NEXT: ldp q3, q2, [x9]26; CHECK-NEXT: cmp x8, #160027; CHECK-NEXT: ldp q5, q4, [x10]28; CHECK-NEXT: fcmla v1.2d, v5.2d, v3.2d, #029; CHECK-NEXT: fcmla v0.2d, v4.2d, v2.2d, #030; CHECK-NEXT: fcmla v1.2d, v5.2d, v3.2d, #9031; CHECK-NEXT: fcmla v0.2d, v4.2d, v2.2d, #9032; CHECK-NEXT: b.ne .LBB0_133; CHECK-NEXT: // %bb.2: // %middle.block34; CHECK-NEXT: zip2 v2.2d, v1.2d, v0.2d35; CHECK-NEXT: zip1 v0.2d, v1.2d, v0.2d36; CHECK-NEXT: faddp d0, v0.2d37; CHECK-NEXT: faddp d1, v2.2d38; CHECK-NEXT: ret39entry:40 br label %vector.body41 42vector.body: ; preds = %vector.body, %entry43 %lsr.iv = phi i64 [ %lsr.iv.next, %vector.body ], [ 0, %entry ]44 %vec.phi = phi <2 x double> [ zeroinitializer, %entry ], [ %7, %vector.body ]45 %vec.phi27 = phi <2 x double> [ zeroinitializer, %entry ], [ %5, %vector.body ]46 %scevgep = getelementptr i8, ptr %a, i64 %lsr.iv47 %scevgep35 = getelementptr i8, ptr %b, i64 %lsr.iv48 %wide.vec = load <4 x double>, ptr %scevgep, align 849 %strided.vec = shufflevector <4 x double> %wide.vec, <4 x double> poison, <2 x i32> <i32 0, i32 2>50 %strided.vec28 = shufflevector <4 x double> %wide.vec, <4 x double> poison, <2 x i32> <i32 1, i32 3>51 %wide.vec29 = load <4 x double>, ptr %scevgep35, align 852 %strided.vec30 = shufflevector <4 x double> %wide.vec29, <4 x double> poison, <2 x i32> <i32 0, i32 2>53 %strided.vec31 = shufflevector <4 x double> %wide.vec29, <4 x double> poison, <2 x i32> <i32 1, i32 3>54 %0 = fmul fast <2 x double> %strided.vec31, %strided.vec55 %1 = fmul fast <2 x double> %strided.vec30, %strided.vec2856 %2 = fmul fast <2 x double> %strided.vec30, %strided.vec57 %3 = fadd fast <2 x double> %2, %vec.phi2758 %4 = fmul fast <2 x double> %strided.vec31, %strided.vec2859 %5 = fsub fast <2 x double> %3, %460 %6 = fadd fast <2 x double> %1, %vec.phi61 %7 = fadd fast <2 x double> %6, %062 %lsr.iv.next = add nuw nsw i64 %lsr.iv, 3263 %8 = icmp eq i64 %lsr.iv.next, 160064 br i1 %8, label %middle.block, label %vector.body65 66middle.block: ; preds = %vector.body67 %9 = tail call fast double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> %5)68 %10 = tail call fast double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> %7)69 %.fca.0.0.insert = insertvalue %"struct.std::complex" poison, double %9, 0, 070 %.fca.0.1.insert = insertvalue %"struct.std::complex" %.fca.0.0.insert, double %10, 0, 171 ret %"struct.std::complex" %.fca.0.1.insert72}73 74; Fixed value initialized reduction75;76; complex<double> x = 2.0 + 1.0i;77; for (int i = 0; i < 100; ++i)78; x += a[i] * b[i];79;80define %"struct.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) {81; CHECK-LABEL: complex_mul_nonzero_init_v2f64:82; CHECK: // %bb.0: // %entry83; CHECK-NEXT: movi v0.2d, #000000000000000084; CHECK-NEXT: adrp x8, .LCPI1_085; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI1_0]86; CHECK-NEXT: mov x8, xzr87; CHECK-NEXT: .LBB1_1: // %vector.body88; CHECK-NEXT: // =>This Inner Loop Header: Depth=189; CHECK-NEXT: add x9, x0, x890; CHECK-NEXT: add x10, x1, x891; CHECK-NEXT: add x8, x8, #3292; CHECK-NEXT: ldp q3, q2, [x9]93; CHECK-NEXT: cmp x8, #160094; CHECK-NEXT: ldp q5, q4, [x10]95; CHECK-NEXT: fcmla v1.2d, v5.2d, v3.2d, #096; CHECK-NEXT: fcmla v0.2d, v4.2d, v2.2d, #097; CHECK-NEXT: fcmla v1.2d, v5.2d, v3.2d, #9098; CHECK-NEXT: fcmla v0.2d, v4.2d, v2.2d, #9099; CHECK-NEXT: b.ne .LBB1_1100; CHECK-NEXT: // %bb.2: // %middle.block101; CHECK-NEXT: zip2 v2.2d, v1.2d, v0.2d102; CHECK-NEXT: zip1 v0.2d, v1.2d, v0.2d103; CHECK-NEXT: faddp d0, v0.2d104; CHECK-NEXT: faddp d1, v2.2d105; CHECK-NEXT: ret106entry:107 br label %vector.body108 109vector.body: ; preds = %vector.body, %entry110 %lsr.iv = phi i64 [ %lsr.iv.next, %vector.body ], [ 0, %entry ]111 %vec.phi = phi <2 x double> [ <double 1.000000e+00, double 0.000000e+00>, %entry ], [ %7, %vector.body ]112 %vec.phi27 = phi <2 x double> [ <double 2.000000e+00, double 0.000000e+00>, %entry ], [ %5, %vector.body ]113 %scevgep = getelementptr i8, ptr %a, i64 %lsr.iv114 %scevgep35 = getelementptr i8, ptr %b, i64 %lsr.iv115 %wide.vec = load <4 x double>, ptr %scevgep, align 8116 %strided.vec = shufflevector <4 x double> %wide.vec, <4 x double> poison, <2 x i32> <i32 0, i32 2>117 %strided.vec28 = shufflevector <4 x double> %wide.vec, <4 x double> poison, <2 x i32> <i32 1, i32 3>118 %wide.vec29 = load <4 x double>, ptr %scevgep35, align 8119 %strided.vec30 = shufflevector <4 x double> %wide.vec29, <4 x double> poison, <2 x i32> <i32 0, i32 2>120 %strided.vec31 = shufflevector <4 x double> %wide.vec29, <4 x double> poison, <2 x i32> <i32 1, i32 3>121 %0 = fmul fast <2 x double> %strided.vec31, %strided.vec122 %1 = fmul fast <2 x double> %strided.vec30, %strided.vec28123 %2 = fmul fast <2 x double> %strided.vec30, %strided.vec124 %3 = fadd fast <2 x double> %2, %vec.phi27125 %4 = fmul fast <2 x double> %strided.vec31, %strided.vec28126 %5 = fsub fast <2 x double> %3, %4127 %6 = fadd fast <2 x double> %1, %vec.phi128 %7 = fadd fast <2 x double> %6, %0129 %lsr.iv.next = add nuw nsw i64 %lsr.iv, 32130 %8 = icmp eq i64 %lsr.iv.next, 1600131 br i1 %8, label %middle.block, label %vector.body132 133middle.block: ; preds = %vector.body134 %9 = tail call fast double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> %5)135 %10 = tail call fast double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> %7)136 %.fca.0.0.insert = insertvalue %"struct.std::complex" poison, double %9, 0, 0137 %.fca.0.1.insert = insertvalue %"struct.std::complex" %.fca.0.0.insert, double %10, 0, 1138 ret %"struct.std::complex" %.fca.0.1.insert139}140 141; Loop unrolled with factor 2142;143define %"struct.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {144; CHECK-LABEL: complex_mul_v2f64_unrolled:145; CHECK: // %bb.0: // %entry146; CHECK-NEXT: movi v0.2d, #0000000000000000147; CHECK-NEXT: movi v1.2d, #0000000000000000148; CHECK-NEXT: adrp x8, .LCPI2_0149; CHECK-NEXT: movi v3.2d, #0000000000000000150; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI2_0]151; CHECK-NEXT: add x8, x0, #32152; CHECK-NEXT: add x9, x1, #32153; CHECK-NEXT: mov x10, #-100 // =0xffffffffffffff9c154; CHECK-NEXT: .LBB2_1: // %vector.body155; CHECK-NEXT: // =>This Inner Loop Header: Depth=1156; CHECK-NEXT: ldp q5, q4, [x8, #-32]157; CHECK-NEXT: adds x10, x10, #4158; CHECK-NEXT: ldp q7, q6, [x9, #-32]159; CHECK-NEXT: ldp q17, q16, [x8], #64160; CHECK-NEXT: ldp q19, q18, [x9], #64161; CHECK-NEXT: fcmla v2.2d, v7.2d, v5.2d, #0162; CHECK-NEXT: fcmla v1.2d, v6.2d, v4.2d, #0163; CHECK-NEXT: fcmla v0.2d, v19.2d, v17.2d, #0164; CHECK-NEXT: fcmla v3.2d, v18.2d, v16.2d, #0165; CHECK-NEXT: fcmla v2.2d, v7.2d, v5.2d, #90166; CHECK-NEXT: fcmla v1.2d, v6.2d, v4.2d, #90167; CHECK-NEXT: fcmla v0.2d, v19.2d, v17.2d, #90168; CHECK-NEXT: fcmla v3.2d, v18.2d, v16.2d, #90169; CHECK-NEXT: b.ne .LBB2_1170; CHECK-NEXT: // %bb.2: // %middle.block171; CHECK-NEXT: zip2 v4.2d, v0.2d, v3.2d172; CHECK-NEXT: zip1 v0.2d, v0.2d, v3.2d173; CHECK-NEXT: zip2 v3.2d, v2.2d, v1.2d174; CHECK-NEXT: zip1 v1.2d, v2.2d, v1.2d175; CHECK-NEXT: fadd v0.2d, v0.2d, v1.2d176; CHECK-NEXT: fadd v1.2d, v4.2d, v3.2d177; CHECK-NEXT: faddp d0, v0.2d178; CHECK-NEXT: faddp d1, v1.2d179; CHECK-NEXT: ret180entry:181 %scevgep = getelementptr i8, ptr %a, i64 32182 %scevgep49 = getelementptr i8, ptr %b, i64 32183 br label %vector.body184 185vector.body: ; preds = %vector.body, %entry186 %lsr.iv54 = phi i64 [ %lsr.iv.next, %vector.body ], [ 100, %entry ]187 %lsr.iv50 = phi ptr [ %scevgep51, %vector.body ], [ %scevgep49, %entry ]188 %lsr.iv = phi ptr [ %scevgep48, %vector.body ], [ %scevgep, %entry ]189 %vec.phi = phi <2 x double> [ <double 1.000000e+00, double 0.000000e+00>, %entry ], [ %14, %vector.body ]190 %vec.phi27 = phi <2 x double> [ zeroinitializer, %entry ], [ %15, %vector.body ]191 %vec.phi28 = phi <2 x double> [ <double 2.000000e+00, double 0.000000e+00>, %entry ], [ %10, %vector.body ]192 %vec.phi29 = phi <2 x double> [ zeroinitializer, %entry ], [ %11, %vector.body ]193 %scevgep52 = getelementptr i8, ptr %lsr.iv, i64 -32194 %scevgep53 = getelementptr i8, ptr %lsr.iv50, i64 -32195 %wide.vec = load <4 x double>, ptr %scevgep52, align 8196 %wide.vec30 = load <4 x double>, ptr %lsr.iv, align 8197 %strided.vec = shufflevector <4 x double> %wide.vec, <4 x double> poison, <2 x i32> <i32 0, i32 2>198 %strided.vec31 = shufflevector <4 x double> %wide.vec30, <4 x double> poison, <2 x i32> <i32 0, i32 2>199 %strided.vec32 = shufflevector <4 x double> %wide.vec, <4 x double> poison, <2 x i32> <i32 1, i32 3>200 %strided.vec33 = shufflevector <4 x double> %wide.vec30, <4 x double> poison, <2 x i32> <i32 1, i32 3>201 %wide.vec34 = load <4 x double>, ptr %scevgep53, align 8202 %wide.vec35 = load <4 x double>, ptr %lsr.iv50, align 8203 %strided.vec36 = shufflevector <4 x double> %wide.vec34, <4 x double> poison, <2 x i32> <i32 0, i32 2>204 %strided.vec37 = shufflevector <4 x double> %wide.vec35, <4 x double> poison, <2 x i32> <i32 0, i32 2>205 %strided.vec38 = shufflevector <4 x double> %wide.vec34, <4 x double> poison, <2 x i32> <i32 1, i32 3>206 %strided.vec39 = shufflevector <4 x double> %wide.vec35, <4 x double> poison, <2 x i32> <i32 1, i32 3>207 %0 = fmul fast <2 x double> %strided.vec38, %strided.vec208 %1 = fmul fast <2 x double> %strided.vec39, %strided.vec31209 %2 = fmul fast <2 x double> %strided.vec36, %strided.vec32210 %3 = fmul fast <2 x double> %strided.vec37, %strided.vec33211 %4 = fmul fast <2 x double> %strided.vec36, %strided.vec212 %5 = fmul fast <2 x double> %strided.vec37, %strided.vec31213 %6 = fadd fast <2 x double> %4, %vec.phi28214 %7 = fadd fast <2 x double> %5, %vec.phi29215 %8 = fmul fast <2 x double> %strided.vec38, %strided.vec32216 %9 = fmul fast <2 x double> %strided.vec39, %strided.vec33217 %10 = fsub fast <2 x double> %6, %8218 %11 = fsub fast <2 x double> %7, %9219 %12 = fadd fast <2 x double> %2, %vec.phi220 %13 = fadd fast <2 x double> %3, %vec.phi27221 %14 = fadd fast <2 x double> %12, %0222 %15 = fadd fast <2 x double> %13, %1223 %scevgep48 = getelementptr i8, ptr %lsr.iv, i64 64224 %scevgep51 = getelementptr i8, ptr %lsr.iv50, i64 64225 %lsr.iv.next = add nsw i64 %lsr.iv54, -4226 %16 = icmp eq i64 %lsr.iv.next, 0227 br i1 %16, label %middle.block, label %vector.body228 229middle.block: ; preds = %vector.body230 %bin.rdx40 = fadd fast <2 x double> %11, %10231 %17 = tail call fast double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> %bin.rdx40)232 %bin.rdx = fadd fast <2 x double> %15, %14233 %18 = tail call fast double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> %bin.rdx)234 %.fca.0.0.insert = insertvalue %"struct.std::complex" poison, double %17, 0, 0235 %.fca.0.1.insert = insertvalue %"struct.std::complex" %.fca.0.0.insert, double %18, 0, 1236 ret %"struct.std::complex" %.fca.0.1.insert237}238 239; The reduced bug from D153355. Shows that reduction was detected where it did not exist.240define void @incorrect_reduction_pattern(i1 %exitcond.not) {241; CHECK-LABEL: incorrect_reduction_pattern:242; CHECK: // %bb.0: // %entry243; CHECK-NEXT: .LBB3_1: // %for.body244; CHECK-NEXT: // =>This Inner Loop Header: Depth=1245; CHECK-NEXT: tbz w0, #0, .LBB3_1246; CHECK-NEXT: // %bb.2: // %for.end.loopexit247; CHECK-NEXT: ret248entry:249 br label %for.body250 251for.body: ; preds = %for.body, %entry252 %vec_r = phi <4 x float> [ zeroinitializer, %entry ], [ %lane_r, %for.body ]253 %vec_i = phi <4 x float> [ zeroinitializer, %entry ], [ %lane_i, %for.body ]254 %add = fadd <4 x float> %vec_r, %vec_i255 %lane_r = shufflevector <4 x float> <float 1.000000e+00, float undef, float undef, float undef>, <4 x float> zeroinitializer, <4 x i32> zeroinitializer256 %lane_i = shufflevector <4 x float> <float 1.000000e+00, float undef, float undef, float undef>, <4 x float> zeroinitializer, <4 x i32> zeroinitializer257 br i1 %exitcond.not, label %for.end.loopexit, label %for.body258 259for.end.loopexit: ; preds = %for.body260 %mul.r = fadd <4 x float> %lane_r, %add261 %mul.i = fadd <4 x float> %lane_i, %add262 ret void263}264 265declare double @llvm.vector.reduce.fadd.v2f64(double, <2 x double>)266