602 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "aba" --filter "abd" --filter "add" --version 52; RUN: opt -passes=loop-unroll %s -o - | llc -O3 - -mtriple=aarch64-unknown-unknown -mcpu=neoverse-v2 -o - | FileCheck %s3 4define i64 @sabalb_i32_to_i64_accumulation(ptr %ptr1, ptr %ptr2) {5; CHECK-LABEL: sabalb_i32_to_i64_accumulation:6; CHECK: sabdlb z1.d, z1.s, z2.s7; CHECK: sabalb z0.d, z3.s, z4.s8; CHECK: sabalb z1.d, z4.s, z2.s9; CHECK: sabdlb z2.d, z3.s, z2.s10; CHECK: sabalb z2.d, z4.s, z3.s11; CHECK: sabalb z0.d, z3.s, z4.s12; CHECK: sabalb z1.d, z3.s, z4.s13; CHECK: sabalb z2.d, z3.s, z4.s14; CHECK: sabalb z0.d, z3.s, z4.s15; CHECK: sabalb z1.d, z3.s, z4.s16; CHECK: sabalb z2.d, z3.s, z4.s17; CHECK: sabalb z0.d, z3.s, z4.s18; CHECK: sabalb z1.d, z3.s, z4.s19; CHECK: sabalb z2.d, z3.s, z4.s20; CHECK: sabalb z0.d, z3.s, z4.s21; CHECK: add z0.d, z2.d, z0.d22; CHECK: sabalb z1.d, z3.s, z4.s23; CHECK: add z0.d, z0.d, z1.d24; CHECK: uaddv d0, p0, z0.d25entry:26 br label %loop27loop:28 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]29 %acc_phi = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %acc_next, %loop ]30 %ptr1_i = getelementptr <vscale x 4 x i32>, ptr %ptr1, i32 %i31 %ptr2_i = getelementptr <vscale x 4 x i32>, ptr %ptr2, i32 %i32 %a = load <vscale x 4 x i32>, ptr %ptr1_i, align 133 %b = load <vscale x 4 x i32>, ptr %ptr2_i, align 134 %acc_next = call <vscale x 2 x i64> @llvm.aarch64.sve.sabalb.nxv2i64(<vscale x 2 x i64> %acc_phi,35 <vscale x 4 x i32> %a,36 <vscale x 4 x i32> %b)37 38 %next_i = add i32 %i, 139 %cmp = icmp slt i32 %next_i, 1640 br i1 %cmp, label %loop, label %exit41exit:42 %reduce = tail call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %acc_next)43 ret i64 %reduce44}45 46declare <vscale x 2 x i64> @llvm.aarch64.sve.sabalb.nxv2i64(<vscale x 2 x i64>, <vscale x 4 x i32>, <vscale x 4 x i32>)47declare i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64>)48 49define i32 @sabalb_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {50; CHECK-LABEL: sabalb_i16_to_i32_accumulation:51; CHECK: sabdlb z1.s, z1.h, z2.h52; CHECK: sabalb z0.s, z3.h, z4.h53; CHECK: sabalb z1.s, z4.h, z2.h54; CHECK: sabdlb z2.s, z3.h, z2.h55; CHECK: sabalb z2.s, z4.h, z3.h56; CHECK: sabalb z0.s, z3.h, z4.h57; CHECK: sabalb z1.s, z3.h, z4.h58; CHECK: sabalb z2.s, z3.h, z4.h59; CHECK: sabalb z0.s, z3.h, z4.h60; CHECK: sabalb z1.s, z3.h, z4.h61; CHECK: sabalb z2.s, z3.h, z4.h62; CHECK: sabalb z0.s, z3.h, z4.h63; CHECK: sabalb z1.s, z3.h, z4.h64; CHECK: sabalb z2.s, z3.h, z4.h65; CHECK: sabalb z0.s, z3.h, z4.h66; CHECK: add z0.s, z2.s, z0.s67; CHECK: sabalb z1.s, z3.h, z4.h68; CHECK: add z0.s, z0.s, z1.s69; CHECK: uaddv d0, p0, z0.s70entry:71 br label %loop72loop:73 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]74 %acc_phi = phi <vscale x 4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]75 %ptr1_i = getelementptr <vscale x 8 x i16>, ptr %ptr1, i32 %i76 %ptr2_i = getelementptr <vscale x 8 x i16>, ptr %ptr2, i32 %i77 %a = load <vscale x 8 x i16>, ptr %ptr1_i, align 178 %b = load <vscale x 8 x i16>, ptr %ptr2_i, align 179 %acc_next = call <vscale x 4 x i32> @llvm.aarch64.sve.sabalb.nxv4i32(<vscale x 4 x i32> %acc_phi,80 <vscale x 8 x i16> %a,81 <vscale x 8 x i16> %b)82 83 %next_i = add i32 %i, 184 %cmp = icmp slt i32 %next_i, 1685 br i1 %cmp, label %loop, label %exit86exit:87 %reduce = tail call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %acc_next)88 ret i32 %reduce89}90 91declare <vscale x 4 x i32> @llvm.aarch64.sve.sabalb.nxv4i32(<vscale x 4 x i32>, <vscale x 8 x i16>, <vscale x 8 x i16>)92declare i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32>)93 94define i16 @sabalb_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {95; CHECK-LABEL: sabalb_i8_to_i16_accumulation:96; CHECK: sabdlb z1.h, z1.b, z2.b97; CHECK: sabalb z0.h, z3.b, z4.b98; CHECK: sabalb z1.h, z4.b, z2.b99; CHECK: sabdlb z2.h, z3.b, z2.b100; CHECK: sabalb z2.h, z4.b, z3.b101; CHECK: sabalb z0.h, z3.b, z4.b102; CHECK: sabalb z1.h, z3.b, z4.b103; CHECK: sabalb z2.h, z3.b, z4.b104; CHECK: sabalb z0.h, z3.b, z4.b105; CHECK: sabalb z1.h, z3.b, z4.b106; CHECK: sabalb z2.h, z3.b, z4.b107; CHECK: sabalb z0.h, z3.b, z4.b108; CHECK: sabalb z1.h, z3.b, z4.b109; CHECK: sabalb z2.h, z3.b, z4.b110; CHECK: sabalb z0.h, z3.b, z4.b111; CHECK: add z0.h, z2.h, z0.h112; CHECK: sabalb z1.h, z3.b, z4.b113; CHECK: add z0.h, z0.h, z1.h114; CHECK: uaddv d0, p0, z0.h115entry:116 br label %loop117loop:118 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]119 %acc_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]120 %ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i121 %ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i122 %a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1123 %b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1124 %acc_next = call <vscale x 8 x i16> @llvm.aarch64.sve.sabalb.nxv8i16(<vscale x 8 x i16> %acc_phi,125 <vscale x 16 x i8> %a,126 <vscale x 16 x i8> %b)127 128 %next_i = add i32 %i, 1129 %cmp = icmp slt i32 %next_i, 16130 br i1 %cmp, label %loop, label %exit131exit:132 %reduce = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %acc_next)133 ret i16 %reduce134}135 136declare <vscale x 8 x i16> @llvm.aarch64.sve.sabalb.nxv8i16(<vscale x 8 x i16>, <vscale x 16 x i8>, <vscale x 16 x i8>)137declare i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16>)138 139define i64 @sabalt_i32_to_i64_accumulation(ptr %ptr1, ptr %ptr2) {140; CHECK-LABEL: sabalt_i32_to_i64_accumulation:141; CHECK: sabdlt z1.d, z1.s, z2.s142; CHECK: sabalt z0.d, z3.s, z4.s143; CHECK: sabalt z1.d, z4.s, z2.s144; CHECK: sabdlt z2.d, z3.s, z2.s145; CHECK: sabalt z2.d, z4.s, z3.s146; CHECK: sabalt z0.d, z3.s, z4.s147; CHECK: sabalt z1.d, z3.s, z4.s148; CHECK: sabalt z2.d, z3.s, z4.s149; CHECK: sabalt z0.d, z3.s, z4.s150; CHECK: sabalt z1.d, z3.s, z4.s151; CHECK: sabalt z2.d, z3.s, z4.s152; CHECK: sabalt z0.d, z3.s, z4.s153; CHECK: sabalt z1.d, z3.s, z4.s154; CHECK: sabalt z2.d, z3.s, z4.s155; CHECK: sabalt z0.d, z3.s, z4.s156; CHECK: add z0.d, z2.d, z0.d157; CHECK: sabalt z1.d, z3.s, z4.s158; CHECK: add z0.d, z0.d, z1.d159; CHECK: uaddv d0, p0, z0.d160entry:161 br label %loop162loop:163 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]164 %acc_phi = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %acc_next, %loop ]165 %ptr1_i = getelementptr <vscale x 4 x i32>, ptr %ptr1, i32 %i166 %ptr2_i = getelementptr <vscale x 4 x i32>, ptr %ptr2, i32 %i167 %a = load <vscale x 4 x i32>, ptr %ptr1_i, align 1168 %b = load <vscale x 4 x i32>, ptr %ptr2_i, align 1169 %acc_next = call <vscale x 2 x i64> @llvm.aarch64.sve.sabalt.nxv2i64(<vscale x 2 x i64> %acc_phi,170 <vscale x 4 x i32> %a,171 <vscale x 4 x i32> %b)172 173 %next_i = add i32 %i, 1174 %cmp = icmp slt i32 %next_i, 16175 br i1 %cmp, label %loop, label %exit176exit:177 %reduce = tail call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %acc_next)178 ret i64 %reduce179}180 181declare <vscale x 2 x i64> @llvm.aarch64.sve.sabalt.nxv2i64(<vscale x 2 x i64>, <vscale x 4 x i32>, <vscale x 4 x i32>)182 183define i32 @sabalt_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {184; CHECK-LABEL: sabalt_i16_to_i32_accumulation:185; CHECK: sabdlt z1.s, z1.h, z2.h186; CHECK: sabalt z0.s, z3.h, z4.h187; CHECK: sabalt z1.s, z4.h, z2.h188; CHECK: sabdlt z2.s, z3.h, z2.h189; CHECK: sabalt z2.s, z4.h, z3.h190; CHECK: sabalt z0.s, z3.h, z4.h191; CHECK: sabalt z1.s, z3.h, z4.h192; CHECK: sabalt z2.s, z3.h, z4.h193; CHECK: sabalt z0.s, z3.h, z4.h194; CHECK: sabalt z1.s, z3.h, z4.h195; CHECK: sabalt z2.s, z3.h, z4.h196; CHECK: sabalt z0.s, z3.h, z4.h197; CHECK: sabalt z1.s, z3.h, z4.h198; CHECK: sabalt z2.s, z3.h, z4.h199; CHECK: sabalt z0.s, z3.h, z4.h200; CHECK: add z0.s, z2.s, z0.s201; CHECK: sabalt z1.s, z3.h, z4.h202; CHECK: add z0.s, z0.s, z1.s203; CHECK: uaddv d0, p0, z0.s204entry:205 br label %loop206loop:207 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]208 %acc_phi = phi <vscale x 4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]209 %ptr1_i = getelementptr <vscale x 8 x i16>, ptr %ptr1, i32 %i210 %ptr2_i = getelementptr <vscale x 8 x i16>, ptr %ptr2, i32 %i211 %a = load <vscale x 8 x i16>, ptr %ptr1_i, align 1212 %b = load <vscale x 8 x i16>, ptr %ptr2_i, align 1213 %acc_next = call <vscale x 4 x i32> @llvm.aarch64.sve.sabalt.nxv4i32(<vscale x 4 x i32> %acc_phi,214 <vscale x 8 x i16> %a,215 <vscale x 8 x i16> %b)216 217 %next_i = add i32 %i, 1218 %cmp = icmp slt i32 %next_i, 16219 br i1 %cmp, label %loop, label %exit220exit:221 %reduce = tail call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %acc_next)222 ret i32 %reduce223}224 225declare <vscale x 4 x i32> @llvm.aarch64.sve.sabalt.nxv4i32(<vscale x 4 x i32>, <vscale x 8 x i16>, <vscale x 8 x i16>)226 227define i16 @sabalt_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {228; CHECK-LABEL: sabalt_i8_to_i16_accumulation:229; CHECK: sabdlt z1.h, z1.b, z2.b230; CHECK: sabalt z0.h, z3.b, z4.b231; CHECK: sabalt z1.h, z4.b, z2.b232; CHECK: sabdlt z2.h, z3.b, z2.b233; CHECK: sabalt z2.h, z4.b, z3.b234; CHECK: sabalt z0.h, z3.b, z4.b235; CHECK: sabalt z1.h, z3.b, z4.b236; CHECK: sabalt z2.h, z3.b, z4.b237; CHECK: sabalt z0.h, z3.b, z4.b238; CHECK: sabalt z1.h, z3.b, z4.b239; CHECK: sabalt z2.h, z3.b, z4.b240; CHECK: sabalt z0.h, z3.b, z4.b241; CHECK: sabalt z1.h, z3.b, z4.b242; CHECK: sabalt z2.h, z3.b, z4.b243; CHECK: sabalt z0.h, z3.b, z4.b244; CHECK: add z0.h, z2.h, z0.h245; CHECK: sabalt z1.h, z3.b, z4.b246; CHECK: add z0.h, z0.h, z1.h247; CHECK: uaddv d0, p0, z0.h248entry:249 br label %loop250loop:251 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]252 %acc_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]253 %ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i254 %ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i255 %a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1256 %b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1257 %acc_next = call <vscale x 8 x i16> @llvm.aarch64.sve.sabalt.nxv8i16(<vscale x 8 x i16> %acc_phi,258 <vscale x 16 x i8> %a,259 <vscale x 16 x i8> %b)260 261 %next_i = add i32 %i, 1262 %cmp = icmp slt i32 %next_i, 16263 br i1 %cmp, label %loop, label %exit264exit:265 %reduce = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %acc_next)266 ret i16 %reduce267}268 269declare <vscale x 8 x i16> @llvm.aarch64.sve.sabalt.nxv8i16(<vscale x 8 x i16>, <vscale x 16 x i8>, <vscale x 16 x i8>)270 271define i64 @uabalb_i32_to_i64_accumulation(ptr %ptr1, ptr %ptr2) {272; CHECK-LABEL: uabalb_i32_to_i64_accumulation:273; CHECK: uabdlb z1.d, z1.s, z2.s274; CHECK: uabalb z0.d, z3.s, z4.s275; CHECK: uabalb z1.d, z4.s, z2.s276; CHECK: uabdlb z2.d, z3.s, z2.s277; CHECK: uabalb z2.d, z4.s, z3.s278; CHECK: uabalb z0.d, z3.s, z4.s279; CHECK: uabalb z1.d, z3.s, z4.s280; CHECK: uabalb z2.d, z3.s, z4.s281; CHECK: uabalb z0.d, z3.s, z4.s282; CHECK: uabalb z1.d, z3.s, z4.s283; CHECK: uabalb z2.d, z3.s, z4.s284; CHECK: uabalb z0.d, z3.s, z4.s285; CHECK: uabalb z1.d, z3.s, z4.s286; CHECK: uabalb z2.d, z3.s, z4.s287; CHECK: uabalb z0.d, z3.s, z4.s288; CHECK: add z0.d, z2.d, z0.d289; CHECK: uabalb z1.d, z3.s, z4.s290; CHECK: add z0.d, z0.d, z1.d291; CHECK: uaddv d0, p0, z0.d292entry:293 br label %loop294loop:295 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]296 %acc_phi = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %acc_next, %loop ]297 %ptr1_i = getelementptr <vscale x 4 x i32>, ptr %ptr1, i32 %i298 %ptr2_i = getelementptr <vscale x 4 x i32>, ptr %ptr2, i32 %i299 %a = load <vscale x 4 x i32>, ptr %ptr1_i, align 1300 %b = load <vscale x 4 x i32>, ptr %ptr2_i, align 1301 %acc_next = call <vscale x 2 x i64> @llvm.aarch64.sve.uabalb.nxv2i64(<vscale x 2 x i64> %acc_phi,302 <vscale x 4 x i32> %a,303 <vscale x 4 x i32> %b)304 305 %next_i = add i32 %i, 1306 %cmp = icmp slt i32 %next_i, 16307 br i1 %cmp, label %loop, label %exit308exit:309 %reduce = tail call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %acc_next)310 ret i64 %reduce311}312 313declare <vscale x 2 x i64> @llvm.aarch64.sve.uabalb.nxv2i64(<vscale x 2 x i64>, <vscale x 4 x i32>, <vscale x 4 x i32>)314 315define i32 @uabalb_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {316; CHECK-LABEL: uabalb_i16_to_i32_accumulation:317; CHECK: uabdlb z1.s, z1.h, z2.h318; CHECK: uabalb z0.s, z3.h, z4.h319; CHECK: uabalb z1.s, z4.h, z2.h320; CHECK: uabdlb z2.s, z3.h, z2.h321; CHECK: uabalb z2.s, z4.h, z3.h322; CHECK: uabalb z0.s, z3.h, z4.h323; CHECK: uabalb z1.s, z3.h, z4.h324; CHECK: uabalb z2.s, z3.h, z4.h325; CHECK: uabalb z0.s, z3.h, z4.h326; CHECK: uabalb z1.s, z3.h, z4.h327; CHECK: uabalb z2.s, z3.h, z4.h328; CHECK: uabalb z0.s, z3.h, z4.h329; CHECK: uabalb z1.s, z3.h, z4.h330; CHECK: uabalb z2.s, z3.h, z4.h331; CHECK: uabalb z0.s, z3.h, z4.h332; CHECK: add z0.s, z2.s, z0.s333; CHECK: uabalb z1.s, z3.h, z4.h334; CHECK: add z0.s, z0.s, z1.s335; CHECK: uaddv d0, p0, z0.s336entry:337 br label %loop338loop:339 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]340 %acc_phi = phi <vscale x 4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]341 %ptr1_i = getelementptr <vscale x 8 x i16>, ptr %ptr1, i32 %i342 %ptr2_i = getelementptr <vscale x 8 x i16>, ptr %ptr2, i32 %i343 %a = load <vscale x 8 x i16>, ptr %ptr1_i, align 1344 %b = load <vscale x 8 x i16>, ptr %ptr2_i, align 1345 %acc_next = call <vscale x 4 x i32> @llvm.aarch64.sve.uabalb.nxv4i32(<vscale x 4 x i32> %acc_phi,346 <vscale x 8 x i16> %a,347 <vscale x 8 x i16> %b)348 349 %next_i = add i32 %i, 1350 %cmp = icmp slt i32 %next_i, 16351 br i1 %cmp, label %loop, label %exit352exit:353 %reduce = tail call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %acc_next)354 ret i32 %reduce355}356 357declare <vscale x 4 x i32> @llvm.aarch64.sve.uabalb.nxv4i32(<vscale x 4 x i32>, <vscale x 8 x i16>, <vscale x 8 x i16>)358 359define i16 @uabalb_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {360; CHECK-LABEL: uabalb_i8_to_i16_accumulation:361; CHECK: uabdlb z1.h, z1.b, z2.b362; CHECK: uabalb z0.h, z3.b, z4.b363; CHECK: uabalb z1.h, z4.b, z2.b364; CHECK: uabdlb z2.h, z3.b, z2.b365; CHECK: uabalb z2.h, z4.b, z3.b366; CHECK: uabalb z0.h, z3.b, z4.b367; CHECK: uabalb z1.h, z3.b, z4.b368; CHECK: uabalb z2.h, z3.b, z4.b369; CHECK: uabalb z0.h, z3.b, z4.b370; CHECK: uabalb z1.h, z3.b, z4.b371; CHECK: uabalb z2.h, z3.b, z4.b372; CHECK: uabalb z0.h, z3.b, z4.b373; CHECK: uabalb z1.h, z3.b, z4.b374; CHECK: uabalb z2.h, z3.b, z4.b375; CHECK: uabalb z0.h, z3.b, z4.b376; CHECK: add z0.h, z2.h, z0.h377; CHECK: uabalb z1.h, z3.b, z4.b378; CHECK: add z0.h, z0.h, z1.h379; CHECK: uaddv d0, p0, z0.h380entry:381 br label %loop382loop:383 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]384 %acc_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]385 %ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i386 %ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i387 %a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1388 %b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1389 %acc_next = call <vscale x 8 x i16> @llvm.aarch64.sve.uabalb.nxv8i16(<vscale x 8 x i16> %acc_phi,390 <vscale x 16 x i8> %a,391 <vscale x 16 x i8> %b)392 393 %next_i = add i32 %i, 1394 %cmp = icmp slt i32 %next_i, 16395 br i1 %cmp, label %loop, label %exit396exit:397 %reduce = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %acc_next)398 ret i16 %reduce399}400 401declare <vscale x 8 x i16> @llvm.aarch64.sve.uabalb.nxv8i16(<vscale x 8 x i16>, <vscale x 16 x i8>, <vscale x 16 x i8>)402 403define i64 @uabalt_i32_to_i64_accumulation(ptr %ptr1, ptr %ptr2) {404; CHECK-LABEL: uabalt_i32_to_i64_accumulation:405; CHECK: uabdlt z1.d, z1.s, z2.s406; CHECK: uabalt z0.d, z3.s, z4.s407; CHECK: uabalt z1.d, z4.s, z2.s408; CHECK: uabdlt z2.d, z3.s, z2.s409; CHECK: uabalt z2.d, z4.s, z3.s410; CHECK: uabalt z0.d, z3.s, z4.s411; CHECK: uabalt z1.d, z3.s, z4.s412; CHECK: uabalt z2.d, z3.s, z4.s413; CHECK: uabalt z0.d, z3.s, z4.s414; CHECK: uabalt z1.d, z3.s, z4.s415; CHECK: uabalt z2.d, z3.s, z4.s416; CHECK: uabalt z0.d, z3.s, z4.s417; CHECK: uabalt z1.d, z3.s, z4.s418; CHECK: uabalt z2.d, z3.s, z4.s419; CHECK: uabalt z0.d, z3.s, z4.s420; CHECK: add z0.d, z2.d, z0.d421; CHECK: uabalt z1.d, z3.s, z4.s422; CHECK: add z0.d, z0.d, z1.d423; CHECK: uaddv d0, p0, z0.d424entry:425 br label %loop426loop:427 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]428 %acc_phi = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %acc_next, %loop ]429 %ptr1_i = getelementptr <vscale x 4 x i32>, ptr %ptr1, i32 %i430 %ptr2_i = getelementptr <vscale x 4 x i32>, ptr %ptr2, i32 %i431 %a = load <vscale x 4 x i32>, ptr %ptr1_i, align 1432 %b = load <vscale x 4 x i32>, ptr %ptr2_i, align 1433 %acc_next = call <vscale x 2 x i64> @llvm.aarch64.sve.uabalt.nxv2i64(<vscale x 2 x i64> %acc_phi,434 <vscale x 4 x i32> %a,435 <vscale x 4 x i32> %b)436 437 %next_i = add i32 %i, 1438 %cmp = icmp slt i32 %next_i, 16439 br i1 %cmp, label %loop, label %exit440exit:441 %reduce = tail call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> %acc_next)442 ret i64 %reduce443}444 445declare <vscale x 2 x i64> @llvm.aarch64.sve.uabalt.nxv2i64(<vscale x 2 x i64>, <vscale x 4 x i32>, <vscale x 4 x i32>)446 447define i32 @uabalt_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {448; CHECK-LABEL: uabalt_i16_to_i32_accumulation:449; CHECK: uabdlt z1.s, z1.h, z2.h450; CHECK: uabalt z0.s, z3.h, z4.h451; CHECK: uabalt z1.s, z4.h, z2.h452; CHECK: uabdlt z2.s, z3.h, z2.h453; CHECK: uabalt z2.s, z4.h, z3.h454; CHECK: uabalt z0.s, z3.h, z4.h455; CHECK: uabalt z1.s, z3.h, z4.h456; CHECK: uabalt z2.s, z3.h, z4.h457; CHECK: uabalt z0.s, z3.h, z4.h458; CHECK: uabalt z1.s, z3.h, z4.h459; CHECK: uabalt z2.s, z3.h, z4.h460; CHECK: uabalt z0.s, z3.h, z4.h461; CHECK: uabalt z1.s, z3.h, z4.h462; CHECK: uabalt z2.s, z3.h, z4.h463; CHECK: uabalt z0.s, z3.h, z4.h464; CHECK: add z0.s, z2.s, z0.s465; CHECK: uabalt z1.s, z3.h, z4.h466; CHECK: add z0.s, z0.s, z1.s467; CHECK: uaddv d0, p0, z0.s468entry:469 br label %loop470loop:471 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]472 %acc_phi = phi <vscale x 4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]473 %ptr1_i = getelementptr <vscale x 8 x i16>, ptr %ptr1, i32 %i474 %ptr2_i = getelementptr <vscale x 8 x i16>, ptr %ptr2, i32 %i475 %a = load <vscale x 8 x i16>, ptr %ptr1_i, align 1476 %b = load <vscale x 8 x i16>, ptr %ptr2_i, align 1477 %acc_next = call <vscale x 4 x i32> @llvm.aarch64.sve.uabalt.nxv4i32(<vscale x 4 x i32> %acc_phi,478 <vscale x 8 x i16> %a,479 <vscale x 8 x i16> %b)480 481 %next_i = add i32 %i, 1482 %cmp = icmp slt i32 %next_i, 16483 br i1 %cmp, label %loop, label %exit484exit:485 %reduce = tail call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> %acc_next)486 ret i32 %reduce487}488 489declare <vscale x 4 x i32> @llvm.aarch64.sve.uabalt.nxv4i32(<vscale x 4 x i32>, <vscale x 8 x i16>, <vscale x 8 x i16>)490 491define i16 @uabalt_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {492; CHECK-LABEL: uabalt_i8_to_i16_accumulation:493; CHECK: uabdlt z1.h, z1.b, z2.b494; CHECK: uabalt z0.h, z3.b, z4.b495; CHECK: uabalt z1.h, z4.b, z2.b496; CHECK: uabdlt z2.h, z3.b, z2.b497; CHECK: uabalt z2.h, z4.b, z3.b498; CHECK: uabalt z0.h, z3.b, z4.b499; CHECK: uabalt z1.h, z3.b, z4.b500; CHECK: uabalt z2.h, z3.b, z4.b501; CHECK: uabalt z0.h, z3.b, z4.b502; CHECK: uabalt z1.h, z3.b, z4.b503; CHECK: uabalt z2.h, z3.b, z4.b504; CHECK: uabalt z0.h, z3.b, z4.b505; CHECK: uabalt z1.h, z3.b, z4.b506; CHECK: uabalt z2.h, z3.b, z4.b507; CHECK: uabalt z0.h, z3.b, z4.b508; CHECK: add z0.h, z2.h, z0.h509; CHECK: uabalt z1.h, z3.b, z4.b510; CHECK: add z0.h, z0.h, z1.h511; CHECK: uaddv d0, p0, z0.h512entry:513 br label %loop514loop:515 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]516 %acc_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]517 %ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i518 %ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i519 %a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1520 %b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1521 %acc_next = call <vscale x 8 x i16> @llvm.aarch64.sve.uabalt.nxv8i16(<vscale x 8 x i16> %acc_phi,522 <vscale x 16 x i8> %a,523 <vscale x 16 x i8> %b)524 525 %next_i = add i32 %i, 1526 %cmp = icmp slt i32 %next_i, 16527 br i1 %cmp, label %loop, label %exit528exit:529 %reduce = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %acc_next)530 ret i16 %reduce531}532 533declare <vscale x 8 x i16> @llvm.aarch64.sve.uabalt.nxv8i16(<vscale x 8 x i16>, <vscale x 16 x i8>, <vscale x 16 x i8>)534 535define i16 @uabalt_and_uabalb_accumulation(ptr %ptr1, ptr %ptr2) {536; CHECK-LABEL: uabalt_and_uabalb_accumulation:537; CHECK: uabalb z1.h, z2.b, z3.b538; CHECK: uabalt z0.h, z2.b, z3.b539; CHECK: uabalb z1.h, z6.b, z7.b540; CHECK: uabalt z0.h, z6.b, z7.b541; CHECK: uabdlb z7.h, z4.b, z5.b542; CHECK: uabdlt z4.h, z4.b, z5.b543; CHECK: uabalb z7.h, z6.b, z5.b544; CHECK: uabalt z4.h, z6.b, z5.b545; CHECK: uabdlb z6.h, z2.b, z3.b546; CHECK: uabdlt z2.h, z2.b, z3.b547; CHECK: uabalb z6.h, z5.b, z3.b548; CHECK: uabalt z2.h, z5.b, z3.b549; CHECK: uabalb z1.h, z3.b, z5.b550; CHECK: uabalt z0.h, z3.b, z5.b551; CHECK: uabalb z7.h, z3.b, z5.b552; CHECK: uabalt z4.h, z3.b, z5.b553; CHECK: uabalb z6.h, z3.b, z5.b554; CHECK: uabalt z2.h, z3.b, z5.b555; CHECK: uabalb z1.h, z3.b, z5.b556; CHECK: uabalt z0.h, z3.b, z5.b557; CHECK: uabalb z7.h, z3.b, z5.b558; CHECK: uabalt z4.h, z3.b, z5.b559; CHECK: uabalb z6.h, z3.b, z5.b560; CHECK: uabalt z2.h, z3.b, z5.b561; CHECK: uabalb z1.h, z3.b, z5.b562; CHECK: uabalt z0.h, z3.b, z5.b563; CHECK: uabalb z7.h, z3.b, z5.b564; CHECK: uabalt z4.h, z3.b, z5.b565; CHECK: uabalb z6.h, z3.b, z5.b566; CHECK: uabalt z2.h, z3.b, z5.b567; CHECK: add z2.h, z4.h, z2.h568; CHECK: uabalb z1.h, z3.b, z5.b569; CHECK: uabalt z0.h, z3.b, z5.b570; CHECK: add z3.h, z7.h, z6.h571; CHECK: add z1.h, z3.h, z1.h572; CHECK: add z0.h, z2.h, z0.h573; CHECK: add z1.h, p0/m, z1.h, z0.h574; CHECK: uaddv d0, p0, z1.h575entry:576 br label %loop577loop:578 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]579 %acc_hi_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next_hi, %loop ]580 %acc_lo_phi = phi <vscale x 8 x i16> [ zeroinitializer, %entry ], [ %acc_next_lo, %loop ]581 %ptr1_i = getelementptr <vscale x 16 x i8>, ptr %ptr1, i32 %i582 %ptr2_i = getelementptr <vscale x 16 x i8>, ptr %ptr2, i32 %i583 %a = load <vscale x 16 x i8>, ptr %ptr1_i, align 1584 %b = load <vscale x 16 x i8>, ptr %ptr2_i, align 1585 %acc_next_lo = call <vscale x 8 x i16> @llvm.aarch64.sve.uabalb.nxv8i16(<vscale x 8 x i16> %acc_lo_phi,586 <vscale x 16 x i8> %a,587 <vscale x 16 x i8> %b)588 %acc_next_hi = call <vscale x 8 x i16> @llvm.aarch64.sve.uabalt.nxv8i16(<vscale x 8 x i16> %acc_hi_phi,589 <vscale x 16 x i8> %a,590 <vscale x 16 x i8> %b)591 %next_i = add i32 %i, 1592 %cmp = icmp slt i32 %next_i, 16593 br i1 %cmp, label %loop, label %exit594exit:595 %mask = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)596 %acc_next = tail call <vscale x 8 x i16> @llvm.aarch64.sve.add.nxv8i16(<vscale x 8 x i1> %mask, <vscale x 8 x i16> %acc_next_lo, <vscale x 8 x i16> %acc_next_hi)597 %reduce = tail call i16 @llvm.vector.reduce.add.nxv8i16(<vscale x 8 x i16> %acc_next)598 ret i16 %reduce599}600 601declare <vscale x 8 x i16> @llvm.aarch64.sve.add.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)602