532 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "aba" --filter "abd" --filter "add" --version 52; RUN: opt -passes=loop-unroll %s -o - | llc -O3 - -mtriple=arm64e-apple-darwin -o - | FileCheck %s3 4 5define i16 @sabal_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {6; CHECK-LABEL: sabal_i8_to_i16_accumulation:7; CHECK: sabdl.8h v1, v1, v38; CHECK: sabdl.8h v0, v0, v29; CHECK: sabdl.8h v2, v3, v510; CHECK: sabal.8h v1, v4, v611; CHECK: sabal.8h v0, v7, v1712; CHECK: sabal.8h v2, v16, v1813; CHECK: sabal.8h v1, v19, v2114; CHECK: sabal.8h v0, v20, v2215; CHECK: add.8h v1, v2, v116; CHECK: add.8h v0, v1, v017; CHECK: addv.8h h0, v018entry:19 br label %loop20 21loop:22 23 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]24 %acc_phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]25 %ptr1_i = getelementptr i8, ptr %ptr1, i32 %i26 %ptr2_i = getelementptr i8, ptr %ptr2, i32 %i27 %a = load <8 x i8>, ptr %ptr1_i, align 128 %b = load <8 x i8>, ptr %ptr2_i, align 129 %vabd = call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %a, <8 x i8> %b)30 %vabd_ext = zext <8 x i8> %vabd to <8 x i16>31 %acc_next = add <8 x i16> %vabd_ext, %acc_phi32 %next_i = add i32 %i, 833 %cmp = icmp slt i32 %next_i, 6434 br i1 %cmp, label %loop, label %exit35 36exit:37 %reduce = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %acc_next)38 ret i16 %reduce39}40 41; Declare the signed absolute difference intrinsic42declare <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8>, <8 x i8>)43declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)44 45 46define i32 @sabal_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {47; CHECK-LABEL: sabal_i16_to_i32_accumulation:48; CHECK: sabdl.4s v1, v1, v349; CHECK: sabdl.4s v0, v0, v250; CHECK: sabdl.4s v2, v3, v551; CHECK: sabal.4s v1, v4, v652; CHECK: sabal.4s v0, v7, v1753; CHECK: sabal.4s v2, v16, v1854; CHECK: sabal.4s v1, v19, v2155; CHECK: sabal.4s v0, v20, v2256; CHECK: add.4s v1, v2, v157; CHECK: add.4s v0, v1, v058; CHECK: addv.4s s0, v059entry:60 br label %loop61 62loop:63 64 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]65 %acc_phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]66 %ptr1_i = getelementptr i16, ptr %ptr1, i32 %i67 %ptr2_i = getelementptr i16, ptr %ptr2, i32 %i68 %a = load <4 x i16>, ptr %ptr1_i, align 169 %b = load <4 x i16>, ptr %ptr2_i, align 170 %vabd = tail call <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16> %a, <4 x i16> %b)71 %vmov = zext <4 x i16> %vabd to <4 x i32>72 %acc_next = add <4 x i32> %vmov, %acc_phi73 %next_i = add i32 %i, 474 %cmp = icmp slt i32 %next_i, 3275 br i1 %cmp, label %loop, label %exit76 77exit:78 %reduce = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %acc_next)79 ret i32 %reduce80}81 82declare <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16>, <4 x i16>)83declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)84 85define i16 @uabal2_accumulation(ptr %ptr1, ptr %ptr2) {86; CHECK-LABEL: uabal2_accumulation:87; CHECK: uabdl2.8h v4, v1, v388; CHECK: uabdl.8h v1, v1, v389; CHECK: uabdl2.8h v24, v0, v290; CHECK: uabdl2.8h v25, v3, v691; CHECK: uabal2.8h v4, v5, v792; CHECK: uabal2.8h v24, v16, v1893; CHECK: uabal2.8h v25, v17, v1994; CHECK: uabal2.8h v4, v20, v2295; CHECK: uabal2.8h v24, v21, v2396; CHECK: add.8h v4, v25, v497; CHECK: add.8h v4, v4, v2498; CHECK: uabdl.8h v0, v0, v299; CHECK: uabdl.8h v2, v3, v6100; CHECK: uabal.8h v1, v5, v7101; CHECK: uabal.8h v0, v16, v18102; CHECK: uabal.8h v2, v17, v19103; CHECK: uabal.8h v1, v20, v22104; CHECK: uabal.8h v0, v21, v23105; CHECK: add.8h v1, v2, v1106; CHECK: add.8h v0, v1, v0107; CHECK: add.8h v0, v4, v0108; CHECK: addv.8h h0, v0109entry:110 br label %loop111 112loop:113 114 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]115 %acc_phi_hi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next_hi, %loop ]116 %acc_phi_lo = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next_lo, %loop ]117 %ptr1_i = getelementptr i8, ptr %ptr1, i32 %i118 %ptr2_i = getelementptr i8, ptr %ptr2, i32 %i119 %a = load <16 x i8>, ptr %ptr1_i, align 1120 %b = load <16 x i8>, ptr %ptr2_i, align 1121 %a_hi = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>122 %b_hi = shufflevector <16 x i8> %b, <16 x i8> zeroinitializer, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>123 %a_lo = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>124 %b_lo = shufflevector <16 x i8> %b, <16 x i8> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>125 %vabd_hi = tail call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %a_hi, <8 x i8> %b_hi)126 %vabd_lo = tail call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %a_lo, <8 x i8> %b_lo)127 %vmov_hi = zext <8 x i8> %vabd_hi to <8 x i16>128 %vmov_lo = zext <8 x i8> %vabd_lo to <8 x i16>129 %acc_next_hi = add <8 x i16> %vmov_hi, %acc_phi_hi130 %acc_next_lo = add <8 x i16> %vmov_lo, %acc_phi_lo131 %next_i = add i32 %i, 16132 %cmp = icmp slt i32 %next_i, 128133 br i1 %cmp, label %loop, label %exit134 135exit:136 %hi_plus_lo = add <8 x i16> %acc_next_hi, %acc_next_lo137 %reduce = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %hi_plus_lo)138 ret i16 %reduce139}140 141define i32 @uaba_accumulation(ptr %ptr1, ptr %ptr2) {142; CHECK-LABEL: uaba_accumulation:143; CHECK: uabd.4s v0, v0, v2144; CHECK: uabd.4s v1, v1, v3145; CHECK: uabd.4s v2, v2, v5146; CHECK: uaba.4s v0, v4, v6147; CHECK: uaba.4s v1, v7, v17148; CHECK: uaba.4s v2, v16, v18149; CHECK: uaba.4s v0, v19, v21150; CHECK: uaba.4s v1, v20, v22151; CHECK: add.4s v0, v2, v0152; CHECK: add.4s v0, v0, v1153; CHECK: addv.4s s0, v0154entry:155 br label %loop156 157loop:158 159 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]160 %acc_phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]161 %ptr1_i = getelementptr i32, ptr %ptr1, i32 %i162 %ptr2_i = getelementptr i32, ptr %ptr2, i32 %i163 %a = load <4 x i32>, ptr %ptr1_i, align 1164 %b = load <4 x i32>, ptr %ptr2_i, align 1165 %vabd = tail call <4 x i32> @llvm.aarch64.neon.uabd.v4i32(<4 x i32> %a, <4 x i32> %b)166 %acc_next = add <4 x i32> %acc_phi, %vabd167 %next_i = add i32 %i, 4168 %cmp = icmp slt i32 %next_i, 32169 br i1 %cmp, label %loop, label %exit170exit:171 172 %reduce = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %acc_next)173 ret i32 %reduce174}175 176declare <4 x i32> @llvm.aarch64.neon.uabd.v4i32(<4 x i32>, <4 x i32>) nounwind readnone177 178define i32 @saba_accumulation(ptr %ptr1, ptr %ptr2) {179; CHECK-LABEL: saba_accumulation:180; CHECK: sabd.4s v0, v0, v2181; CHECK: sabd.4s v1, v1, v3182; CHECK: sabd.4s v2, v2, v5183; CHECK: saba.4s v0, v4, v6184; CHECK: saba.4s v1, v7, v17185; CHECK: saba.4s v2, v16, v18186; CHECK: saba.4s v0, v19, v21187; CHECK: saba.4s v1, v20, v22188; CHECK: add.4s v0, v2, v0189; CHECK: add.4s v0, v0, v1190; CHECK: addv.4s s0, v0191entry:192 br label %loop193 194loop:195 196 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]197 %acc_phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]198 ; Load values from ptr1 and ptr2199 %ptr1_i = getelementptr i32, ptr %ptr1, i32 %i200 %ptr2_i = getelementptr i32, ptr %ptr2, i32 %i201 %a = load <4 x i32>, ptr %ptr1_i, align 1202 %b = load <4 x i32>, ptr %ptr2_i, align 1203 ; Perform the intrinsic operation204 %vabd = tail call <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32> %a, <4 x i32> %b)205 %acc_next = add <4 x i32> %acc_phi, %vabd206 ; Increment loop counter and check the bound207 %next_i = add i32 %i, 4208 %cmp = icmp slt i32 %next_i, 32209 br i1 %cmp, label %loop, label %exit210 211exit:212 %reduce = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %acc_next)213 ret i32 %reduce214}215 216declare <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32>, <4 x i32>) nounwind readnone217 218define i32 @uaba_v2i32_accumulation(ptr %ptr1, ptr %ptr2) {219; CHECK-LABEL: uaba_v2i32_accumulation:220; CHECK: uabd.2s v0, v0, v2221; CHECK: uabd.2s v1, v1, v3222; CHECK: uabd.2s v2, v2, v5223; CHECK: uaba.2s v0, v4, v6224; CHECK: uaba.2s v1, v7, v17225; CHECK: uaba.2s v2, v16, v18226; CHECK: uaba.2s v0, v19, v21227; CHECK: uaba.2s v1, v20, v22228; CHECK: add.2s v0, v2, v0229; CHECK: add.2s v0, v0, v1230; CHECK: addp.2s v0, v0, v0231entry:232 br label %loop233 234loop:235 236 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]237 %acc_phi = phi <2 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]238 %ptr1_i = getelementptr i32, ptr %ptr1, i32 %i239 %ptr2_i = getelementptr i32, ptr %ptr2, i32 %i240 %a = load <2 x i32>, ptr %ptr1_i, align 1241 %b = load <2 x i32>, ptr %ptr2_i, align 1242 %vabd = tail call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %a, <2 x i32> %b)243 %acc_next = add <2 x i32> %acc_phi, %vabd244 %next_i = add i32 %i, 2245 %cmp = icmp slt i32 %next_i, 16246 br i1 %cmp, label %loop, label %exit247 248exit:249 %reduce = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %acc_next)250 ret i32 %reduce251}252 253define i8 @uaba_v8i8_accumulation(ptr %ptr1, ptr %ptr2) {254; CHECK-LABEL: uaba_v8i8_accumulation:255; CHECK: uabd.8b v0, v0, v2256; CHECK: uabd.8b v1, v1, v3257; CHECK: uabd.8b v2, v2, v5258; CHECK: uaba.8b v0, v4, v6259; CHECK: uaba.8b v1, v7, v17260; CHECK: uaba.8b v2, v16, v18261; CHECK: uaba.8b v0, v19, v21262; CHECK: uaba.8b v1, v20, v22263; CHECK: add.8b v0, v2, v0264; CHECK: add.8b v0, v0, v1265; CHECK: addv.8b b0, v0266entry:267 br label %loop268 269loop:270 271 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]272 %acc_phi = phi <8 x i8> [ zeroinitializer, %entry ], [ %acc_next, %loop ]273 %ptr1_i = getelementptr i8, ptr %ptr1, i32 %i274 %ptr2_i = getelementptr i8, ptr %ptr2, i32 %i275 %a = load <8 x i8>, ptr %ptr1_i, align 1276 %b = load <8 x i8>, ptr %ptr2_i, align 1277 %vabd = tail call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %a, <8 x i8> %b)278 %acc_next = add <8 x i8> %acc_phi, %vabd279 %next_i = add i32 %i, 8280 %cmp = icmp slt i32 %next_i, 64281 br i1 %cmp, label %loop, label %exit282 283exit:284 %reduce = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %acc_next)285 ret i8 %reduce286}287 288define i8 @uaba_v16i8_accumulation(ptr %ptr1, ptr %ptr2) {289; CHECK-LABEL: uaba_v16i8_accumulation:290; CHECK: uabd.16b v0, v0, v2291; CHECK: uabd.16b v1, v1, v3292; CHECK: uabd.16b v2, v2, v5293; CHECK: uaba.16b v0, v4, v6294; CHECK: uaba.16b v1, v7, v17295; CHECK: uaba.16b v2, v16, v18296; CHECK: uaba.16b v0, v19, v21297; CHECK: uaba.16b v1, v20, v22298; CHECK: add.16b v0, v2, v0299; CHECK: add.16b v0, v0, v1300; CHECK: addv.16b b0, v0301entry:302 br label %loop303 304loop:305 306 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]307 %acc_phi = phi <16 x i8> [ zeroinitializer, %entry ], [ %acc_next, %loop ]308 %ptr1_i = getelementptr i8, ptr %ptr1, i32 %i309 %ptr2_i = getelementptr i8, ptr %ptr2, i32 %i310 %a = load <16 x i8>, ptr %ptr1_i, align 1311 %b = load <16 x i8>, ptr %ptr2_i, align 1312 %vabd = tail call <16 x i8> @llvm.aarch64.neon.uabd.v16i8(<16 x i8> %a, <16 x i8> %b)313 %acc_next = add <16 x i8> %acc_phi, %vabd314 %next_i = add i32 %i, 16315 %cmp = icmp slt i32 %next_i, 128316 br i1 %cmp, label %loop, label %exit317 318exit:319 %reduce = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %acc_next)320 ret i8 %reduce321}322 323define i16 @uaba_v8i16_accumulation(ptr %ptr1, ptr %ptr2) {324; CHECK-LABEL: uaba_v8i16_accumulation:325; CHECK: uabd.8h v0, v0, v2326; CHECK: uabd.8h v1, v1, v3327; CHECK: uabd.8h v2, v2, v5328; CHECK: uaba.8h v0, v4, v6329; CHECK: uaba.8h v1, v7, v17330; CHECK: uaba.8h v2, v16, v18331; CHECK: uaba.8h v0, v19, v21332; CHECK: uaba.8h v1, v20, v22333; CHECK: add.8h v0, v2, v0334; CHECK: add.8h v0, v0, v1335; CHECK: addv.8h h0, v0336entry:337 br label %loop338 339loop:340 341 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]342 %acc_phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]343 %ptr1_i = getelementptr i16, ptr %ptr1, i32 %i344 %ptr2_i = getelementptr i16, ptr %ptr2, i32 %i345 %a = load <8 x i16>, ptr %ptr1_i, align 1346 %b = load <8 x i16>, ptr %ptr2_i, align 1347 %vabd = tail call <8 x i16> @llvm.aarch64.neon.uabd.v8i16(<8 x i16> %a, <8 x i16> %b)348 %acc_next = add <8 x i16> %acc_phi, %vabd349 %next_i = add i32 %i, 8350 %cmp = icmp slt i32 %next_i, 64351 br i1 %cmp, label %loop, label %exit352 353exit:354 %reduce = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %acc_next)355 ret i16 %reduce356}357 358define i8 @saba_v8i8_accumulation(ptr %ptr1, ptr %ptr2) {359; CHECK-LABEL: saba_v8i8_accumulation:360; CHECK: sabd.8b v0, v0, v2361; CHECK: sabd.8b v1, v1, v3362; CHECK: sabd.8b v2, v2, v5363; CHECK: saba.8b v0, v4, v6364; CHECK: saba.8b v1, v7, v17365; CHECK: saba.8b v2, v16, v18366; CHECK: saba.8b v0, v19, v21367; CHECK: saba.8b v1, v20, v22368; CHECK: add.8b v0, v2, v0369; CHECK: add.8b v0, v0, v1370; CHECK: addv.8b b0, v0371entry:372 br label %loop373 374loop:375 376 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]377 %acc_phi = phi <8 x i8> [ zeroinitializer, %entry ], [ %acc_next, %loop ]378 %ptr1_i = getelementptr i8, ptr %ptr1, i32 %i379 %ptr2_i = getelementptr i8, ptr %ptr2, i32 %i380 %a = load <8 x i8>, ptr %ptr1_i, align 1381 %b = load <8 x i8>, ptr %ptr2_i, align 1382 %vabd = tail call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %a, <8 x i8> %b)383 %acc_next = add <8 x i8> %acc_phi, %vabd384 %next_i = add i32 %i, 8385 %cmp = icmp slt i32 %next_i, 64386 br i1 %cmp, label %loop, label %exit387 388exit:389 %reduce = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %acc_next)390 ret i8 %reduce391}392 393define i16 @saba_v4i16_accumulation(ptr %ptr1, ptr %ptr2) {394; CHECK-LABEL: saba_v4i16_accumulation:395; CHECK: sabd.4h v0, v0, v2396; CHECK: sabd.4h v1, v1, v3397; CHECK: sabd.4h v2, v2, v5398; CHECK: saba.4h v0, v4, v6399; CHECK: saba.4h v1, v7, v17400; CHECK: saba.4h v2, v16, v18401; CHECK: saba.4h v0, v19, v21402; CHECK: saba.4h v1, v20, v22403; CHECK: add.4h v0, v2, v0404; CHECK: add.4h v0, v0, v1405; CHECK: addv.4h h0, v0406entry:407 br label %loop408loop:409 410 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]411 %acc_phi = phi <4 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]412 %ptr1_i = getelementptr i16, ptr %ptr1, i32 %i413 %ptr2_i = getelementptr i16, ptr %ptr2, i32 %i414 %a = load <4 x i16>, ptr %ptr1_i, align 1415 %b = load <4 x i16>, ptr %ptr2_i, align 1416 %vabd = tail call <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16> %a, <4 x i16> %b)417 %acc_next = add <4 x i16> %acc_phi, %vabd418 %next_i = add i32 %i, 4419 %cmp = icmp slt i32 %next_i, 32420 br i1 %cmp, label %loop, label %exit421exit:422 %reduce = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %acc_next)423 ret i16 %reduce424}425 426define i16 @saba_v8i16_accumulation(ptr %ptr1, ptr %ptr2) {427; CHECK-LABEL: saba_v8i16_accumulation:428; CHECK: sabd.8h v0, v0, v2429; CHECK: sabd.8h v1, v1, v3430; CHECK: sabd.8h v2, v2, v5431; CHECK: saba.8h v0, v4, v6432; CHECK: saba.8h v1, v7, v17433; CHECK: saba.8h v2, v16, v18434; CHECK: saba.8h v0, v19, v21435; CHECK: saba.8h v1, v20, v22436; CHECK: add.8h v0, v2, v0437; CHECK: add.8h v0, v0, v1438; CHECK: addv.8h h0, v0439entry:440 br label %loop441 442loop:443 444 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]445 %acc_phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]446 %ptr1_i = getelementptr i16, ptr %ptr1, i32 %i447 %ptr2_i = getelementptr i16, ptr %ptr2, i32 %i448 %a = load <8 x i16>, ptr %ptr1_i, align 1449 %b = load <8 x i16>, ptr %ptr2_i, align 1450 %vabd = tail call <8 x i16> @llvm.aarch64.neon.sabd.v8i16(<8 x i16> %a, <8 x i16> %b)451 %acc_next = add <8 x i16> %acc_phi, %vabd452 %next_i = add i32 %i, 8453 %cmp = icmp slt i32 %next_i, 64454 br i1 %cmp, label %loop, label %exit455 456exit:457 %reduce = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %acc_next)458 ret i16 %reduce459}460 461define i16 @uabal_i8_to_i16_accumulation(ptr %ptr1, ptr %ptr2) {462; CHECK-LABEL: uabal_i8_to_i16_accumulation:463; CHECK: uabdl.8h v1, v1, v3464; CHECK: uabdl.8h v0, v0, v2465; CHECK: uabdl.8h v2, v3, v5466; CHECK: uabal.8h v1, v4, v6467; CHECK: uabal.8h v0, v7, v17468; CHECK: uabal.8h v2, v16, v18469; CHECK: uabal.8h v1, v19, v21470; CHECK: uabal.8h v0, v20, v22471; CHECK: add.8h v1, v2, v1472; CHECK: add.8h v0, v1, v0473; CHECK: addv.8h h0, v0474entry:475 br label %loop476 477loop:478 479 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]480 %acc_phi = phi <8 x i16> [ zeroinitializer, %entry ], [ %acc_next, %loop ]481 %ptr1_i = getelementptr i8, ptr %ptr1, i32 %i482 %ptr2_i = getelementptr i8, ptr %ptr2, i32 %i483 %a = load <8 x i8>, ptr %ptr1_i, align 1484 %b = load <8 x i8>, ptr %ptr2_i, align 1485 %vabd = tail call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %a, <8 x i8> %b)486 %vmov = zext <8 x i8> %vabd to <8 x i16>487 %acc_next = add <8 x i16> %vmov, %acc_phi488 %next_i = add i32 %i, 8489 %cmp = icmp slt i32 %next_i, 64490 br i1 %cmp, label %loop, label %exit491 492exit:493 %reduce = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %acc_next)494 ret i16 %reduce495}496 497define i32 @uabal_i16_to_i32_accumulation(ptr %ptr1, ptr %ptr2) {498; CHECK-LABEL: uabal_i16_to_i32_accumulation:499; CHECK: uabdl.4s v1, v1, v3500; CHECK: uabdl.4s v0, v0, v2501; CHECK: uabdl.4s v2, v3, v5502; CHECK: uabal.4s v1, v4, v6503; CHECK: uabal.4s v0, v7, v17504; CHECK: uabal.4s v2, v16, v18505; CHECK: uabal.4s v1, v19, v21506; CHECK: uabal.4s v0, v20, v22507; CHECK: add.4s v1, v2, v1508; CHECK: add.4s v0, v1, v0509; CHECK: addv.4s s0, v0510entry:511 br label %loop512 513loop:514 515 %i = phi i32 [ 0, %entry ], [ %next_i, %loop ]516 %acc_phi = phi <4 x i32> [ zeroinitializer, %entry ], [ %acc_next, %loop ]517 %ptr1_i = getelementptr i16, ptr %ptr1, i32 %i518 %ptr2_i = getelementptr i16, ptr %ptr2, i32 %i519 %a = load <4 x i16>, ptr %ptr1_i, align 1520 %b = load <4 x i16>, ptr %ptr2_i, align 1521 %vabd = tail call <4 x i16> @llvm.aarch64.neon.uabd.v4i16(<4 x i16> %a, <4 x i16> %b)522 %vmov = zext <4 x i16> %vabd to <4 x i32>523 %acc_next = add <4 x i32> %vmov, %acc_phi524 %next_i = add i32 %i, 4525 %cmp = icmp slt i32 %next_i, 32526 br i1 %cmp, label %loop, label %exit527 528exit:529 %reduce = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %acc_next)530 ret i32 %reduce531}532