3416 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s3 4; Various reductions generated fro SLP vectorizing unrolled loops. Generated5; from https://godbolt.org/z/ebxdPh1Kz with some less interesting cases removed.6 7define i32 @addv2i32i32(ptr %x) {8; CHECK-LABEL: addv2i32i32:9; CHECK: @ %bb.0: @ %entry10; CHECK-NEXT: ldrd r0, r1, [r0]11; CHECK-NEXT: add r0, r112; CHECK-NEXT: bx lr13entry:14 %0 = load i32, ptr %x, align 415 %arrayidx.1 = getelementptr inbounds i32, ptr %x, i32 116 %1 = load i32, ptr %arrayidx.1, align 417 %add.1 = add nsw i32 %1, %018 ret i32 %add.119}20 21define i32 @addv4i32i32(ptr %x) {22; CHECK-LABEL: addv4i32i32:23; CHECK: @ %bb.0: @ %entry24; CHECK-NEXT: vldrw.u32 q0, [r0]25; CHECK-NEXT: vaddv.u32 r0, q026; CHECK-NEXT: bx lr27entry:28 %0 = load <4 x i32>, ptr %x, align 429 %1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %0)30 ret i32 %131}32 33define i32 @addv8i32i32(ptr %x) {34; CHECK-LABEL: addv8i32i32:35; CHECK: @ %bb.0: @ %entry36; CHECK-NEXT: vldrw.u32 q1, [r0]37; CHECK-NEXT: vldrw.u32 q0, [r0, #16]38; CHECK-NEXT: vaddv.u32 r0, q139; CHECK-NEXT: vaddva.u32 r0, q040; CHECK-NEXT: bx lr41entry:42 %0 = load <8 x i32>, ptr %x, align 443 %1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %0)44 ret i32 %145}46 47define i32 @addv16i32i32(ptr %x) {48; CHECK-LABEL: addv16i32i32:49; CHECK: @ %bb.0: @ %entry50; CHECK-NEXT: vldrw.u32 q1, [r0]51; CHECK-NEXT: vldrw.u32 q0, [r0, #16]52; CHECK-NEXT: vaddv.u32 r2, q153; CHECK-NEXT: vaddva.u32 r2, q054; CHECK-NEXT: vldrw.u32 q0, [r0, #32]55; CHECK-NEXT: vaddva.u32 r2, q056; CHECK-NEXT: vldrw.u32 q0, [r0, #48]57; CHECK-NEXT: vaddva.u32 r2, q058; CHECK-NEXT: mov r0, r259; CHECK-NEXT: bx lr60entry:61 %0 = load <16 x i32>, ptr %x, align 462 %1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %0)63 ret i32 %164}65 66define i32 @addv24i32i32(ptr %x) {67; CHECK-LABEL: addv24i32i32:68; CHECK: @ %bb.0: @ %entry69; CHECK-NEXT: vldrw.u32 q1, [r0]70; CHECK-NEXT: vldrw.u32 q0, [r0, #16]71; CHECK-NEXT: vaddv.u32 r2, q172; CHECK-NEXT: vaddva.u32 r2, q073; CHECK-NEXT: vldrw.u32 q0, [r0, #32]74; CHECK-NEXT: vaddva.u32 r2, q075; CHECK-NEXT: vldrw.u32 q0, [r0, #48]76; CHECK-NEXT: vaddva.u32 r2, q077; CHECK-NEXT: vldrw.u32 q0, [r0, #64]78; CHECK-NEXT: vaddva.u32 r2, q079; CHECK-NEXT: vldrw.u32 q0, [r0, #80]80; CHECK-NEXT: vaddva.u32 r2, q081; CHECK-NEXT: mov r0, r282; CHECK-NEXT: bx lr83entry:84 %0 = load <8 x i32>, ptr %x, align 485 %arrayidx.8 = getelementptr inbounds i32, ptr %x, i32 886 %1 = load <16 x i32>, ptr %arrayidx.8, align 487 %2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)88 %3 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %0)89 %op.rdx = add nsw i32 %2, %390 ret i32 %op.rdx91}92 93define i32 @addv32i32i32(ptr %x) {94; CHECK-LABEL: addv32i32i32:95; CHECK: @ %bb.0: @ %entry96; CHECK-NEXT: vldrw.u32 q1, [r0]97; CHECK-NEXT: vldrw.u32 q0, [r0, #16]98; CHECK-NEXT: mov r1, r099; CHECK-NEXT: vaddv.u32 r0, q1100; CHECK-NEXT: vaddva.u32 r0, q0101; CHECK-NEXT: vldrw.u32 q0, [r1, #32]102; CHECK-NEXT: vaddva.u32 r0, q0103; CHECK-NEXT: vldrw.u32 q0, [r1, #48]104; CHECK-NEXT: vaddva.u32 r0, q0105; CHECK-NEXT: vldrw.u32 q0, [r1, #64]106; CHECK-NEXT: vaddva.u32 r0, q0107; CHECK-NEXT: vldrw.u32 q0, [r1, #80]108; CHECK-NEXT: vaddva.u32 r0, q0109; CHECK-NEXT: vldrw.u32 q0, [r1, #96]110; CHECK-NEXT: vaddva.u32 r0, q0111; CHECK-NEXT: vldrw.u32 q0, [r1, #112]112; CHECK-NEXT: vaddva.u32 r0, q0113; CHECK-NEXT: bx lr114entry:115 %0 = load <32 x i32>, ptr %x, align 4116 %1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %0)117 ret i32 %1118}119 120define i32 @addv64i32i32(ptr %x) {121; CHECK-LABEL: addv64i32i32:122; CHECK: @ %bb.0: @ %entry123; CHECK-NEXT: vldrw.u32 q1, [r0]124; CHECK-NEXT: vldrw.u32 q0, [r0, #16]125; CHECK-NEXT: vaddv.u32 r2, q1126; CHECK-NEXT: vaddva.u32 r2, q0127; CHECK-NEXT: vldrw.u32 q0, [r0, #32]128; CHECK-NEXT: vaddva.u32 r2, q0129; CHECK-NEXT: vldrw.u32 q0, [r0, #48]130; CHECK-NEXT: vaddva.u32 r2, q0131; CHECK-NEXT: vldrw.u32 q0, [r0, #64]132; CHECK-NEXT: vaddva.u32 r2, q0133; CHECK-NEXT: vldrw.u32 q0, [r0, #80]134; CHECK-NEXT: vaddva.u32 r2, q0135; CHECK-NEXT: vldrw.u32 q0, [r0, #96]136; CHECK-NEXT: vaddva.u32 r2, q0137; CHECK-NEXT: vldrw.u32 q0, [r0, #112]138; CHECK-NEXT: vaddva.u32 r2, q0139; CHECK-NEXT: vldrw.u32 q0, [r0, #128]140; CHECK-NEXT: vaddva.u32 r2, q0141; CHECK-NEXT: vldrw.u32 q0, [r0, #144]142; CHECK-NEXT: vaddva.u32 r2, q0143; CHECK-NEXT: vldrw.u32 q0, [r0, #160]144; CHECK-NEXT: vaddva.u32 r2, q0145; CHECK-NEXT: vldrw.u32 q0, [r0, #176]146; CHECK-NEXT: vaddva.u32 r2, q0147; CHECK-NEXT: vldrw.u32 q0, [r0, #192]148; CHECK-NEXT: vaddva.u32 r2, q0149; CHECK-NEXT: vldrw.u32 q0, [r0, #208]150; CHECK-NEXT: vaddva.u32 r2, q0151; CHECK-NEXT: vldrw.u32 q0, [r0, #224]152; CHECK-NEXT: vaddva.u32 r2, q0153; CHECK-NEXT: vldrw.u32 q0, [r0, #240]154; CHECK-NEXT: vaddva.u32 r2, q0155; CHECK-NEXT: mov r0, r2156; CHECK-NEXT: bx lr157entry:158 %0 = load <64 x i32>, ptr %x, align 4159 %1 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %0)160 ret i32 %1161}162 163define i32 @addv128i32i32(ptr %x) {164; CHECK-LABEL: addv128i32i32:165; CHECK: @ %bb.0: @ %entry166; CHECK-NEXT: vldrw.u32 q1, [r0]167; CHECK-NEXT: vldrw.u32 q0, [r0, #16]168; CHECK-NEXT: vaddv.u32 r2, q1169; CHECK-NEXT: vaddva.u32 r2, q0170; CHECK-NEXT: vldrw.u32 q0, [r0, #32]171; CHECK-NEXT: vaddva.u32 r2, q0172; CHECK-NEXT: vldrw.u32 q0, [r0, #48]173; CHECK-NEXT: vaddva.u32 r2, q0174; CHECK-NEXT: vldrw.u32 q0, [r0, #64]175; CHECK-NEXT: vaddva.u32 r2, q0176; CHECK-NEXT: vldrw.u32 q0, [r0, #80]177; CHECK-NEXT: vaddva.u32 r2, q0178; CHECK-NEXT: vldrw.u32 q0, [r0, #96]179; CHECK-NEXT: vaddva.u32 r2, q0180; CHECK-NEXT: vldrw.u32 q0, [r0, #112]181; CHECK-NEXT: vaddva.u32 r2, q0182; CHECK-NEXT: vldrw.u32 q0, [r0, #128]183; CHECK-NEXT: vaddva.u32 r2, q0184; CHECK-NEXT: vldrw.u32 q0, [r0, #144]185; CHECK-NEXT: vaddva.u32 r2, q0186; CHECK-NEXT: vldrw.u32 q0, [r0, #160]187; CHECK-NEXT: vaddva.u32 r2, q0188; CHECK-NEXT: vldrw.u32 q0, [r0, #176]189; CHECK-NEXT: vaddva.u32 r2, q0190; CHECK-NEXT: vldrw.u32 q0, [r0, #192]191; CHECK-NEXT: vaddva.u32 r2, q0192; CHECK-NEXT: vldrw.u32 q0, [r0, #208]193; CHECK-NEXT: vaddva.u32 r2, q0194; CHECK-NEXT: vldrw.u32 q0, [r0, #224]195; CHECK-NEXT: vaddva.u32 r2, q0196; CHECK-NEXT: vldrw.u32 q0, [r0, #240]197; CHECK-NEXT: vaddva.u32 r2, q0198; CHECK-NEXT: vldrw.u32 q0, [r0, #256]199; CHECK-NEXT: vaddva.u32 r2, q0200; CHECK-NEXT: vldrw.u32 q0, [r0, #272]201; CHECK-NEXT: vaddva.u32 r2, q0202; CHECK-NEXT: vldrw.u32 q0, [r0, #288]203; CHECK-NEXT: vaddva.u32 r2, q0204; CHECK-NEXT: vldrw.u32 q0, [r0, #304]205; CHECK-NEXT: vaddva.u32 r2, q0206; CHECK-NEXT: vldrw.u32 q0, [r0, #320]207; CHECK-NEXT: vaddva.u32 r2, q0208; CHECK-NEXT: vldrw.u32 q0, [r0, #336]209; CHECK-NEXT: vaddva.u32 r2, q0210; CHECK-NEXT: vldrw.u32 q0, [r0, #352]211; CHECK-NEXT: vaddva.u32 r2, q0212; CHECK-NEXT: vldrw.u32 q0, [r0, #368]213; CHECK-NEXT: vaddva.u32 r2, q0214; CHECK-NEXT: vldrw.u32 q0, [r0, #384]215; CHECK-NEXT: vaddva.u32 r2, q0216; CHECK-NEXT: vldrw.u32 q0, [r0, #400]217; CHECK-NEXT: vaddva.u32 r2, q0218; CHECK-NEXT: vldrw.u32 q0, [r0, #416]219; CHECK-NEXT: vaddva.u32 r2, q0220; CHECK-NEXT: vldrw.u32 q0, [r0, #432]221; CHECK-NEXT: vaddva.u32 r2, q0222; CHECK-NEXT: vldrw.u32 q0, [r0, #448]223; CHECK-NEXT: vaddva.u32 r2, q0224; CHECK-NEXT: vldrw.u32 q0, [r0, #464]225; CHECK-NEXT: vaddva.u32 r2, q0226; CHECK-NEXT: vldrw.u32 q0, [r0, #480]227; CHECK-NEXT: vaddva.u32 r2, q0228; CHECK-NEXT: vldrw.u32 q0, [r0, #496]229; CHECK-NEXT: vaddva.u32 r2, q0230; CHECK-NEXT: mov r0, r2231; CHECK-NEXT: bx lr232entry:233 %wide.load = load <4 x i32>, ptr %x, align 4234 %0 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load)235 %1 = getelementptr inbounds i32, ptr %x, i32 4236 %wide.load.1 = load <4 x i32>, ptr %1, align 4237 %2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.1)238 %3 = add i32 %2, %0239 %4 = getelementptr inbounds i32, ptr %x, i32 8240 %wide.load.2 = load <4 x i32>, ptr %4, align 4241 %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.2)242 %6 = add i32 %5, %3243 %7 = getelementptr inbounds i32, ptr %x, i32 12244 %wide.load.3 = load <4 x i32>, ptr %7, align 4245 %8 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.3)246 %9 = add i32 %8, %6247 %10 = getelementptr inbounds i32, ptr %x, i32 16248 %wide.load.4 = load <4 x i32>, ptr %10, align 4249 %11 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.4)250 %12 = add i32 %11, %9251 %13 = getelementptr inbounds i32, ptr %x, i32 20252 %wide.load.5 = load <4 x i32>, ptr %13, align 4253 %14 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.5)254 %15 = add i32 %14, %12255 %16 = getelementptr inbounds i32, ptr %x, i32 24256 %wide.load.6 = load <4 x i32>, ptr %16, align 4257 %17 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.6)258 %18 = add i32 %17, %15259 %19 = getelementptr inbounds i32, ptr %x, i32 28260 %wide.load.7 = load <4 x i32>, ptr %19, align 4261 %20 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.7)262 %21 = add i32 %20, %18263 %22 = getelementptr inbounds i32, ptr %x, i32 32264 %wide.load.8 = load <4 x i32>, ptr %22, align 4265 %23 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.8)266 %24 = add i32 %23, %21267 %25 = getelementptr inbounds i32, ptr %x, i32 36268 %wide.load.9 = load <4 x i32>, ptr %25, align 4269 %26 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.9)270 %27 = add i32 %26, %24271 %28 = getelementptr inbounds i32, ptr %x, i32 40272 %wide.load.10 = load <4 x i32>, ptr %28, align 4273 %29 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.10)274 %30 = add i32 %29, %27275 %31 = getelementptr inbounds i32, ptr %x, i32 44276 %wide.load.11 = load <4 x i32>, ptr %31, align 4277 %32 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.11)278 %33 = add i32 %32, %30279 %34 = getelementptr inbounds i32, ptr %x, i32 48280 %wide.load.12 = load <4 x i32>, ptr %34, align 4281 %35 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.12)282 %36 = add i32 %35, %33283 %37 = getelementptr inbounds i32, ptr %x, i32 52284 %wide.load.13 = load <4 x i32>, ptr %37, align 4285 %38 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.13)286 %39 = add i32 %38, %36287 %40 = getelementptr inbounds i32, ptr %x, i32 56288 %wide.load.14 = load <4 x i32>, ptr %40, align 4289 %41 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.14)290 %42 = add i32 %41, %39291 %43 = getelementptr inbounds i32, ptr %x, i32 60292 %wide.load.15 = load <4 x i32>, ptr %43, align 4293 %44 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.15)294 %45 = add i32 %44, %42295 %46 = getelementptr inbounds i32, ptr %x, i32 64296 %wide.load.16 = load <4 x i32>, ptr %46, align 4297 %47 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.16)298 %48 = add i32 %47, %45299 %49 = getelementptr inbounds i32, ptr %x, i32 68300 %wide.load.17 = load <4 x i32>, ptr %49, align 4301 %50 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.17)302 %51 = add i32 %50, %48303 %52 = getelementptr inbounds i32, ptr %x, i32 72304 %wide.load.18 = load <4 x i32>, ptr %52, align 4305 %53 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.18)306 %54 = add i32 %53, %51307 %55 = getelementptr inbounds i32, ptr %x, i32 76308 %wide.load.19 = load <4 x i32>, ptr %55, align 4309 %56 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.19)310 %57 = add i32 %56, %54311 %58 = getelementptr inbounds i32, ptr %x, i32 80312 %wide.load.20 = load <4 x i32>, ptr %58, align 4313 %59 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.20)314 %60 = add i32 %59, %57315 %61 = getelementptr inbounds i32, ptr %x, i32 84316 %wide.load.21 = load <4 x i32>, ptr %61, align 4317 %62 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.21)318 %63 = add i32 %62, %60319 %64 = getelementptr inbounds i32, ptr %x, i32 88320 %wide.load.22 = load <4 x i32>, ptr %64, align 4321 %65 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.22)322 %66 = add i32 %65, %63323 %67 = getelementptr inbounds i32, ptr %x, i32 92324 %wide.load.23 = load <4 x i32>, ptr %67, align 4325 %68 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.23)326 %69 = add i32 %68, %66327 %70 = getelementptr inbounds i32, ptr %x, i32 96328 %wide.load.24 = load <4 x i32>, ptr %70, align 4329 %71 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.24)330 %72 = add i32 %71, %69331 %73 = getelementptr inbounds i32, ptr %x, i32 100332 %wide.load.25 = load <4 x i32>, ptr %73, align 4333 %74 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.25)334 %75 = add i32 %74, %72335 %76 = getelementptr inbounds i32, ptr %x, i32 104336 %wide.load.26 = load <4 x i32>, ptr %76, align 4337 %77 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.26)338 %78 = add i32 %77, %75339 %79 = getelementptr inbounds i32, ptr %x, i32 108340 %wide.load.27 = load <4 x i32>, ptr %79, align 4341 %80 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.27)342 %81 = add i32 %80, %78343 %82 = getelementptr inbounds i32, ptr %x, i32 112344 %wide.load.28 = load <4 x i32>, ptr %82, align 4345 %83 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.28)346 %84 = add i32 %83, %81347 %85 = getelementptr inbounds i32, ptr %x, i32 116348 %wide.load.29 = load <4 x i32>, ptr %85, align 4349 %86 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.29)350 %87 = add i32 %86, %84351 %88 = getelementptr inbounds i32, ptr %x, i32 120352 %wide.load.30 = load <4 x i32>, ptr %88, align 4353 %89 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.30)354 %90 = add i32 %89, %87355 %91 = getelementptr inbounds i32, ptr %x, i32 124356 %wide.load.31 = load <4 x i32>, ptr %91, align 4357 %92 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %wide.load.31)358 %93 = add i32 %92, %90359 ret i32 %93360}361 362define i32 @addv2i32i16(ptr %x) {363; CHECK-LABEL: addv2i32i16:364; CHECK: @ %bb.0: @ %entry365; CHECK-NEXT: ldrsh.w r1, [r0]366; CHECK-NEXT: ldrsh.w r0, [r0, #2]367; CHECK-NEXT: add r0, r1368; CHECK-NEXT: bx lr369entry:370 %0 = load i16, ptr %x, align 2371 %conv = sext i16 %0 to i32372 %arrayidx.1 = getelementptr inbounds i16, ptr %x, i32 1373 %1 = load i16, ptr %arrayidx.1, align 2374 %conv.1 = sext i16 %1 to i32375 %add.1 = add nsw i32 %conv, %conv.1376 ret i32 %add.1377}378 379define i32 @addv4i32i16(ptr %x) {380; CHECK-LABEL: addv4i32i16:381; CHECK: @ %bb.0: @ %entry382; CHECK-NEXT: vldrh.s32 q0, [r0]383; CHECK-NEXT: vaddv.u32 r0, q0384; CHECK-NEXT: bx lr385entry:386 %0 = load <4 x i16>, ptr %x, align 2387 %1 = sext <4 x i16> %0 to <4 x i32>388 %2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)389 ret i32 %2390}391 392define i32 @addv8i32i16(ptr %x) {393; CHECK-LABEL: addv8i32i16:394; CHECK: @ %bb.0: @ %entry395; CHECK-NEXT: vldrh.u16 q0, [r0]396; CHECK-NEXT: vaddv.s16 r0, q0397; CHECK-NEXT: bx lr398entry:399 %0 = load <8 x i16>, ptr %x, align 2400 %1 = sext <8 x i16> %0 to <8 x i32>401 %2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)402 ret i32 %2403}404 405define i32 @addv16i32i16(ptr %x) {406; CHECK-LABEL: addv16i32i16:407; CHECK: @ %bb.0: @ %entry408; CHECK-NEXT: vldrh.s32 q1, [r0]409; CHECK-NEXT: vldrh.s32 q0, [r0, #8]410; CHECK-NEXT: vaddv.u32 r2, q1411; CHECK-NEXT: vaddva.u32 r2, q0412; CHECK-NEXT: vldrh.s32 q0, [r0, #16]413; CHECK-NEXT: vaddva.u32 r2, q0414; CHECK-NEXT: vldrh.s32 q0, [r0, #24]415; CHECK-NEXT: vaddva.u32 r2, q0416; CHECK-NEXT: mov r0, r2417; CHECK-NEXT: bx lr418entry:419 %0 = load <16 x i16>, ptr %x, align 2420 %1 = sext <16 x i16> %0 to <16 x i32>421 %2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)422 ret i32 %2423}424 425define i32 @addv24i32i16(ptr %x) {426; CHECK-LABEL: addv24i32i16:427; CHECK: @ %bb.0: @ %entry428; CHECK-NEXT: vldrh.s32 q1, [r0]429; CHECK-NEXT: vldrh.s32 q0, [r0, #8]430; CHECK-NEXT: vaddv.u32 r2, q1431; CHECK-NEXT: vaddva.u32 r2, q0432; CHECK-NEXT: vldrh.s32 q0, [r0, #16]433; CHECK-NEXT: vaddva.u32 r2, q0434; CHECK-NEXT: vldrh.s32 q0, [r0, #24]435; CHECK-NEXT: vaddva.u32 r2, q0436; CHECK-NEXT: vldrh.u16 q0, [r0, #32]437; CHECK-NEXT: vaddva.s16 r2, q0438; CHECK-NEXT: mov r0, r2439; CHECK-NEXT: bx lr440entry:441 %0 = load <16 x i16>, ptr %x, align 2442 %1 = sext <16 x i16> %0 to <16 x i32>443 %arrayidx.16 = getelementptr inbounds i16, ptr %x, i32 16444 %2 = load <8 x i16>, ptr %arrayidx.16, align 2445 %3 = sext <8 x i16> %2 to <8 x i32>446 %4 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)447 %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %3)448 %op.rdx = add nsw i32 %4, %5449 ret i32 %op.rdx450}451 452define i32 @addv32i32i16(ptr %x) {453; CHECK-LABEL: addv32i32i16:454; CHECK: @ %bb.0: @ %entry455; CHECK-NEXT: vldrh.s32 q1, [r0]456; CHECK-NEXT: vldrh.s32 q0, [r0, #8]457; CHECK-NEXT: vaddv.u32 r2, q1458; CHECK-NEXT: vaddva.u32 r2, q0459; CHECK-NEXT: vldrh.s32 q0, [r0, #16]460; CHECK-NEXT: vaddva.u32 r2, q0461; CHECK-NEXT: vldrh.s32 q0, [r0, #24]462; CHECK-NEXT: vaddva.u32 r2, q0463; CHECK-NEXT: vldrh.s32 q0, [r0, #32]464; CHECK-NEXT: vaddva.u32 r2, q0465; CHECK-NEXT: vldrh.s32 q0, [r0, #40]466; CHECK-NEXT: vaddva.u32 r2, q0467; CHECK-NEXT: vldrh.s32 q0, [r0, #48]468; CHECK-NEXT: vaddva.u32 r2, q0469; CHECK-NEXT: vldrh.s32 q0, [r0, #56]470; CHECK-NEXT: vaddva.u32 r2, q0471; CHECK-NEXT: mov r0, r2472; CHECK-NEXT: bx lr473entry:474 %0 = load <32 x i16>, ptr %x, align 2475 %1 = sext <32 x i16> %0 to <32 x i32>476 %2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %1)477 ret i32 %2478}479 480define i32 @addv64i32i16(ptr %x) {481; CHECK-LABEL: addv64i32i16:482; CHECK: @ %bb.0: @ %entry483; CHECK-NEXT: vldrh.s32 q1, [r0]484; CHECK-NEXT: vldrh.s32 q0, [r0, #8]485; CHECK-NEXT: ldrsh.w r1, [r0, #120]486; CHECK-NEXT: vaddv.u32 r2, q1487; CHECK-NEXT: ldrsh.w r3, [r0, #122]488; CHECK-NEXT: vaddva.u32 r2, q0489; CHECK-NEXT: vldrh.s32 q0, [r0, #16]490; CHECK-NEXT: ldrsh.w r12, [r0, #124]491; CHECK-NEXT: vaddva.u32 r2, q0492; CHECK-NEXT: vldrh.s32 q0, [r0, #24]493; CHECK-NEXT: vaddva.u32 r2, q0494; CHECK-NEXT: vldrh.s32 q0, [r0, #32]495; CHECK-NEXT: vaddva.u32 r2, q0496; CHECK-NEXT: vldrh.s32 q0, [r0, #40]497; CHECK-NEXT: vaddva.u32 r2, q0498; CHECK-NEXT: vldrh.s32 q0, [r0, #48]499; CHECK-NEXT: vaddva.u32 r2, q0500; CHECK-NEXT: vldrh.s32 q0, [r0, #56]501; CHECK-NEXT: vaddva.u32 r2, q0502; CHECK-NEXT: vldrh.s32 q0, [r0, #64]503; CHECK-NEXT: vaddva.u32 r2, q0504; CHECK-NEXT: vldrh.s32 q0, [r0, #72]505; CHECK-NEXT: vaddva.u32 r2, q0506; CHECK-NEXT: vldrh.s32 q0, [r0, #80]507; CHECK-NEXT: vaddva.u32 r2, q0508; CHECK-NEXT: vldrh.s32 q0, [r0, #88]509; CHECK-NEXT: vaddva.u32 r2, q0510; CHECK-NEXT: vldrh.u16 q0, [r0, #96]511; CHECK-NEXT: vaddva.s16 r2, q0512; CHECK-NEXT: vldrh.s32 q0, [r0, #112]513; CHECK-NEXT: ldrsh.w r0, [r0, #126]514; CHECK-NEXT: vaddva.u32 r2, q0515; CHECK-NEXT: add r1, r2516; CHECK-NEXT: add r1, r3517; CHECK-NEXT: add r1, r12518; CHECK-NEXT: add r0, r1519; CHECK-NEXT: bx lr520entry:521 %0 = load <32 x i16>, ptr %x, align 2522 %1 = sext <32 x i16> %0 to <32 x i32>523 %arrayidx.32 = getelementptr inbounds i16, ptr %x, i32 32524 %2 = load <16 x i16>, ptr %arrayidx.32, align 2525 %3 = sext <16 x i16> %2 to <16 x i32>526 %arrayidx.48 = getelementptr inbounds i16, ptr %x, i32 48527 %4 = load <8 x i16>, ptr %arrayidx.48, align 2528 %5 = sext <8 x i16> %4 to <8 x i32>529 %arrayidx.56 = getelementptr inbounds i16, ptr %x, i32 56530 %6 = load <4 x i16>, ptr %arrayidx.56, align 2531 %7 = sext <4 x i16> %6 to <4 x i32>532 %arrayidx.60 = getelementptr inbounds i16, ptr %x, i32 60533 %8 = load i16, ptr %arrayidx.60, align 2534 %conv.60 = sext i16 %8 to i32535 %arrayidx.61 = getelementptr inbounds i16, ptr %x, i32 61536 %9 = load i16, ptr %arrayidx.61, align 2537 %conv.61 = sext i16 %9 to i32538 %arrayidx.62 = getelementptr inbounds i16, ptr %x, i32 62539 %10 = load i16, ptr %arrayidx.62, align 2540 %conv.62 = sext i16 %10 to i32541 %11 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %1)542 %12 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %3)543 %op.rdx = add nsw i32 %11, %12544 %13 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %5)545 %op.rdx8 = add nsw i32 %op.rdx, %13546 %14 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %7)547 %op.rdx9 = add nsw i32 %op.rdx8, %14548 %15 = add nsw i32 %op.rdx9, %conv.60549 %16 = add nsw i32 %15, %conv.61550 %17 = add nsw i32 %16, %conv.62551 %arrayidx.63 = getelementptr inbounds i16, ptr %x, i32 63552 %18 = load i16, ptr %arrayidx.63, align 2553 %conv.63 = sext i16 %18 to i32554 %add.63 = add nsw i32 %17, %conv.63555 ret i32 %add.63556}557 558define i32 @addv128i32i16(ptr %x) {559; CHECK-LABEL: addv128i32i16:560; CHECK: @ %bb.0: @ %entry561; CHECK-NEXT: vldrh.u16 q1, [r0]562; CHECK-NEXT: vldrh.u16 q0, [r0, #16]563; CHECK-NEXT: vaddv.s16 r2, q1564; CHECK-NEXT: vaddva.s16 r2, q0565; CHECK-NEXT: vldrh.u16 q0, [r0, #32]566; CHECK-NEXT: vaddva.s16 r2, q0567; CHECK-NEXT: vldrh.u16 q0, [r0, #48]568; CHECK-NEXT: vaddva.s16 r2, q0569; CHECK-NEXT: vldrh.u16 q0, [r0, #64]570; CHECK-NEXT: vaddva.s16 r2, q0571; CHECK-NEXT: vldrh.u16 q0, [r0, #80]572; CHECK-NEXT: vaddva.s16 r2, q0573; CHECK-NEXT: vldrh.u16 q0, [r0, #96]574; CHECK-NEXT: vaddva.s16 r2, q0575; CHECK-NEXT: vldrh.u16 q0, [r0, #112]576; CHECK-NEXT: vaddva.s16 r2, q0577; CHECK-NEXT: vldrh.u16 q0, [r0, #128]578; CHECK-NEXT: vaddva.s16 r2, q0579; CHECK-NEXT: vldrh.u16 q0, [r0, #144]580; CHECK-NEXT: vaddva.s16 r2, q0581; CHECK-NEXT: vldrh.u16 q0, [r0, #160]582; CHECK-NEXT: vaddva.s16 r2, q0583; CHECK-NEXT: vldrh.u16 q0, [r0, #176]584; CHECK-NEXT: vaddva.s16 r2, q0585; CHECK-NEXT: vldrh.u16 q0, [r0, #192]586; CHECK-NEXT: vaddva.s16 r2, q0587; CHECK-NEXT: vldrh.u16 q0, [r0, #208]588; CHECK-NEXT: vaddva.s16 r2, q0589; CHECK-NEXT: vldrh.u16 q0, [r0, #224]590; CHECK-NEXT: vaddva.s16 r2, q0591; CHECK-NEXT: vldrh.u16 q0, [r0, #240]592; CHECK-NEXT: vaddva.s16 r2, q0593; CHECK-NEXT: mov r0, r2594; CHECK-NEXT: bx lr595entry:596 %wide.load = load <8 x i16>, ptr %x, align 2597 %0 = sext <8 x i16> %wide.load to <8 x i32>598 %1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %0)599 %2 = getelementptr inbounds i16, ptr %x, i32 8600 %wide.load.1 = load <8 x i16>, ptr %2, align 2601 %3 = sext <8 x i16> %wide.load.1 to <8 x i32>602 %4 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %3)603 %5 = add i32 %4, %1604 %6 = getelementptr inbounds i16, ptr %x, i32 16605 %wide.load.2 = load <8 x i16>, ptr %6, align 2606 %7 = sext <8 x i16> %wide.load.2 to <8 x i32>607 %8 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %7)608 %9 = add i32 %8, %5609 %10 = getelementptr inbounds i16, ptr %x, i32 24610 %wide.load.3 = load <8 x i16>, ptr %10, align 2611 %11 = sext <8 x i16> %wide.load.3 to <8 x i32>612 %12 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %11)613 %13 = add i32 %12, %9614 %14 = getelementptr inbounds i16, ptr %x, i32 32615 %wide.load.4 = load <8 x i16>, ptr %14, align 2616 %15 = sext <8 x i16> %wide.load.4 to <8 x i32>617 %16 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %15)618 %17 = add i32 %16, %13619 %18 = getelementptr inbounds i16, ptr %x, i32 40620 %wide.load.5 = load <8 x i16>, ptr %18, align 2621 %19 = sext <8 x i16> %wide.load.5 to <8 x i32>622 %20 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %19)623 %21 = add i32 %20, %17624 %22 = getelementptr inbounds i16, ptr %x, i32 48625 %wide.load.6 = load <8 x i16>, ptr %22, align 2626 %23 = sext <8 x i16> %wide.load.6 to <8 x i32>627 %24 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %23)628 %25 = add i32 %24, %21629 %26 = getelementptr inbounds i16, ptr %x, i32 56630 %wide.load.7 = load <8 x i16>, ptr %26, align 2631 %27 = sext <8 x i16> %wide.load.7 to <8 x i32>632 %28 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %27)633 %29 = add i32 %28, %25634 %30 = getelementptr inbounds i16, ptr %x, i32 64635 %wide.load.8 = load <8 x i16>, ptr %30, align 2636 %31 = sext <8 x i16> %wide.load.8 to <8 x i32>637 %32 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %31)638 %33 = add i32 %32, %29639 %34 = getelementptr inbounds i16, ptr %x, i32 72640 %wide.load.9 = load <8 x i16>, ptr %34, align 2641 %35 = sext <8 x i16> %wide.load.9 to <8 x i32>642 %36 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %35)643 %37 = add i32 %36, %33644 %38 = getelementptr inbounds i16, ptr %x, i32 80645 %wide.load.10 = load <8 x i16>, ptr %38, align 2646 %39 = sext <8 x i16> %wide.load.10 to <8 x i32>647 %40 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %39)648 %41 = add i32 %40, %37649 %42 = getelementptr inbounds i16, ptr %x, i32 88650 %wide.load.11 = load <8 x i16>, ptr %42, align 2651 %43 = sext <8 x i16> %wide.load.11 to <8 x i32>652 %44 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %43)653 %45 = add i32 %44, %41654 %46 = getelementptr inbounds i16, ptr %x, i32 96655 %wide.load.12 = load <8 x i16>, ptr %46, align 2656 %47 = sext <8 x i16> %wide.load.12 to <8 x i32>657 %48 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %47)658 %49 = add i32 %48, %45659 %50 = getelementptr inbounds i16, ptr %x, i32 104660 %wide.load.13 = load <8 x i16>, ptr %50, align 2661 %51 = sext <8 x i16> %wide.load.13 to <8 x i32>662 %52 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %51)663 %53 = add i32 %52, %49664 %54 = getelementptr inbounds i16, ptr %x, i32 112665 %wide.load.14 = load <8 x i16>, ptr %54, align 2666 %55 = sext <8 x i16> %wide.load.14 to <8 x i32>667 %56 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %55)668 %57 = add i32 %56, %53669 %58 = getelementptr inbounds i16, ptr %x, i32 120670 %wide.load.15 = load <8 x i16>, ptr %58, align 2671 %59 = sext <8 x i16> %wide.load.15 to <8 x i32>672 %60 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %59)673 %61 = add i32 %60, %57674 ret i32 %61675}676 677define i32 @addv2i32i8(ptr %x) {678; CHECK-LABEL: addv2i32i8:679; CHECK: @ %bb.0: @ %entry680; CHECK-NEXT: ldrb r1, [r0]681; CHECK-NEXT: ldrb r0, [r0, #1]682; CHECK-NEXT: add r0, r1683; CHECK-NEXT: bx lr684entry:685 %0 = load i8, ptr %x, align 1686 %conv = zext i8 %0 to i32687 %arrayidx.1 = getelementptr inbounds i8, ptr %x, i32 1688 %1 = load i8, ptr %arrayidx.1, align 1689 %conv.1 = zext i8 %1 to i32690 %add.1 = add nuw nsw i32 %conv, %conv.1691 ret i32 %add.1692}693 694define i32 @addv4i32i8(ptr %x) {695; CHECK-LABEL: addv4i32i8:696; CHECK: @ %bb.0: @ %entry697; CHECK-NEXT: vldrb.u32 q0, [r0]698; CHECK-NEXT: vaddv.u32 r0, q0699; CHECK-NEXT: bx lr700entry:701 %0 = load <4 x i8>, ptr %x, align 1702 %1 = zext <4 x i8> %0 to <4 x i32>703 %2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)704 ret i32 %2705}706 707define i32 @addv8i32i8(ptr %x) {708; CHECK-LABEL: addv8i32i8:709; CHECK: @ %bb.0: @ %entry710; CHECK-NEXT: vldrb.u16 q0, [r0]711; CHECK-NEXT: vaddv.u16 r0, q0712; CHECK-NEXT: bx lr713entry:714 %0 = load <8 x i8>, ptr %x, align 1715 %1 = zext <8 x i8> %0 to <8 x i32>716 %2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)717 ret i32 %2718}719 720define i32 @addv16i32i8(ptr %x) {721; CHECK-LABEL: addv16i32i8:722; CHECK: @ %bb.0: @ %entry723; CHECK-NEXT: vldrb.u8 q0, [r0]724; CHECK-NEXT: vaddv.u8 r0, q0725; CHECK-NEXT: bx lr726entry:727 %0 = load <16 x i8>, ptr %x, align 1728 %1 = zext <16 x i8> %0 to <16 x i32>729 %2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)730 ret i32 %2731}732 733define i32 @addv24i32i8(ptr %x) {734; CHECK-LABEL: addv24i32i8:735; CHECK: @ %bb.0: @ %entry736; CHECK-NEXT: vldrb.u8 q1, [r0]737; CHECK-NEXT: vldrb.u16 q0, [r0, #16]738; CHECK-NEXT: vaddv.u8 r0, q1739; CHECK-NEXT: vaddva.u16 r0, q0740; CHECK-NEXT: bx lr741entry:742 %0 = load <16 x i8>, ptr %x, align 1743 %1 = zext <16 x i8> %0 to <16 x i32>744 %arrayidx.16 = getelementptr inbounds i8, ptr %x, i32 16745 %2 = load <8 x i8>, ptr %arrayidx.16, align 1746 %3 = zext <8 x i8> %2 to <8 x i32>747 %4 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)748 %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %3)749 %op.rdx = add nuw nsw i32 %4, %5750 ret i32 %op.rdx751}752 753define i32 @addv32i32i8(ptr %x) {754; CHECK-LABEL: addv32i32i8:755; CHECK: @ %bb.0: @ %entry756; CHECK-NEXT: vldrb.u32 q1, [r0]757; CHECK-NEXT: vldrb.u32 q0, [r0, #4]758; CHECK-NEXT: vaddv.u32 r2, q1759; CHECK-NEXT: vaddva.u32 r2, q0760; CHECK-NEXT: vldrb.u32 q0, [r0, #8]761; CHECK-NEXT: vaddva.u32 r2, q0762; CHECK-NEXT: vldrb.u32 q0, [r0, #12]763; CHECK-NEXT: vaddva.u32 r2, q0764; CHECK-NEXT: vldrb.u32 q0, [r0, #16]765; CHECK-NEXT: vaddva.u32 r2, q0766; CHECK-NEXT: vldrb.u32 q0, [r0, #20]767; CHECK-NEXT: vaddva.u32 r2, q0768; CHECK-NEXT: vldrb.u32 q0, [r0, #24]769; CHECK-NEXT: vaddva.u32 r2, q0770; CHECK-NEXT: vldrb.u32 q0, [r0, #28]771; CHECK-NEXT: vaddva.u32 r2, q0772; CHECK-NEXT: mov r0, r2773; CHECK-NEXT: bx lr774entry:775 %0 = load <32 x i8>, ptr %x, align 1776 %1 = zext <32 x i8> %0 to <32 x i32>777 %2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %1)778 ret i32 %2779}780 781define i32 @addv64i32i8(ptr %x) {782; CHECK-LABEL: addv64i32i8:783; CHECK: @ %bb.0: @ %entry784; CHECK-NEXT: vldrb.u32 q1, [r0]785; CHECK-NEXT: vldrb.u32 q0, [r0, #4]786; CHECK-NEXT: ldrb.w r1, [r0, #60]787; CHECK-NEXT: vaddv.u32 r2, q1788; CHECK-NEXT: ldrb.w r3, [r0, #61]789; CHECK-NEXT: vaddva.u32 r2, q0790; CHECK-NEXT: vldrb.u32 q0, [r0, #8]791; CHECK-NEXT: ldrb.w r12, [r0, #62]792; CHECK-NEXT: vaddva.u32 r2, q0793; CHECK-NEXT: vldrb.u32 q0, [r0, #12]794; CHECK-NEXT: vaddva.u32 r2, q0795; CHECK-NEXT: vldrb.u32 q0, [r0, #16]796; CHECK-NEXT: vaddva.u32 r2, q0797; CHECK-NEXT: vldrb.u32 q0, [r0, #20]798; CHECK-NEXT: vaddva.u32 r2, q0799; CHECK-NEXT: vldrb.u32 q0, [r0, #24]800; CHECK-NEXT: vaddva.u32 r2, q0801; CHECK-NEXT: vldrb.u32 q0, [r0, #28]802; CHECK-NEXT: vaddva.u32 r2, q0803; CHECK-NEXT: vldrb.u8 q0, [r0, #32]804; CHECK-NEXT: vaddva.u8 r2, q0805; CHECK-NEXT: vldrb.u16 q0, [r0, #48]806; CHECK-NEXT: vaddva.u16 r2, q0807; CHECK-NEXT: vldrb.u32 q0, [r0, #56]808; CHECK-NEXT: ldrb.w r0, [r0, #63]809; CHECK-NEXT: vaddva.u32 r2, q0810; CHECK-NEXT: add r1, r2811; CHECK-NEXT: add r1, r3812; CHECK-NEXT: add r1, r12813; CHECK-NEXT: add r0, r1814; CHECK-NEXT: bx lr815entry:816 %0 = load <32 x i8>, ptr %x, align 1817 %1 = zext <32 x i8> %0 to <32 x i32>818 %arrayidx.32 = getelementptr inbounds i8, ptr %x, i32 32819 %2 = load <16 x i8>, ptr %arrayidx.32, align 1820 %3 = zext <16 x i8> %2 to <16 x i32>821 %arrayidx.48 = getelementptr inbounds i8, ptr %x, i32 48822 %4 = load <8 x i8>, ptr %arrayidx.48, align 1823 %5 = zext <8 x i8> %4 to <8 x i32>824 %arrayidx.56 = getelementptr inbounds i8, ptr %x, i32 56825 %6 = load <4 x i8>, ptr %arrayidx.56, align 1826 %7 = zext <4 x i8> %6 to <4 x i32>827 %arrayidx.60 = getelementptr inbounds i8, ptr %x, i32 60828 %8 = load i8, ptr %arrayidx.60, align 1829 %conv.60 = zext i8 %8 to i32830 %arrayidx.61 = getelementptr inbounds i8, ptr %x, i32 61831 %9 = load i8, ptr %arrayidx.61, align 1832 %conv.61 = zext i8 %9 to i32833 %arrayidx.62 = getelementptr inbounds i8, ptr %x, i32 62834 %10 = load i8, ptr %arrayidx.62, align 1835 %conv.62 = zext i8 %10 to i32836 %11 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %1)837 %12 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %3)838 %op.rdx = add nuw nsw i32 %11, %12839 %13 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %5)840 %op.rdx8 = add nuw nsw i32 %op.rdx, %13841 %14 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %7)842 %op.rdx9 = add nuw nsw i32 %op.rdx8, %14843 %15 = add nuw nsw i32 %op.rdx9, %conv.60844 %16 = add nuw nsw i32 %15, %conv.61845 %17 = add nuw nsw i32 %16, %conv.62846 %arrayidx.63 = getelementptr inbounds i8, ptr %x, i32 63847 %18 = load i8, ptr %arrayidx.63, align 1848 %conv.63 = zext i8 %18 to i32849 %add.63 = add nuw nsw i32 %17, %conv.63850 ret i32 %add.63851}852 853define i32 @addv128i32i8(ptr %x) {854; CHECK-LABEL: addv128i32i8:855; CHECK: @ %bb.0: @ %entry856; CHECK-NEXT: vldrb.u8 q1, [r0]857; CHECK-NEXT: vldrb.u8 q0, [r0, #16]858; CHECK-NEXT: mov r1, r0859; CHECK-NEXT: vaddv.u8 r0, q1860; CHECK-NEXT: vaddva.u8 r0, q0861; CHECK-NEXT: vldrb.u8 q0, [r1, #32]862; CHECK-NEXT: vaddva.u8 r0, q0863; CHECK-NEXT: vldrb.u8 q0, [r1, #48]864; CHECK-NEXT: vaddva.u8 r0, q0865; CHECK-NEXT: vldrb.u8 q0, [r1, #64]866; CHECK-NEXT: vaddva.u8 r0, q0867; CHECK-NEXT: vldrb.u8 q0, [r1, #80]868; CHECK-NEXT: vaddva.u8 r0, q0869; CHECK-NEXT: vldrb.u8 q0, [r1, #96]870; CHECK-NEXT: vaddva.u8 r0, q0871; CHECK-NEXT: vldrb.u8 q0, [r1, #112]872; CHECK-NEXT: vaddva.u8 r0, q0873; CHECK-NEXT: bx lr874entry:875 %wide.load = load <16 x i8>, ptr %x, align 1876 %0 = zext <16 x i8> %wide.load to <16 x i32>877 %1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %0)878 %2 = getelementptr inbounds i8, ptr %x, i32 16879 %wide.load.1 = load <16 x i8>, ptr %2, align 1880 %3 = zext <16 x i8> %wide.load.1 to <16 x i32>881 %4 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %3)882 %5 = add i32 %4, %1883 %6 = getelementptr inbounds i8, ptr %x, i32 32884 %wide.load.2 = load <16 x i8>, ptr %6, align 1885 %7 = zext <16 x i8> %wide.load.2 to <16 x i32>886 %8 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %7)887 %9 = add i32 %8, %5888 %10 = getelementptr inbounds i8, ptr %x, i32 48889 %wide.load.3 = load <16 x i8>, ptr %10, align 1890 %11 = zext <16 x i8> %wide.load.3 to <16 x i32>891 %12 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %11)892 %13 = add i32 %12, %9893 %14 = getelementptr inbounds i8, ptr %x, i32 64894 %wide.load.4 = load <16 x i8>, ptr %14, align 1895 %15 = zext <16 x i8> %wide.load.4 to <16 x i32>896 %16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %15)897 %17 = add i32 %16, %13898 %18 = getelementptr inbounds i8, ptr %x, i32 80899 %wide.load.5 = load <16 x i8>, ptr %18, align 1900 %19 = zext <16 x i8> %wide.load.5 to <16 x i32>901 %20 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %19)902 %21 = add i32 %20, %17903 %22 = getelementptr inbounds i8, ptr %x, i32 96904 %wide.load.6 = load <16 x i8>, ptr %22, align 1905 %23 = zext <16 x i8> %wide.load.6 to <16 x i32>906 %24 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %23)907 %25 = add i32 %24, %21908 %26 = getelementptr inbounds i8, ptr %x, i32 112909 %wide.load.7 = load <16 x i8>, ptr %26, align 1910 %27 = zext <16 x i8> %wide.load.7 to <16 x i32>911 %28 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %27)912 %29 = add i32 %28, %25913 ret i32 %29914}915 916define signext i16 @addv2i16i16(ptr %x) {917; CHECK-LABEL: addv2i16i16:918; CHECK: @ %bb.0: @ %entry919; CHECK-NEXT: ldrh r1, [r0]920; CHECK-NEXT: ldrh r0, [r0, #2]921; CHECK-NEXT: add r0, r1922; CHECK-NEXT: sxth r0, r0923; CHECK-NEXT: bx lr924entry:925 %0 = load i16, ptr %x, align 2926 %arrayidx.1 = getelementptr inbounds i16, ptr %x, i32 1927 %1 = load i16, ptr %arrayidx.1, align 2928 %add.1 = add i16 %1, %0929 ret i16 %add.1930}931 932define signext i16 @addv4i16i16(ptr %x) {933; CHECK-LABEL: addv4i16i16:934; CHECK: @ %bb.0: @ %entry935; CHECK-NEXT: vldrh.u32 q0, [r0]936; CHECK-NEXT: vaddv.u32 r0, q0937; CHECK-NEXT: sxth r0, r0938; CHECK-NEXT: bx lr939entry:940 %0 = load <4 x i16>, ptr %x, align 2941 %1 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %0)942 ret i16 %1943}944 945define signext i16 @addv8i16i16(ptr %x) {946; CHECK-LABEL: addv8i16i16:947; CHECK: @ %bb.0: @ %entry948; CHECK-NEXT: vldrh.u16 q0, [r0]949; CHECK-NEXT: vaddv.u16 r0, q0950; CHECK-NEXT: sxth r0, r0951; CHECK-NEXT: bx lr952entry:953 %0 = load <8 x i16>, ptr %x, align 2954 %1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %0)955 ret i16 %1956}957 958define signext i16 @addv16i16i16(ptr %x) {959; CHECK-LABEL: addv16i16i16:960; CHECK: @ %bb.0: @ %entry961; CHECK-NEXT: vldrh.u16 q1, [r0]962; CHECK-NEXT: vldrh.u16 q0, [r0, #16]963; CHECK-NEXT: vaddv.u16 r0, q1964; CHECK-NEXT: vaddva.u16 r0, q0965; CHECK-NEXT: sxth r0, r0966; CHECK-NEXT: bx lr967entry:968 %0 = load <16 x i16>, ptr %x, align 2969 %1 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %0)970 ret i16 %1971}972 973define signext i16 @addv24i16i16(ptr %x) {974; CHECK-LABEL: addv24i16i16:975; CHECK: @ %bb.0: @ %entry976; CHECK-NEXT: vldrh.u16 q1, [r0]977; CHECK-NEXT: vldrh.u16 q0, [r0, #16]978; CHECK-NEXT: vaddv.u16 r2, q1979; CHECK-NEXT: vaddva.u16 r2, q0980; CHECK-NEXT: vldrh.u16 q0, [r0, #32]981; CHECK-NEXT: vaddva.u16 r2, q0982; CHECK-NEXT: sxth r0, r2983; CHECK-NEXT: bx lr984entry:985 %0 = load <8 x i16>, ptr %x, align 2986 %arrayidx.8 = getelementptr inbounds i16, ptr %x, i32 8987 %1 = load <16 x i16>, ptr %arrayidx.8, align 2988 %2 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %1)989 %3 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %0)990 %op.rdx = add i16 %2, %3991 ret i16 %op.rdx992}993 994define signext i16 @addv32i16i16(ptr %x) {995; CHECK-LABEL: addv32i16i16:996; CHECK: @ %bb.0: @ %entry997; CHECK-NEXT: vldrh.u16 q1, [r0]998; CHECK-NEXT: vldrh.u16 q0, [r0, #16]999; CHECK-NEXT: vaddv.u16 r2, q11000; CHECK-NEXT: vaddva.u16 r2, q01001; CHECK-NEXT: vldrh.u16 q0, [r0, #32]1002; CHECK-NEXT: vaddva.u16 r2, q01003; CHECK-NEXT: vldrh.u16 q0, [r0, #48]1004; CHECK-NEXT: vaddva.u16 r2, q01005; CHECK-NEXT: sxth r0, r21006; CHECK-NEXT: bx lr1007entry:1008 %0 = load <32 x i16>, ptr %x, align 21009 %1 = call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> %0)1010 ret i16 %11011}1012 1013define signext i16 @addv64i16i16(ptr %x) {1014; CHECK-LABEL: addv64i16i16:1015; CHECK: @ %bb.0: @ %entry1016; CHECK-NEXT: vldrh.u16 q1, [r0]1017; CHECK-NEXT: vldrh.u16 q0, [r0, #16]1018; CHECK-NEXT: vaddv.u16 r2, q11019; CHECK-NEXT: vaddva.u16 r2, q01020; CHECK-NEXT: vldrh.u16 q0, [r0, #32]1021; CHECK-NEXT: vaddva.u16 r2, q01022; CHECK-NEXT: vldrh.u16 q0, [r0, #48]1023; CHECK-NEXT: vaddva.u16 r2, q01024; CHECK-NEXT: vldrh.u16 q0, [r0, #64]1025; CHECK-NEXT: vaddva.u16 r2, q01026; CHECK-NEXT: vldrh.u16 q0, [r0, #80]1027; CHECK-NEXT: vaddva.u16 r2, q01028; CHECK-NEXT: vldrh.u16 q0, [r0, #96]1029; CHECK-NEXT: vaddva.u16 r2, q01030; CHECK-NEXT: vldrh.u16 q0, [r0, #112]1031; CHECK-NEXT: vaddva.u16 r2, q01032; CHECK-NEXT: sxth r0, r21033; CHECK-NEXT: bx lr1034entry:1035 %0 = load <64 x i16>, ptr %x, align 21036 %1 = call i16 @llvm.vector.reduce.add.v64i16(<64 x i16> %0)1037 ret i16 %11038}1039 1040define signext i16 @addv128i16i16(ptr %x) {1041; CHECK-LABEL: addv128i16i16:1042; CHECK: @ %bb.0: @ %entry1043; CHECK-NEXT: vldrh.u16 q1, [r0]1044; CHECK-NEXT: vldrh.u16 q0, [r0, #16]1045; CHECK-NEXT: vaddv.u16 r2, q11046; CHECK-NEXT: vaddva.u16 r2, q01047; CHECK-NEXT: vldrh.u16 q0, [r0, #32]1048; CHECK-NEXT: vaddva.u16 r2, q01049; CHECK-NEXT: vldrh.u16 q0, [r0, #48]1050; CHECK-NEXT: vaddva.u16 r2, q01051; CHECK-NEXT: vldrh.u16 q0, [r0, #64]1052; CHECK-NEXT: vaddva.u16 r2, q01053; CHECK-NEXT: vldrh.u16 q0, [r0, #80]1054; CHECK-NEXT: vaddva.u16 r2, q01055; CHECK-NEXT: vldrh.u16 q0, [r0, #96]1056; CHECK-NEXT: vaddva.u16 r2, q01057; CHECK-NEXT: vldrh.u16 q0, [r0, #112]1058; CHECK-NEXT: vaddva.u16 r2, q01059; CHECK-NEXT: vldrh.u16 q0, [r0, #128]1060; CHECK-NEXT: vaddva.u16 r2, q01061; CHECK-NEXT: vldrh.u16 q0, [r0, #144]1062; CHECK-NEXT: vaddva.u16 r2, q01063; CHECK-NEXT: vldrh.u16 q0, [r0, #160]1064; CHECK-NEXT: vaddva.u16 r2, q01065; CHECK-NEXT: vldrh.u16 q0, [r0, #176]1066; CHECK-NEXT: vaddva.u16 r2, q01067; CHECK-NEXT: vldrh.u16 q0, [r0, #192]1068; CHECK-NEXT: vaddva.u16 r2, q01069; CHECK-NEXT: vldrh.u16 q0, [r0, #208]1070; CHECK-NEXT: vaddva.u16 r2, q01071; CHECK-NEXT: vldrh.u16 q0, [r0, #224]1072; CHECK-NEXT: vaddva.u16 r2, q01073; CHECK-NEXT: vldrh.u16 q0, [r0, #240]1074; CHECK-NEXT: vaddva.u16 r2, q01075; CHECK-NEXT: sxth r0, r21076; CHECK-NEXT: bx lr1077entry:1078 %wide.load = load <8 x i16>, ptr %x, align 21079 %0 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load)1080 %1 = getelementptr inbounds i16, ptr %x, i32 81081 %wide.load.1 = load <8 x i16>, ptr %1, align 21082 %2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.1)1083 %3 = add i16 %2, %01084 %4 = getelementptr inbounds i16, ptr %x, i32 161085 %wide.load.2 = load <8 x i16>, ptr %4, align 21086 %5 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.2)1087 %6 = add i16 %5, %31088 %7 = getelementptr inbounds i16, ptr %x, i32 241089 %wide.load.3 = load <8 x i16>, ptr %7, align 21090 %8 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.3)1091 %9 = add i16 %8, %61092 %10 = getelementptr inbounds i16, ptr %x, i32 321093 %wide.load.4 = load <8 x i16>, ptr %10, align 21094 %11 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.4)1095 %12 = add i16 %11, %91096 %13 = getelementptr inbounds i16, ptr %x, i32 401097 %wide.load.5 = load <8 x i16>, ptr %13, align 21098 %14 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.5)1099 %15 = add i16 %14, %121100 %16 = getelementptr inbounds i16, ptr %x, i32 481101 %wide.load.6 = load <8 x i16>, ptr %16, align 21102 %17 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.6)1103 %18 = add i16 %17, %151104 %19 = getelementptr inbounds i16, ptr %x, i32 561105 %wide.load.7 = load <8 x i16>, ptr %19, align 21106 %20 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.7)1107 %21 = add i16 %20, %181108 %22 = getelementptr inbounds i16, ptr %x, i32 641109 %wide.load.8 = load <8 x i16>, ptr %22, align 21110 %23 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.8)1111 %24 = add i16 %23, %211112 %25 = getelementptr inbounds i16, ptr %x, i32 721113 %wide.load.9 = load <8 x i16>, ptr %25, align 21114 %26 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.9)1115 %27 = add i16 %26, %241116 %28 = getelementptr inbounds i16, ptr %x, i32 801117 %wide.load.10 = load <8 x i16>, ptr %28, align 21118 %29 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.10)1119 %30 = add i16 %29, %271120 %31 = getelementptr inbounds i16, ptr %x, i32 881121 %wide.load.11 = load <8 x i16>, ptr %31, align 21122 %32 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.11)1123 %33 = add i16 %32, %301124 %34 = getelementptr inbounds i16, ptr %x, i32 961125 %wide.load.12 = load <8 x i16>, ptr %34, align 21126 %35 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.12)1127 %36 = add i16 %35, %331128 %37 = getelementptr inbounds i16, ptr %x, i32 1041129 %wide.load.13 = load <8 x i16>, ptr %37, align 21130 %38 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.13)1131 %39 = add i16 %38, %361132 %40 = getelementptr inbounds i16, ptr %x, i32 1121133 %wide.load.14 = load <8 x i16>, ptr %40, align 21134 %41 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.14)1135 %42 = add i16 %41, %391136 %43 = getelementptr inbounds i16, ptr %x, i32 1201137 %wide.load.15 = load <8 x i16>, ptr %43, align 21138 %44 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %wide.load.15)1139 %45 = add i16 %44, %421140 ret i16 %451141}1142 1143define zeroext i8 @addv2i8i8(ptr %x) {1144; CHECK-LABEL: addv2i8i8:1145; CHECK: @ %bb.0: @ %entry1146; CHECK-NEXT: ldrb r1, [r0]1147; CHECK-NEXT: ldrb r0, [r0, #1]1148; CHECK-NEXT: add r0, r11149; CHECK-NEXT: uxtb r0, r01150; CHECK-NEXT: bx lr1151entry:1152 %0 = load i8, ptr %x, align 11153 %arrayidx.1 = getelementptr inbounds i8, ptr %x, i32 11154 %1 = load i8, ptr %arrayidx.1, align 11155 %add.1 = add i8 %1, %01156 ret i8 %add.11157}1158 1159define zeroext i8 @addv4i8i8(ptr %x) {1160; CHECK-LABEL: addv4i8i8:1161; CHECK: @ %bb.0: @ %entry1162; CHECK-NEXT: vldrb.u32 q0, [r0]1163; CHECK-NEXT: vaddv.u32 r0, q01164; CHECK-NEXT: uxtb r0, r01165; CHECK-NEXT: bx lr1166entry:1167 %0 = load <4 x i8>, ptr %x, align 11168 %1 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> %0)1169 ret i8 %11170}1171 1172define zeroext i8 @addv8i8i8(ptr %x) {1173; CHECK-LABEL: addv8i8i8:1174; CHECK: @ %bb.0: @ %entry1175; CHECK-NEXT: vldrb.u16 q0, [r0]1176; CHECK-NEXT: vaddv.u16 r0, q01177; CHECK-NEXT: uxtb r0, r01178; CHECK-NEXT: bx lr1179entry:1180 %0 = load <8 x i8>, ptr %x, align 11181 %1 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %0)1182 ret i8 %11183}1184 1185define zeroext i8 @addv16i8i8(ptr %x) {1186; CHECK-LABEL: addv16i8i8:1187; CHECK: @ %bb.0: @ %entry1188; CHECK-NEXT: vldrb.u8 q0, [r0]1189; CHECK-NEXT: vaddv.u8 r0, q01190; CHECK-NEXT: uxtb r0, r01191; CHECK-NEXT: bx lr1192entry:1193 %0 = load <16 x i8>, ptr %x, align 11194 %1 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %0)1195 ret i8 %11196}1197 1198define zeroext i8 @addv24i8i8(ptr %x) {1199; CHECK-LABEL: addv24i8i8:1200; CHECK: @ %bb.0: @ %entry1201; CHECK-NEXT: vldrb.u16 q1, [r0]1202; CHECK-NEXT: vldrb.u8 q0, [r0, #8]1203; CHECK-NEXT: vaddv.u16 r0, q11204; CHECK-NEXT: vaddva.u8 r0, q01205; CHECK-NEXT: uxtb r0, r01206; CHECK-NEXT: bx lr1207entry:1208 %0 = load <8 x i8>, ptr %x, align 11209 %arrayidx.8 = getelementptr inbounds i8, ptr %x, i32 81210 %1 = load <16 x i8>, ptr %arrayidx.8, align 11211 %2 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %1)1212 %3 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %0)1213 %op.rdx = add i8 %2, %31214 ret i8 %op.rdx1215}1216 1217define zeroext i8 @addv32i8i8(ptr %x) {1218; CHECK-LABEL: addv32i8i8:1219; CHECK: @ %bb.0: @ %entry1220; CHECK-NEXT: vldrb.u8 q1, [r0]1221; CHECK-NEXT: vldrb.u8 q0, [r0, #16]1222; CHECK-NEXT: vaddv.u8 r0, q11223; CHECK-NEXT: vaddva.u8 r0, q01224; CHECK-NEXT: uxtb r0, r01225; CHECK-NEXT: bx lr1226entry:1227 %0 = load <32 x i8>, ptr %x, align 11228 %1 = call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> %0)1229 ret i8 %11230}1231 1232define zeroext i8 @addv64i8i8(ptr %x) {1233; CHECK-LABEL: addv64i8i8:1234; CHECK: @ %bb.0: @ %entry1235; CHECK-NEXT: vldrb.u8 q1, [r0]1236; CHECK-NEXT: vldrb.u8 q0, [r0, #16]1237; CHECK-NEXT: vaddv.u8 r2, q11238; CHECK-NEXT: vaddva.u8 r2, q01239; CHECK-NEXT: vldrb.u8 q0, [r0, #32]1240; CHECK-NEXT: vaddva.u8 r2, q01241; CHECK-NEXT: vldrb.u8 q0, [r0, #48]1242; CHECK-NEXT: vaddva.u8 r2, q01243; CHECK-NEXT: uxtb r0, r21244; CHECK-NEXT: bx lr1245entry:1246 %0 = load <64 x i8>, ptr %x, align 11247 %1 = call i8 @llvm.vector.reduce.add.v64i8(<64 x i8> %0)1248 ret i8 %11249}1250 1251define zeroext i8 @addv128i8i8(ptr %x) {1252; CHECK-LABEL: addv128i8i8:1253; CHECK: @ %bb.0: @ %entry1254; CHECK-NEXT: vldrb.u8 q1, [r0]1255; CHECK-NEXT: vldrb.u8 q0, [r0, #16]1256; CHECK-NEXT: vaddv.u8 r2, q11257; CHECK-NEXT: vaddva.u8 r2, q01258; CHECK-NEXT: vldrb.u8 q0, [r0, #32]1259; CHECK-NEXT: vaddva.u8 r2, q01260; CHECK-NEXT: vldrb.u8 q0, [r0, #48]1261; CHECK-NEXT: vaddva.u8 r2, q01262; CHECK-NEXT: vldrb.u8 q0, [r0, #64]1263; CHECK-NEXT: vaddva.u8 r2, q01264; CHECK-NEXT: vldrb.u8 q0, [r0, #80]1265; CHECK-NEXT: vaddva.u8 r2, q01266; CHECK-NEXT: vldrb.u8 q0, [r0, #96]1267; CHECK-NEXT: vaddva.u8 r2, q01268; CHECK-NEXT: vldrb.u8 q0, [r0, #112]1269; CHECK-NEXT: vaddva.u8 r2, q01270; CHECK-NEXT: uxtb r0, r21271; CHECK-NEXT: bx lr1272entry:1273 %wide.load = load <16 x i8>, ptr %x, align 11274 %0 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %wide.load)1275 %1 = getelementptr inbounds i8, ptr %x, i32 161276 %wide.load.1 = load <16 x i8>, ptr %1, align 11277 %2 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %wide.load.1)1278 %3 = add i8 %2, %01279 %4 = getelementptr inbounds i8, ptr %x, i32 321280 %wide.load.2 = load <16 x i8>, ptr %4, align 11281 %5 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %wide.load.2)1282 %6 = add i8 %5, %31283 %7 = getelementptr inbounds i8, ptr %x, i32 481284 %wide.load.3 = load <16 x i8>, ptr %7, align 11285 %8 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %wide.load.3)1286 %9 = add i8 %8, %61287 %10 = getelementptr inbounds i8, ptr %x, i32 641288 %wide.load.4 = load <16 x i8>, ptr %10, align 11289 %11 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %wide.load.4)1290 %12 = add i8 %11, %91291 %13 = getelementptr inbounds i8, ptr %x, i32 801292 %wide.load.5 = load <16 x i8>, ptr %13, align 11293 %14 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %wide.load.5)1294 %15 = add i8 %14, %121295 %16 = getelementptr inbounds i8, ptr %x, i32 961296 %wide.load.6 = load <16 x i8>, ptr %16, align 11297 %17 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %wide.load.6)1298 %18 = add i8 %17, %151299 %19 = getelementptr inbounds i8, ptr %x, i32 1121300 %wide.load.7 = load <16 x i8>, ptr %19, align 11301 %20 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %wide.load.7)1302 %21 = add i8 %20, %181303 ret i8 %211304}1305 1306 1307 1308define i32 @mlav2i32i32(ptr %x, ptr %y) {1309; CHECK-LABEL: mlav2i32i32:1310; CHECK: @ %bb.0: @ %entry1311; CHECK-NEXT: ldrd r0, r2, [r0]1312; CHECK-NEXT: ldrd r1, r3, [r1]1313; CHECK-NEXT: muls r0, r1, r01314; CHECK-NEXT: mla r0, r3, r2, r01315; CHECK-NEXT: bx lr1316entry:1317 %0 = load i32, ptr %x, align 41318 %1 = load i32, ptr %y, align 41319 %mul = mul nsw i32 %1, %01320 %arrayidx.1 = getelementptr inbounds i32, ptr %x, i32 11321 %2 = load i32, ptr %arrayidx.1, align 41322 %arrayidx1.1 = getelementptr inbounds i32, ptr %y, i32 11323 %3 = load i32, ptr %arrayidx1.1, align 41324 %mul.1 = mul nsw i32 %3, %21325 %add.1 = add nsw i32 %mul.1, %mul1326 ret i32 %add.11327}1328 1329define i32 @mlav4i32i32(ptr %x, ptr %y) {1330; CHECK-LABEL: mlav4i32i32:1331; CHECK: @ %bb.0: @ %entry1332; CHECK-NEXT: vldrw.u32 q0, [r0]1333; CHECK-NEXT: vldrw.u32 q1, [r1]1334; CHECK-NEXT: vmlav.u32 r0, q1, q01335; CHECK-NEXT: bx lr1336entry:1337 %0 = load <4 x i32>, ptr %x, align 41338 %1 = load <4 x i32>, ptr %y, align 41339 %2 = mul nsw <4 x i32> %1, %01340 %3 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %2)1341 ret i32 %31342}1343 1344define i32 @mlav8i32i32(ptr %x, ptr %y) {1345; CHECK-LABEL: mlav8i32i32:1346; CHECK: @ %bb.0: @ %entry1347; CHECK-NEXT: vldrw.u32 q0, [r0]1348; CHECK-NEXT: vldrw.u32 q1, [r1]1349; CHECK-NEXT: vmlav.u32 r2, q1, q01350; CHECK-NEXT: vldrw.u32 q0, [r0, #16]1351; CHECK-NEXT: vldrw.u32 q1, [r1, #16]1352; CHECK-NEXT: vmlava.u32 r2, q1, q01353; CHECK-NEXT: mov r0, r21354; CHECK-NEXT: bx lr1355entry:1356 %0 = load <8 x i32>, ptr %x, align 41357 %1 = load <8 x i32>, ptr %y, align 41358 %2 = mul nsw <8 x i32> %1, %01359 %3 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %2)1360 ret i32 %31361}1362 1363define i32 @mlav16i32i32(ptr %x, ptr %y) {1364; CHECK-LABEL: mlav16i32i32:1365; CHECK: @ %bb.0: @ %entry1366; CHECK-NEXT: vldrw.u32 q0, [r0]1367; CHECK-NEXT: vldrw.u32 q1, [r1]1368; CHECK-NEXT: vmlav.u32 r2, q1, q01369; CHECK-NEXT: vldrw.u32 q0, [r0, #16]1370; CHECK-NEXT: vldrw.u32 q1, [r1, #16]1371; CHECK-NEXT: vmlava.u32 r2, q1, q01372; CHECK-NEXT: vldrw.u32 q0, [r0, #32]1373; CHECK-NEXT: vldrw.u32 q1, [r1, #32]1374; CHECK-NEXT: vmlava.u32 r2, q1, q01375; CHECK-NEXT: vldrw.u32 q0, [r0, #48]1376; CHECK-NEXT: vldrw.u32 q1, [r1, #48]1377; CHECK-NEXT: vmlava.u32 r2, q1, q01378; CHECK-NEXT: mov r0, r21379; CHECK-NEXT: bx lr1380entry:1381 %0 = load <16 x i32>, ptr %x, align 41382 %1 = load <16 x i32>, ptr %y, align 41383 %2 = mul nsw <16 x i32> %1, %01384 %3 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %2)1385 ret i32 %31386}1387 1388define i32 @mlav24i32i32(ptr %x, ptr %y) {1389; CHECK-LABEL: mlav24i32i32:1390; CHECK: @ %bb.0: @ %entry1391; CHECK-NEXT: vldrw.u32 q0, [r0]1392; CHECK-NEXT: vldrw.u32 q1, [r1]1393; CHECK-NEXT: mov r2, r01394; CHECK-NEXT: vmlav.u32 r0, q1, q01395; CHECK-NEXT: vldrw.u32 q0, [r2, #16]1396; CHECK-NEXT: vldrw.u32 q1, [r1, #16]1397; CHECK-NEXT: vmlava.u32 r0, q1, q01398; CHECK-NEXT: vldrw.u32 q0, [r2, #32]1399; CHECK-NEXT: vldrw.u32 q1, [r1, #32]1400; CHECK-NEXT: vmlava.u32 r0, q1, q01401; CHECK-NEXT: vldrw.u32 q0, [r2, #48]1402; CHECK-NEXT: vldrw.u32 q1, [r1, #48]1403; CHECK-NEXT: vmlava.u32 r0, q1, q01404; CHECK-NEXT: vldrw.u32 q0, [r2, #64]1405; CHECK-NEXT: vldrw.u32 q1, [r1, #64]1406; CHECK-NEXT: vmlava.u32 r0, q1, q01407; CHECK-NEXT: vldrw.u32 q0, [r2, #80]1408; CHECK-NEXT: vldrw.u32 q1, [r1, #80]1409; CHECK-NEXT: vmlava.u32 r0, q1, q01410; CHECK-NEXT: bx lr1411entry:1412 %0 = load <8 x i32>, ptr %x, align 41413 %1 = load <8 x i32>, ptr %y, align 41414 %2 = mul nsw <8 x i32> %1, %01415 %arrayidx.8 = getelementptr inbounds i32, ptr %x, i32 81416 %arrayidx1.8 = getelementptr inbounds i32, ptr %y, i32 81417 %3 = load <16 x i32>, ptr %arrayidx.8, align 41418 %4 = load <16 x i32>, ptr %arrayidx1.8, align 41419 %5 = mul nsw <16 x i32> %4, %31420 %6 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %5)1421 %7 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %2)1422 %op.rdx = add nsw i32 %6, %71423 ret i32 %op.rdx1424}1425 1426define i32 @mlav32i32i32(ptr %x, ptr %y) {1427; CHECK-LABEL: mlav32i32i32:1428; CHECK: @ %bb.0: @ %entry1429; CHECK-NEXT: vldrw.u32 q0, [r0]1430; CHECK-NEXT: vldrw.u32 q1, [r1]1431; CHECK-NEXT: mov r2, r01432; CHECK-NEXT: vmlav.u32 r0, q1, q01433; CHECK-NEXT: vldrw.u32 q0, [r2, #16]1434; CHECK-NEXT: vldrw.u32 q1, [r1, #16]1435; CHECK-NEXT: vmlava.u32 r0, q1, q01436; CHECK-NEXT: vldrw.u32 q0, [r2, #32]1437; CHECK-NEXT: vldrw.u32 q1, [r1, #32]1438; CHECK-NEXT: vmlava.u32 r0, q1, q01439; CHECK-NEXT: vldrw.u32 q0, [r2, #48]1440; CHECK-NEXT: vldrw.u32 q1, [r1, #48]1441; CHECK-NEXT: vmlava.u32 r0, q1, q01442; CHECK-NEXT: vldrw.u32 q0, [r2, #64]1443; CHECK-NEXT: vldrw.u32 q1, [r1, #64]1444; CHECK-NEXT: vmlava.u32 r0, q1, q01445; CHECK-NEXT: vldrw.u32 q0, [r2, #80]1446; CHECK-NEXT: vldrw.u32 q1, [r1, #80]1447; CHECK-NEXT: vmlava.u32 r0, q1, q01448; CHECK-NEXT: vldrw.u32 q0, [r2, #96]1449; CHECK-NEXT: vldrw.u32 q1, [r1, #96]1450; CHECK-NEXT: vmlava.u32 r0, q1, q01451; CHECK-NEXT: vldrw.u32 q0, [r2, #112]1452; CHECK-NEXT: vldrw.u32 q1, [r1, #112]1453; CHECK-NEXT: vmlava.u32 r0, q1, q01454; CHECK-NEXT: bx lr1455entry:1456 %0 = load <32 x i32>, ptr %x, align 41457 %1 = load <32 x i32>, ptr %y, align 41458 %2 = mul nsw <32 x i32> %1, %01459 %3 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %2)1460 ret i32 %31461}1462 1463define i32 @mlav64i32i32(ptr %x, ptr %y) {1464; CHECK-LABEL: mlav64i32i32:1465; CHECK: @ %bb.0: @ %entry1466; CHECK-NEXT: vldrw.u32 q0, [r0]1467; CHECK-NEXT: vldrw.u32 q1, [r1]1468; CHECK-NEXT: mov r2, r01469; CHECK-NEXT: vmlav.u32 r0, q1, q01470; CHECK-NEXT: vldrw.u32 q0, [r2, #16]1471; CHECK-NEXT: vldrw.u32 q1, [r1, #16]1472; CHECK-NEXT: vmlava.u32 r0, q1, q01473; CHECK-NEXT: vldrw.u32 q0, [r2, #32]1474; CHECK-NEXT: vldrw.u32 q1, [r1, #32]1475; CHECK-NEXT: vmlava.u32 r0, q1, q01476; CHECK-NEXT: vldrw.u32 q0, [r2, #48]1477; CHECK-NEXT: vldrw.u32 q1, [r1, #48]1478; CHECK-NEXT: vmlava.u32 r0, q1, q01479; CHECK-NEXT: vldrw.u32 q0, [r2, #64]1480; CHECK-NEXT: vldrw.u32 q1, [r1, #64]1481; CHECK-NEXT: vmlava.u32 r0, q1, q01482; CHECK-NEXT: vldrw.u32 q0, [r2, #80]1483; CHECK-NEXT: vldrw.u32 q1, [r1, #80]1484; CHECK-NEXT: vmlava.u32 r0, q1, q01485; CHECK-NEXT: vldrw.u32 q0, [r2, #96]1486; CHECK-NEXT: vldrw.u32 q1, [r1, #96]1487; CHECK-NEXT: vmlava.u32 r0, q1, q01488; CHECK-NEXT: vldrw.u32 q0, [r2, #112]1489; CHECK-NEXT: vldrw.u32 q1, [r1, #112]1490; CHECK-NEXT: vmlava.u32 r0, q1, q01491; CHECK-NEXT: vldrw.u32 q0, [r2, #128]1492; CHECK-NEXT: vldrw.u32 q1, [r1, #128]1493; CHECK-NEXT: vmlava.u32 r0, q1, q01494; CHECK-NEXT: vldrw.u32 q0, [r2, #144]1495; CHECK-NEXT: vldrw.u32 q1, [r1, #144]1496; CHECK-NEXT: vmlava.u32 r0, q1, q01497; CHECK-NEXT: vldrw.u32 q0, [r2, #160]1498; CHECK-NEXT: vldrw.u32 q1, [r1, #160]1499; CHECK-NEXT: vmlava.u32 r0, q1, q01500; CHECK-NEXT: vldrw.u32 q0, [r2, #176]1501; CHECK-NEXT: vldrw.u32 q1, [r1, #176]1502; CHECK-NEXT: vmlava.u32 r0, q1, q01503; CHECK-NEXT: vldrw.u32 q0, [r2, #192]1504; CHECK-NEXT: vldrw.u32 q1, [r1, #192]1505; CHECK-NEXT: vmlava.u32 r0, q1, q01506; CHECK-NEXT: vldrw.u32 q0, [r2, #208]1507; CHECK-NEXT: vldrw.u32 q1, [r1, #208]1508; CHECK-NEXT: vmlava.u32 r0, q1, q01509; CHECK-NEXT: vldrw.u32 q0, [r2, #224]1510; CHECK-NEXT: vldrw.u32 q1, [r1, #224]1511; CHECK-NEXT: vmlava.u32 r0, q1, q01512; CHECK-NEXT: vldrw.u32 q0, [r2, #240]1513; CHECK-NEXT: vldrw.u32 q1, [r1, #240]1514; CHECK-NEXT: vmlava.u32 r0, q1, q01515; CHECK-NEXT: bx lr1516entry:1517 %wide.load = load <4 x i32>, ptr %x, align 41518 %wide.load10 = load <4 x i32>, ptr %y, align 41519 %0 = mul nsw <4 x i32> %wide.load10, %wide.load1520 %1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %0)1521 %2 = getelementptr inbounds i32, ptr %x, i32 41522 %wide.load.1 = load <4 x i32>, ptr %2, align 41523 %3 = getelementptr inbounds i32, ptr %y, i32 41524 %wide.load10.1 = load <4 x i32>, ptr %3, align 41525 %4 = mul nsw <4 x i32> %wide.load10.1, %wide.load.11526 %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)1527 %6 = add i32 %5, %11528 %7 = getelementptr inbounds i32, ptr %x, i32 81529 %wide.load.2 = load <4 x i32>, ptr %7, align 41530 %8 = getelementptr inbounds i32, ptr %y, i32 81531 %wide.load10.2 = load <4 x i32>, ptr %8, align 41532 %9 = mul nsw <4 x i32> %wide.load10.2, %wide.load.21533 %10 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %9)1534 %11 = add i32 %10, %61535 %12 = getelementptr inbounds i32, ptr %x, i32 121536 %wide.load.3 = load <4 x i32>, ptr %12, align 41537 %13 = getelementptr inbounds i32, ptr %y, i32 121538 %wide.load10.3 = load <4 x i32>, ptr %13, align 41539 %14 = mul nsw <4 x i32> %wide.load10.3, %wide.load.31540 %15 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %14)1541 %16 = add i32 %15, %111542 %17 = getelementptr inbounds i32, ptr %x, i32 161543 %wide.load.4 = load <4 x i32>, ptr %17, align 41544 %18 = getelementptr inbounds i32, ptr %y, i32 161545 %wide.load10.4 = load <4 x i32>, ptr %18, align 41546 %19 = mul nsw <4 x i32> %wide.load10.4, %wide.load.41547 %20 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %19)1548 %21 = add i32 %20, %161549 %22 = getelementptr inbounds i32, ptr %x, i32 201550 %wide.load.5 = load <4 x i32>, ptr %22, align 41551 %23 = getelementptr inbounds i32, ptr %y, i32 201552 %wide.load10.5 = load <4 x i32>, ptr %23, align 41553 %24 = mul nsw <4 x i32> %wide.load10.5, %wide.load.51554 %25 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %24)1555 %26 = add i32 %25, %211556 %27 = getelementptr inbounds i32, ptr %x, i32 241557 %wide.load.6 = load <4 x i32>, ptr %27, align 41558 %28 = getelementptr inbounds i32, ptr %y, i32 241559 %wide.load10.6 = load <4 x i32>, ptr %28, align 41560 %29 = mul nsw <4 x i32> %wide.load10.6, %wide.load.61561 %30 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %29)1562 %31 = add i32 %30, %261563 %32 = getelementptr inbounds i32, ptr %x, i32 281564 %wide.load.7 = load <4 x i32>, ptr %32, align 41565 %33 = getelementptr inbounds i32, ptr %y, i32 281566 %wide.load10.7 = load <4 x i32>, ptr %33, align 41567 %34 = mul nsw <4 x i32> %wide.load10.7, %wide.load.71568 %35 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %34)1569 %36 = add i32 %35, %311570 %37 = getelementptr inbounds i32, ptr %x, i32 321571 %wide.load.8 = load <4 x i32>, ptr %37, align 41572 %38 = getelementptr inbounds i32, ptr %y, i32 321573 %wide.load10.8 = load <4 x i32>, ptr %38, align 41574 %39 = mul nsw <4 x i32> %wide.load10.8, %wide.load.81575 %40 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %39)1576 %41 = add i32 %40, %361577 %42 = getelementptr inbounds i32, ptr %x, i32 361578 %wide.load.9 = load <4 x i32>, ptr %42, align 41579 %43 = getelementptr inbounds i32, ptr %y, i32 361580 %wide.load10.9 = load <4 x i32>, ptr %43, align 41581 %44 = mul nsw <4 x i32> %wide.load10.9, %wide.load.91582 %45 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %44)1583 %46 = add i32 %45, %411584 %47 = getelementptr inbounds i32, ptr %x, i32 401585 %wide.load.10 = load <4 x i32>, ptr %47, align 41586 %48 = getelementptr inbounds i32, ptr %y, i32 401587 %wide.load10.10 = load <4 x i32>, ptr %48, align 41588 %49 = mul nsw <4 x i32> %wide.load10.10, %wide.load.101589 %50 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %49)1590 %51 = add i32 %50, %461591 %52 = getelementptr inbounds i32, ptr %x, i32 441592 %wide.load.11 = load <4 x i32>, ptr %52, align 41593 %53 = getelementptr inbounds i32, ptr %y, i32 441594 %wide.load10.11 = load <4 x i32>, ptr %53, align 41595 %54 = mul nsw <4 x i32> %wide.load10.11, %wide.load.111596 %55 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %54)1597 %56 = add i32 %55, %511598 %57 = getelementptr inbounds i32, ptr %x, i32 481599 %wide.load.12 = load <4 x i32>, ptr %57, align 41600 %58 = getelementptr inbounds i32, ptr %y, i32 481601 %wide.load10.12 = load <4 x i32>, ptr %58, align 41602 %59 = mul nsw <4 x i32> %wide.load10.12, %wide.load.121603 %60 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %59)1604 %61 = add i32 %60, %561605 %62 = getelementptr inbounds i32, ptr %x, i32 521606 %wide.load.13 = load <4 x i32>, ptr %62, align 41607 %63 = getelementptr inbounds i32, ptr %y, i32 521608 %wide.load10.13 = load <4 x i32>, ptr %63, align 41609 %64 = mul nsw <4 x i32> %wide.load10.13, %wide.load.131610 %65 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %64)1611 %66 = add i32 %65, %611612 %67 = getelementptr inbounds i32, ptr %x, i32 561613 %wide.load.14 = load <4 x i32>, ptr %67, align 41614 %68 = getelementptr inbounds i32, ptr %y, i32 561615 %wide.load10.14 = load <4 x i32>, ptr %68, align 41616 %69 = mul nsw <4 x i32> %wide.load10.14, %wide.load.141617 %70 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %69)1618 %71 = add i32 %70, %661619 %72 = getelementptr inbounds i32, ptr %x, i32 601620 %wide.load.15 = load <4 x i32>, ptr %72, align 41621 %73 = getelementptr inbounds i32, ptr %y, i32 601622 %wide.load10.15 = load <4 x i32>, ptr %73, align 41623 %74 = mul nsw <4 x i32> %wide.load10.15, %wide.load.151624 %75 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %74)1625 %76 = add i32 %75, %711626 ret i32 %761627}1628 1629define i32 @mlav128i32i32(ptr %x, ptr %y) {1630; CHECK-LABEL: mlav128i32i32:1631; CHECK: @ %bb.0: @ %entry1632; CHECK-NEXT: vldrw.u32 q0, [r0]1633; CHECK-NEXT: vldrw.u32 q1, [r1]1634; CHECK-NEXT: mov r2, r01635; CHECK-NEXT: vmlav.u32 r0, q1, q01636; CHECK-NEXT: vldrw.u32 q0, [r2, #16]1637; CHECK-NEXT: vldrw.u32 q1, [r1, #16]1638; CHECK-NEXT: vmlava.u32 r0, q1, q01639; CHECK-NEXT: vldrw.u32 q0, [r2, #32]1640; CHECK-NEXT: vldrw.u32 q1, [r1, #32]1641; CHECK-NEXT: vmlava.u32 r0, q1, q01642; CHECK-NEXT: vldrw.u32 q0, [r2, #48]1643; CHECK-NEXT: vldrw.u32 q1, [r1, #48]1644; CHECK-NEXT: vmlava.u32 r0, q1, q01645; CHECK-NEXT: vldrw.u32 q0, [r2, #64]1646; CHECK-NEXT: vldrw.u32 q1, [r1, #64]1647; CHECK-NEXT: vmlava.u32 r0, q1, q01648; CHECK-NEXT: vldrw.u32 q0, [r2, #80]1649; CHECK-NEXT: vldrw.u32 q1, [r1, #80]1650; CHECK-NEXT: vmlava.u32 r0, q1, q01651; CHECK-NEXT: vldrw.u32 q0, [r2, #96]1652; CHECK-NEXT: vldrw.u32 q1, [r1, #96]1653; CHECK-NEXT: vmlava.u32 r0, q1, q01654; CHECK-NEXT: vldrw.u32 q0, [r2, #112]1655; CHECK-NEXT: vldrw.u32 q1, [r1, #112]1656; CHECK-NEXT: vmlava.u32 r0, q1, q01657; CHECK-NEXT: vldrw.u32 q0, [r2, #128]1658; CHECK-NEXT: vldrw.u32 q1, [r1, #128]1659; CHECK-NEXT: vmlava.u32 r0, q1, q01660; CHECK-NEXT: vldrw.u32 q0, [r2, #144]1661; CHECK-NEXT: vldrw.u32 q1, [r1, #144]1662; CHECK-NEXT: vmlava.u32 r0, q1, q01663; CHECK-NEXT: vldrw.u32 q0, [r2, #160]1664; CHECK-NEXT: vldrw.u32 q1, [r1, #160]1665; CHECK-NEXT: vmlava.u32 r0, q1, q01666; CHECK-NEXT: vldrw.u32 q0, [r2, #176]1667; CHECK-NEXT: vldrw.u32 q1, [r1, #176]1668; CHECK-NEXT: vmlava.u32 r0, q1, q01669; CHECK-NEXT: vldrw.u32 q0, [r2, #192]1670; CHECK-NEXT: vldrw.u32 q1, [r1, #192]1671; CHECK-NEXT: vmlava.u32 r0, q1, q01672; CHECK-NEXT: vldrw.u32 q0, [r2, #208]1673; CHECK-NEXT: vldrw.u32 q1, [r1, #208]1674; CHECK-NEXT: vmlava.u32 r0, q1, q01675; CHECK-NEXT: vldrw.u32 q0, [r2, #224]1676; CHECK-NEXT: vldrw.u32 q1, [r1, #224]1677; CHECK-NEXT: vmlava.u32 r0, q1, q01678; CHECK-NEXT: vldrw.u32 q0, [r2, #240]1679; CHECK-NEXT: vldrw.u32 q1, [r1, #240]1680; CHECK-NEXT: vmlava.u32 r0, q1, q01681; CHECK-NEXT: vldrw.u32 q0, [r2, #256]1682; CHECK-NEXT: vldrw.u32 q1, [r1, #256]1683; CHECK-NEXT: vmlava.u32 r0, q1, q01684; CHECK-NEXT: vldrw.u32 q0, [r2, #272]1685; CHECK-NEXT: vldrw.u32 q1, [r1, #272]1686; CHECK-NEXT: vmlava.u32 r0, q1, q01687; CHECK-NEXT: vldrw.u32 q0, [r2, #288]1688; CHECK-NEXT: vldrw.u32 q1, [r1, #288]1689; CHECK-NEXT: vmlava.u32 r0, q1, q01690; CHECK-NEXT: vldrw.u32 q0, [r2, #304]1691; CHECK-NEXT: vldrw.u32 q1, [r1, #304]1692; CHECK-NEXT: vmlava.u32 r0, q1, q01693; CHECK-NEXT: vldrw.u32 q0, [r2, #320]1694; CHECK-NEXT: vldrw.u32 q1, [r1, #320]1695; CHECK-NEXT: vmlava.u32 r0, q1, q01696; CHECK-NEXT: vldrw.u32 q0, [r2, #336]1697; CHECK-NEXT: vldrw.u32 q1, [r1, #336]1698; CHECK-NEXT: vmlava.u32 r0, q1, q01699; CHECK-NEXT: vldrw.u32 q0, [r2, #352]1700; CHECK-NEXT: vldrw.u32 q1, [r1, #352]1701; CHECK-NEXT: vmlava.u32 r0, q1, q01702; CHECK-NEXT: vldrw.u32 q0, [r2, #368]1703; CHECK-NEXT: vldrw.u32 q1, [r1, #368]1704; CHECK-NEXT: vmlava.u32 r0, q1, q01705; CHECK-NEXT: vldrw.u32 q0, [r2, #384]1706; CHECK-NEXT: vldrw.u32 q1, [r1, #384]1707; CHECK-NEXT: vmlava.u32 r0, q1, q01708; CHECK-NEXT: vldrw.u32 q0, [r2, #400]1709; CHECK-NEXT: vldrw.u32 q1, [r1, #400]1710; CHECK-NEXT: vmlava.u32 r0, q1, q01711; CHECK-NEXT: vldrw.u32 q0, [r2, #416]1712; CHECK-NEXT: vldrw.u32 q1, [r1, #416]1713; CHECK-NEXT: vmlava.u32 r0, q1, q01714; CHECK-NEXT: vldrw.u32 q0, [r2, #432]1715; CHECK-NEXT: vldrw.u32 q1, [r1, #432]1716; CHECK-NEXT: vmlava.u32 r0, q1, q01717; CHECK-NEXT: vldrw.u32 q0, [r2, #448]1718; CHECK-NEXT: vldrw.u32 q1, [r1, #448]1719; CHECK-NEXT: vmlava.u32 r0, q1, q01720; CHECK-NEXT: vldrw.u32 q0, [r2, #464]1721; CHECK-NEXT: vldrw.u32 q1, [r1, #464]1722; CHECK-NEXT: vmlava.u32 r0, q1, q01723; CHECK-NEXT: vldrw.u32 q0, [r2, #480]1724; CHECK-NEXT: vldrw.u32 q1, [r1, #480]1725; CHECK-NEXT: vmlava.u32 r0, q1, q01726; CHECK-NEXT: vldrw.u32 q0, [r2, #496]1727; CHECK-NEXT: vldrw.u32 q1, [r1, #496]1728; CHECK-NEXT: vmlava.u32 r0, q1, q01729; CHECK-NEXT: bx lr1730entry:1731 %wide.load = load <4 x i32>, ptr %x, align 41732 %wide.load10 = load <4 x i32>, ptr %y, align 41733 %0 = mul nsw <4 x i32> %wide.load10, %wide.load1734 %1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %0)1735 %2 = getelementptr inbounds i32, ptr %x, i32 41736 %wide.load.1 = load <4 x i32>, ptr %2, align 41737 %3 = getelementptr inbounds i32, ptr %y, i32 41738 %wide.load10.1 = load <4 x i32>, ptr %3, align 41739 %4 = mul nsw <4 x i32> %wide.load10.1, %wide.load.11740 %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)1741 %6 = add i32 %5, %11742 %7 = getelementptr inbounds i32, ptr %x, i32 81743 %wide.load.2 = load <4 x i32>, ptr %7, align 41744 %8 = getelementptr inbounds i32, ptr %y, i32 81745 %wide.load10.2 = load <4 x i32>, ptr %8, align 41746 %9 = mul nsw <4 x i32> %wide.load10.2, %wide.load.21747 %10 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %9)1748 %11 = add i32 %10, %61749 %12 = getelementptr inbounds i32, ptr %x, i32 121750 %wide.load.3 = load <4 x i32>, ptr %12, align 41751 %13 = getelementptr inbounds i32, ptr %y, i32 121752 %wide.load10.3 = load <4 x i32>, ptr %13, align 41753 %14 = mul nsw <4 x i32> %wide.load10.3, %wide.load.31754 %15 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %14)1755 %16 = add i32 %15, %111756 %17 = getelementptr inbounds i32, ptr %x, i32 161757 %wide.load.4 = load <4 x i32>, ptr %17, align 41758 %18 = getelementptr inbounds i32, ptr %y, i32 161759 %wide.load10.4 = load <4 x i32>, ptr %18, align 41760 %19 = mul nsw <4 x i32> %wide.load10.4, %wide.load.41761 %20 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %19)1762 %21 = add i32 %20, %161763 %22 = getelementptr inbounds i32, ptr %x, i32 201764 %wide.load.5 = load <4 x i32>, ptr %22, align 41765 %23 = getelementptr inbounds i32, ptr %y, i32 201766 %wide.load10.5 = load <4 x i32>, ptr %23, align 41767 %24 = mul nsw <4 x i32> %wide.load10.5, %wide.load.51768 %25 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %24)1769 %26 = add i32 %25, %211770 %27 = getelementptr inbounds i32, ptr %x, i32 241771 %wide.load.6 = load <4 x i32>, ptr %27, align 41772 %28 = getelementptr inbounds i32, ptr %y, i32 241773 %wide.load10.6 = load <4 x i32>, ptr %28, align 41774 %29 = mul nsw <4 x i32> %wide.load10.6, %wide.load.61775 %30 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %29)1776 %31 = add i32 %30, %261777 %32 = getelementptr inbounds i32, ptr %x, i32 281778 %wide.load.7 = load <4 x i32>, ptr %32, align 41779 %33 = getelementptr inbounds i32, ptr %y, i32 281780 %wide.load10.7 = load <4 x i32>, ptr %33, align 41781 %34 = mul nsw <4 x i32> %wide.load10.7, %wide.load.71782 %35 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %34)1783 %36 = add i32 %35, %311784 %37 = getelementptr inbounds i32, ptr %x, i32 321785 %wide.load.8 = load <4 x i32>, ptr %37, align 41786 %38 = getelementptr inbounds i32, ptr %y, i32 321787 %wide.load10.8 = load <4 x i32>, ptr %38, align 41788 %39 = mul nsw <4 x i32> %wide.load10.8, %wide.load.81789 %40 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %39)1790 %41 = add i32 %40, %361791 %42 = getelementptr inbounds i32, ptr %x, i32 361792 %wide.load.9 = load <4 x i32>, ptr %42, align 41793 %43 = getelementptr inbounds i32, ptr %y, i32 361794 %wide.load10.9 = load <4 x i32>, ptr %43, align 41795 %44 = mul nsw <4 x i32> %wide.load10.9, %wide.load.91796 %45 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %44)1797 %46 = add i32 %45, %411798 %47 = getelementptr inbounds i32, ptr %x, i32 401799 %wide.load.10 = load <4 x i32>, ptr %47, align 41800 %48 = getelementptr inbounds i32, ptr %y, i32 401801 %wide.load10.10 = load <4 x i32>, ptr %48, align 41802 %49 = mul nsw <4 x i32> %wide.load10.10, %wide.load.101803 %50 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %49)1804 %51 = add i32 %50, %461805 %52 = getelementptr inbounds i32, ptr %x, i32 441806 %wide.load.11 = load <4 x i32>, ptr %52, align 41807 %53 = getelementptr inbounds i32, ptr %y, i32 441808 %wide.load10.11 = load <4 x i32>, ptr %53, align 41809 %54 = mul nsw <4 x i32> %wide.load10.11, %wide.load.111810 %55 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %54)1811 %56 = add i32 %55, %511812 %57 = getelementptr inbounds i32, ptr %x, i32 481813 %wide.load.12 = load <4 x i32>, ptr %57, align 41814 %58 = getelementptr inbounds i32, ptr %y, i32 481815 %wide.load10.12 = load <4 x i32>, ptr %58, align 41816 %59 = mul nsw <4 x i32> %wide.load10.12, %wide.load.121817 %60 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %59)1818 %61 = add i32 %60, %561819 %62 = getelementptr inbounds i32, ptr %x, i32 521820 %wide.load.13 = load <4 x i32>, ptr %62, align 41821 %63 = getelementptr inbounds i32, ptr %y, i32 521822 %wide.load10.13 = load <4 x i32>, ptr %63, align 41823 %64 = mul nsw <4 x i32> %wide.load10.13, %wide.load.131824 %65 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %64)1825 %66 = add i32 %65, %611826 %67 = getelementptr inbounds i32, ptr %x, i32 561827 %wide.load.14 = load <4 x i32>, ptr %67, align 41828 %68 = getelementptr inbounds i32, ptr %y, i32 561829 %wide.load10.14 = load <4 x i32>, ptr %68, align 41830 %69 = mul nsw <4 x i32> %wide.load10.14, %wide.load.141831 %70 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %69)1832 %71 = add i32 %70, %661833 %72 = getelementptr inbounds i32, ptr %x, i32 601834 %wide.load.15 = load <4 x i32>, ptr %72, align 41835 %73 = getelementptr inbounds i32, ptr %y, i32 601836 %wide.load10.15 = load <4 x i32>, ptr %73, align 41837 %74 = mul nsw <4 x i32> %wide.load10.15, %wide.load.151838 %75 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %74)1839 %76 = add i32 %75, %711840 %77 = getelementptr inbounds i32, ptr %x, i32 641841 %wide.load.16 = load <4 x i32>, ptr %77, align 41842 %78 = getelementptr inbounds i32, ptr %y, i32 641843 %wide.load10.16 = load <4 x i32>, ptr %78, align 41844 %79 = mul nsw <4 x i32> %wide.load10.16, %wide.load.161845 %80 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %79)1846 %81 = add i32 %80, %761847 %82 = getelementptr inbounds i32, ptr %x, i32 681848 %wide.load.17 = load <4 x i32>, ptr %82, align 41849 %83 = getelementptr inbounds i32, ptr %y, i32 681850 %wide.load10.17 = load <4 x i32>, ptr %83, align 41851 %84 = mul nsw <4 x i32> %wide.load10.17, %wide.load.171852 %85 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %84)1853 %86 = add i32 %85, %811854 %87 = getelementptr inbounds i32, ptr %x, i32 721855 %wide.load.18 = load <4 x i32>, ptr %87, align 41856 %88 = getelementptr inbounds i32, ptr %y, i32 721857 %wide.load10.18 = load <4 x i32>, ptr %88, align 41858 %89 = mul nsw <4 x i32> %wide.load10.18, %wide.load.181859 %90 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %89)1860 %91 = add i32 %90, %861861 %92 = getelementptr inbounds i32, ptr %x, i32 761862 %wide.load.19 = load <4 x i32>, ptr %92, align 41863 %93 = getelementptr inbounds i32, ptr %y, i32 761864 %wide.load10.19 = load <4 x i32>, ptr %93, align 41865 %94 = mul nsw <4 x i32> %wide.load10.19, %wide.load.191866 %95 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %94)1867 %96 = add i32 %95, %911868 %97 = getelementptr inbounds i32, ptr %x, i32 801869 %wide.load.20 = load <4 x i32>, ptr %97, align 41870 %98 = getelementptr inbounds i32, ptr %y, i32 801871 %wide.load10.20 = load <4 x i32>, ptr %98, align 41872 %99 = mul nsw <4 x i32> %wide.load10.20, %wide.load.201873 %100 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %99)1874 %101 = add i32 %100, %961875 %102 = getelementptr inbounds i32, ptr %x, i32 841876 %wide.load.21 = load <4 x i32>, ptr %102, align 41877 %103 = getelementptr inbounds i32, ptr %y, i32 841878 %wide.load10.21 = load <4 x i32>, ptr %103, align 41879 %104 = mul nsw <4 x i32> %wide.load10.21, %wide.load.211880 %105 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %104)1881 %106 = add i32 %105, %1011882 %107 = getelementptr inbounds i32, ptr %x, i32 881883 %wide.load.22 = load <4 x i32>, ptr %107, align 41884 %108 = getelementptr inbounds i32, ptr %y, i32 881885 %wide.load10.22 = load <4 x i32>, ptr %108, align 41886 %109 = mul nsw <4 x i32> %wide.load10.22, %wide.load.221887 %110 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %109)1888 %111 = add i32 %110, %1061889 %112 = getelementptr inbounds i32, ptr %x, i32 921890 %wide.load.23 = load <4 x i32>, ptr %112, align 41891 %113 = getelementptr inbounds i32, ptr %y, i32 921892 %wide.load10.23 = load <4 x i32>, ptr %113, align 41893 %114 = mul nsw <4 x i32> %wide.load10.23, %wide.load.231894 %115 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %114)1895 %116 = add i32 %115, %1111896 %117 = getelementptr inbounds i32, ptr %x, i32 961897 %wide.load.24 = load <4 x i32>, ptr %117, align 41898 %118 = getelementptr inbounds i32, ptr %y, i32 961899 %wide.load10.24 = load <4 x i32>, ptr %118, align 41900 %119 = mul nsw <4 x i32> %wide.load10.24, %wide.load.241901 %120 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %119)1902 %121 = add i32 %120, %1161903 %122 = getelementptr inbounds i32, ptr %x, i32 1001904 %wide.load.25 = load <4 x i32>, ptr %122, align 41905 %123 = getelementptr inbounds i32, ptr %y, i32 1001906 %wide.load10.25 = load <4 x i32>, ptr %123, align 41907 %124 = mul nsw <4 x i32> %wide.load10.25, %wide.load.251908 %125 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %124)1909 %126 = add i32 %125, %1211910 %127 = getelementptr inbounds i32, ptr %x, i32 1041911 %wide.load.26 = load <4 x i32>, ptr %127, align 41912 %128 = getelementptr inbounds i32, ptr %y, i32 1041913 %wide.load10.26 = load <4 x i32>, ptr %128, align 41914 %129 = mul nsw <4 x i32> %wide.load10.26, %wide.load.261915 %130 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %129)1916 %131 = add i32 %130, %1261917 %132 = getelementptr inbounds i32, ptr %x, i32 1081918 %wide.load.27 = load <4 x i32>, ptr %132, align 41919 %133 = getelementptr inbounds i32, ptr %y, i32 1081920 %wide.load10.27 = load <4 x i32>, ptr %133, align 41921 %134 = mul nsw <4 x i32> %wide.load10.27, %wide.load.271922 %135 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %134)1923 %136 = add i32 %135, %1311924 %137 = getelementptr inbounds i32, ptr %x, i32 1121925 %wide.load.28 = load <4 x i32>, ptr %137, align 41926 %138 = getelementptr inbounds i32, ptr %y, i32 1121927 %wide.load10.28 = load <4 x i32>, ptr %138, align 41928 %139 = mul nsw <4 x i32> %wide.load10.28, %wide.load.281929 %140 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %139)1930 %141 = add i32 %140, %1361931 %142 = getelementptr inbounds i32, ptr %x, i32 1161932 %wide.load.29 = load <4 x i32>, ptr %142, align 41933 %143 = getelementptr inbounds i32, ptr %y, i32 1161934 %wide.load10.29 = load <4 x i32>, ptr %143, align 41935 %144 = mul nsw <4 x i32> %wide.load10.29, %wide.load.291936 %145 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %144)1937 %146 = add i32 %145, %1411938 %147 = getelementptr inbounds i32, ptr %x, i32 1201939 %wide.load.30 = load <4 x i32>, ptr %147, align 41940 %148 = getelementptr inbounds i32, ptr %y, i32 1201941 %wide.load10.30 = load <4 x i32>, ptr %148, align 41942 %149 = mul nsw <4 x i32> %wide.load10.30, %wide.load.301943 %150 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %149)1944 %151 = add i32 %150, %1461945 %152 = getelementptr inbounds i32, ptr %x, i32 1241946 %wide.load.31 = load <4 x i32>, ptr %152, align 41947 %153 = getelementptr inbounds i32, ptr %y, i32 1241948 %wide.load10.31 = load <4 x i32>, ptr %153, align 41949 %154 = mul nsw <4 x i32> %wide.load10.31, %wide.load.311950 %155 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %154)1951 %156 = add i32 %155, %1511952 ret i32 %1561953}1954 1955define i32 @mlav2i32i16(ptr %x, ptr %y) {1956; CHECK-LABEL: mlav2i32i16:1957; CHECK: @ %bb.0: @ %entry1958; CHECK-NEXT: ldrsh.w r2, [r0]1959; CHECK-NEXT: ldrsh.w r3, [r1]1960; CHECK-NEXT: ldrsh.w r0, [r0, #2]1961; CHECK-NEXT: ldrsh.w r1, [r1, #2]1962; CHECK-NEXT: muls r0, r1, r01963; CHECK-NEXT: smlabb r0, r3, r2, r01964; CHECK-NEXT: bx lr1965entry:1966 %0 = load i16, ptr %x, align 21967 %conv = sext i16 %0 to i321968 %1 = load i16, ptr %y, align 21969 %conv2 = sext i16 %1 to i321970 %mul = mul nsw i32 %conv2, %conv1971 %arrayidx.1 = getelementptr inbounds i16, ptr %x, i32 11972 %2 = load i16, ptr %arrayidx.1, align 21973 %conv.1 = sext i16 %2 to i321974 %arrayidx1.1 = getelementptr inbounds i16, ptr %y, i32 11975 %3 = load i16, ptr %arrayidx1.1, align 21976 %conv2.1 = sext i16 %3 to i321977 %mul.1 = mul nsw i32 %conv2.1, %conv.11978 %add.1 = add nsw i32 %mul.1, %mul1979 ret i32 %add.11980}1981 1982define i32 @mlav4i32i16(ptr %x, ptr %y) {1983; CHECK-LABEL: mlav4i32i16:1984; CHECK: @ %bb.0: @ %entry1985; CHECK-NEXT: vldrh.s32 q0, [r0]1986; CHECK-NEXT: vldrh.s32 q1, [r1]1987; CHECK-NEXT: vmlav.u32 r0, q1, q01988; CHECK-NEXT: bx lr1989entry:1990 %0 = load <4 x i16>, ptr %x, align 21991 %1 = sext <4 x i16> %0 to <4 x i32>1992 %2 = load <4 x i16>, ptr %y, align 21993 %3 = sext <4 x i16> %2 to <4 x i32>1994 %4 = mul nsw <4 x i32> %3, %11995 %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)1996 ret i32 %51997}1998 1999define i32 @mlav8i32i16(ptr %x, ptr %y) {2000; CHECK-LABEL: mlav8i32i16:2001; CHECK: @ %bb.0: @ %entry2002; CHECK-NEXT: vldrh.u16 q0, [r0]2003; CHECK-NEXT: vldrh.u16 q1, [r1]2004; CHECK-NEXT: vmlav.s16 r0, q1, q02005; CHECK-NEXT: bx lr2006entry:2007 %0 = load <8 x i16>, ptr %x, align 22008 %1 = sext <8 x i16> %0 to <8 x i32>2009 %2 = load <8 x i16>, ptr %y, align 22010 %3 = sext <8 x i16> %2 to <8 x i32>2011 %4 = mul nsw <8 x i32> %3, %12012 %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)2013 ret i32 %52014}2015 2016define i32 @mlav16i32i16(ptr %x, ptr %y) {2017; CHECK-LABEL: mlav16i32i16:2018; CHECK: @ %bb.0: @ %entry2019; CHECK-NEXT: vldrh.s32 q0, [r0]2020; CHECK-NEXT: vldrh.s32 q1, [r1]2021; CHECK-NEXT: vmlav.u32 r2, q1, q02022; CHECK-NEXT: vldrh.s32 q0, [r0, #8]2023; CHECK-NEXT: vldrh.s32 q1, [r1, #8]2024; CHECK-NEXT: vmlava.u32 r2, q1, q02025; CHECK-NEXT: vldrh.s32 q0, [r0, #16]2026; CHECK-NEXT: vldrh.s32 q1, [r1, #16]2027; CHECK-NEXT: vmlava.u32 r2, q1, q02028; CHECK-NEXT: vldrh.s32 q0, [r0, #24]2029; CHECK-NEXT: vldrh.s32 q1, [r1, #24]2030; CHECK-NEXT: vmlava.u32 r2, q1, q02031; CHECK-NEXT: mov r0, r22032; CHECK-NEXT: bx lr2033entry:2034 %0 = load <16 x i16>, ptr %x, align 22035 %1 = sext <16 x i16> %0 to <16 x i32>2036 %2 = load <16 x i16>, ptr %y, align 22037 %3 = sext <16 x i16> %2 to <16 x i32>2038 %4 = mul nsw <16 x i32> %3, %12039 %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)2040 ret i32 %52041}2042 2043define i32 @mlav24i32i16(ptr %x, ptr %y) {2044; CHECK-LABEL: mlav24i32i16:2045; CHECK: @ %bb.0: @ %entry2046; CHECK-NEXT: vldrh.u16 q0, [r0]2047; CHECK-NEXT: vldrh.u16 q1, [r1]2048; CHECK-NEXT: mov r2, r02049; CHECK-NEXT: vmlav.s16 r0, q1, q02050; CHECK-NEXT: vldrh.s32 q0, [r2, #16]2051; CHECK-NEXT: vldrh.s32 q1, [r1, #16]2052; CHECK-NEXT: vmlava.u32 r0, q1, q02053; CHECK-NEXT: vldrh.s32 q0, [r2, #24]2054; CHECK-NEXT: vldrh.s32 q1, [r1, #24]2055; CHECK-NEXT: vmlava.u32 r0, q1, q02056; CHECK-NEXT: vldrh.s32 q0, [r2, #32]2057; CHECK-NEXT: vldrh.s32 q1, [r1, #32]2058; CHECK-NEXT: vmlava.u32 r0, q1, q02059; CHECK-NEXT: vldrh.s32 q0, [r2, #40]2060; CHECK-NEXT: vldrh.s32 q1, [r1, #40]2061; CHECK-NEXT: vmlava.u32 r0, q1, q02062; CHECK-NEXT: bx lr2063entry:2064 %0 = load <8 x i16>, ptr %x, align 22065 %1 = sext <8 x i16> %0 to <8 x i32>2066 %2 = load <8 x i16>, ptr %y, align 22067 %3 = sext <8 x i16> %2 to <8 x i32>2068 %4 = mul nsw <8 x i32> %3, %12069 %arrayidx.8 = getelementptr inbounds i16, ptr %x, i32 82070 %arrayidx1.8 = getelementptr inbounds i16, ptr %y, i32 82071 %5 = load <16 x i16>, ptr %arrayidx.8, align 22072 %6 = sext <16 x i16> %5 to <16 x i32>2073 %7 = load <16 x i16>, ptr %arrayidx1.8, align 22074 %8 = sext <16 x i16> %7 to <16 x i32>2075 %9 = mul nsw <16 x i32> %8, %62076 %10 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %9)2077 %11 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)2078 %op.rdx = add nsw i32 %10, %112079 ret i32 %op.rdx2080}2081 2082define i32 @mlav32i32i16(ptr %x, ptr %y) {2083; CHECK-LABEL: mlav32i32i16:2084; CHECK: @ %bb.0: @ %entry2085; CHECK-NEXT: vldrh.s32 q0, [r0]2086; CHECK-NEXT: vldrh.s32 q1, [r1]2087; CHECK-NEXT: mov r2, r02088; CHECK-NEXT: vmlav.u32 r0, q1, q02089; CHECK-NEXT: vldrh.s32 q0, [r2, #8]2090; CHECK-NEXT: vldrh.s32 q1, [r1, #8]2091; CHECK-NEXT: vmlava.u32 r0, q1, q02092; CHECK-NEXT: vldrh.s32 q0, [r2, #16]2093; CHECK-NEXT: vldrh.s32 q1, [r1, #16]2094; CHECK-NEXT: vmlava.u32 r0, q1, q02095; CHECK-NEXT: vldrh.s32 q0, [r2, #24]2096; CHECK-NEXT: vldrh.s32 q1, [r1, #24]2097; CHECK-NEXT: vmlava.u32 r0, q1, q02098; CHECK-NEXT: vldrh.s32 q0, [r2, #32]2099; CHECK-NEXT: vldrh.s32 q1, [r1, #32]2100; CHECK-NEXT: vmlava.u32 r0, q1, q02101; CHECK-NEXT: vldrh.s32 q0, [r2, #40]2102; CHECK-NEXT: vldrh.s32 q1, [r1, #40]2103; CHECK-NEXT: vmlava.u32 r0, q1, q02104; CHECK-NEXT: vldrh.s32 q0, [r2, #48]2105; CHECK-NEXT: vldrh.s32 q1, [r1, #48]2106; CHECK-NEXT: vmlava.u32 r0, q1, q02107; CHECK-NEXT: vldrh.s32 q0, [r2, #56]2108; CHECK-NEXT: vldrh.s32 q1, [r1, #56]2109; CHECK-NEXT: vmlava.u32 r0, q1, q02110; CHECK-NEXT: bx lr2111entry:2112 %0 = load <32 x i16>, ptr %x, align 22113 %1 = sext <32 x i16> %0 to <32 x i32>2114 %2 = load <32 x i16>, ptr %y, align 22115 %3 = sext <32 x i16> %2 to <32 x i32>2116 %4 = mul nsw <32 x i32> %3, %12117 %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)2118 ret i32 %52119}2120 2121define i32 @mlav64i32i16(ptr %x, ptr %y) {2122; CHECK-LABEL: mlav64i32i16:2123; CHECK: @ %bb.0: @ %entry2124; CHECK-NEXT: vldrh.u16 q0, [r0]2125; CHECK-NEXT: vldrh.u16 q1, [r1]2126; CHECK-NEXT: mov r2, r02127; CHECK-NEXT: vmlav.s16 r0, q1, q02128; CHECK-NEXT: vldrh.u16 q0, [r2, #16]2129; CHECK-NEXT: vldrh.u16 q1, [r1, #16]2130; CHECK-NEXT: vmlava.s16 r0, q1, q02131; CHECK-NEXT: vldrh.u16 q0, [r2, #32]2132; CHECK-NEXT: vldrh.u16 q1, [r1, #32]2133; CHECK-NEXT: vmlava.s16 r0, q1, q02134; CHECK-NEXT: vldrh.u16 q0, [r2, #48]2135; CHECK-NEXT: vldrh.u16 q1, [r1, #48]2136; CHECK-NEXT: vmlava.s16 r0, q1, q02137; CHECK-NEXT: vldrh.u16 q0, [r2, #64]2138; CHECK-NEXT: vldrh.u16 q1, [r1, #64]2139; CHECK-NEXT: vmlava.s16 r0, q1, q02140; CHECK-NEXT: vldrh.u16 q0, [r2, #80]2141; CHECK-NEXT: vldrh.u16 q1, [r1, #80]2142; CHECK-NEXT: vmlava.s16 r0, q1, q02143; CHECK-NEXT: vldrh.u16 q0, [r2, #96]2144; CHECK-NEXT: vldrh.u16 q1, [r1, #96]2145; CHECK-NEXT: vmlava.s16 r0, q1, q02146; CHECK-NEXT: vldrh.u16 q0, [r2, #112]2147; CHECK-NEXT: vldrh.u16 q1, [r1, #112]2148; CHECK-NEXT: vmlava.s16 r0, q1, q02149; CHECK-NEXT: bx lr2150entry:2151 %wide.load = load <8 x i16>, ptr %x, align 22152 %0 = sext <8 x i16> %wide.load to <8 x i32>2153 %wide.load11 = load <8 x i16>, ptr %y, align 22154 %1 = sext <8 x i16> %wide.load11 to <8 x i32>2155 %2 = mul nsw <8 x i32> %1, %02156 %3 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %2)2157 %4 = getelementptr inbounds i16, ptr %x, i32 82158 %wide.load.1 = load <8 x i16>, ptr %4, align 22159 %5 = sext <8 x i16> %wide.load.1 to <8 x i32>2160 %6 = getelementptr inbounds i16, ptr %y, i32 82161 %wide.load11.1 = load <8 x i16>, ptr %6, align 22162 %7 = sext <8 x i16> %wide.load11.1 to <8 x i32>2163 %8 = mul nsw <8 x i32> %7, %52164 %9 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %8)2165 %10 = add i32 %9, %32166 %11 = getelementptr inbounds i16, ptr %x, i32 162167 %wide.load.2 = load <8 x i16>, ptr %11, align 22168 %12 = sext <8 x i16> %wide.load.2 to <8 x i32>2169 %13 = getelementptr inbounds i16, ptr %y, i32 162170 %wide.load11.2 = load <8 x i16>, ptr %13, align 22171 %14 = sext <8 x i16> %wide.load11.2 to <8 x i32>2172 %15 = mul nsw <8 x i32> %14, %122173 %16 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %15)2174 %17 = add i32 %16, %102175 %18 = getelementptr inbounds i16, ptr %x, i32 242176 %wide.load.3 = load <8 x i16>, ptr %18, align 22177 %19 = sext <8 x i16> %wide.load.3 to <8 x i32>2178 %20 = getelementptr inbounds i16, ptr %y, i32 242179 %wide.load11.3 = load <8 x i16>, ptr %20, align 22180 %21 = sext <8 x i16> %wide.load11.3 to <8 x i32>2181 %22 = mul nsw <8 x i32> %21, %192182 %23 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %22)2183 %24 = add i32 %23, %172184 %25 = getelementptr inbounds i16, ptr %x, i32 322185 %wide.load.4 = load <8 x i16>, ptr %25, align 22186 %26 = sext <8 x i16> %wide.load.4 to <8 x i32>2187 %27 = getelementptr inbounds i16, ptr %y, i32 322188 %wide.load11.4 = load <8 x i16>, ptr %27, align 22189 %28 = sext <8 x i16> %wide.load11.4 to <8 x i32>2190 %29 = mul nsw <8 x i32> %28, %262191 %30 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %29)2192 %31 = add i32 %30, %242193 %32 = getelementptr inbounds i16, ptr %x, i32 402194 %wide.load.5 = load <8 x i16>, ptr %32, align 22195 %33 = sext <8 x i16> %wide.load.5 to <8 x i32>2196 %34 = getelementptr inbounds i16, ptr %y, i32 402197 %wide.load11.5 = load <8 x i16>, ptr %34, align 22198 %35 = sext <8 x i16> %wide.load11.5 to <8 x i32>2199 %36 = mul nsw <8 x i32> %35, %332200 %37 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %36)2201 %38 = add i32 %37, %312202 %39 = getelementptr inbounds i16, ptr %x, i32 482203 %wide.load.6 = load <8 x i16>, ptr %39, align 22204 %40 = sext <8 x i16> %wide.load.6 to <8 x i32>2205 %41 = getelementptr inbounds i16, ptr %y, i32 482206 %wide.load11.6 = load <8 x i16>, ptr %41, align 22207 %42 = sext <8 x i16> %wide.load11.6 to <8 x i32>2208 %43 = mul nsw <8 x i32> %42, %402209 %44 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %43)2210 %45 = add i32 %44, %382211 %46 = getelementptr inbounds i16, ptr %x, i32 562212 %wide.load.7 = load <8 x i16>, ptr %46, align 22213 %47 = sext <8 x i16> %wide.load.7 to <8 x i32>2214 %48 = getelementptr inbounds i16, ptr %y, i32 562215 %wide.load11.7 = load <8 x i16>, ptr %48, align 22216 %49 = sext <8 x i16> %wide.load11.7 to <8 x i32>2217 %50 = mul nsw <8 x i32> %49, %472218 %51 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %50)2219 %52 = add i32 %51, %452220 ret i32 %522221}2222 2223define i32 @mlav128i32i16(ptr %x, ptr %y) {2224; CHECK-LABEL: mlav128i32i16:2225; CHECK: @ %bb.0: @ %entry2226; CHECK-NEXT: vldrh.u16 q0, [r0]2227; CHECK-NEXT: vldrh.u16 q1, [r1]2228; CHECK-NEXT: mov r2, r02229; CHECK-NEXT: vmlav.s16 r0, q1, q02230; CHECK-NEXT: vldrh.u16 q0, [r2, #16]2231; CHECK-NEXT: vldrh.u16 q1, [r1, #16]2232; CHECK-NEXT: vmlava.s16 r0, q1, q02233; CHECK-NEXT: vldrh.u16 q0, [r2, #32]2234; CHECK-NEXT: vldrh.u16 q1, [r1, #32]2235; CHECK-NEXT: vmlava.s16 r0, q1, q02236; CHECK-NEXT: vldrh.u16 q0, [r2, #48]2237; CHECK-NEXT: vldrh.u16 q1, [r1, #48]2238; CHECK-NEXT: vmlava.s16 r0, q1, q02239; CHECK-NEXT: vldrh.u16 q0, [r2, #64]2240; CHECK-NEXT: vldrh.u16 q1, [r1, #64]2241; CHECK-NEXT: vmlava.s16 r0, q1, q02242; CHECK-NEXT: vldrh.u16 q0, [r2, #80]2243; CHECK-NEXT: vldrh.u16 q1, [r1, #80]2244; CHECK-NEXT: vmlava.s16 r0, q1, q02245; CHECK-NEXT: vldrh.u16 q0, [r2, #96]2246; CHECK-NEXT: vldrh.u16 q1, [r1, #96]2247; CHECK-NEXT: vmlava.s16 r0, q1, q02248; CHECK-NEXT: vldrh.u16 q0, [r2, #112]2249; CHECK-NEXT: vldrh.u16 q1, [r1, #112]2250; CHECK-NEXT: vmlava.s16 r0, q1, q02251; CHECK-NEXT: vldrh.u16 q0, [r2, #128]2252; CHECK-NEXT: vldrh.u16 q1, [r1, #128]2253; CHECK-NEXT: vmlava.s16 r0, q1, q02254; CHECK-NEXT: vldrh.u16 q0, [r2, #144]2255; CHECK-NEXT: vldrh.u16 q1, [r1, #144]2256; CHECK-NEXT: vmlava.s16 r0, q1, q02257; CHECK-NEXT: vldrh.u16 q0, [r2, #160]2258; CHECK-NEXT: vldrh.u16 q1, [r1, #160]2259; CHECK-NEXT: vmlava.s16 r0, q1, q02260; CHECK-NEXT: vldrh.u16 q0, [r2, #176]2261; CHECK-NEXT: vldrh.u16 q1, [r1, #176]2262; CHECK-NEXT: vmlava.s16 r0, q1, q02263; CHECK-NEXT: vldrh.u16 q0, [r2, #192]2264; CHECK-NEXT: vldrh.u16 q1, [r1, #192]2265; CHECK-NEXT: vmlava.s16 r0, q1, q02266; CHECK-NEXT: vldrh.u16 q0, [r2, #208]2267; CHECK-NEXT: vldrh.u16 q1, [r1, #208]2268; CHECK-NEXT: vmlava.s16 r0, q1, q02269; CHECK-NEXT: vldrh.u16 q0, [r2, #224]2270; CHECK-NEXT: vldrh.u16 q1, [r1, #224]2271; CHECK-NEXT: vmlava.s16 r0, q1, q02272; CHECK-NEXT: vldrh.u16 q0, [r2, #240]2273; CHECK-NEXT: vldrh.u16 q1, [r1, #240]2274; CHECK-NEXT: vmlava.s16 r0, q1, q02275; CHECK-NEXT: bx lr2276entry:2277 %wide.load = load <8 x i16>, ptr %x, align 22278 %0 = sext <8 x i16> %wide.load to <8 x i32>2279 %wide.load11 = load <8 x i16>, ptr %y, align 22280 %1 = sext <8 x i16> %wide.load11 to <8 x i32>2281 %2 = mul nsw <8 x i32> %1, %02282 %3 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %2)2283 %4 = getelementptr inbounds i16, ptr %x, i32 82284 %wide.load.1 = load <8 x i16>, ptr %4, align 22285 %5 = sext <8 x i16> %wide.load.1 to <8 x i32>2286 %6 = getelementptr inbounds i16, ptr %y, i32 82287 %wide.load11.1 = load <8 x i16>, ptr %6, align 22288 %7 = sext <8 x i16> %wide.load11.1 to <8 x i32>2289 %8 = mul nsw <8 x i32> %7, %52290 %9 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %8)2291 %10 = add i32 %9, %32292 %11 = getelementptr inbounds i16, ptr %x, i32 162293 %wide.load.2 = load <8 x i16>, ptr %11, align 22294 %12 = sext <8 x i16> %wide.load.2 to <8 x i32>2295 %13 = getelementptr inbounds i16, ptr %y, i32 162296 %wide.load11.2 = load <8 x i16>, ptr %13, align 22297 %14 = sext <8 x i16> %wide.load11.2 to <8 x i32>2298 %15 = mul nsw <8 x i32> %14, %122299 %16 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %15)2300 %17 = add i32 %16, %102301 %18 = getelementptr inbounds i16, ptr %x, i32 242302 %wide.load.3 = load <8 x i16>, ptr %18, align 22303 %19 = sext <8 x i16> %wide.load.3 to <8 x i32>2304 %20 = getelementptr inbounds i16, ptr %y, i32 242305 %wide.load11.3 = load <8 x i16>, ptr %20, align 22306 %21 = sext <8 x i16> %wide.load11.3 to <8 x i32>2307 %22 = mul nsw <8 x i32> %21, %192308 %23 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %22)2309 %24 = add i32 %23, %172310 %25 = getelementptr inbounds i16, ptr %x, i32 322311 %wide.load.4 = load <8 x i16>, ptr %25, align 22312 %26 = sext <8 x i16> %wide.load.4 to <8 x i32>2313 %27 = getelementptr inbounds i16, ptr %y, i32 322314 %wide.load11.4 = load <8 x i16>, ptr %27, align 22315 %28 = sext <8 x i16> %wide.load11.4 to <8 x i32>2316 %29 = mul nsw <8 x i32> %28, %262317 %30 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %29)2318 %31 = add i32 %30, %242319 %32 = getelementptr inbounds i16, ptr %x, i32 402320 %wide.load.5 = load <8 x i16>, ptr %32, align 22321 %33 = sext <8 x i16> %wide.load.5 to <8 x i32>2322 %34 = getelementptr inbounds i16, ptr %y, i32 402323 %wide.load11.5 = load <8 x i16>, ptr %34, align 22324 %35 = sext <8 x i16> %wide.load11.5 to <8 x i32>2325 %36 = mul nsw <8 x i32> %35, %332326 %37 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %36)2327 %38 = add i32 %37, %312328 %39 = getelementptr inbounds i16, ptr %x, i32 482329 %wide.load.6 = load <8 x i16>, ptr %39, align 22330 %40 = sext <8 x i16> %wide.load.6 to <8 x i32>2331 %41 = getelementptr inbounds i16, ptr %y, i32 482332 %wide.load11.6 = load <8 x i16>, ptr %41, align 22333 %42 = sext <8 x i16> %wide.load11.6 to <8 x i32>2334 %43 = mul nsw <8 x i32> %42, %402335 %44 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %43)2336 %45 = add i32 %44, %382337 %46 = getelementptr inbounds i16, ptr %x, i32 562338 %wide.load.7 = load <8 x i16>, ptr %46, align 22339 %47 = sext <8 x i16> %wide.load.7 to <8 x i32>2340 %48 = getelementptr inbounds i16, ptr %y, i32 562341 %wide.load11.7 = load <8 x i16>, ptr %48, align 22342 %49 = sext <8 x i16> %wide.load11.7 to <8 x i32>2343 %50 = mul nsw <8 x i32> %49, %472344 %51 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %50)2345 %52 = add i32 %51, %452346 %53 = getelementptr inbounds i16, ptr %x, i32 642347 %wide.load.8 = load <8 x i16>, ptr %53, align 22348 %54 = sext <8 x i16> %wide.load.8 to <8 x i32>2349 %55 = getelementptr inbounds i16, ptr %y, i32 642350 %wide.load11.8 = load <8 x i16>, ptr %55, align 22351 %56 = sext <8 x i16> %wide.load11.8 to <8 x i32>2352 %57 = mul nsw <8 x i32> %56, %542353 %58 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %57)2354 %59 = add i32 %58, %522355 %60 = getelementptr inbounds i16, ptr %x, i32 722356 %wide.load.9 = load <8 x i16>, ptr %60, align 22357 %61 = sext <8 x i16> %wide.load.9 to <8 x i32>2358 %62 = getelementptr inbounds i16, ptr %y, i32 722359 %wide.load11.9 = load <8 x i16>, ptr %62, align 22360 %63 = sext <8 x i16> %wide.load11.9 to <8 x i32>2361 %64 = mul nsw <8 x i32> %63, %612362 %65 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %64)2363 %66 = add i32 %65, %592364 %67 = getelementptr inbounds i16, ptr %x, i32 802365 %wide.load.10 = load <8 x i16>, ptr %67, align 22366 %68 = sext <8 x i16> %wide.load.10 to <8 x i32>2367 %69 = getelementptr inbounds i16, ptr %y, i32 802368 %wide.load11.10 = load <8 x i16>, ptr %69, align 22369 %70 = sext <8 x i16> %wide.load11.10 to <8 x i32>2370 %71 = mul nsw <8 x i32> %70, %682371 %72 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %71)2372 %73 = add i32 %72, %662373 %74 = getelementptr inbounds i16, ptr %x, i32 882374 %wide.load.11 = load <8 x i16>, ptr %74, align 22375 %75 = sext <8 x i16> %wide.load.11 to <8 x i32>2376 %76 = getelementptr inbounds i16, ptr %y, i32 882377 %wide.load11.11 = load <8 x i16>, ptr %76, align 22378 %77 = sext <8 x i16> %wide.load11.11 to <8 x i32>2379 %78 = mul nsw <8 x i32> %77, %752380 %79 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %78)2381 %80 = add i32 %79, %732382 %81 = getelementptr inbounds i16, ptr %x, i32 962383 %wide.load.12 = load <8 x i16>, ptr %81, align 22384 %82 = sext <8 x i16> %wide.load.12 to <8 x i32>2385 %83 = getelementptr inbounds i16, ptr %y, i32 962386 %wide.load11.12 = load <8 x i16>, ptr %83, align 22387 %84 = sext <8 x i16> %wide.load11.12 to <8 x i32>2388 %85 = mul nsw <8 x i32> %84, %822389 %86 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %85)2390 %87 = add i32 %86, %802391 %88 = getelementptr inbounds i16, ptr %x, i32 1042392 %wide.load.13 = load <8 x i16>, ptr %88, align 22393 %89 = sext <8 x i16> %wide.load.13 to <8 x i32>2394 %90 = getelementptr inbounds i16, ptr %y, i32 1042395 %wide.load11.13 = load <8 x i16>, ptr %90, align 22396 %91 = sext <8 x i16> %wide.load11.13 to <8 x i32>2397 %92 = mul nsw <8 x i32> %91, %892398 %93 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %92)2399 %94 = add i32 %93, %872400 %95 = getelementptr inbounds i16, ptr %x, i32 1122401 %wide.load.14 = load <8 x i16>, ptr %95, align 22402 %96 = sext <8 x i16> %wide.load.14 to <8 x i32>2403 %97 = getelementptr inbounds i16, ptr %y, i32 1122404 %wide.load11.14 = load <8 x i16>, ptr %97, align 22405 %98 = sext <8 x i16> %wide.load11.14 to <8 x i32>2406 %99 = mul nsw <8 x i32> %98, %962407 %100 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %99)2408 %101 = add i32 %100, %942409 %102 = getelementptr inbounds i16, ptr %x, i32 1202410 %wide.load.15 = load <8 x i16>, ptr %102, align 22411 %103 = sext <8 x i16> %wide.load.15 to <8 x i32>2412 %104 = getelementptr inbounds i16, ptr %y, i32 1202413 %wide.load11.15 = load <8 x i16>, ptr %104, align 22414 %105 = sext <8 x i16> %wide.load11.15 to <8 x i32>2415 %106 = mul nsw <8 x i32> %105, %1032416 %107 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %106)2417 %108 = add i32 %107, %1012418 ret i32 %1082419}2420 2421define i32 @mlav2i32i8(ptr %x, ptr %y) {2422; CHECK-LABEL: mlav2i32i8:2423; CHECK: @ %bb.0: @ %entry2424; CHECK-NEXT: ldrb r2, [r0]2425; CHECK-NEXT: ldrb r3, [r1]2426; CHECK-NEXT: ldrb r0, [r0, #1]2427; CHECK-NEXT: ldrb r1, [r1, #1]2428; CHECK-NEXT: muls r0, r1, r02429; CHECK-NEXT: smlabb r0, r3, r2, r02430; CHECK-NEXT: bx lr2431entry:2432 %0 = load i8, ptr %x, align 12433 %conv = zext i8 %0 to i322434 %1 = load i8, ptr %y, align 12435 %conv2 = zext i8 %1 to i322436 %mul = mul nuw nsw i32 %conv2, %conv2437 %arrayidx.1 = getelementptr inbounds i8, ptr %x, i32 12438 %2 = load i8, ptr %arrayidx.1, align 12439 %conv.1 = zext i8 %2 to i322440 %arrayidx1.1 = getelementptr inbounds i8, ptr %y, i32 12441 %3 = load i8, ptr %arrayidx1.1, align 12442 %conv2.1 = zext i8 %3 to i322443 %mul.1 = mul nuw nsw i32 %conv2.1, %conv.12444 %add.1 = add nuw nsw i32 %mul.1, %mul2445 ret i32 %add.12446}2447 2448define i32 @mlav4i32i8(ptr %x, ptr %y) {2449; CHECK-LABEL: mlav4i32i8:2450; CHECK: @ %bb.0: @ %entry2451; CHECK-NEXT: vldrb.u32 q0, [r0]2452; CHECK-NEXT: vldrb.u32 q1, [r1]2453; CHECK-NEXT: vmlav.u32 r0, q1, q02454; CHECK-NEXT: bx lr2455entry:2456 %0 = load <4 x i8>, ptr %x, align 12457 %1 = zext <4 x i8> %0 to <4 x i32>2458 %2 = load <4 x i8>, ptr %y, align 12459 %3 = zext <4 x i8> %2 to <4 x i32>2460 %4 = mul nuw nsw <4 x i32> %3, %12461 %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)2462 ret i32 %52463}2464 2465define i32 @mlav8i32i8(ptr %x, ptr %y) {2466; CHECK-LABEL: mlav8i32i8:2467; CHECK: @ %bb.0: @ %entry2468; CHECK-NEXT: vldrb.u16 q0, [r0]2469; CHECK-NEXT: vldrb.u16 q1, [r1]2470; CHECK-NEXT: vmlav.u16 r0, q1, q02471; CHECK-NEXT: bx lr2472entry:2473 %0 = load <8 x i8>, ptr %x, align 12474 %1 = zext <8 x i8> %0 to <8 x i32>2475 %2 = load <8 x i8>, ptr %y, align 12476 %3 = zext <8 x i8> %2 to <8 x i32>2477 %4 = mul nuw nsw <8 x i32> %3, %12478 %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)2479 ret i32 %52480}2481 2482define i32 @mlav16i32i8(ptr %x, ptr %y) {2483; CHECK-LABEL: mlav16i32i8:2484; CHECK: @ %bb.0: @ %entry2485; CHECK-NEXT: vldrb.u8 q0, [r0]2486; CHECK-NEXT: vldrb.u8 q1, [r1]2487; CHECK-NEXT: vmlav.u8 r0, q1, q02488; CHECK-NEXT: bx lr2489entry:2490 %0 = load <16 x i8>, ptr %x, align 12491 %1 = zext <16 x i8> %0 to <16 x i32>2492 %2 = load <16 x i8>, ptr %y, align 12493 %3 = zext <16 x i8> %2 to <16 x i32>2494 %4 = mul nuw nsw <16 x i32> %3, %12495 %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)2496 ret i32 %52497}2498 2499define i32 @mlav24i32i8(ptr %x, ptr %y) {2500; CHECK-LABEL: mlav24i32i8:2501; CHECK: @ %bb.0: @ %entry2502; CHECK-NEXT: vldrb.u16 q0, [r0]2503; CHECK-NEXT: vldrb.u16 q1, [r1]2504; CHECK-NEXT: vmlav.u16 r2, q1, q02505; CHECK-NEXT: vldrb.u8 q0, [r0, #8]2506; CHECK-NEXT: vldrb.u8 q1, [r1, #8]2507; CHECK-NEXT: vmlava.u8 r2, q1, q02508; CHECK-NEXT: mov r0, r22509; CHECK-NEXT: bx lr2510entry:2511 %0 = load <8 x i8>, ptr %x, align 12512 %1 = zext <8 x i8> %0 to <8 x i32>2513 %2 = load <8 x i8>, ptr %y, align 12514 %3 = zext <8 x i8> %2 to <8 x i32>2515 %4 = mul nuw nsw <8 x i32> %3, %12516 %arrayidx.8 = getelementptr inbounds i8, ptr %x, i32 82517 %arrayidx1.8 = getelementptr inbounds i8, ptr %y, i32 82518 %5 = load <16 x i8>, ptr %arrayidx.8, align 12519 %6 = zext <16 x i8> %5 to <16 x i32>2520 %7 = load <16 x i8>, ptr %arrayidx1.8, align 12521 %8 = zext <16 x i8> %7 to <16 x i32>2522 %9 = mul nuw nsw <16 x i32> %8, %62523 %10 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %9)2524 %11 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)2525 %op.rdx = add nuw nsw i32 %10, %112526 ret i32 %op.rdx2527}2528 2529define i32 @mlav32i32i8(ptr %x, ptr %y) {2530; CHECK-LABEL: mlav32i32i8:2531; CHECK: @ %bb.0: @ %entry2532; CHECK-NEXT: vldrb.u32 q0, [r0]2533; CHECK-NEXT: vldrb.u32 q1, [r1]2534; CHECK-NEXT: mov r2, r02535; CHECK-NEXT: vmlav.u32 r0, q1, q02536; CHECK-NEXT: vldrb.u32 q0, [r2, #4]2537; CHECK-NEXT: vldrb.u32 q1, [r1, #4]2538; CHECK-NEXT: vmlava.u32 r0, q1, q02539; CHECK-NEXT: vldrb.u32 q0, [r2, #8]2540; CHECK-NEXT: vldrb.u32 q1, [r1, #8]2541; CHECK-NEXT: vmlava.u32 r0, q1, q02542; CHECK-NEXT: vldrb.u32 q0, [r2, #12]2543; CHECK-NEXT: vldrb.u32 q1, [r1, #12]2544; CHECK-NEXT: vmlava.u32 r0, q1, q02545; CHECK-NEXT: vldrb.u32 q0, [r2, #16]2546; CHECK-NEXT: vldrb.u32 q1, [r1, #16]2547; CHECK-NEXT: vmlava.u32 r0, q1, q02548; CHECK-NEXT: vldrb.u32 q0, [r2, #20]2549; CHECK-NEXT: vldrb.u32 q1, [r1, #20]2550; CHECK-NEXT: vmlava.u32 r0, q1, q02551; CHECK-NEXT: vldrb.u32 q0, [r2, #24]2552; CHECK-NEXT: vldrb.u32 q1, [r1, #24]2553; CHECK-NEXT: vmlava.u32 r0, q1, q02554; CHECK-NEXT: vldrb.u32 q0, [r2, #28]2555; CHECK-NEXT: vldrb.u32 q1, [r1, #28]2556; CHECK-NEXT: vmlava.u32 r0, q1, q02557; CHECK-NEXT: bx lr2558entry:2559 %0 = load <32 x i8>, ptr %x, align 12560 %1 = zext <32 x i8> %0 to <32 x i32>2561 %2 = load <32 x i8>, ptr %y, align 12562 %3 = zext <32 x i8> %2 to <32 x i32>2563 %4 = mul nuw nsw <32 x i32> %3, %12564 %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)2565 ret i32 %52566}2567 2568define i32 @mlav64i32i8(ptr %x, ptr %y) {2569; CHECK-LABEL: mlav64i32i8:2570; CHECK: @ %bb.0: @ %entry2571; CHECK-NEXT: vldrb.u8 q0, [r0]2572; CHECK-NEXT: vldrb.u8 q1, [r1]2573; CHECK-NEXT: vmlav.u8 r2, q1, q02574; CHECK-NEXT: vldrb.u8 q0, [r0, #16]2575; CHECK-NEXT: vldrb.u8 q1, [r1, #16]2576; CHECK-NEXT: vmlava.u8 r2, q1, q02577; CHECK-NEXT: vldrb.u8 q0, [r0, #32]2578; CHECK-NEXT: vldrb.u8 q1, [r1, #32]2579; CHECK-NEXT: vmlava.u8 r2, q1, q02580; CHECK-NEXT: vldrb.u8 q0, [r0, #48]2581; CHECK-NEXT: vldrb.u8 q1, [r1, #48]2582; CHECK-NEXT: vmlava.u8 r2, q1, q02583; CHECK-NEXT: mov r0, r22584; CHECK-NEXT: bx lr2585entry:2586 %wide.load = load <16 x i8>, ptr %x, align 12587 %0 = zext <16 x i8> %wide.load to <16 x i32>2588 %wide.load11 = load <16 x i8>, ptr %y, align 12589 %1 = zext <16 x i8> %wide.load11 to <16 x i32>2590 %2 = mul nuw nsw <16 x i32> %1, %02591 %3 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %2)2592 %4 = getelementptr inbounds i8, ptr %x, i32 162593 %wide.load.1 = load <16 x i8>, ptr %4, align 12594 %5 = zext <16 x i8> %wide.load.1 to <16 x i32>2595 %6 = getelementptr inbounds i8, ptr %y, i32 162596 %wide.load11.1 = load <16 x i8>, ptr %6, align 12597 %7 = zext <16 x i8> %wide.load11.1 to <16 x i32>2598 %8 = mul nuw nsw <16 x i32> %7, %52599 %9 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %8)2600 %10 = add i32 %9, %32601 %11 = getelementptr inbounds i8, ptr %x, i32 322602 %wide.load.2 = load <16 x i8>, ptr %11, align 12603 %12 = zext <16 x i8> %wide.load.2 to <16 x i32>2604 %13 = getelementptr inbounds i8, ptr %y, i32 322605 %wide.load11.2 = load <16 x i8>, ptr %13, align 12606 %14 = zext <16 x i8> %wide.load11.2 to <16 x i32>2607 %15 = mul nuw nsw <16 x i32> %14, %122608 %16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %15)2609 %17 = add i32 %16, %102610 %18 = getelementptr inbounds i8, ptr %x, i32 482611 %wide.load.3 = load <16 x i8>, ptr %18, align 12612 %19 = zext <16 x i8> %wide.load.3 to <16 x i32>2613 %20 = getelementptr inbounds i8, ptr %y, i32 482614 %wide.load11.3 = load <16 x i8>, ptr %20, align 12615 %21 = zext <16 x i8> %wide.load11.3 to <16 x i32>2616 %22 = mul nuw nsw <16 x i32> %21, %192617 %23 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %22)2618 %24 = add i32 %23, %172619 ret i32 %242620}2621 2622define i32 @mlav128i32i8(ptr %x, ptr %y) {2623; CHECK-LABEL: mlav128i32i8:2624; CHECK: @ %bb.0: @ %entry2625; CHECK-NEXT: vldrb.u8 q0, [r0]2626; CHECK-NEXT: vldrb.u8 q1, [r1]2627; CHECK-NEXT: mov r2, r02628; CHECK-NEXT: vmlav.u8 r0, q1, q02629; CHECK-NEXT: vldrb.u8 q0, [r2, #16]2630; CHECK-NEXT: vldrb.u8 q1, [r1, #16]2631; CHECK-NEXT: vmlava.u8 r0, q1, q02632; CHECK-NEXT: vldrb.u8 q0, [r2, #32]2633; CHECK-NEXT: vldrb.u8 q1, [r1, #32]2634; CHECK-NEXT: vmlava.u8 r0, q1, q02635; CHECK-NEXT: vldrb.u8 q0, [r2, #48]2636; CHECK-NEXT: vldrb.u8 q1, [r1, #48]2637; CHECK-NEXT: vmlava.u8 r0, q1, q02638; CHECK-NEXT: vldrb.u8 q0, [r2, #64]2639; CHECK-NEXT: vldrb.u8 q1, [r1, #64]2640; CHECK-NEXT: vmlava.u8 r0, q1, q02641; CHECK-NEXT: vldrb.u8 q0, [r2, #80]2642; CHECK-NEXT: vldrb.u8 q1, [r1, #80]2643; CHECK-NEXT: vmlava.u8 r0, q1, q02644; CHECK-NEXT: vldrb.u8 q0, [r2, #96]2645; CHECK-NEXT: vldrb.u8 q1, [r1, #96]2646; CHECK-NEXT: vmlava.u8 r0, q1, q02647; CHECK-NEXT: vldrb.u8 q0, [r2, #112]2648; CHECK-NEXT: vldrb.u8 q1, [r1, #112]2649; CHECK-NEXT: vmlava.u8 r0, q1, q02650; CHECK-NEXT: bx lr2651entry:2652 %wide.load = load <16 x i8>, ptr %x, align 12653 %0 = zext <16 x i8> %wide.load to <16 x i32>2654 %wide.load11 = load <16 x i8>, ptr %y, align 12655 %1 = zext <16 x i8> %wide.load11 to <16 x i32>2656 %2 = mul nuw nsw <16 x i32> %1, %02657 %3 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %2)2658 %4 = getelementptr inbounds i8, ptr %x, i32 162659 %wide.load.1 = load <16 x i8>, ptr %4, align 12660 %5 = zext <16 x i8> %wide.load.1 to <16 x i32>2661 %6 = getelementptr inbounds i8, ptr %y, i32 162662 %wide.load11.1 = load <16 x i8>, ptr %6, align 12663 %7 = zext <16 x i8> %wide.load11.1 to <16 x i32>2664 %8 = mul nuw nsw <16 x i32> %7, %52665 %9 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %8)2666 %10 = add i32 %9, %32667 %11 = getelementptr inbounds i8, ptr %x, i32 322668 %wide.load.2 = load <16 x i8>, ptr %11, align 12669 %12 = zext <16 x i8> %wide.load.2 to <16 x i32>2670 %13 = getelementptr inbounds i8, ptr %y, i32 322671 %wide.load11.2 = load <16 x i8>, ptr %13, align 12672 %14 = zext <16 x i8> %wide.load11.2 to <16 x i32>2673 %15 = mul nuw nsw <16 x i32> %14, %122674 %16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %15)2675 %17 = add i32 %16, %102676 %18 = getelementptr inbounds i8, ptr %x, i32 482677 %wide.load.3 = load <16 x i8>, ptr %18, align 12678 %19 = zext <16 x i8> %wide.load.3 to <16 x i32>2679 %20 = getelementptr inbounds i8, ptr %y, i32 482680 %wide.load11.3 = load <16 x i8>, ptr %20, align 12681 %21 = zext <16 x i8> %wide.load11.3 to <16 x i32>2682 %22 = mul nuw nsw <16 x i32> %21, %192683 %23 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %22)2684 %24 = add i32 %23, %172685 %25 = getelementptr inbounds i8, ptr %x, i32 642686 %wide.load.4 = load <16 x i8>, ptr %25, align 12687 %26 = zext <16 x i8> %wide.load.4 to <16 x i32>2688 %27 = getelementptr inbounds i8, ptr %y, i32 642689 %wide.load11.4 = load <16 x i8>, ptr %27, align 12690 %28 = zext <16 x i8> %wide.load11.4 to <16 x i32>2691 %29 = mul nuw nsw <16 x i32> %28, %262692 %30 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %29)2693 %31 = add i32 %30, %242694 %32 = getelementptr inbounds i8, ptr %x, i32 802695 %wide.load.5 = load <16 x i8>, ptr %32, align 12696 %33 = zext <16 x i8> %wide.load.5 to <16 x i32>2697 %34 = getelementptr inbounds i8, ptr %y, i32 802698 %wide.load11.5 = load <16 x i8>, ptr %34, align 12699 %35 = zext <16 x i8> %wide.load11.5 to <16 x i32>2700 %36 = mul nuw nsw <16 x i32> %35, %332701 %37 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %36)2702 %38 = add i32 %37, %312703 %39 = getelementptr inbounds i8, ptr %x, i32 962704 %wide.load.6 = load <16 x i8>, ptr %39, align 12705 %40 = zext <16 x i8> %wide.load.6 to <16 x i32>2706 %41 = getelementptr inbounds i8, ptr %y, i32 962707 %wide.load11.6 = load <16 x i8>, ptr %41, align 12708 %42 = zext <16 x i8> %wide.load11.6 to <16 x i32>2709 %43 = mul nuw nsw <16 x i32> %42, %402710 %44 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %43)2711 %45 = add i32 %44, %382712 %46 = getelementptr inbounds i8, ptr %x, i32 1122713 %wide.load.7 = load <16 x i8>, ptr %46, align 12714 %47 = zext <16 x i8> %wide.load.7 to <16 x i32>2715 %48 = getelementptr inbounds i8, ptr %y, i32 1122716 %wide.load11.7 = load <16 x i8>, ptr %48, align 12717 %49 = zext <16 x i8> %wide.load11.7 to <16 x i32>2718 %50 = mul nuw nsw <16 x i32> %49, %472719 %51 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %50)2720 %52 = add i32 %51, %452721 ret i32 %522722}2723 2724define signext i16 @mlav2i16i16(ptr %x, ptr %y) {2725; CHECK-LABEL: mlav2i16i16:2726; CHECK: @ %bb.0: @ %entry2727; CHECK-NEXT: ldrh r2, [r0]2728; CHECK-NEXT: ldrh r3, [r1]2729; CHECK-NEXT: ldrh r0, [r0, #2]2730; CHECK-NEXT: ldrh r1, [r1, #2]2731; CHECK-NEXT: muls r2, r3, r22732; CHECK-NEXT: mla r0, r1, r0, r22733; CHECK-NEXT: sxth r0, r02734; CHECK-NEXT: bx lr2735entry:2736 %0 = load i16, ptr %x, align 22737 %1 = load i16, ptr %y, align 22738 %mul = mul i16 %1, %02739 %arrayidx.1 = getelementptr inbounds i16, ptr %x, i32 12740 %2 = load i16, ptr %arrayidx.1, align 22741 %arrayidx1.1 = getelementptr inbounds i16, ptr %y, i32 12742 %3 = load i16, ptr %arrayidx1.1, align 22743 %mul.1 = mul i16 %3, %22744 %add.1 = add i16 %mul.1, %mul2745 ret i16 %add.12746}2747 2748define signext i16 @mlav4i16i16(ptr %x, ptr %y) {2749; CHECK-LABEL: mlav4i16i16:2750; CHECK: @ %bb.0: @ %entry2751; CHECK-NEXT: vldrh.u32 q0, [r0]2752; CHECK-NEXT: vldrh.u32 q1, [r1]2753; CHECK-NEXT: vmlav.u32 r0, q1, q02754; CHECK-NEXT: sxth r0, r02755; CHECK-NEXT: bx lr2756entry:2757 %0 = load <4 x i16>, ptr %x, align 22758 %1 = load <4 x i16>, ptr %y, align 22759 %2 = mul <4 x i16> %1, %02760 %3 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %2)2761 ret i16 %32762}2763 2764define signext i16 @mlav8i16i16(ptr %x, ptr %y) {2765; CHECK-LABEL: mlav8i16i16:2766; CHECK: @ %bb.0: @ %entry2767; CHECK-NEXT: vldrh.u16 q0, [r0]2768; CHECK-NEXT: vldrh.u16 q1, [r1]2769; CHECK-NEXT: vmlav.u16 r0, q1, q02770; CHECK-NEXT: sxth r0, r02771; CHECK-NEXT: bx lr2772entry:2773 %0 = load <8 x i16>, ptr %x, align 22774 %1 = load <8 x i16>, ptr %y, align 22775 %2 = mul <8 x i16> %1, %02776 %3 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %2)2777 ret i16 %32778}2779 2780define signext i16 @mlav16i16i16(ptr %x, ptr %y) {2781; CHECK-LABEL: mlav16i16i16:2782; CHECK: @ %bb.0: @ %entry2783; CHECK-NEXT: vldrh.u16 q0, [r0]2784; CHECK-NEXT: vldrh.u16 q1, [r1]2785; CHECK-NEXT: vmlav.u16 r2, q1, q02786; CHECK-NEXT: vldrh.u16 q0, [r0, #16]2787; CHECK-NEXT: vldrh.u16 q1, [r1, #16]2788; CHECK-NEXT: vmlava.u16 r2, q1, q02789; CHECK-NEXT: sxth r0, r22790; CHECK-NEXT: bx lr2791entry:2792 %0 = load <16 x i16>, ptr %x, align 22793 %1 = load <16 x i16>, ptr %y, align 22794 %2 = mul <16 x i16> %1, %02795 %3 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %2)2796 ret i16 %32797}2798 2799define signext i16 @mlav24i16i16(ptr %x, ptr %y) {2800; CHECK-LABEL: mlav24i16i16:2801; CHECK: @ %bb.0: @ %entry2802; CHECK-NEXT: vldrh.u16 q0, [r0]2803; CHECK-NEXT: vldrh.u16 q1, [r1]2804; CHECK-NEXT: vmlav.u16 r2, q1, q02805; CHECK-NEXT: vldrh.u16 q0, [r0, #16]2806; CHECK-NEXT: vldrh.u16 q1, [r1, #16]2807; CHECK-NEXT: vmlava.u16 r2, q1, q02808; CHECK-NEXT: vldrh.u16 q0, [r0, #32]2809; CHECK-NEXT: vldrh.u16 q1, [r1, #32]2810; CHECK-NEXT: vmlava.u16 r2, q1, q02811; CHECK-NEXT: sxth r0, r22812; CHECK-NEXT: bx lr2813entry:2814 %0 = load <8 x i16>, ptr %x, align 22815 %1 = load <8 x i16>, ptr %y, align 22816 %2 = mul <8 x i16> %1, %02817 %arrayidx.8 = getelementptr inbounds i16, ptr %x, i32 82818 %arrayidx1.8 = getelementptr inbounds i16, ptr %y, i32 82819 %3 = load <16 x i16>, ptr %arrayidx.8, align 22820 %4 = load <16 x i16>, ptr %arrayidx1.8, align 22821 %5 = mul <16 x i16> %4, %32822 %6 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %5)2823 %7 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %2)2824 %op.rdx = add i16 %6, %72825 ret i16 %op.rdx2826}2827 2828define signext i16 @mlav32i16i16(ptr %x, ptr %y) {2829; CHECK-LABEL: mlav32i16i16:2830; CHECK: @ %bb.0: @ %entry2831; CHECK-NEXT: vldrh.u16 q0, [r0]2832; CHECK-NEXT: vldrh.u16 q1, [r1]2833; CHECK-NEXT: vmlav.u16 r2, q1, q02834; CHECK-NEXT: vldrh.u16 q0, [r0, #16]2835; CHECK-NEXT: vldrh.u16 q1, [r1, #16]2836; CHECK-NEXT: vmlava.u16 r2, q1, q02837; CHECK-NEXT: vldrh.u16 q0, [r0, #32]2838; CHECK-NEXT: vldrh.u16 q1, [r1, #32]2839; CHECK-NEXT: vmlava.u16 r2, q1, q02840; CHECK-NEXT: vldrh.u16 q0, [r0, #48]2841; CHECK-NEXT: vldrh.u16 q1, [r1, #48]2842; CHECK-NEXT: vmlava.u16 r2, q1, q02843; CHECK-NEXT: sxth r0, r22844; CHECK-NEXT: bx lr2845entry:2846 %0 = load <32 x i16>, ptr %x, align 22847 %1 = load <32 x i16>, ptr %y, align 22848 %2 = mul <32 x i16> %1, %02849 %3 = call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> %2)2850 ret i16 %32851}2852 2853define signext i16 @mlav64i16i16(ptr %x, ptr %y) {2854; CHECK-LABEL: mlav64i16i16:2855; CHECK: @ %bb.0: @ %entry2856; CHECK-NEXT: vldrh.u16 q0, [r0]2857; CHECK-NEXT: vldrh.u16 q1, [r1]2858; CHECK-NEXT: vmlav.u16 r2, q1, q02859; CHECK-NEXT: vldrh.u16 q0, [r0, #16]2860; CHECK-NEXT: vldrh.u16 q1, [r1, #16]2861; CHECK-NEXT: vmlava.u16 r2, q1, q02862; CHECK-NEXT: vldrh.u16 q0, [r0, #32]2863; CHECK-NEXT: vldrh.u16 q1, [r1, #32]2864; CHECK-NEXT: vmlava.u16 r2, q1, q02865; CHECK-NEXT: vldrh.u16 q0, [r0, #48]2866; CHECK-NEXT: vldrh.u16 q1, [r1, #48]2867; CHECK-NEXT: vmlava.u16 r2, q1, q02868; CHECK-NEXT: vldrh.u16 q0, [r0, #64]2869; CHECK-NEXT: vldrh.u16 q1, [r1, #64]2870; CHECK-NEXT: vmlava.u16 r2, q1, q02871; CHECK-NEXT: vldrh.u16 q0, [r0, #80]2872; CHECK-NEXT: vldrh.u16 q1, [r1, #80]2873; CHECK-NEXT: vmlava.u16 r2, q1, q02874; CHECK-NEXT: vldrh.u16 q0, [r0, #96]2875; CHECK-NEXT: vldrh.u16 q1, [r1, #96]2876; CHECK-NEXT: vmlava.u16 r2, q1, q02877; CHECK-NEXT: vldrh.u16 q0, [r0, #112]2878; CHECK-NEXT: vldrh.u16 q1, [r1, #112]2879; CHECK-NEXT: vmlava.u16 r2, q1, q02880; CHECK-NEXT: sxth r0, r22881; CHECK-NEXT: bx lr2882entry:2883 %wide.load = load <8 x i16>, ptr %x, align 22884 %wide.load13 = load <8 x i16>, ptr %y, align 22885 %0 = mul <8 x i16> %wide.load13, %wide.load2886 %1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %0)2887 %2 = getelementptr inbounds i16, ptr %x, i32 82888 %wide.load.1 = load <8 x i16>, ptr %2, align 22889 %3 = getelementptr inbounds i16, ptr %y, i32 82890 %wide.load13.1 = load <8 x i16>, ptr %3, align 22891 %4 = mul <8 x i16> %wide.load13.1, %wide.load.12892 %5 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %4)2893 %6 = add i16 %5, %12894 %7 = getelementptr inbounds i16, ptr %x, i32 162895 %wide.load.2 = load <8 x i16>, ptr %7, align 22896 %8 = getelementptr inbounds i16, ptr %y, i32 162897 %wide.load13.2 = load <8 x i16>, ptr %8, align 22898 %9 = mul <8 x i16> %wide.load13.2, %wide.load.22899 %10 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %9)2900 %11 = add i16 %10, %62901 %12 = getelementptr inbounds i16, ptr %x, i32 242902 %wide.load.3 = load <8 x i16>, ptr %12, align 22903 %13 = getelementptr inbounds i16, ptr %y, i32 242904 %wide.load13.3 = load <8 x i16>, ptr %13, align 22905 %14 = mul <8 x i16> %wide.load13.3, %wide.load.32906 %15 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %14)2907 %16 = add i16 %15, %112908 %17 = getelementptr inbounds i16, ptr %x, i32 322909 %wide.load.4 = load <8 x i16>, ptr %17, align 22910 %18 = getelementptr inbounds i16, ptr %y, i32 322911 %wide.load13.4 = load <8 x i16>, ptr %18, align 22912 %19 = mul <8 x i16> %wide.load13.4, %wide.load.42913 %20 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %19)2914 %21 = add i16 %20, %162915 %22 = getelementptr inbounds i16, ptr %x, i32 402916 %wide.load.5 = load <8 x i16>, ptr %22, align 22917 %23 = getelementptr inbounds i16, ptr %y, i32 402918 %wide.load13.5 = load <8 x i16>, ptr %23, align 22919 %24 = mul <8 x i16> %wide.load13.5, %wide.load.52920 %25 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %24)2921 %26 = add i16 %25, %212922 %27 = getelementptr inbounds i16, ptr %x, i32 482923 %wide.load.6 = load <8 x i16>, ptr %27, align 22924 %28 = getelementptr inbounds i16, ptr %y, i32 482925 %wide.load13.6 = load <8 x i16>, ptr %28, align 22926 %29 = mul <8 x i16> %wide.load13.6, %wide.load.62927 %30 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %29)2928 %31 = add i16 %30, %262929 %32 = getelementptr inbounds i16, ptr %x, i32 562930 %wide.load.7 = load <8 x i16>, ptr %32, align 22931 %33 = getelementptr inbounds i16, ptr %y, i32 562932 %wide.load13.7 = load <8 x i16>, ptr %33, align 22933 %34 = mul <8 x i16> %wide.load13.7, %wide.load.72934 %35 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %34)2935 %36 = add i16 %35, %312936 ret i16 %362937}2938 2939define signext i16 @mlav128i16i16(ptr %x, ptr %y) {2940; CHECK-LABEL: mlav128i16i16:2941; CHECK: @ %bb.0: @ %entry2942; CHECK-NEXT: vldrh.u16 q0, [r0]2943; CHECK-NEXT: vldrh.u16 q1, [r1]2944; CHECK-NEXT: vmlav.u16 r2, q1, q02945; CHECK-NEXT: vldrh.u16 q0, [r0, #16]2946; CHECK-NEXT: vldrh.u16 q1, [r1, #16]2947; CHECK-NEXT: vmlava.u16 r2, q1, q02948; CHECK-NEXT: vldrh.u16 q0, [r0, #32]2949; CHECK-NEXT: vldrh.u16 q1, [r1, #32]2950; CHECK-NEXT: vmlava.u16 r2, q1, q02951; CHECK-NEXT: vldrh.u16 q0, [r0, #48]2952; CHECK-NEXT: vldrh.u16 q1, [r1, #48]2953; CHECK-NEXT: vmlava.u16 r2, q1, q02954; CHECK-NEXT: vldrh.u16 q0, [r0, #64]2955; CHECK-NEXT: vldrh.u16 q1, [r1, #64]2956; CHECK-NEXT: vmlava.u16 r2, q1, q02957; CHECK-NEXT: vldrh.u16 q0, [r0, #80]2958; CHECK-NEXT: vldrh.u16 q1, [r1, #80]2959; CHECK-NEXT: vmlava.u16 r2, q1, q02960; CHECK-NEXT: vldrh.u16 q0, [r0, #96]2961; CHECK-NEXT: vldrh.u16 q1, [r1, #96]2962; CHECK-NEXT: vmlava.u16 r2, q1, q02963; CHECK-NEXT: vldrh.u16 q0, [r0, #112]2964; CHECK-NEXT: vldrh.u16 q1, [r1, #112]2965; CHECK-NEXT: vmlava.u16 r2, q1, q02966; CHECK-NEXT: vldrh.u16 q0, [r0, #128]2967; CHECK-NEXT: vldrh.u16 q1, [r1, #128]2968; CHECK-NEXT: vmlava.u16 r2, q1, q02969; CHECK-NEXT: vldrh.u16 q0, [r0, #144]2970; CHECK-NEXT: vldrh.u16 q1, [r1, #144]2971; CHECK-NEXT: vmlava.u16 r2, q1, q02972; CHECK-NEXT: vldrh.u16 q0, [r0, #160]2973; CHECK-NEXT: vldrh.u16 q1, [r1, #160]2974; CHECK-NEXT: vmlava.u16 r2, q1, q02975; CHECK-NEXT: vldrh.u16 q0, [r0, #176]2976; CHECK-NEXT: vldrh.u16 q1, [r1, #176]2977; CHECK-NEXT: vmlava.u16 r2, q1, q02978; CHECK-NEXT: vldrh.u16 q0, [r0, #192]2979; CHECK-NEXT: vldrh.u16 q1, [r1, #192]2980; CHECK-NEXT: vmlava.u16 r2, q1, q02981; CHECK-NEXT: vldrh.u16 q0, [r0, #208]2982; CHECK-NEXT: vldrh.u16 q1, [r1, #208]2983; CHECK-NEXT: vmlava.u16 r2, q1, q02984; CHECK-NEXT: vldrh.u16 q0, [r0, #224]2985; CHECK-NEXT: vldrh.u16 q1, [r1, #224]2986; CHECK-NEXT: vmlava.u16 r2, q1, q02987; CHECK-NEXT: vldrh.u16 q0, [r0, #240]2988; CHECK-NEXT: vldrh.u16 q1, [r1, #240]2989; CHECK-NEXT: vmlava.u16 r2, q1, q02990; CHECK-NEXT: sxth r0, r22991; CHECK-NEXT: bx lr2992entry:2993 %wide.load = load <8 x i16>, ptr %x, align 22994 %wide.load13 = load <8 x i16>, ptr %y, align 22995 %0 = mul <8 x i16> %wide.load13, %wide.load2996 %1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %0)2997 %2 = getelementptr inbounds i16, ptr %x, i32 82998 %wide.load.1 = load <8 x i16>, ptr %2, align 22999 %3 = getelementptr inbounds i16, ptr %y, i32 83000 %wide.load13.1 = load <8 x i16>, ptr %3, align 23001 %4 = mul <8 x i16> %wide.load13.1, %wide.load.13002 %5 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %4)3003 %6 = add i16 %5, %13004 %7 = getelementptr inbounds i16, ptr %x, i32 163005 %wide.load.2 = load <8 x i16>, ptr %7, align 23006 %8 = getelementptr inbounds i16, ptr %y, i32 163007 %wide.load13.2 = load <8 x i16>, ptr %8, align 23008 %9 = mul <8 x i16> %wide.load13.2, %wide.load.23009 %10 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %9)3010 %11 = add i16 %10, %63011 %12 = getelementptr inbounds i16, ptr %x, i32 243012 %wide.load.3 = load <8 x i16>, ptr %12, align 23013 %13 = getelementptr inbounds i16, ptr %y, i32 243014 %wide.load13.3 = load <8 x i16>, ptr %13, align 23015 %14 = mul <8 x i16> %wide.load13.3, %wide.load.33016 %15 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %14)3017 %16 = add i16 %15, %113018 %17 = getelementptr inbounds i16, ptr %x, i32 323019 %wide.load.4 = load <8 x i16>, ptr %17, align 23020 %18 = getelementptr inbounds i16, ptr %y, i32 323021 %wide.load13.4 = load <8 x i16>, ptr %18, align 23022 %19 = mul <8 x i16> %wide.load13.4, %wide.load.43023 %20 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %19)3024 %21 = add i16 %20, %163025 %22 = getelementptr inbounds i16, ptr %x, i32 403026 %wide.load.5 = load <8 x i16>, ptr %22, align 23027 %23 = getelementptr inbounds i16, ptr %y, i32 403028 %wide.load13.5 = load <8 x i16>, ptr %23, align 23029 %24 = mul <8 x i16> %wide.load13.5, %wide.load.53030 %25 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %24)3031 %26 = add i16 %25, %213032 %27 = getelementptr inbounds i16, ptr %x, i32 483033 %wide.load.6 = load <8 x i16>, ptr %27, align 23034 %28 = getelementptr inbounds i16, ptr %y, i32 483035 %wide.load13.6 = load <8 x i16>, ptr %28, align 23036 %29 = mul <8 x i16> %wide.load13.6, %wide.load.63037 %30 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %29)3038 %31 = add i16 %30, %263039 %32 = getelementptr inbounds i16, ptr %x, i32 563040 %wide.load.7 = load <8 x i16>, ptr %32, align 23041 %33 = getelementptr inbounds i16, ptr %y, i32 563042 %wide.load13.7 = load <8 x i16>, ptr %33, align 23043 %34 = mul <8 x i16> %wide.load13.7, %wide.load.73044 %35 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %34)3045 %36 = add i16 %35, %313046 %37 = getelementptr inbounds i16, ptr %x, i32 643047 %wide.load.8 = load <8 x i16>, ptr %37, align 23048 %38 = getelementptr inbounds i16, ptr %y, i32 643049 %wide.load13.8 = load <8 x i16>, ptr %38, align 23050 %39 = mul <8 x i16> %wide.load13.8, %wide.load.83051 %40 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %39)3052 %41 = add i16 %40, %363053 %42 = getelementptr inbounds i16, ptr %x, i32 723054 %wide.load.9 = load <8 x i16>, ptr %42, align 23055 %43 = getelementptr inbounds i16, ptr %y, i32 723056 %wide.load13.9 = load <8 x i16>, ptr %43, align 23057 %44 = mul <8 x i16> %wide.load13.9, %wide.load.93058 %45 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %44)3059 %46 = add i16 %45, %413060 %47 = getelementptr inbounds i16, ptr %x, i32 803061 %wide.load.10 = load <8 x i16>, ptr %47, align 23062 %48 = getelementptr inbounds i16, ptr %y, i32 803063 %wide.load13.10 = load <8 x i16>, ptr %48, align 23064 %49 = mul <8 x i16> %wide.load13.10, %wide.load.103065 %50 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %49)3066 %51 = add i16 %50, %463067 %52 = getelementptr inbounds i16, ptr %x, i32 883068 %wide.load.11 = load <8 x i16>, ptr %52, align 23069 %53 = getelementptr inbounds i16, ptr %y, i32 883070 %wide.load13.11 = load <8 x i16>, ptr %53, align 23071 %54 = mul <8 x i16> %wide.load13.11, %wide.load.113072 %55 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %54)3073 %56 = add i16 %55, %513074 %57 = getelementptr inbounds i16, ptr %x, i32 963075 %wide.load.12 = load <8 x i16>, ptr %57, align 23076 %58 = getelementptr inbounds i16, ptr %y, i32 963077 %wide.load13.12 = load <8 x i16>, ptr %58, align 23078 %59 = mul <8 x i16> %wide.load13.12, %wide.load.123079 %60 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %59)3080 %61 = add i16 %60, %563081 %62 = getelementptr inbounds i16, ptr %x, i32 1043082 %wide.load.13 = load <8 x i16>, ptr %62, align 23083 %63 = getelementptr inbounds i16, ptr %y, i32 1043084 %wide.load13.13 = load <8 x i16>, ptr %63, align 23085 %64 = mul <8 x i16> %wide.load13.13, %wide.load.133086 %65 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %64)3087 %66 = add i16 %65, %613088 %67 = getelementptr inbounds i16, ptr %x, i32 1123089 %wide.load.14 = load <8 x i16>, ptr %67, align 23090 %68 = getelementptr inbounds i16, ptr %y, i32 1123091 %wide.load13.14 = load <8 x i16>, ptr %68, align 23092 %69 = mul <8 x i16> %wide.load13.14, %wide.load.143093 %70 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %69)3094 %71 = add i16 %70, %663095 %72 = getelementptr inbounds i16, ptr %x, i32 1203096 %wide.load.15 = load <8 x i16>, ptr %72, align 23097 %73 = getelementptr inbounds i16, ptr %y, i32 1203098 %wide.load13.15 = load <8 x i16>, ptr %73, align 23099 %74 = mul <8 x i16> %wide.load13.15, %wide.load.153100 %75 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %74)3101 %76 = add i16 %75, %713102 ret i16 %763103}3104 3105define zeroext i8 @mlav2i8i8(ptr %x, ptr %y) {3106; CHECK-LABEL: mlav2i8i8:3107; CHECK: @ %bb.0: @ %entry3108; CHECK-NEXT: ldrb r2, [r0]3109; CHECK-NEXT: ldrb r3, [r1]3110; CHECK-NEXT: ldrb r0, [r0, #1]3111; CHECK-NEXT: ldrb r1, [r1, #1]3112; CHECK-NEXT: muls r2, r3, r23113; CHECK-NEXT: mla r0, r1, r0, r23114; CHECK-NEXT: uxtb r0, r03115; CHECK-NEXT: bx lr3116entry:3117 %0 = load i8, ptr %x, align 13118 %1 = load i8, ptr %y, align 13119 %mul = mul i8 %1, %03120 %arrayidx.1 = getelementptr inbounds i8, ptr %x, i32 13121 %2 = load i8, ptr %arrayidx.1, align 13122 %arrayidx1.1 = getelementptr inbounds i8, ptr %y, i32 13123 %3 = load i8, ptr %arrayidx1.1, align 13124 %mul.1 = mul i8 %3, %23125 %add.1 = add i8 %mul.1, %mul3126 ret i8 %add.13127}3128 3129define zeroext i8 @mlav4i8i8(ptr %x, ptr %y) {3130; CHECK-LABEL: mlav4i8i8:3131; CHECK: @ %bb.0: @ %entry3132; CHECK-NEXT: vldrb.u32 q0, [r0]3133; CHECK-NEXT: vldrb.u32 q1, [r1]3134; CHECK-NEXT: vmlav.u32 r0, q1, q03135; CHECK-NEXT: uxtb r0, r03136; CHECK-NEXT: bx lr3137entry:3138 %0 = load <4 x i8>, ptr %x, align 13139 %1 = load <4 x i8>, ptr %y, align 13140 %2 = mul <4 x i8> %1, %03141 %3 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> %2)3142 ret i8 %33143}3144 3145define zeroext i8 @mlav8i8i8(ptr %x, ptr %y) {3146; CHECK-LABEL: mlav8i8i8:3147; CHECK: @ %bb.0: @ %entry3148; CHECK-NEXT: vldrb.u16 q0, [r0]3149; CHECK-NEXT: vldrb.u16 q1, [r1]3150; CHECK-NEXT: vmlav.u16 r0, q1, q03151; CHECK-NEXT: uxtb r0, r03152; CHECK-NEXT: bx lr3153entry:3154 %0 = load <8 x i8>, ptr %x, align 13155 %1 = load <8 x i8>, ptr %y, align 13156 %2 = mul <8 x i8> %1, %03157 %3 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %2)3158 ret i8 %33159}3160 3161define zeroext i8 @mlav16i8i8(ptr %x, ptr %y) {3162; CHECK-LABEL: mlav16i8i8:3163; CHECK: @ %bb.0: @ %entry3164; CHECK-NEXT: vldrb.u8 q0, [r0]3165; CHECK-NEXT: vldrb.u8 q1, [r1]3166; CHECK-NEXT: vmlav.u8 r0, q1, q03167; CHECK-NEXT: uxtb r0, r03168; CHECK-NEXT: bx lr3169entry:3170 %0 = load <16 x i8>, ptr %x, align 13171 %1 = load <16 x i8>, ptr %y, align 13172 %2 = mul <16 x i8> %1, %03173 %3 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %2)3174 ret i8 %33175}3176 3177define zeroext i8 @mlav24i8i8(ptr %x, ptr %y) {3178; CHECK-LABEL: mlav24i8i8:3179; CHECK: @ %bb.0: @ %entry3180; CHECK-NEXT: vldrb.u16 q0, [r0]3181; CHECK-NEXT: vldrb.u16 q1, [r1]3182; CHECK-NEXT: vmlav.u16 r2, q1, q03183; CHECK-NEXT: vldrb.u8 q0, [r0, #8]3184; CHECK-NEXT: vldrb.u8 q1, [r1, #8]3185; CHECK-NEXT: vmlava.u8 r2, q1, q03186; CHECK-NEXT: uxtb r0, r23187; CHECK-NEXT: bx lr3188entry:3189 %0 = load <8 x i8>, ptr %x, align 13190 %1 = load <8 x i8>, ptr %y, align 13191 %2 = mul <8 x i8> %1, %03192 %arrayidx.8 = getelementptr inbounds i8, ptr %x, i32 83193 %arrayidx1.8 = getelementptr inbounds i8, ptr %y, i32 83194 %3 = load <16 x i8>, ptr %arrayidx.8, align 13195 %4 = load <16 x i8>, ptr %arrayidx1.8, align 13196 %5 = mul <16 x i8> %4, %33197 %6 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %5)3198 %7 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %2)3199 %op.rdx = add i8 %6, %73200 ret i8 %op.rdx3201}3202 3203define zeroext i8 @mlav32i8i8(ptr %x, ptr %y) {3204; CHECK-LABEL: mlav32i8i8:3205; CHECK: @ %bb.0: @ %entry3206; CHECK-NEXT: vldrb.u8 q0, [r0]3207; CHECK-NEXT: vldrb.u8 q1, [r1]3208; CHECK-NEXT: vmlav.u8 r2, q1, q03209; CHECK-NEXT: vldrb.u8 q0, [r0, #16]3210; CHECK-NEXT: vldrb.u8 q1, [r1, #16]3211; CHECK-NEXT: vmlava.u8 r2, q1, q03212; CHECK-NEXT: uxtb r0, r23213; CHECK-NEXT: bx lr3214entry:3215 %0 = load <32 x i8>, ptr %x, align 13216 %1 = load <32 x i8>, ptr %y, align 13217 %2 = mul <32 x i8> %1, %03218 %3 = call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> %2)3219 ret i8 %33220}3221 3222define zeroext i8 @mlav64i8i8(ptr %x, ptr %y) {3223; CHECK-LABEL: mlav64i8i8:3224; CHECK: @ %bb.0: @ %entry3225; CHECK-NEXT: vldrb.u8 q0, [r0]3226; CHECK-NEXT: vldrb.u8 q1, [r1]3227; CHECK-NEXT: vmlav.u8 r2, q1, q03228; CHECK-NEXT: vldrb.u8 q0, [r0, #16]3229; CHECK-NEXT: vldrb.u8 q1, [r1, #16]3230; CHECK-NEXT: vmlava.u8 r2, q1, q03231; CHECK-NEXT: vldrb.u8 q0, [r0, #32]3232; CHECK-NEXT: vldrb.u8 q1, [r1, #32]3233; CHECK-NEXT: vmlava.u8 r2, q1, q03234; CHECK-NEXT: vldrb.u8 q0, [r0, #48]3235; CHECK-NEXT: vldrb.u8 q1, [r1, #48]3236; CHECK-NEXT: vmlava.u8 r2, q1, q03237; CHECK-NEXT: uxtb r0, r23238; CHECK-NEXT: bx lr3239entry:3240 %wide.load = load <16 x i8>, ptr %x, align 13241 %wide.load12 = load <16 x i8>, ptr %y, align 13242 %0 = mul <16 x i8> %wide.load12, %wide.load3243 %1 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %0)3244 %2 = getelementptr inbounds i8, ptr %x, i32 163245 %wide.load.1 = load <16 x i8>, ptr %2, align 13246 %3 = getelementptr inbounds i8, ptr %y, i32 163247 %wide.load12.1 = load <16 x i8>, ptr %3, align 13248 %4 = mul <16 x i8> %wide.load12.1, %wide.load.13249 %5 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %4)3250 %6 = add i8 %5, %13251 %7 = getelementptr inbounds i8, ptr %x, i32 323252 %wide.load.2 = load <16 x i8>, ptr %7, align 13253 %8 = getelementptr inbounds i8, ptr %y, i32 323254 %wide.load12.2 = load <16 x i8>, ptr %8, align 13255 %9 = mul <16 x i8> %wide.load12.2, %wide.load.23256 %10 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %9)3257 %11 = add i8 %10, %63258 %12 = getelementptr inbounds i8, ptr %x, i32 483259 %wide.load.3 = load <16 x i8>, ptr %12, align 13260 %13 = getelementptr inbounds i8, ptr %y, i32 483261 %wide.load12.3 = load <16 x i8>, ptr %13, align 13262 %14 = mul <16 x i8> %wide.load12.3, %wide.load.33263 %15 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %14)3264 %16 = add i8 %15, %113265 ret i8 %163266}3267 3268define zeroext i8 @mlav128i8i8(ptr %x, ptr %y) {3269; CHECK-LABEL: mlav128i8i8:3270; CHECK: @ %bb.0: @ %entry3271; CHECK-NEXT: vldrb.u8 q0, [r0]3272; CHECK-NEXT: vldrb.u8 q1, [r1]3273; CHECK-NEXT: vmlav.u8 r2, q1, q03274; CHECK-NEXT: vldrb.u8 q0, [r0, #16]3275; CHECK-NEXT: vldrb.u8 q1, [r1, #16]3276; CHECK-NEXT: vmlava.u8 r2, q1, q03277; CHECK-NEXT: vldrb.u8 q0, [r0, #32]3278; CHECK-NEXT: vldrb.u8 q1, [r1, #32]3279; CHECK-NEXT: vmlava.u8 r2, q1, q03280; CHECK-NEXT: vldrb.u8 q0, [r0, #48]3281; CHECK-NEXT: vldrb.u8 q1, [r1, #48]3282; CHECK-NEXT: vmlava.u8 r2, q1, q03283; CHECK-NEXT: vldrb.u8 q0, [r0, #64]3284; CHECK-NEXT: vldrb.u8 q1, [r1, #64]3285; CHECK-NEXT: vmlava.u8 r2, q1, q03286; CHECK-NEXT: vldrb.u8 q0, [r0, #80]3287; CHECK-NEXT: vldrb.u8 q1, [r1, #80]3288; CHECK-NEXT: vmlava.u8 r2, q1, q03289; CHECK-NEXT: vldrb.u8 q0, [r0, #96]3290; CHECK-NEXT: vldrb.u8 q1, [r1, #96]3291; CHECK-NEXT: vmlava.u8 r2, q1, q03292; CHECK-NEXT: vldrb.u8 q0, [r0, #112]3293; CHECK-NEXT: vldrb.u8 q1, [r1, #112]3294; CHECK-NEXT: vmlava.u8 r2, q1, q03295; CHECK-NEXT: uxtb r0, r23296; CHECK-NEXT: bx lr3297entry:3298 %wide.load = load <16 x i8>, ptr %x, align 13299 %wide.load12 = load <16 x i8>, ptr %y, align 13300 %0 = mul <16 x i8> %wide.load12, %wide.load3301 %1 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %0)3302 %2 = getelementptr inbounds i8, ptr %x, i32 163303 %wide.load.1 = load <16 x i8>, ptr %2, align 13304 %3 = getelementptr inbounds i8, ptr %y, i32 163305 %wide.load12.1 = load <16 x i8>, ptr %3, align 13306 %4 = mul <16 x i8> %wide.load12.1, %wide.load.13307 %5 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %4)3308 %6 = add i8 %5, %13309 %7 = getelementptr inbounds i8, ptr %x, i32 323310 %wide.load.2 = load <16 x i8>, ptr %7, align 13311 %8 = getelementptr inbounds i8, ptr %y, i32 323312 %wide.load12.2 = load <16 x i8>, ptr %8, align 13313 %9 = mul <16 x i8> %wide.load12.2, %wide.load.23314 %10 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %9)3315 %11 = add i8 %10, %63316 %12 = getelementptr inbounds i8, ptr %x, i32 483317 %wide.load.3 = load <16 x i8>, ptr %12, align 13318 %13 = getelementptr inbounds i8, ptr %y, i32 483319 %wide.load12.3 = load <16 x i8>, ptr %13, align 13320 %14 = mul <16 x i8> %wide.load12.3, %wide.load.33321 %15 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %14)3322 %16 = add i8 %15, %113323 %17 = getelementptr inbounds i8, ptr %x, i32 643324 %wide.load.4 = load <16 x i8>, ptr %17, align 13325 %18 = getelementptr inbounds i8, ptr %y, i32 643326 %wide.load12.4 = load <16 x i8>, ptr %18, align 13327 %19 = mul <16 x i8> %wide.load12.4, %wide.load.43328 %20 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %19)3329 %21 = add i8 %20, %163330 %22 = getelementptr inbounds i8, ptr %x, i32 803331 %wide.load.5 = load <16 x i8>, ptr %22, align 13332 %23 = getelementptr inbounds i8, ptr %y, i32 803333 %wide.load12.5 = load <16 x i8>, ptr %23, align 13334 %24 = mul <16 x i8> %wide.load12.5, %wide.load.53335 %25 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %24)3336 %26 = add i8 %25, %213337 %27 = getelementptr inbounds i8, ptr %x, i32 963338 %wide.load.6 = load <16 x i8>, ptr %27, align 13339 %28 = getelementptr inbounds i8, ptr %y, i32 963340 %wide.load12.6 = load <16 x i8>, ptr %28, align 13341 %29 = mul <16 x i8> %wide.load12.6, %wide.load.63342 %30 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %29)3343 %31 = add i8 %30, %263344 %32 = getelementptr inbounds i8, ptr %x, i32 1123345 %wide.load.7 = load <16 x i8>, ptr %32, align 13346 %33 = getelementptr inbounds i8, ptr %y, i32 1123347 %wide.load12.7 = load <16 x i8>, ptr %33, align 13348 %34 = mul <16 x i8> %wide.load12.7, %wide.load.73349 %35 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %34)3350 %36 = add i8 %35, %313351 ret i8 %363352}3353 3354 3355define arm_aapcs_vfpcc i32 @add_two_const(<4 x i32> %x, <4 x i32> %y) {3356; CHECK-LABEL: add_two_const:3357; CHECK: @ %bb.0: @ %entry3358; CHECK-NEXT: vaddv.u32 r0, q13359; CHECK-NEXT: vaddva.u32 r0, q03360; CHECK-NEXT: adds r0, #103361; CHECK-NEXT: bx lr3362entry:3363 %a = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %x)3364 %b = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %y)3365 %c = add i32 %a, %b3366 %d = add i32 %c, 103367 ret i32 %d3368}3369 3370define arm_aapcs_vfpcc i32 @add_two_const2(<4 x i32> %x, <4 x i32> %y) {3371; CHECK-LABEL: add_two_const2:3372; CHECK: @ %bb.0: @ %entry3373; CHECK-NEXT: vaddv.u32 r0, q13374; CHECK-NEXT: vaddva.u32 r0, q03375; CHECK-NEXT: adds r0, #103376; CHECK-NEXT: bx lr3377entry:3378 %a = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %x)3379 %b = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %y)3380 %c = add i32 %a, 103381 %d = add i32 %c, %b3382 ret i32 %d3383}3384 3385define arm_aapcs_vfpcc i32 @add_two_const3(<4 x i32> %x, <4 x i32> %y) {3386; CHECK-LABEL: add_two_const3:3387; CHECK: @ %bb.0: @ %entry3388; CHECK-NEXT: vaddv.u32 r0, q03389; CHECK-NEXT: vaddva.u32 r0, q13390; CHECK-NEXT: adds r0, #203391; CHECK-NEXT: bx lr3392entry:3393 %a = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %x)3394 %b = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %y)3395 %c = add i32 %a, 103396 %d = add i32 %b, 103397 %e = add i32 %c, %d3398 ret i32 %e3399}3400 3401declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)3402declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)3403declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)3404declare i32 @llvm.vector.reduce.add.v32i32(<32 x i32>)3405declare i32 @llvm.vector.reduce.add.v64i32(<64 x i32>)3406declare i16 @llvm.vector.reduce.add.v4i16(<4 x i16>)3407declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)3408declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>)3409declare i16 @llvm.vector.reduce.add.v32i16(<32 x i16>)3410declare i16 @llvm.vector.reduce.add.v64i16(<64 x i16>)3411declare i8 @llvm.vector.reduce.add.v4i8(<4 x i8>)3412declare i8 @llvm.vector.reduce.add.v8i8(<8 x i8>)3413declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)3414declare i8 @llvm.vector.reduce.add.v32i8(<32 x i8>)3415declare i8 @llvm.vector.reduce.add.v64i8(<64 x i8>)3416