4862 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-BASE3; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+dotprod %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-DOT4; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-BASE5; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+dotprod -global-isel %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-DOT6 7define i32 @addv_v2i32(<2 x i32> %a) {8; CHECK-LABEL: addv_v2i32:9; CHECK: // %bb.0: // %entry10; CHECK-NEXT: addp v0.2s, v0.2s, v0.2s11; CHECK-NEXT: fmov w0, s012; CHECK-NEXT: ret13entry:14 %arg1 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a)15 ret i32 %arg116}17 18define i16 @addv_v4i16(<4 x i16> %a) {19; CHECK-LABEL: addv_v4i16:20; CHECK: // %bb.0: // %entry21; CHECK-NEXT: addv h0, v0.4h22; CHECK-NEXT: fmov w0, s023; CHECK-NEXT: ret24entry:25 %arg1 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %a)26 ret i16 %arg127}28 29define i32 @add_v4i32_v4i32(<4 x i32> %x) {30; CHECK-LABEL: add_v4i32_v4i32:31; CHECK: // %bb.0: // %entry32; CHECK-NEXT: addv s0, v0.4s33; CHECK-NEXT: fmov w0, s034; CHECK-NEXT: ret35entry:36 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %x)37 ret i32 %z38}39 40define i8 @addv_v8i8(<8 x i8> %a) {41; CHECK-LABEL: addv_v8i8:42; CHECK: // %bb.0: // %entry43; CHECK-NEXT: addv b0, v0.8b44; CHECK-NEXT: fmov w0, s045; CHECK-NEXT: ret46entry:47 %arg1 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %a)48 ret i8 %arg149}50 51define i64 @add_v4i32_v4i64_zext(<4 x i32> %x) {52; CHECK-LABEL: add_v4i32_v4i64_zext:53; CHECK: // %bb.0: // %entry54; CHECK-NEXT: uaddlv d0, v0.4s55; CHECK-NEXT: fmov x0, d056; CHECK-NEXT: ret57entry:58 %xx = zext <4 x i32> %x to <4 x i64>59 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)60 ret i64 %z61}62 63define i64 @add_v4i32_v4i64_sext(<4 x i32> %x) {64; CHECK-LABEL: add_v4i32_v4i64_sext:65; CHECK: // %bb.0: // %entry66; CHECK-NEXT: saddlv d0, v0.4s67; CHECK-NEXT: fmov x0, d068; CHECK-NEXT: ret69entry:70 %xx = sext <4 x i32> %x to <4 x i64>71 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)72 ret i64 %z73}74 75define i64 @add_v4i32_v4i64_zsext(<4 x i32> %xi) {76; CHECK-LABEL: add_v4i32_v4i64_zsext:77; CHECK: // %bb.0: // %entry78; CHECK-NEXT: ushll v1.2d, v0.2s, #079; CHECK-NEXT: saddw2 v0.2d, v1.2d, v0.4s80; CHECK-NEXT: addp d0, v0.2d81; CHECK-NEXT: fmov x0, d082; CHECK-NEXT: ret83entry:84 %x = shufflevector <4 x i32> %xi, <4 x i32> %xi, <2 x i32> <i32 0, i32 1>85 %y = shufflevector <4 x i32> %xi, <4 x i32> %xi, <2 x i32> <i32 2, i32 3>86 %xx = zext <2 x i32> %x to <2 x i64>87 %yy = sext <2 x i32> %y to <2 x i64>88 %zz = add <2 x i64> %xx, %yy89 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %zz)90 ret i64 %z91}92 93define i64 @add_v2i32_v2i64_zext(<2 x i32> %x) {94; CHECK-LABEL: add_v2i32_v2i64_zext:95; CHECK: // %bb.0: // %entry96; CHECK-NEXT: ushll v0.2d, v0.2s, #097; CHECK-NEXT: addp d0, v0.2d98; CHECK-NEXT: fmov x0, d099; CHECK-NEXT: ret100entry:101 %xx = zext <2 x i32> %x to <2 x i64>102 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)103 ret i64 %z104}105 106define i64 @add_v2i32_v2i64_sext(<2 x i32> %x) {107; CHECK-LABEL: add_v2i32_v2i64_sext:108; CHECK: // %bb.0: // %entry109; CHECK-NEXT: sshll v0.2d, v0.2s, #0110; CHECK-NEXT: addp d0, v0.2d111; CHECK-NEXT: fmov x0, d0112; CHECK-NEXT: ret113entry:114 %xx = sext <2 x i32> %x to <2 x i64>115 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)116 ret i64 %z117}118 119define i32 @add_v8i16_v8i32_zext(<8 x i16> %x) {120; CHECK-LABEL: add_v8i16_v8i32_zext:121; CHECK: // %bb.0: // %entry122; CHECK-NEXT: uaddlv s0, v0.8h123; CHECK-NEXT: fmov w0, s0124; CHECK-NEXT: ret125entry:126 %xx = zext <8 x i16> %x to <8 x i32>127 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)128 ret i32 %z129}130 131define i32 @add_v8i16_v8i32_sext(<8 x i16> %x) {132; CHECK-LABEL: add_v8i16_v8i32_sext:133; CHECK: // %bb.0: // %entry134; CHECK-NEXT: saddlv s0, v0.8h135; CHECK-NEXT: fmov w0, s0136; CHECK-NEXT: ret137entry:138 %xx = sext <8 x i16> %x to <8 x i32>139 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)140 ret i32 %z141}142 143define i32 @add_v4i16_v4i32_zext(<4 x i16> %x) {144; CHECK-LABEL: add_v4i16_v4i32_zext:145; CHECK: // %bb.0: // %entry146; CHECK-NEXT: uaddlv s0, v0.4h147; CHECK-NEXT: fmov w0, s0148; CHECK-NEXT: ret149entry:150 %xx = zext <4 x i16> %x to <4 x i32>151 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)152 ret i32 %z153}154 155define i32 @add_v4i16_v4i32_sext(<4 x i16> %x) {156; CHECK-LABEL: add_v4i16_v4i32_sext:157; CHECK: // %bb.0: // %entry158; CHECK-NEXT: saddlv s0, v0.4h159; CHECK-NEXT: fmov w0, s0160; CHECK-NEXT: ret161entry:162 %xx = sext <4 x i16> %x to <4 x i32>163 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)164 ret i32 %z165}166 167define zeroext i16 @add_v8i16_v8i16(<8 x i16> %x) {168; CHECK-SD-LABEL: add_v8i16_v8i16:169; CHECK-SD: // %bb.0: // %entry170; CHECK-SD-NEXT: addv h0, v0.8h171; CHECK-SD-NEXT: fmov w0, s0172; CHECK-SD-NEXT: ret173;174; CHECK-GI-LABEL: add_v8i16_v8i16:175; CHECK-GI: // %bb.0: // %entry176; CHECK-GI-NEXT: addv h0, v0.8h177; CHECK-GI-NEXT: fmov w8, s0178; CHECK-GI-NEXT: uxth w0, w8179; CHECK-GI-NEXT: ret180entry:181 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %x)182 ret i16 %z183}184 185define i64 @add_v8i16_v8i64_zext(<8 x i16> %x) {186; CHECK-SD-LABEL: add_v8i16_v8i64_zext:187; CHECK-SD: // %bb.0: // %entry188; CHECK-SD-NEXT: ushll2 v1.4s, v0.8h, #0189; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0190; CHECK-SD-NEXT: uaddl2 v2.2d, v0.4s, v1.4s191; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s192; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d193; CHECK-SD-NEXT: addp d0, v0.2d194; CHECK-SD-NEXT: fmov x0, d0195; CHECK-SD-NEXT: ret196;197; CHECK-GI-LABEL: add_v8i16_v8i64_zext:198; CHECK-GI: // %bb.0: // %entry199; CHECK-GI-NEXT: uaddlv s0, v0.8h200; CHECK-GI-NEXT: mov w0, v0.s[0]201; CHECK-GI-NEXT: ret202entry:203 %xx = zext <8 x i16> %x to <8 x i64>204 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)205 ret i64 %z206}207 208define i64 @add_v8i16_v8i64_sext(<8 x i16> %x) {209; CHECK-SD-LABEL: add_v8i16_v8i64_sext:210; CHECK-SD: // %bb.0: // %entry211; CHECK-SD-NEXT: sshll2 v1.4s, v0.8h, #0212; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0213; CHECK-SD-NEXT: saddl2 v2.2d, v0.4s, v1.4s214; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v1.2s215; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d216; CHECK-SD-NEXT: addp d0, v0.2d217; CHECK-SD-NEXT: fmov x0, d0218; CHECK-SD-NEXT: ret219;220; CHECK-GI-LABEL: add_v8i16_v8i64_sext:221; CHECK-GI: // %bb.0: // %entry222; CHECK-GI-NEXT: saddlv s0, v0.8h223; CHECK-GI-NEXT: smov x0, v0.s[0]224; CHECK-GI-NEXT: ret225entry:226 %xx = sext <8 x i16> %x to <8 x i64>227 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)228 ret i64 %z229}230 231define i64 @add_v4i16_v4i64_zext(<4 x i16> %x) {232; CHECK-SD-LABEL: add_v4i16_v4i64_zext:233; CHECK-SD: // %bb.0: // %entry234; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0235; CHECK-SD-NEXT: uaddlv d0, v0.4s236; CHECK-SD-NEXT: fmov x0, d0237; CHECK-SD-NEXT: ret238;239; CHECK-GI-LABEL: add_v4i16_v4i64_zext:240; CHECK-GI: // %bb.0: // %entry241; CHECK-GI-NEXT: uaddlv s0, v0.4h242; CHECK-GI-NEXT: mov w0, v0.s[0]243; CHECK-GI-NEXT: ret244entry:245 %xx = zext <4 x i16> %x to <4 x i64>246 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)247 ret i64 %z248}249 250define i64 @add_v4i16_v4i64_sext(<4 x i16> %x) {251; CHECK-SD-LABEL: add_v4i16_v4i64_sext:252; CHECK-SD: // %bb.0: // %entry253; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0254; CHECK-SD-NEXT: saddlv d0, v0.4s255; CHECK-SD-NEXT: fmov x0, d0256; CHECK-SD-NEXT: ret257;258; CHECK-GI-LABEL: add_v4i16_v4i64_sext:259; CHECK-GI: // %bb.0: // %entry260; CHECK-GI-NEXT: saddlv s0, v0.4h261; CHECK-GI-NEXT: smov x0, v0.s[0]262; CHECK-GI-NEXT: ret263entry:264 %xx = sext <4 x i16> %x to <4 x i64>265 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)266 ret i64 %z267}268 269define i64 @add_v2i16_v2i64_zext(<2 x i16> %x) {270; CHECK-SD-LABEL: add_v2i16_v2i64_zext:271; CHECK-SD: // %bb.0: // %entry272; CHECK-SD-NEXT: movi d1, #0x00ffff0000ffff273; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b274; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #0275; CHECK-SD-NEXT: addp d0, v0.2d276; CHECK-SD-NEXT: fmov x0, d0277; CHECK-SD-NEXT: ret278;279; CHECK-GI-LABEL: add_v2i16_v2i64_zext:280; CHECK-GI: // %bb.0: // %entry281; CHECK-GI-NEXT: movi v1.2d, #0x0000000000ffff282; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0283; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b284; CHECK-GI-NEXT: addp d0, v0.2d285; CHECK-GI-NEXT: fmov x0, d0286; CHECK-GI-NEXT: ret287entry:288 %xx = zext <2 x i16> %x to <2 x i64>289 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)290 ret i64 %z291}292 293define i64 @add_v2i16_v2i64_sext(<2 x i16> %x) {294; CHECK-LABEL: add_v2i16_v2i64_sext:295; CHECK: // %bb.0: // %entry296; CHECK-NEXT: ushll v0.2d, v0.2s, #0297; CHECK-NEXT: shl v0.2d, v0.2d, #48298; CHECK-NEXT: sshr v0.2d, v0.2d, #48299; CHECK-NEXT: addp d0, v0.2d300; CHECK-NEXT: fmov x0, d0301; CHECK-NEXT: ret302entry:303 %xx = sext <2 x i16> %x to <2 x i64>304 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)305 ret i64 %z306}307 308define i32 @add_v16i8_v16i32_zext(<16 x i8> %x) {309; CHECK-SD-BASE-LABEL: add_v16i8_v16i32_zext:310; CHECK-SD-BASE: // %bb.0: // %entry311; CHECK-SD-BASE-NEXT: ushll2 v1.8h, v0.16b, #0312; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0313; CHECK-SD-BASE-NEXT: uaddl2 v2.4s, v0.8h, v1.8h314; CHECK-SD-BASE-NEXT: uaddl v0.4s, v0.4h, v1.4h315; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v2.4s316; CHECK-SD-BASE-NEXT: addv s0, v0.4s317; CHECK-SD-BASE-NEXT: fmov w0, s0318; CHECK-SD-BASE-NEXT: ret319;320; CHECK-SD-DOT-LABEL: add_v16i8_v16i32_zext:321; CHECK-SD-DOT: // %bb.0: // %entry322; CHECK-SD-DOT-NEXT: movi v1.16b, #1323; CHECK-SD-DOT-NEXT: movi v2.2d, #0000000000000000324; CHECK-SD-DOT-NEXT: udot v2.4s, v0.16b, v1.16b325; CHECK-SD-DOT-NEXT: addv s0, v2.4s326; CHECK-SD-DOT-NEXT: fmov w0, s0327; CHECK-SD-DOT-NEXT: ret328;329; CHECK-GI-BASE-LABEL: add_v16i8_v16i32_zext:330; CHECK-GI-BASE: // %bb.0: // %entry331; CHECK-GI-BASE-NEXT: uaddlv h0, v0.16b332; CHECK-GI-BASE-NEXT: fmov w8, s0333; CHECK-GI-BASE-NEXT: and w0, w8, #0xffff334; CHECK-GI-BASE-NEXT: ret335;336; CHECK-GI-DOT-LABEL: add_v16i8_v16i32_zext:337; CHECK-GI-DOT: // %bb.0: // %entry338; CHECK-GI-DOT-NEXT: movi v1.16b, #1339; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000340; CHECK-GI-DOT-NEXT: udot v2.4s, v0.16b, v1.16b341; CHECK-GI-DOT-NEXT: addv s0, v2.4s342; CHECK-GI-DOT-NEXT: fmov w0, s0343; CHECK-GI-DOT-NEXT: ret344entry:345 %xx = zext <16 x i8> %x to <16 x i32>346 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx)347 ret i32 %z348}349 350define i32 @add_v16i8_v16i32_sext(<16 x i8> %x) {351; CHECK-SD-BASE-LABEL: add_v16i8_v16i32_sext:352; CHECK-SD-BASE: // %bb.0: // %entry353; CHECK-SD-BASE-NEXT: sshll2 v1.8h, v0.16b, #0354; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0355; CHECK-SD-BASE-NEXT: saddl2 v2.4s, v0.8h, v1.8h356; CHECK-SD-BASE-NEXT: saddl v0.4s, v0.4h, v1.4h357; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v2.4s358; CHECK-SD-BASE-NEXT: addv s0, v0.4s359; CHECK-SD-BASE-NEXT: fmov w0, s0360; CHECK-SD-BASE-NEXT: ret361;362; CHECK-SD-DOT-LABEL: add_v16i8_v16i32_sext:363; CHECK-SD-DOT: // %bb.0: // %entry364; CHECK-SD-DOT-NEXT: movi v1.16b, #1365; CHECK-SD-DOT-NEXT: movi v2.2d, #0000000000000000366; CHECK-SD-DOT-NEXT: sdot v2.4s, v0.16b, v1.16b367; CHECK-SD-DOT-NEXT: addv s0, v2.4s368; CHECK-SD-DOT-NEXT: fmov w0, s0369; CHECK-SD-DOT-NEXT: ret370;371; CHECK-GI-BASE-LABEL: add_v16i8_v16i32_sext:372; CHECK-GI-BASE: // %bb.0: // %entry373; CHECK-GI-BASE-NEXT: saddlv h0, v0.16b374; CHECK-GI-BASE-NEXT: fmov w8, s0375; CHECK-GI-BASE-NEXT: sxth w0, w8376; CHECK-GI-BASE-NEXT: ret377;378; CHECK-GI-DOT-LABEL: add_v16i8_v16i32_sext:379; CHECK-GI-DOT: // %bb.0: // %entry380; CHECK-GI-DOT-NEXT: movi v1.16b, #1381; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000382; CHECK-GI-DOT-NEXT: sdot v2.4s, v0.16b, v1.16b383; CHECK-GI-DOT-NEXT: addv s0, v2.4s384; CHECK-GI-DOT-NEXT: fmov w0, s0385; CHECK-GI-DOT-NEXT: ret386entry:387 %xx = sext <16 x i8> %x to <16 x i32>388 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx)389 ret i32 %z390}391 392define i32 @add_v8i8_v8i32_zext(<8 x i8> %x) {393; CHECK-SD-BASE-LABEL: add_v8i8_v8i32_zext:394; CHECK-SD-BASE: // %bb.0: // %entry395; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0396; CHECK-SD-BASE-NEXT: uaddlv s0, v0.8h397; CHECK-SD-BASE-NEXT: fmov w0, s0398; CHECK-SD-BASE-NEXT: ret399;400; CHECK-SD-DOT-LABEL: add_v8i8_v8i32_zext:401; CHECK-SD-DOT: // %bb.0: // %entry402; CHECK-SD-DOT-NEXT: movi v1.2d, #0000000000000000403; CHECK-SD-DOT-NEXT: movi v2.8b, #1404; CHECK-SD-DOT-NEXT: udot v1.2s, v0.8b, v2.8b405; CHECK-SD-DOT-NEXT: addp v0.2s, v1.2s, v1.2s406; CHECK-SD-DOT-NEXT: fmov w0, s0407; CHECK-SD-DOT-NEXT: ret408;409; CHECK-GI-BASE-LABEL: add_v8i8_v8i32_zext:410; CHECK-GI-BASE: // %bb.0: // %entry411; CHECK-GI-BASE-NEXT: uaddlv h0, v0.8b412; CHECK-GI-BASE-NEXT: fmov w8, s0413; CHECK-GI-BASE-NEXT: and w0, w8, #0xffff414; CHECK-GI-BASE-NEXT: ret415;416; CHECK-GI-DOT-LABEL: add_v8i8_v8i32_zext:417; CHECK-GI-DOT: // %bb.0: // %entry418; CHECK-GI-DOT-NEXT: movi v1.8b, #1419; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000420; CHECK-GI-DOT-NEXT: udot v2.2s, v0.8b, v1.8b421; CHECK-GI-DOT-NEXT: addp v0.2s, v2.2s, v2.2s422; CHECK-GI-DOT-NEXT: fmov w0, s0423; CHECK-GI-DOT-NEXT: ret424entry:425 %xx = zext <8 x i8> %x to <8 x i32>426 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)427 ret i32 %z428}429 430define i32 @add_v8i8_v8i32_sext(<8 x i8> %x) {431; CHECK-SD-BASE-LABEL: add_v8i8_v8i32_sext:432; CHECK-SD-BASE: // %bb.0: // %entry433; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0434; CHECK-SD-BASE-NEXT: saddlv s0, v0.8h435; CHECK-SD-BASE-NEXT: fmov w0, s0436; CHECK-SD-BASE-NEXT: ret437;438; CHECK-SD-DOT-LABEL: add_v8i8_v8i32_sext:439; CHECK-SD-DOT: // %bb.0: // %entry440; CHECK-SD-DOT-NEXT: movi v1.2d, #0000000000000000441; CHECK-SD-DOT-NEXT: movi v2.8b, #1442; CHECK-SD-DOT-NEXT: sdot v1.2s, v0.8b, v2.8b443; CHECK-SD-DOT-NEXT: addp v0.2s, v1.2s, v1.2s444; CHECK-SD-DOT-NEXT: fmov w0, s0445; CHECK-SD-DOT-NEXT: ret446;447; CHECK-GI-BASE-LABEL: add_v8i8_v8i32_sext:448; CHECK-GI-BASE: // %bb.0: // %entry449; CHECK-GI-BASE-NEXT: saddlv h0, v0.8b450; CHECK-GI-BASE-NEXT: fmov w8, s0451; CHECK-GI-BASE-NEXT: sxth w0, w8452; CHECK-GI-BASE-NEXT: ret453;454; CHECK-GI-DOT-LABEL: add_v8i8_v8i32_sext:455; CHECK-GI-DOT: // %bb.0: // %entry456; CHECK-GI-DOT-NEXT: movi v1.8b, #1457; CHECK-GI-DOT-NEXT: movi v2.2d, #0000000000000000458; CHECK-GI-DOT-NEXT: sdot v2.2s, v0.8b, v1.8b459; CHECK-GI-DOT-NEXT: addp v0.2s, v2.2s, v2.2s460; CHECK-GI-DOT-NEXT: fmov w0, s0461; CHECK-GI-DOT-NEXT: ret462entry:463 %xx = sext <8 x i8> %x to <8 x i32>464 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)465 ret i32 %z466}467 468define i32 @add_v4i8_v4i32_zext(<4 x i8> %x) {469; CHECK-SD-LABEL: add_v4i8_v4i32_zext:470; CHECK-SD: // %bb.0: // %entry471; CHECK-SD-NEXT: bic v0.4h, #255, lsl #8472; CHECK-SD-NEXT: uaddlv s0, v0.4h473; CHECK-SD-NEXT: fmov w0, s0474; CHECK-SD-NEXT: ret475;476; CHECK-GI-LABEL: add_v4i8_v4i32_zext:477; CHECK-GI: // %bb.0: // %entry478; CHECK-GI-NEXT: movi d1, #0xff00ff00ff00ff479; CHECK-GI-NEXT: and v0.8b, v0.8b, v1.8b480; CHECK-GI-NEXT: uaddlv s0, v0.4h481; CHECK-GI-NEXT: fmov w8, s0482; CHECK-GI-NEXT: and w0, w8, #0xffff483; CHECK-GI-NEXT: ret484entry:485 %xx = zext <4 x i8> %x to <4 x i32>486 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)487 ret i32 %z488}489 490define i32 @add_v4i8_v4i32_sext(<4 x i8> %x) {491; CHECK-SD-LABEL: add_v4i8_v4i32_sext:492; CHECK-SD: // %bb.0: // %entry493; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0494; CHECK-SD-NEXT: shl v0.4s, v0.4s, #24495; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #24496; CHECK-SD-NEXT: addv s0, v0.4s497; CHECK-SD-NEXT: fmov w0, s0498; CHECK-SD-NEXT: ret499;500; CHECK-GI-LABEL: add_v4i8_v4i32_sext:501; CHECK-GI: // %bb.0: // %entry502; CHECK-GI-NEXT: shl v0.4h, v0.4h, #8503; CHECK-GI-NEXT: sshr v0.4h, v0.4h, #8504; CHECK-GI-NEXT: saddlv s0, v0.4h505; CHECK-GI-NEXT: fmov w8, s0506; CHECK-GI-NEXT: sxth w0, w8507; CHECK-GI-NEXT: ret508entry:509 %xx = sext <4 x i8> %x to <4 x i32>510 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)511 ret i32 %z512}513 514define zeroext i16 @add_v16i8_v16i16_zext(<16 x i8> %x) {515; CHECK-SD-LABEL: add_v16i8_v16i16_zext:516; CHECK-SD: // %bb.0: // %entry517; CHECK-SD-NEXT: uaddlp v0.8h, v0.16b518; CHECK-SD-NEXT: addv h0, v0.8h519; CHECK-SD-NEXT: fmov w0, s0520; CHECK-SD-NEXT: ret521;522; CHECK-GI-LABEL: add_v16i8_v16i16_zext:523; CHECK-GI: // %bb.0: // %entry524; CHECK-GI-NEXT: uaddlv h0, v0.16b525; CHECK-GI-NEXT: fmov w8, s0526; CHECK-GI-NEXT: and w0, w8, #0xffff527; CHECK-GI-NEXT: ret528entry:529 %xx = zext <16 x i8> %x to <16 x i16>530 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)531 ret i16 %z532}533 534define signext i16 @add_v16i8_v16i16_sext(<16 x i8> %x) {535; CHECK-SD-LABEL: add_v16i8_v16i16_sext:536; CHECK-SD: // %bb.0: // %entry537; CHECK-SD-NEXT: saddlp v0.8h, v0.16b538; CHECK-SD-NEXT: addv h0, v0.8h539; CHECK-SD-NEXT: smov w0, v0.h[0]540; CHECK-SD-NEXT: ret541;542; CHECK-GI-LABEL: add_v16i8_v16i16_sext:543; CHECK-GI: // %bb.0: // %entry544; CHECK-GI-NEXT: saddlv h0, v0.16b545; CHECK-GI-NEXT: fmov w8, s0546; CHECK-GI-NEXT: sxth w0, w8547; CHECK-GI-NEXT: ret548entry:549 %xx = sext <16 x i8> %x to <16 x i16>550 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)551 ret i16 %z552}553 554define zeroext i16 @add_v8i8_v8i16_zext(<8 x i8> %x) {555; CHECK-SD-LABEL: add_v8i8_v8i16_zext:556; CHECK-SD: // %bb.0: // %entry557; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0558; CHECK-SD-NEXT: addv h0, v0.8h559; CHECK-SD-NEXT: fmov w0, s0560; CHECK-SD-NEXT: ret561;562; CHECK-GI-LABEL: add_v8i8_v8i16_zext:563; CHECK-GI: // %bb.0: // %entry564; CHECK-GI-NEXT: uaddlv h0, v0.8b565; CHECK-GI-NEXT: fmov w8, s0566; CHECK-GI-NEXT: and w0, w8, #0xffff567; CHECK-GI-NEXT: ret568entry:569 %xx = zext <8 x i8> %x to <8 x i16>570 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)571 ret i16 %z572}573 574define signext i16 @add_v8i8_v8i16_sext(<8 x i8> %x) {575; CHECK-SD-LABEL: add_v8i8_v8i16_sext:576; CHECK-SD: // %bb.0: // %entry577; CHECK-SD-NEXT: saddlv h0, v0.8b578; CHECK-SD-NEXT: smov w0, v0.h[0]579; CHECK-SD-NEXT: ret580;581; CHECK-GI-LABEL: add_v8i8_v8i16_sext:582; CHECK-GI: // %bb.0: // %entry583; CHECK-GI-NEXT: saddlv h0, v0.8b584; CHECK-GI-NEXT: fmov w8, s0585; CHECK-GI-NEXT: sxth w0, w8586; CHECK-GI-NEXT: ret587entry:588 %xx = sext <8 x i8> %x to <8 x i16>589 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)590 ret i16 %z591}592 593define zeroext i8 @add_v16i8_v16i8(<16 x i8> %x) {594; CHECK-SD-LABEL: add_v16i8_v16i8:595; CHECK-SD: // %bb.0: // %entry596; CHECK-SD-NEXT: addv b0, v0.16b597; CHECK-SD-NEXT: fmov w0, s0598; CHECK-SD-NEXT: ret599;600; CHECK-GI-LABEL: add_v16i8_v16i8:601; CHECK-GI: // %bb.0: // %entry602; CHECK-GI-NEXT: addv b0, v0.16b603; CHECK-GI-NEXT: fmov w8, s0604; CHECK-GI-NEXT: uxtb w0, w8605; CHECK-GI-NEXT: ret606entry:607 %z = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %x)608 ret i8 %z609}610 611define i64 @add_v16i8_v16i64_zext(<16 x i8> %x) {612; CHECK-SD-LABEL: add_v16i8_v16i64_zext:613; CHECK-SD: // %bb.0: // %entry614; CHECK-SD-NEXT: ushll2 v1.8h, v0.16b, #0615; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0616; CHECK-SD-NEXT: ushll2 v2.4s, v1.8h, #0617; CHECK-SD-NEXT: ushll2 v3.4s, v0.8h, #0618; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0619; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0620; CHECK-SD-NEXT: uaddl2 v4.2d, v3.4s, v2.4s621; CHECK-SD-NEXT: uaddl v2.2d, v3.2s, v2.2s622; CHECK-SD-NEXT: uaddl2 v5.2d, v0.4s, v1.4s623; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s624; CHECK-SD-NEXT: add v1.2d, v5.2d, v4.2d625; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d626; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d627; CHECK-SD-NEXT: addp d0, v0.2d628; CHECK-SD-NEXT: fmov x0, d0629; CHECK-SD-NEXT: ret630;631; CHECK-GI-LABEL: add_v16i8_v16i64_zext:632; CHECK-GI: // %bb.0: // %entry633; CHECK-GI-NEXT: uaddlv h0, v0.16b634; CHECK-GI-NEXT: fmov w8, s0635; CHECK-GI-NEXT: and x0, x8, #0xffff636; CHECK-GI-NEXT: ret637entry:638 %xx = zext <16 x i8> %x to <16 x i64>639 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx)640 ret i64 %z641}642 643define i64 @add_v16i8_v16i64_sext(<16 x i8> %x) {644; CHECK-SD-LABEL: add_v16i8_v16i64_sext:645; CHECK-SD: // %bb.0: // %entry646; CHECK-SD-NEXT: sshll2 v1.8h, v0.16b, #0647; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0648; CHECK-SD-NEXT: sshll2 v2.4s, v1.8h, #0649; CHECK-SD-NEXT: sshll2 v3.4s, v0.8h, #0650; CHECK-SD-NEXT: sshll v1.4s, v1.4h, #0651; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0652; CHECK-SD-NEXT: saddl2 v4.2d, v3.4s, v2.4s653; CHECK-SD-NEXT: saddl v2.2d, v3.2s, v2.2s654; CHECK-SD-NEXT: saddl2 v5.2d, v0.4s, v1.4s655; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v1.2s656; CHECK-SD-NEXT: add v1.2d, v5.2d, v4.2d657; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d658; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d659; CHECK-SD-NEXT: addp d0, v0.2d660; CHECK-SD-NEXT: fmov x0, d0661; CHECK-SD-NEXT: ret662;663; CHECK-GI-LABEL: add_v16i8_v16i64_sext:664; CHECK-GI: // %bb.0: // %entry665; CHECK-GI-NEXT: saddlv h0, v0.16b666; CHECK-GI-NEXT: fmov w8, s0667; CHECK-GI-NEXT: sxth x0, w8668; CHECK-GI-NEXT: ret669entry:670 %xx = sext <16 x i8> %x to <16 x i64>671 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx)672 ret i64 %z673}674 675define i64 @add_v8i8_v8i64_zext(<8 x i8> %x) {676; CHECK-SD-LABEL: add_v8i8_v8i64_zext:677; CHECK-SD: // %bb.0: // %entry678; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0679; CHECK-SD-NEXT: ushll2 v1.4s, v0.8h, #0680; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0681; CHECK-SD-NEXT: uaddl2 v2.2d, v0.4s, v1.4s682; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s683; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d684; CHECK-SD-NEXT: addp d0, v0.2d685; CHECK-SD-NEXT: fmov x0, d0686; CHECK-SD-NEXT: ret687;688; CHECK-GI-LABEL: add_v8i8_v8i64_zext:689; CHECK-GI: // %bb.0: // %entry690; CHECK-GI-NEXT: uaddlv h0, v0.8b691; CHECK-GI-NEXT: fmov w8, s0692; CHECK-GI-NEXT: and x0, x8, #0xffff693; CHECK-GI-NEXT: ret694entry:695 %xx = zext <8 x i8> %x to <8 x i64>696 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)697 ret i64 %z698}699 700define i64 @add_v8i8_v8i64_sext(<8 x i8> %x) {701; CHECK-SD-LABEL: add_v8i8_v8i64_sext:702; CHECK-SD: // %bb.0: // %entry703; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0704; CHECK-SD-NEXT: sshll2 v1.4s, v0.8h, #0705; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0706; CHECK-SD-NEXT: saddl2 v2.2d, v0.4s, v1.4s707; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v1.2s708; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d709; CHECK-SD-NEXT: addp d0, v0.2d710; CHECK-SD-NEXT: fmov x0, d0711; CHECK-SD-NEXT: ret712;713; CHECK-GI-LABEL: add_v8i8_v8i64_sext:714; CHECK-GI: // %bb.0: // %entry715; CHECK-GI-NEXT: saddlv h0, v0.8b716; CHECK-GI-NEXT: fmov w8, s0717; CHECK-GI-NEXT: sxth x0, w8718; CHECK-GI-NEXT: ret719entry:720 %xx = sext <8 x i8> %x to <8 x i64>721 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)722 ret i64 %z723}724 725define i64 @add_v4i8_v4i64_zext(<4 x i8> %x) {726; CHECK-SD-LABEL: add_v4i8_v4i64_zext:727; CHECK-SD: // %bb.0: // %entry728; CHECK-SD-NEXT: bic v0.4h, #255, lsl #8729; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0730; CHECK-SD-NEXT: uaddlv d0, v0.4s731; CHECK-SD-NEXT: fmov x0, d0732; CHECK-SD-NEXT: ret733;734; CHECK-GI-LABEL: add_v4i8_v4i64_zext:735; CHECK-GI: // %bb.0: // %entry736; CHECK-GI-NEXT: movi d1, #0xff00ff00ff00ff737; CHECK-GI-NEXT: and v0.8b, v0.8b, v1.8b738; CHECK-GI-NEXT: uaddlv s0, v0.4h739; CHECK-GI-NEXT: fmov w8, s0740; CHECK-GI-NEXT: and x0, x8, #0xffff741; CHECK-GI-NEXT: ret742entry:743 %xx = zext <4 x i8> %x to <4 x i64>744 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)745 ret i64 %z746}747 748define i64 @add_v4i8_v4i64_sext(<4 x i8> %x) {749; CHECK-SD-LABEL: add_v4i8_v4i64_sext:750; CHECK-SD: // %bb.0: // %entry751; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0752; CHECK-SD-NEXT: ushll v1.2d, v0.2s, #0753; CHECK-SD-NEXT: ushll2 v0.2d, v0.4s, #0754; CHECK-SD-NEXT: shl v1.2d, v1.2d, #56755; CHECK-SD-NEXT: shl v0.2d, v0.2d, #56756; CHECK-SD-NEXT: sshr v1.2d, v1.2d, #56757; CHECK-SD-NEXT: ssra v1.2d, v0.2d, #56758; CHECK-SD-NEXT: addp d0, v1.2d759; CHECK-SD-NEXT: fmov x0, d0760; CHECK-SD-NEXT: ret761;762; CHECK-GI-LABEL: add_v4i8_v4i64_sext:763; CHECK-GI: // %bb.0: // %entry764; CHECK-GI-NEXT: shl v0.4h, v0.4h, #8765; CHECK-GI-NEXT: sshr v0.4h, v0.4h, #8766; CHECK-GI-NEXT: saddlv s0, v0.4h767; CHECK-GI-NEXT: fmov w8, s0768; CHECK-GI-NEXT: sxth x0, w8769; CHECK-GI-NEXT: ret770entry:771 %xx = sext <4 x i8> %x to <4 x i64>772 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)773 ret i64 %z774}775 776define i64 @add_v2i8_v2i64_zext(<2 x i8> %x) {777; CHECK-SD-LABEL: add_v2i8_v2i64_zext:778; CHECK-SD: // %bb.0: // %entry779; CHECK-SD-NEXT: movi d1, #0x0000ff000000ff780; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b781; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #0782; CHECK-SD-NEXT: addp d0, v0.2d783; CHECK-SD-NEXT: fmov x0, d0784; CHECK-SD-NEXT: ret785;786; CHECK-GI-LABEL: add_v2i8_v2i64_zext:787; CHECK-GI: // %bb.0: // %entry788; CHECK-GI-NEXT: movi v1.2d, #0x000000000000ff789; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #0790; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b791; CHECK-GI-NEXT: addp d0, v0.2d792; CHECK-GI-NEXT: fmov x0, d0793; CHECK-GI-NEXT: ret794entry:795 %xx = zext <2 x i8> %x to <2 x i64>796 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)797 ret i64 %z798}799 800define i64 @add_v2i8_v2i64_sext(<2 x i8> %x) {801; CHECK-LABEL: add_v2i8_v2i64_sext:802; CHECK: // %bb.0: // %entry803; CHECK-NEXT: ushll v0.2d, v0.2s, #0804; CHECK-NEXT: shl v0.2d, v0.2d, #56805; CHECK-NEXT: sshr v0.2d, v0.2d, #56806; CHECK-NEXT: addp d0, v0.2d807; CHECK-NEXT: fmov x0, d0808; CHECK-NEXT: ret809entry:810 %xx = sext <2 x i8> %x to <2 x i64>811 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)812 ret i64 %z813}814 815define i64 @add_v2i64_v2i64(<2 x i64> %x) {816; CHECK-LABEL: add_v2i64_v2i64:817; CHECK: // %bb.0: // %entry818; CHECK-NEXT: addp d0, v0.2d819; CHECK-NEXT: fmov x0, d0820; CHECK-NEXT: ret821entry:822 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %x)823 ret i64 %z824}825 826define i32 @add_v4i32_v4i32_acc(<4 x i32> %x, i32 %a) {827; CHECK-LABEL: add_v4i32_v4i32_acc:828; CHECK: // %bb.0: // %entry829; CHECK-NEXT: addv s0, v0.4s830; CHECK-NEXT: fmov w8, s0831; CHECK-NEXT: add w0, w8, w0832; CHECK-NEXT: ret833entry:834 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %x)835 %r = add i32 %z, %a836 ret i32 %r837}838 839define i64 @add_v4i32_v4i64_acc_zext(<4 x i32> %x, i64 %a) {840; CHECK-LABEL: add_v4i32_v4i64_acc_zext:841; CHECK: // %bb.0: // %entry842; CHECK-NEXT: uaddlv d0, v0.4s843; CHECK-NEXT: fmov x8, d0844; CHECK-NEXT: add x0, x8, x0845; CHECK-NEXT: ret846entry:847 %xx = zext <4 x i32> %x to <4 x i64>848 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)849 %r = add i64 %z, %a850 ret i64 %r851}852 853define i64 @add_v4i32_v4i64_acc_sext(<4 x i32> %x, i64 %a) {854; CHECK-LABEL: add_v4i32_v4i64_acc_sext:855; CHECK: // %bb.0: // %entry856; CHECK-NEXT: saddlv d0, v0.4s857; CHECK-NEXT: fmov x8, d0858; CHECK-NEXT: add x0, x8, x0859; CHECK-NEXT: ret860entry:861 %xx = sext <4 x i32> %x to <4 x i64>862 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)863 %r = add i64 %z, %a864 ret i64 %r865}866 867define i64 @add_v2i32_v2i64_acc_zext(<2 x i32> %x, i64 %a) {868; CHECK-LABEL: add_v2i32_v2i64_acc_zext:869; CHECK: // %bb.0: // %entry870; CHECK-NEXT: ushll v0.2d, v0.2s, #0871; CHECK-NEXT: addp d0, v0.2d872; CHECK-NEXT: fmov x8, d0873; CHECK-NEXT: add x0, x8, x0874; CHECK-NEXT: ret875entry:876 %xx = zext <2 x i32> %x to <2 x i64>877 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)878 %r = add i64 %z, %a879 ret i64 %r880}881 882define i64 @add_v2i32_v2i64_acc_sext(<2 x i32> %x, i64 %a) {883; CHECK-LABEL: add_v2i32_v2i64_acc_sext:884; CHECK: // %bb.0: // %entry885; CHECK-NEXT: sshll v0.2d, v0.2s, #0886; CHECK-NEXT: addp d0, v0.2d887; CHECK-NEXT: fmov x8, d0888; CHECK-NEXT: add x0, x8, x0889; CHECK-NEXT: ret890entry:891 %xx = sext <2 x i32> %x to <2 x i64>892 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)893 %r = add i64 %z, %a894 ret i64 %r895}896 897define i32 @add_v8i16_v8i32_acc_zext(<8 x i16> %x, i32 %a) {898; CHECK-LABEL: add_v8i16_v8i32_acc_zext:899; CHECK: // %bb.0: // %entry900; CHECK-NEXT: uaddlv s0, v0.8h901; CHECK-NEXT: fmov w8, s0902; CHECK-NEXT: add w0, w8, w0903; CHECK-NEXT: ret904entry:905 %xx = zext <8 x i16> %x to <8 x i32>906 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)907 %r = add i32 %z, %a908 ret i32 %r909}910 911define i32 @add_v8i16_v8i32_acc_sext(<8 x i16> %x, i32 %a) {912; CHECK-LABEL: add_v8i16_v8i32_acc_sext:913; CHECK: // %bb.0: // %entry914; CHECK-NEXT: saddlv s0, v0.8h915; CHECK-NEXT: fmov w8, s0916; CHECK-NEXT: add w0, w8, w0917; CHECK-NEXT: ret918entry:919 %xx = sext <8 x i16> %x to <8 x i32>920 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)921 %r = add i32 %z, %a922 ret i32 %r923}924 925define i32 @add_v4i16_v4i32_acc_zext(<4 x i16> %x, i32 %a) {926; CHECK-LABEL: add_v4i16_v4i32_acc_zext:927; CHECK: // %bb.0: // %entry928; CHECK-NEXT: uaddlv s0, v0.4h929; CHECK-NEXT: fmov w8, s0930; CHECK-NEXT: add w0, w8, w0931; CHECK-NEXT: ret932entry:933 %xx = zext <4 x i16> %x to <4 x i32>934 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)935 %r = add i32 %z, %a936 ret i32 %r937}938 939define i32 @add_v4i16_v4i32_acc_sext(<4 x i16> %x, i32 %a) {940; CHECK-LABEL: add_v4i16_v4i32_acc_sext:941; CHECK: // %bb.0: // %entry942; CHECK-NEXT: saddlv s0, v0.4h943; CHECK-NEXT: fmov w8, s0944; CHECK-NEXT: add w0, w8, w0945; CHECK-NEXT: ret946entry:947 %xx = sext <4 x i16> %x to <4 x i32>948 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)949 %r = add i32 %z, %a950 ret i32 %r951}952 953define zeroext i16 @add_v8i16_v8i16_acc(<8 x i16> %x, i16 %a) {954; CHECK-SD-LABEL: add_v8i16_v8i16_acc:955; CHECK-SD: // %bb.0: // %entry956; CHECK-SD-NEXT: addv h0, v0.8h957; CHECK-SD-NEXT: fmov w8, s0958; CHECK-SD-NEXT: add w8, w8, w0959; CHECK-SD-NEXT: and w0, w8, #0xffff960; CHECK-SD-NEXT: ret961;962; CHECK-GI-LABEL: add_v8i16_v8i16_acc:963; CHECK-GI: // %bb.0: // %entry964; CHECK-GI-NEXT: addv h0, v0.8h965; CHECK-GI-NEXT: fmov w8, s0966; CHECK-GI-NEXT: add w8, w0, w8, uxth967; CHECK-GI-NEXT: and w0, w8, #0xffff968; CHECK-GI-NEXT: ret969entry:970 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %x)971 %r = add i16 %z, %a972 ret i16 %r973}974 975define i64 @add_v8i16_v8i64_acc_zext(<8 x i16> %x, i64 %a) {976; CHECK-SD-LABEL: add_v8i16_v8i64_acc_zext:977; CHECK-SD: // %bb.0: // %entry978; CHECK-SD-NEXT: ushll2 v1.4s, v0.8h, #0979; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0980; CHECK-SD-NEXT: uaddl2 v2.2d, v0.4s, v1.4s981; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s982; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d983; CHECK-SD-NEXT: addp d0, v0.2d984; CHECK-SD-NEXT: fmov x8, d0985; CHECK-SD-NEXT: add x0, x8, x0986; CHECK-SD-NEXT: ret987;988; CHECK-GI-LABEL: add_v8i16_v8i64_acc_zext:989; CHECK-GI: // %bb.0: // %entry990; CHECK-GI-NEXT: uaddlv s0, v0.8h991; CHECK-GI-NEXT: fmov w8, s0992; CHECK-GI-NEXT: add x0, x0, w8, uxtw993; CHECK-GI-NEXT: ret994entry:995 %xx = zext <8 x i16> %x to <8 x i64>996 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)997 %r = add i64 %z, %a998 ret i64 %r999}1000 1001define i64 @add_v8i16_v8i64_acc_sext(<8 x i16> %x, i64 %a) {1002; CHECK-SD-LABEL: add_v8i16_v8i64_acc_sext:1003; CHECK-SD: // %bb.0: // %entry1004; CHECK-SD-NEXT: sshll2 v1.4s, v0.8h, #01005; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #01006; CHECK-SD-NEXT: saddl2 v2.2d, v0.4s, v1.4s1007; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v1.2s1008; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d1009; CHECK-SD-NEXT: addp d0, v0.2d1010; CHECK-SD-NEXT: fmov x8, d01011; CHECK-SD-NEXT: add x0, x8, x01012; CHECK-SD-NEXT: ret1013;1014; CHECK-GI-LABEL: add_v8i16_v8i64_acc_sext:1015; CHECK-GI: // %bb.0: // %entry1016; CHECK-GI-NEXT: saddlv s0, v0.8h1017; CHECK-GI-NEXT: fmov w8, s01018; CHECK-GI-NEXT: add x0, x0, w8, sxtw1019; CHECK-GI-NEXT: ret1020entry:1021 %xx = sext <8 x i16> %x to <8 x i64>1022 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)1023 %r = add i64 %z, %a1024 ret i64 %r1025}1026 1027define i64 @add_v4i16_v4i64_acc_zext(<4 x i16> %x, i64 %a) {1028; CHECK-SD-LABEL: add_v4i16_v4i64_acc_zext:1029; CHECK-SD: // %bb.0: // %entry1030; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #01031; CHECK-SD-NEXT: uaddlv d0, v0.4s1032; CHECK-SD-NEXT: fmov x8, d01033; CHECK-SD-NEXT: add x0, x8, x01034; CHECK-SD-NEXT: ret1035;1036; CHECK-GI-LABEL: add_v4i16_v4i64_acc_zext:1037; CHECK-GI: // %bb.0: // %entry1038; CHECK-GI-NEXT: uaddlv s0, v0.4h1039; CHECK-GI-NEXT: fmov w8, s01040; CHECK-GI-NEXT: add x0, x0, w8, uxtw1041; CHECK-GI-NEXT: ret1042entry:1043 %xx = zext <4 x i16> %x to <4 x i64>1044 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)1045 %r = add i64 %z, %a1046 ret i64 %r1047}1048 1049define i64 @add_v4i16_v4i64_acc_sext(<4 x i16> %x, i64 %a) {1050; CHECK-SD-LABEL: add_v4i16_v4i64_acc_sext:1051; CHECK-SD: // %bb.0: // %entry1052; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #01053; CHECK-SD-NEXT: saddlv d0, v0.4s1054; CHECK-SD-NEXT: fmov x8, d01055; CHECK-SD-NEXT: add x0, x8, x01056; CHECK-SD-NEXT: ret1057;1058; CHECK-GI-LABEL: add_v4i16_v4i64_acc_sext:1059; CHECK-GI: // %bb.0: // %entry1060; CHECK-GI-NEXT: saddlv s0, v0.4h1061; CHECK-GI-NEXT: fmov w8, s01062; CHECK-GI-NEXT: add x0, x0, w8, sxtw1063; CHECK-GI-NEXT: ret1064entry:1065 %xx = sext <4 x i16> %x to <4 x i64>1066 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)1067 %r = add i64 %z, %a1068 ret i64 %r1069}1070 1071define i64 @add_v2i16_v2i64_acc_zext(<2 x i16> %x, i64 %a) {1072; CHECK-SD-LABEL: add_v2i16_v2i64_acc_zext:1073; CHECK-SD: // %bb.0: // %entry1074; CHECK-SD-NEXT: movi d1, #0x00ffff0000ffff1075; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b1076; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #01077; CHECK-SD-NEXT: addp d0, v0.2d1078; CHECK-SD-NEXT: fmov x8, d01079; CHECK-SD-NEXT: add x0, x8, x01080; CHECK-SD-NEXT: ret1081;1082; CHECK-GI-LABEL: add_v2i16_v2i64_acc_zext:1083; CHECK-GI: // %bb.0: // %entry1084; CHECK-GI-NEXT: movi v1.2d, #0x0000000000ffff1085; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #01086; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b1087; CHECK-GI-NEXT: addp d0, v0.2d1088; CHECK-GI-NEXT: fmov x8, d01089; CHECK-GI-NEXT: add x0, x8, x01090; CHECK-GI-NEXT: ret1091entry:1092 %xx = zext <2 x i16> %x to <2 x i64>1093 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)1094 %r = add i64 %z, %a1095 ret i64 %r1096}1097 1098define i64 @add_v2i16_v2i64_acc_sext(<2 x i16> %x, i64 %a) {1099; CHECK-LABEL: add_v2i16_v2i64_acc_sext:1100; CHECK: // %bb.0: // %entry1101; CHECK-NEXT: ushll v0.2d, v0.2s, #01102; CHECK-NEXT: shl v0.2d, v0.2d, #481103; CHECK-NEXT: sshr v0.2d, v0.2d, #481104; CHECK-NEXT: addp d0, v0.2d1105; CHECK-NEXT: fmov x8, d01106; CHECK-NEXT: add x0, x8, x01107; CHECK-NEXT: ret1108entry:1109 %xx = sext <2 x i16> %x to <2 x i64>1110 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)1111 %r = add i64 %z, %a1112 ret i64 %r1113}1114 1115define i32 @add_v16i8_v16i32_acc_zext(<16 x i8> %x, i32 %a) {1116; CHECK-SD-BASE-LABEL: add_v16i8_v16i32_acc_zext:1117; CHECK-SD-BASE: // %bb.0: // %entry1118; CHECK-SD-BASE-NEXT: ushll2 v1.8h, v0.16b, #01119; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #01120; CHECK-SD-BASE-NEXT: uaddl2 v2.4s, v0.8h, v1.8h1121; CHECK-SD-BASE-NEXT: uaddl v0.4s, v0.4h, v1.4h1122; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v2.4s1123; CHECK-SD-BASE-NEXT: addv s0, v0.4s1124; CHECK-SD-BASE-NEXT: fmov w8, s01125; CHECK-SD-BASE-NEXT: add w0, w8, w01126; CHECK-SD-BASE-NEXT: ret1127;1128; CHECK-SD-DOT-LABEL: add_v16i8_v16i32_acc_zext:1129; CHECK-SD-DOT: // %bb.0: // %entry1130; CHECK-SD-DOT-NEXT: movi v1.16b, #11131; CHECK-SD-DOT-NEXT: movi v2.2d, #00000000000000001132; CHECK-SD-DOT-NEXT: udot v2.4s, v0.16b, v1.16b1133; CHECK-SD-DOT-NEXT: addv s0, v2.4s1134; CHECK-SD-DOT-NEXT: fmov w8, s01135; CHECK-SD-DOT-NEXT: add w0, w8, w01136; CHECK-SD-DOT-NEXT: ret1137;1138; CHECK-GI-BASE-LABEL: add_v16i8_v16i32_acc_zext:1139; CHECK-GI-BASE: // %bb.0: // %entry1140; CHECK-GI-BASE-NEXT: uaddlv h0, v0.16b1141; CHECK-GI-BASE-NEXT: fmov w8, s01142; CHECK-GI-BASE-NEXT: add w0, w0, w8, uxth1143; CHECK-GI-BASE-NEXT: ret1144;1145; CHECK-GI-DOT-LABEL: add_v16i8_v16i32_acc_zext:1146; CHECK-GI-DOT: // %bb.0: // %entry1147; CHECK-GI-DOT-NEXT: movi v1.16b, #11148; CHECK-GI-DOT-NEXT: movi v2.2d, #00000000000000001149; CHECK-GI-DOT-NEXT: udot v2.4s, v0.16b, v1.16b1150; CHECK-GI-DOT-NEXT: addv s0, v2.4s1151; CHECK-GI-DOT-NEXT: fmov w8, s01152; CHECK-GI-DOT-NEXT: add w0, w8, w01153; CHECK-GI-DOT-NEXT: ret1154entry:1155 %xx = zext <16 x i8> %x to <16 x i32>1156 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx)1157 %r = add i32 %z, %a1158 ret i32 %r1159}1160 1161define i32 @add_v16i8_v16i32_acc_sext(<16 x i8> %x, i32 %a) {1162; CHECK-SD-BASE-LABEL: add_v16i8_v16i32_acc_sext:1163; CHECK-SD-BASE: // %bb.0: // %entry1164; CHECK-SD-BASE-NEXT: sshll2 v1.8h, v0.16b, #01165; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #01166; CHECK-SD-BASE-NEXT: saddl2 v2.4s, v0.8h, v1.8h1167; CHECK-SD-BASE-NEXT: saddl v0.4s, v0.4h, v1.4h1168; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v2.4s1169; CHECK-SD-BASE-NEXT: addv s0, v0.4s1170; CHECK-SD-BASE-NEXT: fmov w8, s01171; CHECK-SD-BASE-NEXT: add w0, w8, w01172; CHECK-SD-BASE-NEXT: ret1173;1174; CHECK-SD-DOT-LABEL: add_v16i8_v16i32_acc_sext:1175; CHECK-SD-DOT: // %bb.0: // %entry1176; CHECK-SD-DOT-NEXT: movi v1.16b, #11177; CHECK-SD-DOT-NEXT: movi v2.2d, #00000000000000001178; CHECK-SD-DOT-NEXT: sdot v2.4s, v0.16b, v1.16b1179; CHECK-SD-DOT-NEXT: addv s0, v2.4s1180; CHECK-SD-DOT-NEXT: fmov w8, s01181; CHECK-SD-DOT-NEXT: add w0, w8, w01182; CHECK-SD-DOT-NEXT: ret1183;1184; CHECK-GI-BASE-LABEL: add_v16i8_v16i32_acc_sext:1185; CHECK-GI-BASE: // %bb.0: // %entry1186; CHECK-GI-BASE-NEXT: saddlv h0, v0.16b1187; CHECK-GI-BASE-NEXT: fmov w8, s01188; CHECK-GI-BASE-NEXT: add w0, w0, w8, sxth1189; CHECK-GI-BASE-NEXT: ret1190;1191; CHECK-GI-DOT-LABEL: add_v16i8_v16i32_acc_sext:1192; CHECK-GI-DOT: // %bb.0: // %entry1193; CHECK-GI-DOT-NEXT: movi v1.16b, #11194; CHECK-GI-DOT-NEXT: movi v2.2d, #00000000000000001195; CHECK-GI-DOT-NEXT: sdot v2.4s, v0.16b, v1.16b1196; CHECK-GI-DOT-NEXT: addv s0, v2.4s1197; CHECK-GI-DOT-NEXT: fmov w8, s01198; CHECK-GI-DOT-NEXT: add w0, w8, w01199; CHECK-GI-DOT-NEXT: ret1200entry:1201 %xx = sext <16 x i8> %x to <16 x i32>1202 %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx)1203 %r = add i32 %z, %a1204 ret i32 %r1205}1206 1207define i32 @add_v8i8_v8i32_acc_zext(<8 x i8> %x, i32 %a) {1208; CHECK-SD-BASE-LABEL: add_v8i8_v8i32_acc_zext:1209; CHECK-SD-BASE: // %bb.0: // %entry1210; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #01211; CHECK-SD-BASE-NEXT: uaddlv s0, v0.8h1212; CHECK-SD-BASE-NEXT: fmov w8, s01213; CHECK-SD-BASE-NEXT: add w0, w8, w01214; CHECK-SD-BASE-NEXT: ret1215;1216; CHECK-SD-DOT-LABEL: add_v8i8_v8i32_acc_zext:1217; CHECK-SD-DOT: // %bb.0: // %entry1218; CHECK-SD-DOT-NEXT: movi v1.2d, #00000000000000001219; CHECK-SD-DOT-NEXT: movi v2.8b, #11220; CHECK-SD-DOT-NEXT: udot v1.2s, v0.8b, v2.8b1221; CHECK-SD-DOT-NEXT: addp v0.2s, v1.2s, v1.2s1222; CHECK-SD-DOT-NEXT: fmov w8, s01223; CHECK-SD-DOT-NEXT: add w0, w8, w01224; CHECK-SD-DOT-NEXT: ret1225;1226; CHECK-GI-BASE-LABEL: add_v8i8_v8i32_acc_zext:1227; CHECK-GI-BASE: // %bb.0: // %entry1228; CHECK-GI-BASE-NEXT: uaddlv h0, v0.8b1229; CHECK-GI-BASE-NEXT: fmov w8, s01230; CHECK-GI-BASE-NEXT: add w0, w0, w8, uxth1231; CHECK-GI-BASE-NEXT: ret1232;1233; CHECK-GI-DOT-LABEL: add_v8i8_v8i32_acc_zext:1234; CHECK-GI-DOT: // %bb.0: // %entry1235; CHECK-GI-DOT-NEXT: movi v1.8b, #11236; CHECK-GI-DOT-NEXT: movi v2.2d, #00000000000000001237; CHECK-GI-DOT-NEXT: udot v2.2s, v0.8b, v1.8b1238; CHECK-GI-DOT-NEXT: addp v0.2s, v2.2s, v2.2s1239; CHECK-GI-DOT-NEXT: fmov w8, s01240; CHECK-GI-DOT-NEXT: add w0, w8, w01241; CHECK-GI-DOT-NEXT: ret1242entry:1243 %xx = zext <8 x i8> %x to <8 x i32>1244 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)1245 %r = add i32 %z, %a1246 ret i32 %r1247}1248 1249define i32 @add_v8i8_v8i32_acc_sext(<8 x i8> %x, i32 %a) {1250; CHECK-SD-BASE-LABEL: add_v8i8_v8i32_acc_sext:1251; CHECK-SD-BASE: // %bb.0: // %entry1252; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #01253; CHECK-SD-BASE-NEXT: saddlv s0, v0.8h1254; CHECK-SD-BASE-NEXT: fmov w8, s01255; CHECK-SD-BASE-NEXT: add w0, w8, w01256; CHECK-SD-BASE-NEXT: ret1257;1258; CHECK-SD-DOT-LABEL: add_v8i8_v8i32_acc_sext:1259; CHECK-SD-DOT: // %bb.0: // %entry1260; CHECK-SD-DOT-NEXT: movi v1.2d, #00000000000000001261; CHECK-SD-DOT-NEXT: movi v2.8b, #11262; CHECK-SD-DOT-NEXT: sdot v1.2s, v0.8b, v2.8b1263; CHECK-SD-DOT-NEXT: addp v0.2s, v1.2s, v1.2s1264; CHECK-SD-DOT-NEXT: fmov w8, s01265; CHECK-SD-DOT-NEXT: add w0, w8, w01266; CHECK-SD-DOT-NEXT: ret1267;1268; CHECK-GI-BASE-LABEL: add_v8i8_v8i32_acc_sext:1269; CHECK-GI-BASE: // %bb.0: // %entry1270; CHECK-GI-BASE-NEXT: saddlv h0, v0.8b1271; CHECK-GI-BASE-NEXT: fmov w8, s01272; CHECK-GI-BASE-NEXT: add w0, w0, w8, sxth1273; CHECK-GI-BASE-NEXT: ret1274;1275; CHECK-GI-DOT-LABEL: add_v8i8_v8i32_acc_sext:1276; CHECK-GI-DOT: // %bb.0: // %entry1277; CHECK-GI-DOT-NEXT: movi v1.8b, #11278; CHECK-GI-DOT-NEXT: movi v2.2d, #00000000000000001279; CHECK-GI-DOT-NEXT: sdot v2.2s, v0.8b, v1.8b1280; CHECK-GI-DOT-NEXT: addp v0.2s, v2.2s, v2.2s1281; CHECK-GI-DOT-NEXT: fmov w8, s01282; CHECK-GI-DOT-NEXT: add w0, w8, w01283; CHECK-GI-DOT-NEXT: ret1284entry:1285 %xx = sext <8 x i8> %x to <8 x i32>1286 %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)1287 %r = add i32 %z, %a1288 ret i32 %r1289}1290 1291define i32 @add_v4i8_v4i32_acc_zext(<4 x i8> %x, i32 %a) {1292; CHECK-SD-LABEL: add_v4i8_v4i32_acc_zext:1293; CHECK-SD: // %bb.0: // %entry1294; CHECK-SD-NEXT: bic v0.4h, #255, lsl #81295; CHECK-SD-NEXT: uaddlv s0, v0.4h1296; CHECK-SD-NEXT: fmov w8, s01297; CHECK-SD-NEXT: add w0, w8, w01298; CHECK-SD-NEXT: ret1299;1300; CHECK-GI-LABEL: add_v4i8_v4i32_acc_zext:1301; CHECK-GI: // %bb.0: // %entry1302; CHECK-GI-NEXT: movi d1, #0xff00ff00ff00ff1303; CHECK-GI-NEXT: and v0.8b, v0.8b, v1.8b1304; CHECK-GI-NEXT: uaddlv s0, v0.4h1305; CHECK-GI-NEXT: fmov w8, s01306; CHECK-GI-NEXT: add w0, w0, w8, uxth1307; CHECK-GI-NEXT: ret1308entry:1309 %xx = zext <4 x i8> %x to <4 x i32>1310 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)1311 %r = add i32 %z, %a1312 ret i32 %r1313}1314 1315define i32 @add_v4i8_v4i32_acc_sext(<4 x i8> %x, i32 %a) {1316; CHECK-SD-LABEL: add_v4i8_v4i32_acc_sext:1317; CHECK-SD: // %bb.0: // %entry1318; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #01319; CHECK-SD-NEXT: shl v0.4s, v0.4s, #241320; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #241321; CHECK-SD-NEXT: addv s0, v0.4s1322; CHECK-SD-NEXT: fmov w8, s01323; CHECK-SD-NEXT: add w0, w8, w01324; CHECK-SD-NEXT: ret1325;1326; CHECK-GI-LABEL: add_v4i8_v4i32_acc_sext:1327; CHECK-GI: // %bb.0: // %entry1328; CHECK-GI-NEXT: shl v0.4h, v0.4h, #81329; CHECK-GI-NEXT: sshr v0.4h, v0.4h, #81330; CHECK-GI-NEXT: saddlv s0, v0.4h1331; CHECK-GI-NEXT: fmov w8, s01332; CHECK-GI-NEXT: add w0, w0, w8, sxth1333; CHECK-GI-NEXT: ret1334entry:1335 %xx = sext <4 x i8> %x to <4 x i32>1336 %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)1337 %r = add i32 %z, %a1338 ret i32 %r1339}1340 1341define zeroext i16 @add_v16i8_v16i16_acc_zext(<16 x i8> %x, i16 %a) {1342; CHECK-LABEL: add_v16i8_v16i16_acc_zext:1343; CHECK: // %bb.0: // %entry1344; CHECK-NEXT: uaddlv h0, v0.16b1345; CHECK-NEXT: fmov w8, s01346; CHECK-NEXT: add w8, w8, w01347; CHECK-NEXT: and w0, w8, #0xffff1348; CHECK-NEXT: ret1349entry:1350 %xx = zext <16 x i8> %x to <16 x i16>1351 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)1352 %r = add i16 %z, %a1353 ret i16 %r1354}1355 1356define signext i16 @add_v16i8_v16i16_acc_sext(<16 x i8> %x, i16 %a) {1357; CHECK-LABEL: add_v16i8_v16i16_acc_sext:1358; CHECK: // %bb.0: // %entry1359; CHECK-NEXT: saddlv h0, v0.16b1360; CHECK-NEXT: fmov w8, s01361; CHECK-NEXT: add w8, w8, w01362; CHECK-NEXT: sxth w0, w81363; CHECK-NEXT: ret1364entry:1365 %xx = sext <16 x i8> %x to <16 x i16>1366 %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)1367 %r = add i16 %z, %a1368 ret i16 %r1369}1370 1371define zeroext i16 @add_v8i8_v8i16_acc_zext(<8 x i8> %x, i16 %a) {1372; CHECK-LABEL: add_v8i8_v8i16_acc_zext:1373; CHECK: // %bb.0: // %entry1374; CHECK-NEXT: uaddlv h0, v0.8b1375; CHECK-NEXT: fmov w8, s01376; CHECK-NEXT: add w8, w8, w01377; CHECK-NEXT: and w0, w8, #0xffff1378; CHECK-NEXT: ret1379entry:1380 %xx = zext <8 x i8> %x to <8 x i16>1381 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)1382 %r = add i16 %z, %a1383 ret i16 %r1384}1385 1386define signext i16 @add_v8i8_v8i16_acc_sext(<8 x i8> %x, i16 %a) {1387; CHECK-LABEL: add_v8i8_v8i16_acc_sext:1388; CHECK: // %bb.0: // %entry1389; CHECK-NEXT: saddlv h0, v0.8b1390; CHECK-NEXT: fmov w8, s01391; CHECK-NEXT: add w8, w8, w01392; CHECK-NEXT: sxth w0, w81393; CHECK-NEXT: ret1394entry:1395 %xx = sext <8 x i8> %x to <8 x i16>1396 %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)1397 %r = add i16 %z, %a1398 ret i16 %r1399}1400 1401define zeroext i8 @add_v16i8_v16i8_acc(<16 x i8> %x, i8 %a) {1402; CHECK-SD-LABEL: add_v16i8_v16i8_acc:1403; CHECK-SD: // %bb.0: // %entry1404; CHECK-SD-NEXT: addv b0, v0.16b1405; CHECK-SD-NEXT: fmov w8, s01406; CHECK-SD-NEXT: add w8, w8, w01407; CHECK-SD-NEXT: and w0, w8, #0xff1408; CHECK-SD-NEXT: ret1409;1410; CHECK-GI-LABEL: add_v16i8_v16i8_acc:1411; CHECK-GI: // %bb.0: // %entry1412; CHECK-GI-NEXT: addv b0, v0.16b1413; CHECK-GI-NEXT: fmov w8, s01414; CHECK-GI-NEXT: add w8, w0, w8, uxtb1415; CHECK-GI-NEXT: and w0, w8, #0xff1416; CHECK-GI-NEXT: ret1417entry:1418 %z = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %x)1419 %r = add i8 %z, %a1420 ret i8 %r1421}1422 1423define i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, i64 %a) {1424; CHECK-SD-LABEL: add_v16i8_v16i64_acc_zext:1425; CHECK-SD: // %bb.0: // %entry1426; CHECK-SD-NEXT: ushll2 v1.8h, v0.16b, #01427; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #01428; CHECK-SD-NEXT: ushll2 v2.4s, v1.8h, #01429; CHECK-SD-NEXT: ushll2 v3.4s, v0.8h, #01430; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #01431; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #01432; CHECK-SD-NEXT: uaddl2 v4.2d, v3.4s, v2.4s1433; CHECK-SD-NEXT: uaddl v2.2d, v3.2s, v2.2s1434; CHECK-SD-NEXT: uaddl2 v5.2d, v0.4s, v1.4s1435; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s1436; CHECK-SD-NEXT: add v1.2d, v5.2d, v4.2d1437; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d1438; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d1439; CHECK-SD-NEXT: addp d0, v0.2d1440; CHECK-SD-NEXT: fmov x8, d01441; CHECK-SD-NEXT: add x0, x8, x01442; CHECK-SD-NEXT: ret1443;1444; CHECK-GI-LABEL: add_v16i8_v16i64_acc_zext:1445; CHECK-GI: // %bb.0: // %entry1446; CHECK-GI-NEXT: uaddlv h0, v0.16b1447; CHECK-GI-NEXT: fmov w8, s01448; CHECK-GI-NEXT: add x0, x0, w8, uxth1449; CHECK-GI-NEXT: ret1450entry:1451 %xx = zext <16 x i8> %x to <16 x i64>1452 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx)1453 %r = add i64 %z, %a1454 ret i64 %r1455}1456 1457define i64 @add_v16i8_v16i64_acc_sext(<16 x i8> %x, i64 %a) {1458; CHECK-SD-LABEL: add_v16i8_v16i64_acc_sext:1459; CHECK-SD: // %bb.0: // %entry1460; CHECK-SD-NEXT: sshll2 v1.8h, v0.16b, #01461; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #01462; CHECK-SD-NEXT: sshll2 v2.4s, v1.8h, #01463; CHECK-SD-NEXT: sshll2 v3.4s, v0.8h, #01464; CHECK-SD-NEXT: sshll v1.4s, v1.4h, #01465; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #01466; CHECK-SD-NEXT: saddl2 v4.2d, v3.4s, v2.4s1467; CHECK-SD-NEXT: saddl v2.2d, v3.2s, v2.2s1468; CHECK-SD-NEXT: saddl2 v5.2d, v0.4s, v1.4s1469; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v1.2s1470; CHECK-SD-NEXT: add v1.2d, v5.2d, v4.2d1471; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d1472; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d1473; CHECK-SD-NEXT: addp d0, v0.2d1474; CHECK-SD-NEXT: fmov x8, d01475; CHECK-SD-NEXT: add x0, x8, x01476; CHECK-SD-NEXT: ret1477;1478; CHECK-GI-LABEL: add_v16i8_v16i64_acc_sext:1479; CHECK-GI: // %bb.0: // %entry1480; CHECK-GI-NEXT: saddlv h0, v0.16b1481; CHECK-GI-NEXT: fmov w8, s01482; CHECK-GI-NEXT: add x0, x0, w8, sxth1483; CHECK-GI-NEXT: ret1484entry:1485 %xx = sext <16 x i8> %x to <16 x i64>1486 %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx)1487 %r = add i64 %z, %a1488 ret i64 %r1489}1490 1491define i64 @add_v8i8_v8i64_acc_zext(<8 x i8> %x, i64 %a) {1492; CHECK-SD-LABEL: add_v8i8_v8i64_acc_zext:1493; CHECK-SD: // %bb.0: // %entry1494; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #01495; CHECK-SD-NEXT: ushll2 v1.4s, v0.8h, #01496; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #01497; CHECK-SD-NEXT: uaddl2 v2.2d, v0.4s, v1.4s1498; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v1.2s1499; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d1500; CHECK-SD-NEXT: addp d0, v0.2d1501; CHECK-SD-NEXT: fmov x8, d01502; CHECK-SD-NEXT: add x0, x8, x01503; CHECK-SD-NEXT: ret1504;1505; CHECK-GI-LABEL: add_v8i8_v8i64_acc_zext:1506; CHECK-GI: // %bb.0: // %entry1507; CHECK-GI-NEXT: uaddlv h0, v0.8b1508; CHECK-GI-NEXT: fmov w8, s01509; CHECK-GI-NEXT: add x0, x0, w8, uxth1510; CHECK-GI-NEXT: ret1511entry:1512 %xx = zext <8 x i8> %x to <8 x i64>1513 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)1514 %r = add i64 %z, %a1515 ret i64 %r1516}1517 1518define i64 @add_v8i8_v8i64_acc_sext(<8 x i8> %x, i64 %a) {1519; CHECK-SD-LABEL: add_v8i8_v8i64_acc_sext:1520; CHECK-SD: // %bb.0: // %entry1521; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #01522; CHECK-SD-NEXT: sshll2 v1.4s, v0.8h, #01523; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #01524; CHECK-SD-NEXT: saddl2 v2.2d, v0.4s, v1.4s1525; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v1.2s1526; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d1527; CHECK-SD-NEXT: addp d0, v0.2d1528; CHECK-SD-NEXT: fmov x8, d01529; CHECK-SD-NEXT: add x0, x8, x01530; CHECK-SD-NEXT: ret1531;1532; CHECK-GI-LABEL: add_v8i8_v8i64_acc_sext:1533; CHECK-GI: // %bb.0: // %entry1534; CHECK-GI-NEXT: saddlv h0, v0.8b1535; CHECK-GI-NEXT: fmov w8, s01536; CHECK-GI-NEXT: add x0, x0, w8, sxth1537; CHECK-GI-NEXT: ret1538entry:1539 %xx = sext <8 x i8> %x to <8 x i64>1540 %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)1541 %r = add i64 %z, %a1542 ret i64 %r1543}1544 1545define i64 @add_v4i8_v4i64_acc_zext(<4 x i8> %x, i64 %a) {1546; CHECK-SD-LABEL: add_v4i8_v4i64_acc_zext:1547; CHECK-SD: // %bb.0: // %entry1548; CHECK-SD-NEXT: bic v0.4h, #255, lsl #81549; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #01550; CHECK-SD-NEXT: uaddlv d0, v0.4s1551; CHECK-SD-NEXT: fmov x8, d01552; CHECK-SD-NEXT: add x0, x8, x01553; CHECK-SD-NEXT: ret1554;1555; CHECK-GI-LABEL: add_v4i8_v4i64_acc_zext:1556; CHECK-GI: // %bb.0: // %entry1557; CHECK-GI-NEXT: movi d1, #0xff00ff00ff00ff1558; CHECK-GI-NEXT: and v0.8b, v0.8b, v1.8b1559; CHECK-GI-NEXT: uaddlv s0, v0.4h1560; CHECK-GI-NEXT: fmov w8, s01561; CHECK-GI-NEXT: add x0, x0, w8, uxth1562; CHECK-GI-NEXT: ret1563entry:1564 %xx = zext <4 x i8> %x to <4 x i64>1565 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)1566 %r = add i64 %z, %a1567 ret i64 %r1568}1569 1570define i64 @add_v4i8_v4i64_acc_sext(<4 x i8> %x, i64 %a) {1571; CHECK-SD-LABEL: add_v4i8_v4i64_acc_sext:1572; CHECK-SD: // %bb.0: // %entry1573; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #01574; CHECK-SD-NEXT: ushll v1.2d, v0.2s, #01575; CHECK-SD-NEXT: ushll2 v0.2d, v0.4s, #01576; CHECK-SD-NEXT: shl v1.2d, v1.2d, #561577; CHECK-SD-NEXT: shl v0.2d, v0.2d, #561578; CHECK-SD-NEXT: sshr v1.2d, v1.2d, #561579; CHECK-SD-NEXT: ssra v1.2d, v0.2d, #561580; CHECK-SD-NEXT: addp d0, v1.2d1581; CHECK-SD-NEXT: fmov x8, d01582; CHECK-SD-NEXT: add x0, x8, x01583; CHECK-SD-NEXT: ret1584;1585; CHECK-GI-LABEL: add_v4i8_v4i64_acc_sext:1586; CHECK-GI: // %bb.0: // %entry1587; CHECK-GI-NEXT: shl v0.4h, v0.4h, #81588; CHECK-GI-NEXT: sshr v0.4h, v0.4h, #81589; CHECK-GI-NEXT: saddlv s0, v0.4h1590; CHECK-GI-NEXT: fmov w8, s01591; CHECK-GI-NEXT: add x0, x0, w8, sxth1592; CHECK-GI-NEXT: ret1593entry:1594 %xx = sext <4 x i8> %x to <4 x i64>1595 %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)1596 %r = add i64 %z, %a1597 ret i64 %r1598}1599 1600define i64 @add_v2i8_v2i64_acc_zext(<2 x i8> %x, i64 %a) {1601; CHECK-SD-LABEL: add_v2i8_v2i64_acc_zext:1602; CHECK-SD: // %bb.0: // %entry1603; CHECK-SD-NEXT: movi d1, #0x0000ff000000ff1604; CHECK-SD-NEXT: and v0.8b, v0.8b, v1.8b1605; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #01606; CHECK-SD-NEXT: addp d0, v0.2d1607; CHECK-SD-NEXT: fmov x8, d01608; CHECK-SD-NEXT: add x0, x8, x01609; CHECK-SD-NEXT: ret1610;1611; CHECK-GI-LABEL: add_v2i8_v2i64_acc_zext:1612; CHECK-GI: // %bb.0: // %entry1613; CHECK-GI-NEXT: movi v1.2d, #0x000000000000ff1614; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #01615; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b1616; CHECK-GI-NEXT: addp d0, v0.2d1617; CHECK-GI-NEXT: fmov x8, d01618; CHECK-GI-NEXT: add x0, x8, x01619; CHECK-GI-NEXT: ret1620entry:1621 %xx = zext <2 x i8> %x to <2 x i64>1622 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)1623 %r = add i64 %z, %a1624 ret i64 %r1625}1626 1627define i64 @add_v2i8_v2i64_acc_sext(<2 x i8> %x, i64 %a) {1628; CHECK-LABEL: add_v2i8_v2i64_acc_sext:1629; CHECK: // %bb.0: // %entry1630; CHECK-NEXT: ushll v0.2d, v0.2s, #01631; CHECK-NEXT: shl v0.2d, v0.2d, #561632; CHECK-NEXT: sshr v0.2d, v0.2d, #561633; CHECK-NEXT: addp d0, v0.2d1634; CHECK-NEXT: fmov x8, d01635; CHECK-NEXT: add x0, x8, x01636; CHECK-NEXT: ret1637entry:1638 %xx = sext <2 x i8> %x to <2 x i64>1639 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)1640 %r = add i64 %z, %a1641 ret i64 %r1642}1643 1644define i64 @add_v2i64_v2i64_acc(<2 x i64> %x, i64 %a) {1645; CHECK-LABEL: add_v2i64_v2i64_acc:1646; CHECK: // %bb.0: // %entry1647; CHECK-NEXT: addp d0, v0.2d1648; CHECK-NEXT: fmov x8, d01649; CHECK-NEXT: add x0, x8, x01650; CHECK-NEXT: ret1651entry:1652 %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %x)1653 %r = add i64 %z, %a1654 ret i64 %r1655}1656 1657define i32 @add_pair_v4i32_v4i32(<4 x i32> %x, <4 x i32> %y) {1658; CHECK-SD-LABEL: add_pair_v4i32_v4i32:1659; CHECK-SD: // %bb.0: // %entry1660; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s1661; CHECK-SD-NEXT: addv s0, v0.4s1662; CHECK-SD-NEXT: fmov w0, s01663; CHECK-SD-NEXT: ret1664;1665; CHECK-GI-LABEL: add_pair_v4i32_v4i32:1666; CHECK-GI: // %bb.0: // %entry1667; CHECK-GI-NEXT: addv s0, v0.4s1668; CHECK-GI-NEXT: addv s1, v1.4s1669; CHECK-GI-NEXT: fmov w8, s01670; CHECK-GI-NEXT: fmov w9, s11671; CHECK-GI-NEXT: add w0, w8, w91672; CHECK-GI-NEXT: ret1673entry:1674 %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %x)1675 %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %y)1676 %z = add i32 %z1, %z21677 ret i32 %z1678}1679 1680define i64 @add_pair_v4i32_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {1681; CHECK-SD-LABEL: add_pair_v4i32_v4i64_zext:1682; CHECK-SD: // %bb.0: // %entry1683; CHECK-SD-NEXT: uaddlp v1.2d, v1.4s1684; CHECK-SD-NEXT: uadalp v1.2d, v0.4s1685; CHECK-SD-NEXT: addp d0, v1.2d1686; CHECK-SD-NEXT: fmov x0, d01687; CHECK-SD-NEXT: ret1688;1689; CHECK-GI-LABEL: add_pair_v4i32_v4i64_zext:1690; CHECK-GI: // %bb.0: // %entry1691; CHECK-GI-NEXT: uaddlv d0, v0.4s1692; CHECK-GI-NEXT: uaddlv d1, v1.4s1693; CHECK-GI-NEXT: fmov x8, d01694; CHECK-GI-NEXT: fmov x9, d11695; CHECK-GI-NEXT: add x0, x8, x91696; CHECK-GI-NEXT: ret1697entry:1698 %xx = zext <4 x i32> %x to <4 x i64>1699 %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)1700 %yy = zext <4 x i32> %y to <4 x i64>1701 %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy)1702 %z = add i64 %z1, %z21703 ret i64 %z1704}1705 1706define i64 @add_pair_v4i32_v4i64_sext(<4 x i32> %x, <4 x i32> %y) {1707; CHECK-SD-LABEL: add_pair_v4i32_v4i64_sext:1708; CHECK-SD: // %bb.0: // %entry1709; CHECK-SD-NEXT: saddlp v1.2d, v1.4s1710; CHECK-SD-NEXT: sadalp v1.2d, v0.4s1711; CHECK-SD-NEXT: addp d0, v1.2d1712; CHECK-SD-NEXT: fmov x0, d01713; CHECK-SD-NEXT: ret1714;1715; CHECK-GI-LABEL: add_pair_v4i32_v4i64_sext:1716; CHECK-GI: // %bb.0: // %entry1717; CHECK-GI-NEXT: saddlv d0, v0.4s1718; CHECK-GI-NEXT: saddlv d1, v1.4s1719; CHECK-GI-NEXT: fmov x8, d01720; CHECK-GI-NEXT: fmov x9, d11721; CHECK-GI-NEXT: add x0, x8, x91722; CHECK-GI-NEXT: ret1723entry:1724 %xx = sext <4 x i32> %x to <4 x i64>1725 %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)1726 %yy = sext <4 x i32> %y to <4 x i64>1727 %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy)1728 %z = add i64 %z1, %z21729 ret i64 %z1730}1731 1732define i64 @add_pair_v2i32_v2i64_zext(<2 x i32> %x, <2 x i32> %y) {1733; CHECK-SD-LABEL: add_pair_v2i32_v2i64_zext:1734; CHECK-SD: // %bb.0: // %entry1735; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q01736; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q11737; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]1738; CHECK-SD-NEXT: uaddlv d0, v0.4s1739; CHECK-SD-NEXT: fmov x0, d01740; CHECK-SD-NEXT: ret1741;1742; CHECK-GI-LABEL: add_pair_v2i32_v2i64_zext:1743; CHECK-GI: // %bb.0: // %entry1744; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #01745; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #01746; CHECK-GI-NEXT: addp d0, v0.2d1747; CHECK-GI-NEXT: addp d1, v1.2d1748; CHECK-GI-NEXT: fmov x8, d01749; CHECK-GI-NEXT: fmov x9, d11750; CHECK-GI-NEXT: add x0, x8, x91751; CHECK-GI-NEXT: ret1752entry:1753 %xx = zext <2 x i32> %x to <2 x i64>1754 %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)1755 %yy = zext <2 x i32> %y to <2 x i64>1756 %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy)1757 %z = add i64 %z1, %z21758 ret i64 %z1759}1760 1761define i64 @add_pair_v2i32_v2i64_sext(<2 x i32> %x, <2 x i32> %y) {1762; CHECK-SD-LABEL: add_pair_v2i32_v2i64_sext:1763; CHECK-SD: // %bb.0: // %entry1764; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v1.2s1765; CHECK-SD-NEXT: addp d0, v0.2d1766; CHECK-SD-NEXT: fmov x0, d01767; CHECK-SD-NEXT: ret1768;1769; CHECK-GI-LABEL: add_pair_v2i32_v2i64_sext:1770; CHECK-GI: // %bb.0: // %entry1771; CHECK-GI-NEXT: sshll v0.2d, v0.2s, #01772; CHECK-GI-NEXT: sshll v1.2d, v1.2s, #01773; CHECK-GI-NEXT: addp d0, v0.2d1774; CHECK-GI-NEXT: addp d1, v1.2d1775; CHECK-GI-NEXT: fmov x8, d01776; CHECK-GI-NEXT: fmov x9, d11777; CHECK-GI-NEXT: add x0, x8, x91778; CHECK-GI-NEXT: ret1779entry:1780 %xx = sext <2 x i32> %x to <2 x i64>1781 %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)1782 %yy = sext <2 x i32> %y to <2 x i64>1783 %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy)1784 %z = add i64 %z1, %z21785 ret i64 %z1786}1787 1788define i32 @add_pair_v8i16_v8i32_zext(<8 x i16> %x, <8 x i16> %y) {1789; CHECK-SD-LABEL: add_pair_v8i16_v8i32_zext:1790; CHECK-SD: // %bb.0: // %entry1791; CHECK-SD-NEXT: uaddlp v1.4s, v1.8h1792; CHECK-SD-NEXT: uadalp v1.4s, v0.8h1793; CHECK-SD-NEXT: addv s0, v1.4s1794; CHECK-SD-NEXT: fmov w0, s01795; CHECK-SD-NEXT: ret1796;1797; CHECK-GI-LABEL: add_pair_v8i16_v8i32_zext:1798; CHECK-GI: // %bb.0: // %entry1799; CHECK-GI-NEXT: uaddlv s0, v0.8h1800; CHECK-GI-NEXT: uaddlv s1, v1.8h1801; CHECK-GI-NEXT: fmov w8, s01802; CHECK-GI-NEXT: fmov w9, s11803; CHECK-GI-NEXT: add w0, w8, w91804; CHECK-GI-NEXT: ret1805entry:1806 %xx = zext <8 x i16> %x to <8 x i32>1807 %z1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)1808 %yy = zext <8 x i16> %y to <8 x i32>1809 %z2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %yy)1810 %z = add i32 %z1, %z21811 ret i32 %z1812}1813 1814define i32 @add_pair_v8i16_v8i32_sext(<8 x i16> %x, <8 x i16> %y) {1815; CHECK-SD-LABEL: add_pair_v8i16_v8i32_sext:1816; CHECK-SD: // %bb.0: // %entry1817; CHECK-SD-NEXT: saddlp v1.4s, v1.8h1818; CHECK-SD-NEXT: sadalp v1.4s, v0.8h1819; CHECK-SD-NEXT: addv s0, v1.4s1820; CHECK-SD-NEXT: fmov w0, s01821; CHECK-SD-NEXT: ret1822;1823; CHECK-GI-LABEL: add_pair_v8i16_v8i32_sext:1824; CHECK-GI: // %bb.0: // %entry1825; CHECK-GI-NEXT: saddlv s0, v0.8h1826; CHECK-GI-NEXT: saddlv s1, v1.8h1827; CHECK-GI-NEXT: fmov w8, s01828; CHECK-GI-NEXT: fmov w9, s11829; CHECK-GI-NEXT: add w0, w8, w91830; CHECK-GI-NEXT: ret1831entry:1832 %xx = sext <8 x i16> %x to <8 x i32>1833 %z1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)1834 %yy = sext <8 x i16> %y to <8 x i32>1835 %z2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %yy)1836 %z = add i32 %z1, %z21837 ret i32 %z1838}1839 1840define i32 @add_pair_v4i16_v4i32_zext(<4 x i16> %x, <4 x i16> %y) {1841; CHECK-SD-LABEL: add_pair_v4i16_v4i32_zext:1842; CHECK-SD: // %bb.0: // %entry1843; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q01844; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q11845; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]1846; CHECK-SD-NEXT: uaddlv s0, v0.8h1847; CHECK-SD-NEXT: fmov w0, s01848; CHECK-SD-NEXT: ret1849;1850; CHECK-GI-LABEL: add_pair_v4i16_v4i32_zext:1851; CHECK-GI: // %bb.0: // %entry1852; CHECK-GI-NEXT: uaddlv s0, v0.4h1853; CHECK-GI-NEXT: uaddlv s1, v1.4h1854; CHECK-GI-NEXT: fmov w8, s01855; CHECK-GI-NEXT: fmov w9, s11856; CHECK-GI-NEXT: add w0, w8, w91857; CHECK-GI-NEXT: ret1858entry:1859 %xx = zext <4 x i16> %x to <4 x i32>1860 %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)1861 %yy = zext <4 x i16> %y to <4 x i32>1862 %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %yy)1863 %z = add i32 %z1, %z21864 ret i32 %z1865}1866 1867define i32 @add_pair_v4i16_v4i32_sext(<4 x i16> %x, <4 x i16> %y) {1868; CHECK-SD-LABEL: add_pair_v4i16_v4i32_sext:1869; CHECK-SD: // %bb.0: // %entry1870; CHECK-SD-NEXT: saddl v0.4s, v0.4h, v1.4h1871; CHECK-SD-NEXT: addv s0, v0.4s1872; CHECK-SD-NEXT: fmov w0, s01873; CHECK-SD-NEXT: ret1874;1875; CHECK-GI-LABEL: add_pair_v4i16_v4i32_sext:1876; CHECK-GI: // %bb.0: // %entry1877; CHECK-GI-NEXT: saddlv s0, v0.4h1878; CHECK-GI-NEXT: saddlv s1, v1.4h1879; CHECK-GI-NEXT: fmov w8, s01880; CHECK-GI-NEXT: fmov w9, s11881; CHECK-GI-NEXT: add w0, w8, w91882; CHECK-GI-NEXT: ret1883entry:1884 %xx = sext <4 x i16> %x to <4 x i32>1885 %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)1886 %yy = sext <4 x i16> %y to <4 x i32>1887 %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %yy)1888 %z = add i32 %z1, %z21889 ret i32 %z1890}1891 1892define i32 @test_udot_v8i8(<8 x i8> %a, <8 x i8> %b) {1893; CHECK-SD-BASE-LABEL: test_udot_v8i8:1894; CHECK-SD-BASE: // %bb.0: // %entry1895; CHECK-SD-BASE-NEXT: umull v0.8h, v1.8b, v0.8b1896; CHECK-SD-BASE-NEXT: uaddlv s0, v0.8h1897; CHECK-SD-BASE-NEXT: fmov w0, s01898; CHECK-SD-BASE-NEXT: ret1899;1900; CHECK-SD-DOT-LABEL: test_udot_v8i8:1901; CHECK-SD-DOT: // %bb.0: // %entry1902; CHECK-SD-DOT-NEXT: movi v2.2d, #00000000000000001903; CHECK-SD-DOT-NEXT: udot v2.2s, v1.8b, v0.8b1904; CHECK-SD-DOT-NEXT: addp v0.2s, v2.2s, v2.2s1905; CHECK-SD-DOT-NEXT: fmov w0, s01906; CHECK-SD-DOT-NEXT: ret1907;1908; CHECK-GI-BASE-LABEL: test_udot_v8i8:1909; CHECK-GI-BASE: // %bb.0: // %entry1910; CHECK-GI-BASE-NEXT: umull v0.8h, v1.8b, v0.8b1911; CHECK-GI-BASE-NEXT: uaddlv s0, v0.8h1912; CHECK-GI-BASE-NEXT: fmov w0, s01913; CHECK-GI-BASE-NEXT: ret1914;1915; CHECK-GI-DOT-LABEL: test_udot_v8i8:1916; CHECK-GI-DOT: // %bb.0: // %entry1917; CHECK-GI-DOT-NEXT: movi v2.2d, #00000000000000001918; CHECK-GI-DOT-NEXT: udot v2.2s, v1.8b, v0.8b1919; CHECK-GI-DOT-NEXT: addp v0.2s, v2.2s, v2.2s1920; CHECK-GI-DOT-NEXT: fmov w0, s01921; CHECK-GI-DOT-NEXT: ret1922entry:1923 %0 = zext <8 x i8> %a to <8 x i32>1924 %1 = zext <8 x i8> %b to <8 x i32>1925 %2 = mul nuw nsw <8 x i32> %1, %01926 %3 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %2)1927 ret i32 %31928}1929 1930define i32 @test_udot_v16i8(<16 x i8> %a, <16 x i8> %b) {1931; CHECK-SD-BASE-LABEL: test_udot_v16i8:1932; CHECK-SD-BASE: // %bb.0: // %entry1933; CHECK-SD-BASE-NEXT: umull2 v2.8h, v1.16b, v0.16b1934; CHECK-SD-BASE-NEXT: umull v0.8h, v1.8b, v0.8b1935; CHECK-SD-BASE-NEXT: uaddl2 v1.4s, v0.8h, v2.8h1936; CHECK-SD-BASE-NEXT: uaddl v0.4s, v0.4h, v2.4h1937; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s1938; CHECK-SD-BASE-NEXT: addv s0, v0.4s1939; CHECK-SD-BASE-NEXT: fmov w0, s01940; CHECK-SD-BASE-NEXT: ret1941;1942; CHECK-SD-DOT-LABEL: test_udot_v16i8:1943; CHECK-SD-DOT: // %bb.0: // %entry1944; CHECK-SD-DOT-NEXT: movi v2.2d, #00000000000000001945; CHECK-SD-DOT-NEXT: udot v2.4s, v1.16b, v0.16b1946; CHECK-SD-DOT-NEXT: addv s0, v2.4s1947; CHECK-SD-DOT-NEXT: fmov w0, s01948; CHECK-SD-DOT-NEXT: ret1949;1950; CHECK-GI-BASE-LABEL: test_udot_v16i8:1951; CHECK-GI-BASE: // %bb.0: // %entry1952; CHECK-GI-BASE-NEXT: umull v2.8h, v1.8b, v0.8b1953; CHECK-GI-BASE-NEXT: umull2 v0.8h, v1.16b, v0.16b1954; CHECK-GI-BASE-NEXT: uaddlv s1, v2.8h1955; CHECK-GI-BASE-NEXT: uaddlv s0, v0.8h1956; CHECK-GI-BASE-NEXT: fmov w8, s11957; CHECK-GI-BASE-NEXT: fmov w9, s01958; CHECK-GI-BASE-NEXT: add w0, w8, w91959; CHECK-GI-BASE-NEXT: ret1960;1961; CHECK-GI-DOT-LABEL: test_udot_v16i8:1962; CHECK-GI-DOT: // %bb.0: // %entry1963; CHECK-GI-DOT-NEXT: movi v2.2d, #00000000000000001964; CHECK-GI-DOT-NEXT: udot v2.4s, v1.16b, v0.16b1965; CHECK-GI-DOT-NEXT: addv s0, v2.4s1966; CHECK-GI-DOT-NEXT: fmov w0, s01967; CHECK-GI-DOT-NEXT: ret1968entry:1969 %0 = zext <16 x i8> %a to <16 x i32>1970 %1 = zext <16 x i8> %b to <16 x i32>1971 %2 = mul nuw nsw <16 x i32> %1, %01972 %3 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %2)1973 ret i32 %31974}1975 1976define i32 @test_udot_v24i8(ptr %p1, ptr %p2) {1977; CHECK-SD-BASE-LABEL: test_udot_v24i8:1978; CHECK-SD-BASE: // %bb.0: // %entry1979; CHECK-SD-BASE-NEXT: ldr q0, [x0]1980; CHECK-SD-BASE-NEXT: ldr q1, [x1]1981; CHECK-SD-BASE-NEXT: ldr d2, [x0, #16]1982; CHECK-SD-BASE-NEXT: ldr d3, [x1, #16]1983; CHECK-SD-BASE-NEXT: umull v2.8h, v3.8b, v2.8b1984; CHECK-SD-BASE-NEXT: umull v3.8h, v1.8b, v0.8b1985; CHECK-SD-BASE-NEXT: umull2 v0.8h, v1.16b, v0.16b1986; CHECK-SD-BASE-NEXT: uaddl2 v1.4s, v3.8h, v2.8h1987; CHECK-SD-BASE-NEXT: uaddl v2.4s, v3.4h, v2.4h1988; CHECK-SD-BASE-NEXT: uaddw2 v1.4s, v1.4s, v0.8h1989; CHECK-SD-BASE-NEXT: uaddw v0.4s, v2.4s, v0.4h1990; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s1991; CHECK-SD-BASE-NEXT: addv s0, v0.4s1992; CHECK-SD-BASE-NEXT: fmov w0, s01993; CHECK-SD-BASE-NEXT: ret1994;1995; CHECK-SD-DOT-LABEL: test_udot_v24i8:1996; CHECK-SD-DOT: // %bb.0: // %entry1997; CHECK-SD-DOT-NEXT: movi v0.2d, #00000000000000001998; CHECK-SD-DOT-NEXT: movi v1.2d, #00000000000000001999; CHECK-SD-DOT-NEXT: ldr q2, [x0]2000; CHECK-SD-DOT-NEXT: ldr q3, [x1]2001; CHECK-SD-DOT-NEXT: ldr d4, [x0, #16]2002; CHECK-SD-DOT-NEXT: ldr d5, [x1, #16]2003; CHECK-SD-DOT-NEXT: udot v1.2s, v5.8b, v4.8b2004; CHECK-SD-DOT-NEXT: udot v0.4s, v3.16b, v2.16b2005; CHECK-SD-DOT-NEXT: addp v1.2s, v1.2s, v1.2s2006; CHECK-SD-DOT-NEXT: addv s0, v0.4s2007; CHECK-SD-DOT-NEXT: fmov w8, s12008; CHECK-SD-DOT-NEXT: fmov w9, s02009; CHECK-SD-DOT-NEXT: add w0, w9, w82010; CHECK-SD-DOT-NEXT: ret2011;2012; CHECK-GI-BASE-LABEL: test_udot_v24i8:2013; CHECK-GI-BASE: // %bb.0: // %entry2014; CHECK-GI-BASE-NEXT: ldr q0, [x0]2015; CHECK-GI-BASE-NEXT: ldr q1, [x1]2016; CHECK-GI-BASE-NEXT: ldr d2, [x0, #16]2017; CHECK-GI-BASE-NEXT: ldr d3, [x1, #16]2018; CHECK-GI-BASE-NEXT: umull v4.8h, v1.8b, v0.8b2019; CHECK-GI-BASE-NEXT: umull2 v0.8h, v1.16b, v0.16b2020; CHECK-GI-BASE-NEXT: umull v1.8h, v3.8b, v2.8b2021; CHECK-GI-BASE-NEXT: uaddlv s2, v4.8h2022; CHECK-GI-BASE-NEXT: uaddlv s0, v0.8h2023; CHECK-GI-BASE-NEXT: uaddlv s1, v1.8h2024; CHECK-GI-BASE-NEXT: fmov w8, s22025; CHECK-GI-BASE-NEXT: fmov w9, s02026; CHECK-GI-BASE-NEXT: add w8, w8, w92027; CHECK-GI-BASE-NEXT: fmov w9, s12028; CHECK-GI-BASE-NEXT: add w0, w8, w92029; CHECK-GI-BASE-NEXT: ret2030;2031; CHECK-GI-DOT-LABEL: test_udot_v24i8:2032; CHECK-GI-DOT: // %bb.0: // %entry2033; CHECK-GI-DOT-NEXT: movi v0.2d, #00000000000000002034; CHECK-GI-DOT-NEXT: movi v1.2d, #00000000000000002035; CHECK-GI-DOT-NEXT: ldr q2, [x0]2036; CHECK-GI-DOT-NEXT: ldr d3, [x0, #16]2037; CHECK-GI-DOT-NEXT: ldr q4, [x1]2038; CHECK-GI-DOT-NEXT: ldr d5, [x1, #16]2039; CHECK-GI-DOT-NEXT: udot v1.4s, v4.16b, v2.16b2040; CHECK-GI-DOT-NEXT: udot v0.4s, v5.16b, v3.16b2041; CHECK-GI-DOT-NEXT: add v0.4s, v1.4s, v0.4s2042; CHECK-GI-DOT-NEXT: addv s0, v0.4s2043; CHECK-GI-DOT-NEXT: fmov w0, s02044; CHECK-GI-DOT-NEXT: ret2045entry:2046 %a = load <24 x i8>, ptr %p12047 %b = load <24 x i8>, ptr %p22048 %0 = zext <24 x i8> %a to <24 x i32>2049 %1 = zext <24 x i8> %b to <24 x i32>2050 %2 = mul nuw nsw <24 x i32> %1, %02051 %3 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %2)2052 ret i32 %32053}2054 2055define i32 @test_udot_v48i8(ptr %p1, ptr %p2) {2056; CHECK-SD-BASE-LABEL: test_udot_v48i8:2057; CHECK-SD-BASE: // %bb.0: // %entry2058; CHECK-SD-BASE-NEXT: ldp q4, q0, [x0, #16]2059; CHECK-SD-BASE-NEXT: ldr q2, [x1, #32]2060; CHECK-SD-BASE-NEXT: ldp q1, q5, [x1]2061; CHECK-SD-BASE-NEXT: ldr q3, [x0]2062; CHECK-SD-BASE-NEXT: umull2 v6.8h, v2.16b, v0.16b2063; CHECK-SD-BASE-NEXT: umull v0.8h, v2.8b, v0.8b2064; CHECK-SD-BASE-NEXT: umull2 v7.8h, v1.16b, v3.16b2065; CHECK-SD-BASE-NEXT: umull v1.8h, v1.8b, v3.8b2066; CHECK-SD-BASE-NEXT: umull2 v2.8h, v5.16b, v4.16b2067; CHECK-SD-BASE-NEXT: umull v3.8h, v5.8b, v4.8b2068; CHECK-SD-BASE-NEXT: uaddl2 v4.4s, v7.8h, v6.8h2069; CHECK-SD-BASE-NEXT: uaddl2 v5.4s, v1.8h, v0.8h2070; CHECK-SD-BASE-NEXT: uaddl v6.4s, v7.4h, v6.4h2071; CHECK-SD-BASE-NEXT: uaddl v0.4s, v1.4h, v0.4h2072; CHECK-SD-BASE-NEXT: uaddw2 v1.4s, v4.4s, v2.8h2073; CHECK-SD-BASE-NEXT: uaddw2 v4.4s, v5.4s, v3.8h2074; CHECK-SD-BASE-NEXT: uaddw v2.4s, v6.4s, v2.4h2075; CHECK-SD-BASE-NEXT: uaddw v0.4s, v0.4s, v3.4h2076; CHECK-SD-BASE-NEXT: add v1.4s, v4.4s, v1.4s2077; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v2.4s2078; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s2079; CHECK-SD-BASE-NEXT: addv s0, v0.4s2080; CHECK-SD-BASE-NEXT: fmov w0, s02081; CHECK-SD-BASE-NEXT: ret2082;2083; CHECK-SD-DOT-LABEL: test_udot_v48i8:2084; CHECK-SD-DOT: // %bb.0: // %entry2085; CHECK-SD-DOT-NEXT: movi v0.2d, #00000000000000002086; CHECK-SD-DOT-NEXT: ldr q1, [x0, #32]2087; CHECK-SD-DOT-NEXT: ldr q2, [x1, #32]2088; CHECK-SD-DOT-NEXT: udot v0.4s, v2.16b, v1.16b2089; CHECK-SD-DOT-NEXT: ldp q3, q1, [x0]2090; CHECK-SD-DOT-NEXT: ldp q4, q2, [x1]2091; CHECK-SD-DOT-NEXT: udot v0.4s, v4.16b, v3.16b2092; CHECK-SD-DOT-NEXT: udot v0.4s, v2.16b, v1.16b2093; CHECK-SD-DOT-NEXT: addv s0, v0.4s2094; CHECK-SD-DOT-NEXT: fmov w0, s02095; CHECK-SD-DOT-NEXT: ret2096;2097; CHECK-GI-BASE-LABEL: test_udot_v48i8:2098; CHECK-GI-BASE: // %bb.0: // %entry2099; CHECK-GI-BASE-NEXT: ldp q0, q1, [x0]2100; CHECK-GI-BASE-NEXT: ldr q3, [x0, #32]2101; CHECK-GI-BASE-NEXT: ldp q2, q4, [x1]2102; CHECK-GI-BASE-NEXT: ldr q5, [x1, #32]2103; CHECK-GI-BASE-NEXT: umull v7.8h, v5.8b, v3.8b2104; CHECK-GI-BASE-NEXT: umull2 v3.8h, v5.16b, v3.16b2105; CHECK-GI-BASE-NEXT: umull v6.8h, v2.8b, v0.8b2106; CHECK-GI-BASE-NEXT: umull2 v0.8h, v2.16b, v0.16b2107; CHECK-GI-BASE-NEXT: umull2 v2.8h, v4.16b, v1.16b2108; CHECK-GI-BASE-NEXT: umull v1.8h, v4.8b, v1.8b2109; CHECK-GI-BASE-NEXT: uaddlv s5, v7.8h2110; CHECK-GI-BASE-NEXT: uaddlv s3, v3.8h2111; CHECK-GI-BASE-NEXT: uaddlv s4, v6.8h2112; CHECK-GI-BASE-NEXT: uaddlv s0, v0.8h2113; CHECK-GI-BASE-NEXT: uaddlv s2, v2.8h2114; CHECK-GI-BASE-NEXT: uaddlv s1, v1.8h2115; CHECK-GI-BASE-NEXT: fmov w11, s52116; CHECK-GI-BASE-NEXT: fmov w8, s42117; CHECK-GI-BASE-NEXT: fmov w9, s02118; CHECK-GI-BASE-NEXT: fmov w10, s22119; CHECK-GI-BASE-NEXT: add w8, w8, w92120; CHECK-GI-BASE-NEXT: fmov w9, s12121; CHECK-GI-BASE-NEXT: add w10, w10, w112122; CHECK-GI-BASE-NEXT: fmov w11, s32123; CHECK-GI-BASE-NEXT: add w8, w8, w92124; CHECK-GI-BASE-NEXT: add w9, w10, w112125; CHECK-GI-BASE-NEXT: add w0, w8, w92126; CHECK-GI-BASE-NEXT: ret2127;2128; CHECK-GI-DOT-LABEL: test_udot_v48i8:2129; CHECK-GI-DOT: // %bb.0: // %entry2130; CHECK-GI-DOT-NEXT: movi d0, #00000000000000002131; CHECK-GI-DOT-NEXT: movi v1.2d, #00000000000000002132; CHECK-GI-DOT-NEXT: ldr q7, [x0, #32]2133; CHECK-GI-DOT-NEXT: movi v2.2d, #00000000000000002134; CHECK-GI-DOT-NEXT: movi v3.2d, #00000000000000002135; CHECK-GI-DOT-NEXT: ldr q17, [x1, #32]2136; CHECK-GI-DOT-NEXT: ldp q4, q5, [x0]2137; CHECK-GI-DOT-NEXT: ldp q6, q16, [x1]2138; CHECK-GI-DOT-NEXT: mov v0.s[1], wzr2139; CHECK-GI-DOT-NEXT: udot v2.4s, v17.16b, v7.16b2140; CHECK-GI-DOT-NEXT: udot v1.4s, v6.16b, v4.16b2141; CHECK-GI-DOT-NEXT: udot v3.4s, v16.16b, v5.16b2142; CHECK-GI-DOT-NEXT: mov v0.s[2], wzr2143; CHECK-GI-DOT-NEXT: add v1.4s, v1.4s, v3.4s2144; CHECK-GI-DOT-NEXT: mov v0.s[3], wzr2145; CHECK-GI-DOT-NEXT: add v0.4s, v2.4s, v0.4s2146; CHECK-GI-DOT-NEXT: add v0.4s, v1.4s, v0.4s2147; CHECK-GI-DOT-NEXT: addv s0, v0.4s2148; CHECK-GI-DOT-NEXT: fmov w0, s02149; CHECK-GI-DOT-NEXT: ret2150entry:2151 %a = load <48 x i8>, ptr %p12152 %b = load <48 x i8>, ptr %p22153 %0 = zext <48 x i8> %a to <48 x i32>2154 %1 = zext <48 x i8> %b to <48 x i32>2155 %2 = mul nuw nsw <48 x i32> %1, %02156 %3 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %2)2157 ret i32 %32158}2159 2160define i32 @test_sdot_v8i8(<8 x i8> %a, <8 x i8> %b) {2161; CHECK-SD-BASE-LABEL: test_sdot_v8i8:2162; CHECK-SD-BASE: // %bb.0: // %entry2163; CHECK-SD-BASE-NEXT: smull v0.8h, v1.8b, v0.8b2164; CHECK-SD-BASE-NEXT: saddlv s0, v0.8h2165; CHECK-SD-BASE-NEXT: fmov w0, s02166; CHECK-SD-BASE-NEXT: ret2167;2168; CHECK-SD-DOT-LABEL: test_sdot_v8i8:2169; CHECK-SD-DOT: // %bb.0: // %entry2170; CHECK-SD-DOT-NEXT: movi v2.2d, #00000000000000002171; CHECK-SD-DOT-NEXT: sdot v2.2s, v1.8b, v0.8b2172; CHECK-SD-DOT-NEXT: addp v0.2s, v2.2s, v2.2s2173; CHECK-SD-DOT-NEXT: fmov w0, s02174; CHECK-SD-DOT-NEXT: ret2175;2176; CHECK-GI-BASE-LABEL: test_sdot_v8i8:2177; CHECK-GI-BASE: // %bb.0: // %entry2178; CHECK-GI-BASE-NEXT: smull v0.8h, v1.8b, v0.8b2179; CHECK-GI-BASE-NEXT: saddlv s0, v0.8h2180; CHECK-GI-BASE-NEXT: fmov w0, s02181; CHECK-GI-BASE-NEXT: ret2182;2183; CHECK-GI-DOT-LABEL: test_sdot_v8i8:2184; CHECK-GI-DOT: // %bb.0: // %entry2185; CHECK-GI-DOT-NEXT: movi v2.2d, #00000000000000002186; CHECK-GI-DOT-NEXT: sdot v2.2s, v1.8b, v0.8b2187; CHECK-GI-DOT-NEXT: addp v0.2s, v2.2s, v2.2s2188; CHECK-GI-DOT-NEXT: fmov w0, s02189; CHECK-GI-DOT-NEXT: ret2190entry:2191 %0 = sext <8 x i8> %a to <8 x i32>2192 %1 = sext <8 x i8> %b to <8 x i32>2193 %2 = mul nuw nsw <8 x i32> %1, %02194 %3 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %2)2195 ret i32 %32196}2197 2198define i32 @test_sdot_v16i8(<16 x i8> %a, <16 x i8> %b) {2199; CHECK-SD-BASE-LABEL: test_sdot_v16i8:2200; CHECK-SD-BASE: // %bb.0: // %entry2201; CHECK-SD-BASE-NEXT: smull2 v2.8h, v1.16b, v0.16b2202; CHECK-SD-BASE-NEXT: smull v0.8h, v1.8b, v0.8b2203; CHECK-SD-BASE-NEXT: saddl2 v1.4s, v0.8h, v2.8h2204; CHECK-SD-BASE-NEXT: saddl v0.4s, v0.4h, v2.4h2205; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s2206; CHECK-SD-BASE-NEXT: addv s0, v0.4s2207; CHECK-SD-BASE-NEXT: fmov w0, s02208; CHECK-SD-BASE-NEXT: ret2209;2210; CHECK-SD-DOT-LABEL: test_sdot_v16i8:2211; CHECK-SD-DOT: // %bb.0: // %entry2212; CHECK-SD-DOT-NEXT: movi v2.2d, #00000000000000002213; CHECK-SD-DOT-NEXT: sdot v2.4s, v1.16b, v0.16b2214; CHECK-SD-DOT-NEXT: addv s0, v2.4s2215; CHECK-SD-DOT-NEXT: fmov w0, s02216; CHECK-SD-DOT-NEXT: ret2217;2218; CHECK-GI-BASE-LABEL: test_sdot_v16i8:2219; CHECK-GI-BASE: // %bb.0: // %entry2220; CHECK-GI-BASE-NEXT: smull v2.8h, v1.8b, v0.8b2221; CHECK-GI-BASE-NEXT: smull2 v0.8h, v1.16b, v0.16b2222; CHECK-GI-BASE-NEXT: saddlv s1, v2.8h2223; CHECK-GI-BASE-NEXT: saddlv s0, v0.8h2224; CHECK-GI-BASE-NEXT: fmov w8, s12225; CHECK-GI-BASE-NEXT: fmov w9, s02226; CHECK-GI-BASE-NEXT: add w0, w8, w92227; CHECK-GI-BASE-NEXT: ret2228;2229; CHECK-GI-DOT-LABEL: test_sdot_v16i8:2230; CHECK-GI-DOT: // %bb.0: // %entry2231; CHECK-GI-DOT-NEXT: movi v2.2d, #00000000000000002232; CHECK-GI-DOT-NEXT: sdot v2.4s, v1.16b, v0.16b2233; CHECK-GI-DOT-NEXT: addv s0, v2.4s2234; CHECK-GI-DOT-NEXT: fmov w0, s02235; CHECK-GI-DOT-NEXT: ret2236entry:2237 %0 = sext <16 x i8> %a to <16 x i32>2238 %1 = sext <16 x i8> %b to <16 x i32>2239 %2 = mul nuw nsw <16 x i32> %1, %02240 %3 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %2)2241 ret i32 %32242}2243 2244define i32 @test_sdot_v24i8(ptr %p1, ptr %p2) {2245; CHECK-SD-BASE-LABEL: test_sdot_v24i8:2246; CHECK-SD-BASE: // %bb.0: // %entry2247; CHECK-SD-BASE-NEXT: ldr q0, [x0]2248; CHECK-SD-BASE-NEXT: ldr q1, [x1]2249; CHECK-SD-BASE-NEXT: ldr d2, [x0, #16]2250; CHECK-SD-BASE-NEXT: ldr d3, [x1, #16]2251; CHECK-SD-BASE-NEXT: smull v2.8h, v3.8b, v2.8b2252; CHECK-SD-BASE-NEXT: smull v3.8h, v1.8b, v0.8b2253; CHECK-SD-BASE-NEXT: smull2 v0.8h, v1.16b, v0.16b2254; CHECK-SD-BASE-NEXT: saddl2 v1.4s, v3.8h, v2.8h2255; CHECK-SD-BASE-NEXT: saddl v2.4s, v3.4h, v2.4h2256; CHECK-SD-BASE-NEXT: saddw2 v1.4s, v1.4s, v0.8h2257; CHECK-SD-BASE-NEXT: saddw v0.4s, v2.4s, v0.4h2258; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s2259; CHECK-SD-BASE-NEXT: addv s0, v0.4s2260; CHECK-SD-BASE-NEXT: fmov w0, s02261; CHECK-SD-BASE-NEXT: ret2262;2263; CHECK-SD-DOT-LABEL: test_sdot_v24i8:2264; CHECK-SD-DOT: // %bb.0: // %entry2265; CHECK-SD-DOT-NEXT: movi v0.2d, #00000000000000002266; CHECK-SD-DOT-NEXT: movi v1.2d, #00000000000000002267; CHECK-SD-DOT-NEXT: ldr q2, [x0]2268; CHECK-SD-DOT-NEXT: ldr q3, [x1]2269; CHECK-SD-DOT-NEXT: ldr d4, [x0, #16]2270; CHECK-SD-DOT-NEXT: ldr d5, [x1, #16]2271; CHECK-SD-DOT-NEXT: sdot v1.2s, v5.8b, v4.8b2272; CHECK-SD-DOT-NEXT: sdot v0.4s, v3.16b, v2.16b2273; CHECK-SD-DOT-NEXT: addp v1.2s, v1.2s, v1.2s2274; CHECK-SD-DOT-NEXT: addv s0, v0.4s2275; CHECK-SD-DOT-NEXT: fmov w8, s12276; CHECK-SD-DOT-NEXT: fmov w9, s02277; CHECK-SD-DOT-NEXT: add w0, w9, w82278; CHECK-SD-DOT-NEXT: ret2279;2280; CHECK-GI-BASE-LABEL: test_sdot_v24i8:2281; CHECK-GI-BASE: // %bb.0: // %entry2282; CHECK-GI-BASE-NEXT: ldr q0, [x0]2283; CHECK-GI-BASE-NEXT: ldr q1, [x1]2284; CHECK-GI-BASE-NEXT: ldr d2, [x0, #16]2285; CHECK-GI-BASE-NEXT: ldr d3, [x1, #16]2286; CHECK-GI-BASE-NEXT: smull v4.8h, v1.8b, v0.8b2287; CHECK-GI-BASE-NEXT: smull2 v0.8h, v1.16b, v0.16b2288; CHECK-GI-BASE-NEXT: smull v1.8h, v3.8b, v2.8b2289; CHECK-GI-BASE-NEXT: saddlv s2, v4.8h2290; CHECK-GI-BASE-NEXT: saddlv s0, v0.8h2291; CHECK-GI-BASE-NEXT: saddlv s1, v1.8h2292; CHECK-GI-BASE-NEXT: fmov w8, s22293; CHECK-GI-BASE-NEXT: fmov w9, s02294; CHECK-GI-BASE-NEXT: add w8, w8, w92295; CHECK-GI-BASE-NEXT: fmov w9, s12296; CHECK-GI-BASE-NEXT: add w0, w8, w92297; CHECK-GI-BASE-NEXT: ret2298;2299; CHECK-GI-DOT-LABEL: test_sdot_v24i8:2300; CHECK-GI-DOT: // %bb.0: // %entry2301; CHECK-GI-DOT-NEXT: movi v0.2d, #00000000000000002302; CHECK-GI-DOT-NEXT: movi v1.2d, #00000000000000002303; CHECK-GI-DOT-NEXT: ldr q2, [x0]2304; CHECK-GI-DOT-NEXT: ldr d3, [x0, #16]2305; CHECK-GI-DOT-NEXT: ldr q4, [x1]2306; CHECK-GI-DOT-NEXT: ldr d5, [x1, #16]2307; CHECK-GI-DOT-NEXT: sdot v1.4s, v4.16b, v2.16b2308; CHECK-GI-DOT-NEXT: sdot v0.4s, v5.16b, v3.16b2309; CHECK-GI-DOT-NEXT: add v0.4s, v1.4s, v0.4s2310; CHECK-GI-DOT-NEXT: addv s0, v0.4s2311; CHECK-GI-DOT-NEXT: fmov w0, s02312; CHECK-GI-DOT-NEXT: ret2313entry:2314 %a = load <24 x i8>, ptr %p12315 %b = load <24 x i8>, ptr %p22316 %0 = sext <24 x i8> %a to <24 x i32>2317 %1 = sext <24 x i8> %b to <24 x i32>2318 %2 = mul nuw nsw <24 x i32> %1, %02319 %3 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %2)2320 ret i32 %32321}2322 2323define i32 @test_sdot_v48i8(ptr %p1, ptr %p2) {2324; CHECK-SD-BASE-LABEL: test_sdot_v48i8:2325; CHECK-SD-BASE: // %bb.0: // %entry2326; CHECK-SD-BASE-NEXT: ldp q4, q0, [x0, #16]2327; CHECK-SD-BASE-NEXT: ldr q2, [x1, #32]2328; CHECK-SD-BASE-NEXT: ldp q1, q5, [x1]2329; CHECK-SD-BASE-NEXT: ldr q3, [x0]2330; CHECK-SD-BASE-NEXT: smull2 v6.8h, v2.16b, v0.16b2331; CHECK-SD-BASE-NEXT: smull v0.8h, v2.8b, v0.8b2332; CHECK-SD-BASE-NEXT: smull2 v7.8h, v1.16b, v3.16b2333; CHECK-SD-BASE-NEXT: smull v1.8h, v1.8b, v3.8b2334; CHECK-SD-BASE-NEXT: smull2 v2.8h, v5.16b, v4.16b2335; CHECK-SD-BASE-NEXT: smull v3.8h, v5.8b, v4.8b2336; CHECK-SD-BASE-NEXT: saddl2 v4.4s, v7.8h, v6.8h2337; CHECK-SD-BASE-NEXT: saddl2 v5.4s, v1.8h, v0.8h2338; CHECK-SD-BASE-NEXT: saddl v6.4s, v7.4h, v6.4h2339; CHECK-SD-BASE-NEXT: saddl v0.4s, v1.4h, v0.4h2340; CHECK-SD-BASE-NEXT: saddw2 v1.4s, v4.4s, v2.8h2341; CHECK-SD-BASE-NEXT: saddw2 v4.4s, v5.4s, v3.8h2342; CHECK-SD-BASE-NEXT: saddw v2.4s, v6.4s, v2.4h2343; CHECK-SD-BASE-NEXT: saddw v0.4s, v0.4s, v3.4h2344; CHECK-SD-BASE-NEXT: add v1.4s, v4.4s, v1.4s2345; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v2.4s2346; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s2347; CHECK-SD-BASE-NEXT: addv s0, v0.4s2348; CHECK-SD-BASE-NEXT: fmov w0, s02349; CHECK-SD-BASE-NEXT: ret2350;2351; CHECK-SD-DOT-LABEL: test_sdot_v48i8:2352; CHECK-SD-DOT: // %bb.0: // %entry2353; CHECK-SD-DOT-NEXT: movi v0.2d, #00000000000000002354; CHECK-SD-DOT-NEXT: ldr q1, [x0, #32]2355; CHECK-SD-DOT-NEXT: ldr q2, [x1, #32]2356; CHECK-SD-DOT-NEXT: sdot v0.4s, v2.16b, v1.16b2357; CHECK-SD-DOT-NEXT: ldp q3, q1, [x0]2358; CHECK-SD-DOT-NEXT: ldp q4, q2, [x1]2359; CHECK-SD-DOT-NEXT: sdot v0.4s, v4.16b, v3.16b2360; CHECK-SD-DOT-NEXT: sdot v0.4s, v2.16b, v1.16b2361; CHECK-SD-DOT-NEXT: addv s0, v0.4s2362; CHECK-SD-DOT-NEXT: fmov w0, s02363; CHECK-SD-DOT-NEXT: ret2364;2365; CHECK-GI-BASE-LABEL: test_sdot_v48i8:2366; CHECK-GI-BASE: // %bb.0: // %entry2367; CHECK-GI-BASE-NEXT: ldp q0, q1, [x0]2368; CHECK-GI-BASE-NEXT: ldr q3, [x0, #32]2369; CHECK-GI-BASE-NEXT: ldp q2, q4, [x1]2370; CHECK-GI-BASE-NEXT: ldr q5, [x1, #32]2371; CHECK-GI-BASE-NEXT: smull v7.8h, v5.8b, v3.8b2372; CHECK-GI-BASE-NEXT: smull2 v3.8h, v5.16b, v3.16b2373; CHECK-GI-BASE-NEXT: smull v6.8h, v2.8b, v0.8b2374; CHECK-GI-BASE-NEXT: smull2 v0.8h, v2.16b, v0.16b2375; CHECK-GI-BASE-NEXT: smull2 v2.8h, v4.16b, v1.16b2376; CHECK-GI-BASE-NEXT: smull v1.8h, v4.8b, v1.8b2377; CHECK-GI-BASE-NEXT: saddlv s5, v7.8h2378; CHECK-GI-BASE-NEXT: saddlv s3, v3.8h2379; CHECK-GI-BASE-NEXT: saddlv s4, v6.8h2380; CHECK-GI-BASE-NEXT: saddlv s0, v0.8h2381; CHECK-GI-BASE-NEXT: saddlv s2, v2.8h2382; CHECK-GI-BASE-NEXT: saddlv s1, v1.8h2383; CHECK-GI-BASE-NEXT: fmov w11, s52384; CHECK-GI-BASE-NEXT: fmov w8, s42385; CHECK-GI-BASE-NEXT: fmov w9, s02386; CHECK-GI-BASE-NEXT: fmov w10, s22387; CHECK-GI-BASE-NEXT: add w8, w8, w92388; CHECK-GI-BASE-NEXT: fmov w9, s12389; CHECK-GI-BASE-NEXT: add w10, w10, w112390; CHECK-GI-BASE-NEXT: fmov w11, s32391; CHECK-GI-BASE-NEXT: add w8, w8, w92392; CHECK-GI-BASE-NEXT: add w9, w10, w112393; CHECK-GI-BASE-NEXT: add w0, w8, w92394; CHECK-GI-BASE-NEXT: ret2395;2396; CHECK-GI-DOT-LABEL: test_sdot_v48i8:2397; CHECK-GI-DOT: // %bb.0: // %entry2398; CHECK-GI-DOT-NEXT: movi d0, #00000000000000002399; CHECK-GI-DOT-NEXT: movi v1.2d, #00000000000000002400; CHECK-GI-DOT-NEXT: ldr q7, [x0, #32]2401; CHECK-GI-DOT-NEXT: movi v2.2d, #00000000000000002402; CHECK-GI-DOT-NEXT: movi v3.2d, #00000000000000002403; CHECK-GI-DOT-NEXT: ldr q17, [x1, #32]2404; CHECK-GI-DOT-NEXT: ldp q4, q5, [x0]2405; CHECK-GI-DOT-NEXT: ldp q6, q16, [x1]2406; CHECK-GI-DOT-NEXT: mov v0.s[1], wzr2407; CHECK-GI-DOT-NEXT: sdot v2.4s, v17.16b, v7.16b2408; CHECK-GI-DOT-NEXT: sdot v1.4s, v6.16b, v4.16b2409; CHECK-GI-DOT-NEXT: sdot v3.4s, v16.16b, v5.16b2410; CHECK-GI-DOT-NEXT: mov v0.s[2], wzr2411; CHECK-GI-DOT-NEXT: add v1.4s, v1.4s, v3.4s2412; CHECK-GI-DOT-NEXT: mov v0.s[3], wzr2413; CHECK-GI-DOT-NEXT: add v0.4s, v2.4s, v0.4s2414; CHECK-GI-DOT-NEXT: add v0.4s, v1.4s, v0.4s2415; CHECK-GI-DOT-NEXT: addv s0, v0.4s2416; CHECK-GI-DOT-NEXT: fmov w0, s02417; CHECK-GI-DOT-NEXT: ret2418entry:2419 %a = load <48 x i8>, ptr %p12420 %b = load <48 x i8>, ptr %p22421 %0 = sext <48 x i8> %a to <48 x i32>2422 %1 = sext <48 x i8> %b to <48 x i32>2423 %2 = mul nuw nsw <48 x i32> %1, %02424 %3 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %2)2425 ret i32 %32426}2427 2428; Test to ensure that if G_MUL has more than 1 use, it should not be combined to UDOT2429define i32 @test_udot_v8i8_multi_use(<8 x i8> %a, <8 x i8> %b) {2430; CHECK-SD-BASE-LABEL: test_udot_v8i8_multi_use:2431; CHECK-SD-BASE: // %bb.0: // %entry2432; CHECK-SD-BASE-NEXT: umull v0.8h, v1.8b, v0.8b2433; CHECK-SD-BASE-NEXT: uaddlv s1, v0.8h2434; CHECK-SD-BASE-NEXT: ushll v0.4s, v0.4h, #02435; CHECK-SD-BASE-NEXT: fmov w9, s02436; CHECK-SD-BASE-NEXT: fmov w8, s12437; CHECK-SD-BASE-NEXT: add w0, w8, w92438; CHECK-SD-BASE-NEXT: ret2439;2440; CHECK-SD-DOT-LABEL: test_udot_v8i8_multi_use:2441; CHECK-SD-DOT: // %bb.0: // %entry2442; CHECK-SD-DOT-NEXT: movi v2.2d, #00000000000000002443; CHECK-SD-DOT-NEXT: umull v3.8h, v1.8b, v0.8b2444; CHECK-SD-DOT-NEXT: udot v2.2s, v1.8b, v0.8b2445; CHECK-SD-DOT-NEXT: ushll v0.4s, v3.4h, #02446; CHECK-SD-DOT-NEXT: fmov w9, s02447; CHECK-SD-DOT-NEXT: addp v1.2s, v2.2s, v2.2s2448; CHECK-SD-DOT-NEXT: fmov w8, s12449; CHECK-SD-DOT-NEXT: add w0, w8, w92450; CHECK-SD-DOT-NEXT: ret2451;2452; CHECK-GI-BASE-LABEL: test_udot_v8i8_multi_use:2453; CHECK-GI-BASE: // %bb.0: // %entry2454; CHECK-GI-BASE-NEXT: umull v0.8h, v1.8b, v0.8b2455; CHECK-GI-BASE-NEXT: uaddlv s1, v0.8h2456; CHECK-GI-BASE-NEXT: ushll v0.4s, v0.4h, #02457; CHECK-GI-BASE-NEXT: fmov w9, s02458; CHECK-GI-BASE-NEXT: fmov w8, s12459; CHECK-GI-BASE-NEXT: add w0, w8, w92460; CHECK-GI-BASE-NEXT: ret2461;2462; CHECK-GI-DOT-LABEL: test_udot_v8i8_multi_use:2463; CHECK-GI-DOT: // %bb.0: // %entry2464; CHECK-GI-DOT-NEXT: movi v2.2d, #00000000000000002465; CHECK-GI-DOT-NEXT: umull v3.8h, v1.8b, v0.8b2466; CHECK-GI-DOT-NEXT: udot v2.2s, v1.8b, v0.8b2467; CHECK-GI-DOT-NEXT: ushll v0.4s, v3.4h, #02468; CHECK-GI-DOT-NEXT: fmov w9, s02469; CHECK-GI-DOT-NEXT: addp v1.2s, v2.2s, v2.2s2470; CHECK-GI-DOT-NEXT: fmov w8, s12471; CHECK-GI-DOT-NEXT: add w0, w8, w92472; CHECK-GI-DOT-NEXT: ret2473entry:2474 %0 = zext <8 x i8> %a to <8 x i32>2475 %1 = zext <8 x i8> %b to <8 x i32>2476 %2 = mul nuw nsw <8 x i32> %1, %02477 %3 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %2)2478 %4 = extractelement <8 x i32> %2, i32 02479 %5 = add nuw nsw i32 %3, %42480 ret i32 %52481}2482 2483define zeroext i16 @add_pair_v8i16_v8i16(<8 x i16> %x, <8 x i16> %y) {2484; CHECK-SD-LABEL: add_pair_v8i16_v8i16:2485; CHECK-SD: // %bb.0: // %entry2486; CHECK-SD-NEXT: add v0.8h, v0.8h, v1.8h2487; CHECK-SD-NEXT: addv h0, v0.8h2488; CHECK-SD-NEXT: fmov w0, s02489; CHECK-SD-NEXT: ret2490;2491; CHECK-GI-LABEL: add_pair_v8i16_v8i16:2492; CHECK-GI: // %bb.0: // %entry2493; CHECK-GI-NEXT: addv h0, v0.8h2494; CHECK-GI-NEXT: addv h1, v1.8h2495; CHECK-GI-NEXT: fmov w8, s02496; CHECK-GI-NEXT: fmov w9, s12497; CHECK-GI-NEXT: add w8, w9, w8, uxth2498; CHECK-GI-NEXT: and w0, w8, #0xffff2499; CHECK-GI-NEXT: ret2500entry:2501 %z1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %x)2502 %z2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %y)2503 %z = add i16 %z1, %z22504 ret i16 %z2505}2506 2507define i64 @add_pair_v8i16_v8i64_zext(<8 x i16> %x, <8 x i16> %y) {2508; CHECK-SD-LABEL: add_pair_v8i16_v8i64_zext:2509; CHECK-SD: // %bb.0: // %entry2510; CHECK-SD-NEXT: ushll2 v2.4s, v0.8h, #02511; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #02512; CHECK-SD-NEXT: ushll2 v3.4s, v1.8h, #02513; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #02514; CHECK-SD-NEXT: uaddl2 v4.2d, v0.4s, v2.4s2515; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v2.2s2516; CHECK-SD-NEXT: uaddl2 v2.2d, v1.4s, v3.4s2517; CHECK-SD-NEXT: uaddl v1.2d, v1.2s, v3.2s2518; CHECK-SD-NEXT: add v0.2d, v0.2d, v4.2d2519; CHECK-SD-NEXT: add v1.2d, v1.2d, v2.2d2520; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d2521; CHECK-SD-NEXT: addp d0, v0.2d2522; CHECK-SD-NEXT: fmov x0, d02523; CHECK-SD-NEXT: ret2524;2525; CHECK-GI-LABEL: add_pair_v8i16_v8i64_zext:2526; CHECK-GI: // %bb.0: // %entry2527; CHECK-GI-NEXT: uaddlv s1, v1.8h2528; CHECK-GI-NEXT: uaddlv s0, v0.8h2529; CHECK-GI-NEXT: mov w8, v1.s[0]2530; CHECK-GI-NEXT: fmov w9, s02531; CHECK-GI-NEXT: add x0, x8, w9, uxtw2532; CHECK-GI-NEXT: ret2533entry:2534 %xx = zext <8 x i16> %x to <8 x i64>2535 %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)2536 %yy = zext <8 x i16> %y to <8 x i64>2537 %z2 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %yy)2538 %z = add i64 %z1, %z22539 ret i64 %z2540}2541 2542define i64 @add_pair_v8i16_v8i64_sext(<8 x i16> %x, <8 x i16> %y) {2543; CHECK-SD-LABEL: add_pair_v8i16_v8i64_sext:2544; CHECK-SD: // %bb.0: // %entry2545; CHECK-SD-NEXT: sshll2 v2.4s, v0.8h, #02546; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #02547; CHECK-SD-NEXT: sshll2 v3.4s, v1.8h, #02548; CHECK-SD-NEXT: sshll v1.4s, v1.4h, #02549; CHECK-SD-NEXT: saddl2 v4.2d, v0.4s, v2.4s2550; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v2.2s2551; CHECK-SD-NEXT: saddl2 v2.2d, v1.4s, v3.4s2552; CHECK-SD-NEXT: saddl v1.2d, v1.2s, v3.2s2553; CHECK-SD-NEXT: add v0.2d, v0.2d, v4.2d2554; CHECK-SD-NEXT: add v1.2d, v1.2d, v2.2d2555; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d2556; CHECK-SD-NEXT: addp d0, v0.2d2557; CHECK-SD-NEXT: fmov x0, d02558; CHECK-SD-NEXT: ret2559;2560; CHECK-GI-LABEL: add_pair_v8i16_v8i64_sext:2561; CHECK-GI: // %bb.0: // %entry2562; CHECK-GI-NEXT: saddlv s1, v1.8h2563; CHECK-GI-NEXT: saddlv s0, v0.8h2564; CHECK-GI-NEXT: smov x8, v1.s[0]2565; CHECK-GI-NEXT: fmov w9, s02566; CHECK-GI-NEXT: add x0, x8, w9, sxtw2567; CHECK-GI-NEXT: ret2568entry:2569 %xx = sext <8 x i16> %x to <8 x i64>2570 %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)2571 %yy = sext <8 x i16> %y to <8 x i64>2572 %z2 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %yy)2573 %z = add i64 %z1, %z22574 ret i64 %z2575}2576 2577define i64 @add_pair_v4i16_v4i64_zext(<4 x i16> %x, <4 x i16> %y) {2578; CHECK-SD-LABEL: add_pair_v4i16_v4i64_zext:2579; CHECK-SD: // %bb.0: // %entry2580; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #02581; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #02582; CHECK-SD-NEXT: uaddlp v1.2d, v1.4s2583; CHECK-SD-NEXT: uadalp v1.2d, v0.4s2584; CHECK-SD-NEXT: addp d0, v1.2d2585; CHECK-SD-NEXT: fmov x0, d02586; CHECK-SD-NEXT: ret2587;2588; CHECK-GI-LABEL: add_pair_v4i16_v4i64_zext:2589; CHECK-GI: // %bb.0: // %entry2590; CHECK-GI-NEXT: uaddlv s1, v1.4h2591; CHECK-GI-NEXT: uaddlv s0, v0.4h2592; CHECK-GI-NEXT: mov w8, v1.s[0]2593; CHECK-GI-NEXT: fmov w9, s02594; CHECK-GI-NEXT: add x0, x8, w9, uxtw2595; CHECK-GI-NEXT: ret2596entry:2597 %xx = zext <4 x i16> %x to <4 x i64>2598 %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)2599 %yy = zext <4 x i16> %y to <4 x i64>2600 %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy)2601 %z = add i64 %z1, %z22602 ret i64 %z2603}2604 2605define i64 @add_pair_v4i16_v4i64_sext(<4 x i16> %x, <4 x i16> %y) {2606; CHECK-SD-LABEL: add_pair_v4i16_v4i64_sext:2607; CHECK-SD: // %bb.0: // %entry2608; CHECK-SD-NEXT: sshll v1.4s, v1.4h, #02609; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #02610; CHECK-SD-NEXT: saddlp v1.2d, v1.4s2611; CHECK-SD-NEXT: sadalp v1.2d, v0.4s2612; CHECK-SD-NEXT: addp d0, v1.2d2613; CHECK-SD-NEXT: fmov x0, d02614; CHECK-SD-NEXT: ret2615;2616; CHECK-GI-LABEL: add_pair_v4i16_v4i64_sext:2617; CHECK-GI: // %bb.0: // %entry2618; CHECK-GI-NEXT: saddlv s1, v1.4h2619; CHECK-GI-NEXT: saddlv s0, v0.4h2620; CHECK-GI-NEXT: smov x8, v1.s[0]2621; CHECK-GI-NEXT: fmov w9, s02622; CHECK-GI-NEXT: add x0, x8, w9, sxtw2623; CHECK-GI-NEXT: ret2624entry:2625 %xx = sext <4 x i16> %x to <4 x i64>2626 %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)2627 %yy = sext <4 x i16> %y to <4 x i64>2628 %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy)2629 %z = add i64 %z1, %z22630 ret i64 %z2631}2632 2633define i64 @add_pair_v2i16_v2i64_zext(<2 x i16> %x, <2 x i16> %y) {2634; CHECK-SD-LABEL: add_pair_v2i16_v2i64_zext:2635; CHECK-SD: // %bb.0: // %entry2636; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q02637; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q12638; CHECK-SD-NEXT: movi v2.2d, #0x00ffff0000ffff2639; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]2640; CHECK-SD-NEXT: and v0.16b, v0.16b, v2.16b2641; CHECK-SD-NEXT: uaddlv d0, v0.4s2642; CHECK-SD-NEXT: fmov x0, d02643; CHECK-SD-NEXT: ret2644;2645; CHECK-GI-LABEL: add_pair_v2i16_v2i64_zext:2646; CHECK-GI: // %bb.0: // %entry2647; CHECK-GI-NEXT: movi v2.2d, #0x0000000000ffff2648; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #02649; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #02650; CHECK-GI-NEXT: and v0.16b, v0.16b, v2.16b2651; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b2652; CHECK-GI-NEXT: addp d0, v0.2d2653; CHECK-GI-NEXT: addp d1, v1.2d2654; CHECK-GI-NEXT: fmov x8, d02655; CHECK-GI-NEXT: fmov x9, d12656; CHECK-GI-NEXT: add x0, x8, x92657; CHECK-GI-NEXT: ret2658entry:2659 %xx = zext <2 x i16> %x to <2 x i64>2660 %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)2661 %yy = zext <2 x i16> %y to <2 x i64>2662 %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy)2663 %z = add i64 %z1, %z22664 ret i64 %z2665}2666 2667define i64 @add_pair_v2i16_v2i64_sext(<2 x i16> %x, <2 x i16> %y) {2668; CHECK-SD-LABEL: add_pair_v2i16_v2i64_sext:2669; CHECK-SD: // %bb.0: // %entry2670; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #02671; CHECK-SD-NEXT: ushll v1.2d, v1.2s, #02672; CHECK-SD-NEXT: shl v0.2d, v0.2d, #482673; CHECK-SD-NEXT: shl v1.2d, v1.2d, #482674; CHECK-SD-NEXT: sshr v0.2d, v0.2d, #482675; CHECK-SD-NEXT: ssra v0.2d, v1.2d, #482676; CHECK-SD-NEXT: addp d0, v0.2d2677; CHECK-SD-NEXT: fmov x0, d02678; CHECK-SD-NEXT: ret2679;2680; CHECK-GI-LABEL: add_pair_v2i16_v2i64_sext:2681; CHECK-GI: // %bb.0: // %entry2682; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #02683; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #02684; CHECK-GI-NEXT: shl v0.2d, v0.2d, #482685; CHECK-GI-NEXT: shl v1.2d, v1.2d, #482686; CHECK-GI-NEXT: sshr v0.2d, v0.2d, #482687; CHECK-GI-NEXT: sshr v1.2d, v1.2d, #482688; CHECK-GI-NEXT: addp d0, v0.2d2689; CHECK-GI-NEXT: addp d1, v1.2d2690; CHECK-GI-NEXT: fmov x8, d02691; CHECK-GI-NEXT: fmov x9, d12692; CHECK-GI-NEXT: add x0, x8, x92693; CHECK-GI-NEXT: ret2694entry:2695 %xx = sext <2 x i16> %x to <2 x i64>2696 %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)2697 %yy = sext <2 x i16> %y to <2 x i64>2698 %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy)2699 %z = add i64 %z1, %z22700 ret i64 %z2701}2702 2703define i32 @add_pair_v16i8_v16i32_zext(<16 x i8> %x, <16 x i8> %y) {2704; CHECK-SD-BASE-LABEL: add_pair_v16i8_v16i32_zext:2705; CHECK-SD-BASE: // %bb.0: // %entry2706; CHECK-SD-BASE-NEXT: ushll2 v2.8h, v0.16b, #02707; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #02708; CHECK-SD-BASE-NEXT: ushll2 v3.8h, v1.16b, #02709; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #02710; CHECK-SD-BASE-NEXT: uaddl2 v4.4s, v0.8h, v2.8h2711; CHECK-SD-BASE-NEXT: uaddl v0.4s, v0.4h, v2.4h2712; CHECK-SD-BASE-NEXT: uaddl2 v2.4s, v1.8h, v3.8h2713; CHECK-SD-BASE-NEXT: uaddl v1.4s, v1.4h, v3.4h2714; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v4.4s2715; CHECK-SD-BASE-NEXT: add v1.4s, v1.4s, v2.4s2716; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s2717; CHECK-SD-BASE-NEXT: addv s0, v0.4s2718; CHECK-SD-BASE-NEXT: fmov w0, s02719; CHECK-SD-BASE-NEXT: ret2720;2721; CHECK-SD-DOT-LABEL: add_pair_v16i8_v16i32_zext:2722; CHECK-SD-DOT: // %bb.0: // %entry2723; CHECK-SD-DOT-NEXT: movi v2.16b, #12724; CHECK-SD-DOT-NEXT: movi v3.2d, #00000000000000002725; CHECK-SD-DOT-NEXT: udot v3.4s, v1.16b, v2.16b2726; CHECK-SD-DOT-NEXT: udot v3.4s, v0.16b, v2.16b2727; CHECK-SD-DOT-NEXT: addv s0, v3.4s2728; CHECK-SD-DOT-NEXT: fmov w0, s02729; CHECK-SD-DOT-NEXT: ret2730;2731; CHECK-GI-BASE-LABEL: add_pair_v16i8_v16i32_zext:2732; CHECK-GI-BASE: // %bb.0: // %entry2733; CHECK-GI-BASE-NEXT: uaddlv h1, v1.16b2734; CHECK-GI-BASE-NEXT: uaddlv h0, v0.16b2735; CHECK-GI-BASE-NEXT: fmov w8, s12736; CHECK-GI-BASE-NEXT: fmov w9, s02737; CHECK-GI-BASE-NEXT: and w8, w8, #0xffff2738; CHECK-GI-BASE-NEXT: add w0, w8, w9, uxth2739; CHECK-GI-BASE-NEXT: ret2740;2741; CHECK-GI-DOT-LABEL: add_pair_v16i8_v16i32_zext:2742; CHECK-GI-DOT: // %bb.0: // %entry2743; CHECK-GI-DOT-NEXT: movi v2.16b, #12744; CHECK-GI-DOT-NEXT: movi v3.2d, #00000000000000002745; CHECK-GI-DOT-NEXT: movi v4.2d, #00000000000000002746; CHECK-GI-DOT-NEXT: udot v4.4s, v0.16b, v2.16b2747; CHECK-GI-DOT-NEXT: udot v3.4s, v1.16b, v2.16b2748; CHECK-GI-DOT-NEXT: addv s0, v4.4s2749; CHECK-GI-DOT-NEXT: addv s1, v3.4s2750; CHECK-GI-DOT-NEXT: fmov w8, s02751; CHECK-GI-DOT-NEXT: fmov w9, s12752; CHECK-GI-DOT-NEXT: add w0, w8, w92753; CHECK-GI-DOT-NEXT: ret2754entry:2755 %xx = zext <16 x i8> %x to <16 x i32>2756 %z1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx)2757 %yy = zext <16 x i8> %y to <16 x i32>2758 %z2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %yy)2759 %z = add i32 %z1, %z22760 ret i32 %z2761}2762 2763define i32 @add_pair_v16i8_v16i32_sext(<16 x i8> %x, <16 x i8> %y) {2764; CHECK-SD-BASE-LABEL: add_pair_v16i8_v16i32_sext:2765; CHECK-SD-BASE: // %bb.0: // %entry2766; CHECK-SD-BASE-NEXT: sshll2 v2.8h, v0.16b, #02767; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #02768; CHECK-SD-BASE-NEXT: sshll2 v3.8h, v1.16b, #02769; CHECK-SD-BASE-NEXT: sshll v1.8h, v1.8b, #02770; CHECK-SD-BASE-NEXT: saddl2 v4.4s, v0.8h, v2.8h2771; CHECK-SD-BASE-NEXT: saddl v0.4s, v0.4h, v2.4h2772; CHECK-SD-BASE-NEXT: saddl2 v2.4s, v1.8h, v3.8h2773; CHECK-SD-BASE-NEXT: saddl v1.4s, v1.4h, v3.4h2774; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v4.4s2775; CHECK-SD-BASE-NEXT: add v1.4s, v1.4s, v2.4s2776; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s2777; CHECK-SD-BASE-NEXT: addv s0, v0.4s2778; CHECK-SD-BASE-NEXT: fmov w0, s02779; CHECK-SD-BASE-NEXT: ret2780;2781; CHECK-SD-DOT-LABEL: add_pair_v16i8_v16i32_sext:2782; CHECK-SD-DOT: // %bb.0: // %entry2783; CHECK-SD-DOT-NEXT: movi v2.16b, #12784; CHECK-SD-DOT-NEXT: movi v3.2d, #00000000000000002785; CHECK-SD-DOT-NEXT: sdot v3.4s, v1.16b, v2.16b2786; CHECK-SD-DOT-NEXT: sdot v3.4s, v0.16b, v2.16b2787; CHECK-SD-DOT-NEXT: addv s0, v3.4s2788; CHECK-SD-DOT-NEXT: fmov w0, s02789; CHECK-SD-DOT-NEXT: ret2790;2791; CHECK-GI-BASE-LABEL: add_pair_v16i8_v16i32_sext:2792; CHECK-GI-BASE: // %bb.0: // %entry2793; CHECK-GI-BASE-NEXT: saddlv h1, v1.16b2794; CHECK-GI-BASE-NEXT: saddlv h0, v0.16b2795; CHECK-GI-BASE-NEXT: fmov w8, s12796; CHECK-GI-BASE-NEXT: fmov w9, s02797; CHECK-GI-BASE-NEXT: sxth w8, w82798; CHECK-GI-BASE-NEXT: add w0, w8, w9, sxth2799; CHECK-GI-BASE-NEXT: ret2800;2801; CHECK-GI-DOT-LABEL: add_pair_v16i8_v16i32_sext:2802; CHECK-GI-DOT: // %bb.0: // %entry2803; CHECK-GI-DOT-NEXT: movi v2.16b, #12804; CHECK-GI-DOT-NEXT: movi v3.2d, #00000000000000002805; CHECK-GI-DOT-NEXT: movi v4.2d, #00000000000000002806; CHECK-GI-DOT-NEXT: sdot v4.4s, v0.16b, v2.16b2807; CHECK-GI-DOT-NEXT: sdot v3.4s, v1.16b, v2.16b2808; CHECK-GI-DOT-NEXT: addv s0, v4.4s2809; CHECK-GI-DOT-NEXT: addv s1, v3.4s2810; CHECK-GI-DOT-NEXT: fmov w8, s02811; CHECK-GI-DOT-NEXT: fmov w9, s12812; CHECK-GI-DOT-NEXT: add w0, w8, w92813; CHECK-GI-DOT-NEXT: ret2814entry:2815 %xx = sext <16 x i8> %x to <16 x i32>2816 %z1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx)2817 %yy = sext <16 x i8> %y to <16 x i32>2818 %z2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %yy)2819 %z = add i32 %z1, %z22820 ret i32 %z2821}2822 2823define i32 @add_pair_v8i8_v8i32_zext(<8 x i8> %x, <8 x i8> %y) {2824; CHECK-SD-BASE-LABEL: add_pair_v8i8_v8i32_zext:2825; CHECK-SD-BASE: // %bb.0: // %entry2826; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #02827; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #02828; CHECK-SD-BASE-NEXT: uaddlp v1.4s, v1.8h2829; CHECK-SD-BASE-NEXT: uadalp v1.4s, v0.8h2830; CHECK-SD-BASE-NEXT: addv s0, v1.4s2831; CHECK-SD-BASE-NEXT: fmov w0, s02832; CHECK-SD-BASE-NEXT: ret2833;2834; CHECK-SD-DOT-LABEL: add_pair_v8i8_v8i32_zext:2835; CHECK-SD-DOT: // %bb.0: // %entry2836; CHECK-SD-DOT-NEXT: movi v2.2d, #00000000000000002837; CHECK-SD-DOT-NEXT: movi v3.8b, #12838; CHECK-SD-DOT-NEXT: udot v2.2s, v1.8b, v3.8b2839; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b2840; CHECK-SD-DOT-NEXT: addp v0.2s, v2.2s, v2.2s2841; CHECK-SD-DOT-NEXT: fmov w0, s02842; CHECK-SD-DOT-NEXT: ret2843;2844; CHECK-GI-BASE-LABEL: add_pair_v8i8_v8i32_zext:2845; CHECK-GI-BASE: // %bb.0: // %entry2846; CHECK-GI-BASE-NEXT: uaddlv h1, v1.8b2847; CHECK-GI-BASE-NEXT: uaddlv h0, v0.8b2848; CHECK-GI-BASE-NEXT: fmov w8, s12849; CHECK-GI-BASE-NEXT: fmov w9, s02850; CHECK-GI-BASE-NEXT: and w8, w8, #0xffff2851; CHECK-GI-BASE-NEXT: add w0, w8, w9, uxth2852; CHECK-GI-BASE-NEXT: ret2853;2854; CHECK-GI-DOT-LABEL: add_pair_v8i8_v8i32_zext:2855; CHECK-GI-DOT: // %bb.0: // %entry2856; CHECK-GI-DOT-NEXT: movi v2.8b, #12857; CHECK-GI-DOT-NEXT: movi v3.2d, #00000000000000002858; CHECK-GI-DOT-NEXT: movi v4.2d, #00000000000000002859; CHECK-GI-DOT-NEXT: udot v4.2s, v0.8b, v2.8b2860; CHECK-GI-DOT-NEXT: udot v3.2s, v1.8b, v2.8b2861; CHECK-GI-DOT-NEXT: addp v0.2s, v4.2s, v4.2s2862; CHECK-GI-DOT-NEXT: addp v1.2s, v3.2s, v3.2s2863; CHECK-GI-DOT-NEXT: fmov w8, s02864; CHECK-GI-DOT-NEXT: fmov w9, s12865; CHECK-GI-DOT-NEXT: add w0, w8, w92866; CHECK-GI-DOT-NEXT: ret2867entry:2868 %xx = zext <8 x i8> %x to <8 x i32>2869 %z1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)2870 %yy = zext <8 x i8> %y to <8 x i32>2871 %z2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %yy)2872 %z = add i32 %z1, %z22873 ret i32 %z2874}2875 2876define i32 @add_pair_v8i8_v8i32_sext(<8 x i8> %x, <8 x i8> %y) {2877; CHECK-SD-BASE-LABEL: add_pair_v8i8_v8i32_sext:2878; CHECK-SD-BASE: // %bb.0: // %entry2879; CHECK-SD-BASE-NEXT: sshll v1.8h, v1.8b, #02880; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #02881; CHECK-SD-BASE-NEXT: saddlp v1.4s, v1.8h2882; CHECK-SD-BASE-NEXT: sadalp v1.4s, v0.8h2883; CHECK-SD-BASE-NEXT: addv s0, v1.4s2884; CHECK-SD-BASE-NEXT: fmov w0, s02885; CHECK-SD-BASE-NEXT: ret2886;2887; CHECK-SD-DOT-LABEL: add_pair_v8i8_v8i32_sext:2888; CHECK-SD-DOT: // %bb.0: // %entry2889; CHECK-SD-DOT-NEXT: movi v2.2d, #00000000000000002890; CHECK-SD-DOT-NEXT: movi v3.8b, #12891; CHECK-SD-DOT-NEXT: sdot v2.2s, v1.8b, v3.8b2892; CHECK-SD-DOT-NEXT: sdot v2.2s, v0.8b, v3.8b2893; CHECK-SD-DOT-NEXT: addp v0.2s, v2.2s, v2.2s2894; CHECK-SD-DOT-NEXT: fmov w0, s02895; CHECK-SD-DOT-NEXT: ret2896;2897; CHECK-GI-BASE-LABEL: add_pair_v8i8_v8i32_sext:2898; CHECK-GI-BASE: // %bb.0: // %entry2899; CHECK-GI-BASE-NEXT: saddlv h1, v1.8b2900; CHECK-GI-BASE-NEXT: saddlv h0, v0.8b2901; CHECK-GI-BASE-NEXT: fmov w8, s12902; CHECK-GI-BASE-NEXT: fmov w9, s02903; CHECK-GI-BASE-NEXT: sxth w8, w82904; CHECK-GI-BASE-NEXT: add w0, w8, w9, sxth2905; CHECK-GI-BASE-NEXT: ret2906;2907; CHECK-GI-DOT-LABEL: add_pair_v8i8_v8i32_sext:2908; CHECK-GI-DOT: // %bb.0: // %entry2909; CHECK-GI-DOT-NEXT: movi v2.8b, #12910; CHECK-GI-DOT-NEXT: movi v3.2d, #00000000000000002911; CHECK-GI-DOT-NEXT: movi v4.2d, #00000000000000002912; CHECK-GI-DOT-NEXT: sdot v4.2s, v0.8b, v2.8b2913; CHECK-GI-DOT-NEXT: sdot v3.2s, v1.8b, v2.8b2914; CHECK-GI-DOT-NEXT: addp v0.2s, v4.2s, v4.2s2915; CHECK-GI-DOT-NEXT: addp v1.2s, v3.2s, v3.2s2916; CHECK-GI-DOT-NEXT: fmov w8, s02917; CHECK-GI-DOT-NEXT: fmov w9, s12918; CHECK-GI-DOT-NEXT: add w0, w8, w92919; CHECK-GI-DOT-NEXT: ret2920entry:2921 %xx = sext <8 x i8> %x to <8 x i32>2922 %z1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)2923 %yy = sext <8 x i8> %y to <8 x i32>2924 %z2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %yy)2925 %z = add i32 %z1, %z22926 ret i32 %z2927}2928 2929define i32 @add_pair_v4i8_v4i32_zext(<4 x i8> %x, <4 x i8> %y) {2930; CHECK-SD-LABEL: add_pair_v4i8_v4i32_zext:2931; CHECK-SD: // %bb.0: // %entry2932; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q12933; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q02934; CHECK-SD-NEXT: bic v0.4h, #255, lsl #82935; CHECK-SD-NEXT: bic v1.4h, #255, lsl #82936; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]2937; CHECK-SD-NEXT: uaddlv s0, v0.8h2938; CHECK-SD-NEXT: fmov w0, s02939; CHECK-SD-NEXT: ret2940;2941; CHECK-GI-LABEL: add_pair_v4i8_v4i32_zext:2942; CHECK-GI: // %bb.0: // %entry2943; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff2944; CHECK-GI-NEXT: and v1.8b, v1.8b, v2.8b2945; CHECK-GI-NEXT: and v0.8b, v0.8b, v2.8b2946; CHECK-GI-NEXT: uaddlv s1, v1.4h2947; CHECK-GI-NEXT: uaddlv s0, v0.4h2948; CHECK-GI-NEXT: fmov w8, s12949; CHECK-GI-NEXT: fmov w9, s02950; CHECK-GI-NEXT: and w8, w8, #0xffff2951; CHECK-GI-NEXT: add w0, w8, w9, uxth2952; CHECK-GI-NEXT: ret2953entry:2954 %xx = zext <4 x i8> %x to <4 x i32>2955 %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)2956 %yy = zext <4 x i8> %y to <4 x i32>2957 %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %yy)2958 %z = add i32 %z1, %z22959 ret i32 %z2960}2961 2962define i32 @add_pair_v4i8_v4i32_sext(<4 x i8> %x, <4 x i8> %y) {2963; CHECK-SD-LABEL: add_pair_v4i8_v4i32_sext:2964; CHECK-SD: // %bb.0: // %entry2965; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #02966; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #02967; CHECK-SD-NEXT: shl v0.4s, v0.4s, #242968; CHECK-SD-NEXT: shl v1.4s, v1.4s, #242969; CHECK-SD-NEXT: sshr v0.4s, v0.4s, #242970; CHECK-SD-NEXT: ssra v0.4s, v1.4s, #242971; CHECK-SD-NEXT: addv s0, v0.4s2972; CHECK-SD-NEXT: fmov w0, s02973; CHECK-SD-NEXT: ret2974;2975; CHECK-GI-LABEL: add_pair_v4i8_v4i32_sext:2976; CHECK-GI: // %bb.0: // %entry2977; CHECK-GI-NEXT: shl v1.4h, v1.4h, #82978; CHECK-GI-NEXT: shl v0.4h, v0.4h, #82979; CHECK-GI-NEXT: sshr v1.4h, v1.4h, #82980; CHECK-GI-NEXT: sshr v0.4h, v0.4h, #82981; CHECK-GI-NEXT: saddlv s1, v1.4h2982; CHECK-GI-NEXT: saddlv s0, v0.4h2983; CHECK-GI-NEXT: fmov w8, s12984; CHECK-GI-NEXT: fmov w9, s02985; CHECK-GI-NEXT: sxth w8, w82986; CHECK-GI-NEXT: add w0, w8, w9, sxth2987; CHECK-GI-NEXT: ret2988entry:2989 %xx = sext <4 x i8> %x to <4 x i32>2990 %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)2991 %yy = sext <4 x i8> %y to <4 x i32>2992 %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %yy)2993 %z = add i32 %z1, %z22994 ret i32 %z2995}2996 2997define zeroext i16 @add_pair_v16i8_v16i16_zext(<16 x i8> %x, <16 x i8> %y) {2998; CHECK-SD-LABEL: add_pair_v16i8_v16i16_zext:2999; CHECK-SD: // %bb.0: // %entry3000; CHECK-SD-NEXT: uaddlp v1.8h, v1.16b3001; CHECK-SD-NEXT: uadalp v1.8h, v0.16b3002; CHECK-SD-NEXT: addv h0, v1.8h3003; CHECK-SD-NEXT: fmov w0, s03004; CHECK-SD-NEXT: ret3005;3006; CHECK-GI-LABEL: add_pair_v16i8_v16i16_zext:3007; CHECK-GI: // %bb.0: // %entry3008; CHECK-GI-NEXT: uaddlv h0, v0.16b3009; CHECK-GI-NEXT: uaddlv h1, v1.16b3010; CHECK-GI-NEXT: fmov w8, s03011; CHECK-GI-NEXT: fmov w9, s13012; CHECK-GI-NEXT: add w8, w8, w93013; CHECK-GI-NEXT: and w0, w8, #0xffff3014; CHECK-GI-NEXT: ret3015entry:3016 %xx = zext <16 x i8> %x to <16 x i16>3017 %z1 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)3018 %yy = zext <16 x i8> %y to <16 x i16>3019 %z2 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %yy)3020 %z = add i16 %z1, %z23021 ret i16 %z3022}3023 3024define signext i16 @add_pair_v16i8_v16i16_sext(<16 x i8> %x, <16 x i8> %y) {3025; CHECK-SD-LABEL: add_pair_v16i8_v16i16_sext:3026; CHECK-SD: // %bb.0: // %entry3027; CHECK-SD-NEXT: saddlp v1.8h, v1.16b3028; CHECK-SD-NEXT: sadalp v1.8h, v0.16b3029; CHECK-SD-NEXT: addv h0, v1.8h3030; CHECK-SD-NEXT: smov w0, v0.h[0]3031; CHECK-SD-NEXT: ret3032;3033; CHECK-GI-LABEL: add_pair_v16i8_v16i16_sext:3034; CHECK-GI: // %bb.0: // %entry3035; CHECK-GI-NEXT: saddlv h0, v0.16b3036; CHECK-GI-NEXT: saddlv h1, v1.16b3037; CHECK-GI-NEXT: fmov w8, s03038; CHECK-GI-NEXT: fmov w9, s13039; CHECK-GI-NEXT: add w8, w8, w93040; CHECK-GI-NEXT: sxth w0, w83041; CHECK-GI-NEXT: ret3042entry:3043 %xx = sext <16 x i8> %x to <16 x i16>3044 %z1 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)3045 %yy = sext <16 x i8> %y to <16 x i16>3046 %z2 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %yy)3047 %z = add i16 %z1, %z23048 ret i16 %z3049}3050 3051define zeroext i16 @add_pair_v8i8_v8i16_zext(<8 x i8> %x, <8 x i8> %y) {3052; CHECK-SD-LABEL: add_pair_v8i8_v8i16_zext:3053; CHECK-SD: // %bb.0: // %entry3054; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q03055; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q13056; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]3057; CHECK-SD-NEXT: uaddlv h0, v0.16b3058; CHECK-SD-NEXT: umov w0, v0.h[0]3059; CHECK-SD-NEXT: ret3060;3061; CHECK-GI-LABEL: add_pair_v8i8_v8i16_zext:3062; CHECK-GI: // %bb.0: // %entry3063; CHECK-GI-NEXT: uaddlv h0, v0.8b3064; CHECK-GI-NEXT: uaddlv h1, v1.8b3065; CHECK-GI-NEXT: fmov w8, s03066; CHECK-GI-NEXT: fmov w9, s13067; CHECK-GI-NEXT: add w8, w8, w93068; CHECK-GI-NEXT: and w0, w8, #0xffff3069; CHECK-GI-NEXT: ret3070entry:3071 %xx = zext <8 x i8> %x to <8 x i16>3072 %z1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)3073 %yy = zext <8 x i8> %y to <8 x i16>3074 %z2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %yy)3075 %z = add i16 %z1, %z23076 ret i16 %z3077}3078 3079define signext i16 @add_pair_v8i8_v8i16_sext(<8 x i8> %x, <8 x i8> %y) {3080; CHECK-SD-LABEL: add_pair_v8i8_v8i16_sext:3081; CHECK-SD: // %bb.0: // %entry3082; CHECK-SD-NEXT: saddl v0.8h, v0.8b, v1.8b3083; CHECK-SD-NEXT: addv h0, v0.8h3084; CHECK-SD-NEXT: smov w0, v0.h[0]3085; CHECK-SD-NEXT: ret3086;3087; CHECK-GI-LABEL: add_pair_v8i8_v8i16_sext:3088; CHECK-GI: // %bb.0: // %entry3089; CHECK-GI-NEXT: saddlv h0, v0.8b3090; CHECK-GI-NEXT: saddlv h1, v1.8b3091; CHECK-GI-NEXT: fmov w8, s03092; CHECK-GI-NEXT: fmov w9, s13093; CHECK-GI-NEXT: add w8, w8, w93094; CHECK-GI-NEXT: sxth w0, w83095; CHECK-GI-NEXT: ret3096entry:3097 %xx = sext <8 x i8> %x to <8 x i16>3098 %z1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)3099 %yy = sext <8 x i8> %y to <8 x i16>3100 %z2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %yy)3101 %z = add i16 %z1, %z23102 ret i16 %z3103}3104 3105define zeroext i8 @add_pair_v16i8_v16i8(<16 x i8> %x, <16 x i8> %y) {3106; CHECK-SD-LABEL: add_pair_v16i8_v16i8:3107; CHECK-SD: // %bb.0: // %entry3108; CHECK-SD-NEXT: add v0.16b, v0.16b, v1.16b3109; CHECK-SD-NEXT: addv b0, v0.16b3110; CHECK-SD-NEXT: fmov w0, s03111; CHECK-SD-NEXT: ret3112;3113; CHECK-GI-LABEL: add_pair_v16i8_v16i8:3114; CHECK-GI: // %bb.0: // %entry3115; CHECK-GI-NEXT: addv b0, v0.16b3116; CHECK-GI-NEXT: addv b1, v1.16b3117; CHECK-GI-NEXT: fmov w8, s03118; CHECK-GI-NEXT: fmov w9, s13119; CHECK-GI-NEXT: add w8, w9, w8, uxtb3120; CHECK-GI-NEXT: and w0, w8, #0xff3121; CHECK-GI-NEXT: ret3122entry:3123 %z1 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %x)3124 %z2 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %y)3125 %z = add i8 %z1, %z23126 ret i8 %z3127}3128 3129define i64 @add_pair_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %y) {3130; CHECK-SD-LABEL: add_pair_v16i8_v16i64_zext:3131; CHECK-SD: // %bb.0: // %entry3132; CHECK-SD-NEXT: ushll2 v2.8h, v0.16b, #03133; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #03134; CHECK-SD-NEXT: ushll2 v3.8h, v1.16b, #03135; CHECK-SD-NEXT: ushll v1.8h, v1.8b, #03136; CHECK-SD-NEXT: ushll v4.4s, v2.4h, #03137; CHECK-SD-NEXT: ushll2 v2.4s, v2.8h, #03138; CHECK-SD-NEXT: ushll2 v5.4s, v0.8h, #03139; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #03140; CHECK-SD-NEXT: ushll2 v6.4s, v3.8h, #03141; CHECK-SD-NEXT: ushll2 v7.4s, v1.8h, #03142; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #03143; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #03144; CHECK-SD-NEXT: uaddl2 v16.2d, v5.4s, v2.4s3145; CHECK-SD-NEXT: uaddl v2.2d, v5.2s, v2.2s3146; CHECK-SD-NEXT: uaddl2 v5.2d, v0.4s, v4.4s3147; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v4.2s3148; CHECK-SD-NEXT: uaddl2 v4.2d, v7.4s, v6.4s3149; CHECK-SD-NEXT: uaddl v6.2d, v7.2s, v6.2s3150; CHECK-SD-NEXT: uaddl2 v7.2d, v1.4s, v3.4s3151; CHECK-SD-NEXT: uaddl v1.2d, v1.2s, v3.2s3152; CHECK-SD-NEXT: add v3.2d, v5.2d, v16.2d3153; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d3154; CHECK-SD-NEXT: add v2.2d, v7.2d, v4.2d3155; CHECK-SD-NEXT: add v1.2d, v1.2d, v6.2d3156; CHECK-SD-NEXT: add v0.2d, v0.2d, v3.2d3157; CHECK-SD-NEXT: add v1.2d, v1.2d, v2.2d3158; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d3159; CHECK-SD-NEXT: addp d0, v0.2d3160; CHECK-SD-NEXT: fmov x0, d03161; CHECK-SD-NEXT: ret3162;3163; CHECK-GI-LABEL: add_pair_v16i8_v16i64_zext:3164; CHECK-GI: // %bb.0: // %entry3165; CHECK-GI-NEXT: uaddlv h1, v1.16b3166; CHECK-GI-NEXT: uaddlv h0, v0.16b3167; CHECK-GI-NEXT: fmov w8, s13168; CHECK-GI-NEXT: fmov w9, s03169; CHECK-GI-NEXT: and x8, x8, #0xffff3170; CHECK-GI-NEXT: add x0, x8, w9, uxth3171; CHECK-GI-NEXT: ret3172entry:3173 %xx = zext <16 x i8> %x to <16 x i64>3174 %z1 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx)3175 %yy = zext <16 x i8> %y to <16 x i64>3176 %z2 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %yy)3177 %z = add i64 %z1, %z23178 ret i64 %z3179}3180 3181define i64 @add_pair_v16i8_v16i64_sext(<16 x i8> %x, <16 x i8> %y) {3182; CHECK-SD-LABEL: add_pair_v16i8_v16i64_sext:3183; CHECK-SD: // %bb.0: // %entry3184; CHECK-SD-NEXT: sshll2 v2.8h, v0.16b, #03185; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #03186; CHECK-SD-NEXT: sshll2 v3.8h, v1.16b, #03187; CHECK-SD-NEXT: sshll v1.8h, v1.8b, #03188; CHECK-SD-NEXT: sshll v4.4s, v2.4h, #03189; CHECK-SD-NEXT: sshll2 v2.4s, v2.8h, #03190; CHECK-SD-NEXT: sshll2 v5.4s, v0.8h, #03191; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #03192; CHECK-SD-NEXT: sshll2 v6.4s, v3.8h, #03193; CHECK-SD-NEXT: sshll2 v7.4s, v1.8h, #03194; CHECK-SD-NEXT: sshll v3.4s, v3.4h, #03195; CHECK-SD-NEXT: sshll v1.4s, v1.4h, #03196; CHECK-SD-NEXT: saddl2 v16.2d, v5.4s, v2.4s3197; CHECK-SD-NEXT: saddl v2.2d, v5.2s, v2.2s3198; CHECK-SD-NEXT: saddl2 v5.2d, v0.4s, v4.4s3199; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v4.2s3200; CHECK-SD-NEXT: saddl2 v4.2d, v7.4s, v6.4s3201; CHECK-SD-NEXT: saddl v6.2d, v7.2s, v6.2s3202; CHECK-SD-NEXT: saddl2 v7.2d, v1.4s, v3.4s3203; CHECK-SD-NEXT: saddl v1.2d, v1.2s, v3.2s3204; CHECK-SD-NEXT: add v3.2d, v5.2d, v16.2d3205; CHECK-SD-NEXT: add v0.2d, v0.2d, v2.2d3206; CHECK-SD-NEXT: add v2.2d, v7.2d, v4.2d3207; CHECK-SD-NEXT: add v1.2d, v1.2d, v6.2d3208; CHECK-SD-NEXT: add v0.2d, v0.2d, v3.2d3209; CHECK-SD-NEXT: add v1.2d, v1.2d, v2.2d3210; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d3211; CHECK-SD-NEXT: addp d0, v0.2d3212; CHECK-SD-NEXT: fmov x0, d03213; CHECK-SD-NEXT: ret3214;3215; CHECK-GI-LABEL: add_pair_v16i8_v16i64_sext:3216; CHECK-GI: // %bb.0: // %entry3217; CHECK-GI-NEXT: saddlv h1, v1.16b3218; CHECK-GI-NEXT: saddlv h0, v0.16b3219; CHECK-GI-NEXT: fmov w8, s13220; CHECK-GI-NEXT: fmov w9, s03221; CHECK-GI-NEXT: sxth x8, w83222; CHECK-GI-NEXT: add x0, x8, w9, sxth3223; CHECK-GI-NEXT: ret3224entry:3225 %xx = sext <16 x i8> %x to <16 x i64>3226 %z1 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx)3227 %yy = sext <16 x i8> %y to <16 x i64>3228 %z2 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %yy)3229 %z = add i64 %z1, %z23230 ret i64 %z3231}3232 3233define i64 @add_pair_v8i8_v8i64_zext(<8 x i8> %x, <8 x i8> %y) {3234; CHECK-SD-LABEL: add_pair_v8i8_v8i64_zext:3235; CHECK-SD: // %bb.0: // %entry3236; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #03237; CHECK-SD-NEXT: ushll v1.8h, v1.8b, #03238; CHECK-SD-NEXT: ushll2 v2.4s, v0.8h, #03239; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #03240; CHECK-SD-NEXT: ushll2 v3.4s, v1.8h, #03241; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #03242; CHECK-SD-NEXT: uaddl2 v4.2d, v0.4s, v2.4s3243; CHECK-SD-NEXT: uaddl v0.2d, v0.2s, v2.2s3244; CHECK-SD-NEXT: uaddl2 v2.2d, v1.4s, v3.4s3245; CHECK-SD-NEXT: uaddl v1.2d, v1.2s, v3.2s3246; CHECK-SD-NEXT: add v0.2d, v0.2d, v4.2d3247; CHECK-SD-NEXT: add v1.2d, v1.2d, v2.2d3248; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d3249; CHECK-SD-NEXT: addp d0, v0.2d3250; CHECK-SD-NEXT: fmov x0, d03251; CHECK-SD-NEXT: ret3252;3253; CHECK-GI-LABEL: add_pair_v8i8_v8i64_zext:3254; CHECK-GI: // %bb.0: // %entry3255; CHECK-GI-NEXT: uaddlv h1, v1.8b3256; CHECK-GI-NEXT: uaddlv h0, v0.8b3257; CHECK-GI-NEXT: fmov w8, s13258; CHECK-GI-NEXT: fmov w9, s03259; CHECK-GI-NEXT: and x8, x8, #0xffff3260; CHECK-GI-NEXT: add x0, x8, w9, uxth3261; CHECK-GI-NEXT: ret3262entry:3263 %xx = zext <8 x i8> %x to <8 x i64>3264 %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)3265 %yy = zext <8 x i8> %y to <8 x i64>3266 %z2 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %yy)3267 %z = add i64 %z1, %z23268 ret i64 %z3269}3270 3271define i64 @add_pair_v8i8_v8i64_sext(<8 x i8> %x, <8 x i8> %y) {3272; CHECK-SD-LABEL: add_pair_v8i8_v8i64_sext:3273; CHECK-SD: // %bb.0: // %entry3274; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #03275; CHECK-SD-NEXT: sshll v1.8h, v1.8b, #03276; CHECK-SD-NEXT: sshll2 v2.4s, v0.8h, #03277; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #03278; CHECK-SD-NEXT: sshll2 v3.4s, v1.8h, #03279; CHECK-SD-NEXT: sshll v1.4s, v1.4h, #03280; CHECK-SD-NEXT: saddl2 v4.2d, v0.4s, v2.4s3281; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v2.2s3282; CHECK-SD-NEXT: saddl2 v2.2d, v1.4s, v3.4s3283; CHECK-SD-NEXT: saddl v1.2d, v1.2s, v3.2s3284; CHECK-SD-NEXT: add v0.2d, v0.2d, v4.2d3285; CHECK-SD-NEXT: add v1.2d, v1.2d, v2.2d3286; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d3287; CHECK-SD-NEXT: addp d0, v0.2d3288; CHECK-SD-NEXT: fmov x0, d03289; CHECK-SD-NEXT: ret3290;3291; CHECK-GI-LABEL: add_pair_v8i8_v8i64_sext:3292; CHECK-GI: // %bb.0: // %entry3293; CHECK-GI-NEXT: saddlv h1, v1.8b3294; CHECK-GI-NEXT: saddlv h0, v0.8b3295; CHECK-GI-NEXT: fmov w8, s13296; CHECK-GI-NEXT: fmov w9, s03297; CHECK-GI-NEXT: sxth x8, w83298; CHECK-GI-NEXT: add x0, x8, w9, sxth3299; CHECK-GI-NEXT: ret3300entry:3301 %xx = sext <8 x i8> %x to <8 x i64>3302 %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)3303 %yy = sext <8 x i8> %y to <8 x i64>3304 %z2 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %yy)3305 %z = add i64 %z1, %z23306 ret i64 %z3307}3308 3309define i64 @add_pair_v4i8_v4i64_zext(<4 x i8> %x, <4 x i8> %y) {3310; CHECK-SD-LABEL: add_pair_v4i8_v4i64_zext:3311; CHECK-SD: // %bb.0: // %entry3312; CHECK-SD-NEXT: bic v1.4h, #255, lsl #83313; CHECK-SD-NEXT: bic v0.4h, #255, lsl #83314; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #03315; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #03316; CHECK-SD-NEXT: uaddlp v1.2d, v1.4s3317; CHECK-SD-NEXT: uadalp v1.2d, v0.4s3318; CHECK-SD-NEXT: addp d0, v1.2d3319; CHECK-SD-NEXT: fmov x0, d03320; CHECK-SD-NEXT: ret3321;3322; CHECK-GI-LABEL: add_pair_v4i8_v4i64_zext:3323; CHECK-GI: // %bb.0: // %entry3324; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff3325; CHECK-GI-NEXT: and v1.8b, v1.8b, v2.8b3326; CHECK-GI-NEXT: and v0.8b, v0.8b, v2.8b3327; CHECK-GI-NEXT: uaddlv s1, v1.4h3328; CHECK-GI-NEXT: uaddlv s0, v0.4h3329; CHECK-GI-NEXT: fmov w8, s13330; CHECK-GI-NEXT: fmov w9, s03331; CHECK-GI-NEXT: and x8, x8, #0xffff3332; CHECK-GI-NEXT: add x0, x8, w9, uxth3333; CHECK-GI-NEXT: ret3334entry:3335 %xx = zext <4 x i8> %x to <4 x i64>3336 %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)3337 %yy = zext <4 x i8> %y to <4 x i64>3338 %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy)3339 %z = add i64 %z1, %z23340 ret i64 %z3341}3342 3343define i64 @add_pair_v4i8_v4i64_sext(<4 x i8> %x, <4 x i8> %y) {3344; CHECK-SD-LABEL: add_pair_v4i8_v4i64_sext:3345; CHECK-SD: // %bb.0: // %entry3346; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #03347; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #03348; CHECK-SD-NEXT: ushll v2.2d, v1.2s, #03349; CHECK-SD-NEXT: ushll v3.2d, v0.2s, #03350; CHECK-SD-NEXT: ushll2 v0.2d, v0.4s, #03351; CHECK-SD-NEXT: ushll2 v1.2d, v1.4s, #03352; CHECK-SD-NEXT: shl v3.2d, v3.2d, #563353; CHECK-SD-NEXT: shl v2.2d, v2.2d, #563354; CHECK-SD-NEXT: shl v0.2d, v0.2d, #563355; CHECK-SD-NEXT: shl v1.2d, v1.2d, #563356; CHECK-SD-NEXT: sshr v3.2d, v3.2d, #563357; CHECK-SD-NEXT: sshr v2.2d, v2.2d, #563358; CHECK-SD-NEXT: ssra v3.2d, v0.2d, #563359; CHECK-SD-NEXT: ssra v2.2d, v1.2d, #563360; CHECK-SD-NEXT: add v0.2d, v3.2d, v2.2d3361; CHECK-SD-NEXT: addp d0, v0.2d3362; CHECK-SD-NEXT: fmov x0, d03363; CHECK-SD-NEXT: ret3364;3365; CHECK-GI-LABEL: add_pair_v4i8_v4i64_sext:3366; CHECK-GI: // %bb.0: // %entry3367; CHECK-GI-NEXT: shl v1.4h, v1.4h, #83368; CHECK-GI-NEXT: shl v0.4h, v0.4h, #83369; CHECK-GI-NEXT: sshr v1.4h, v1.4h, #83370; CHECK-GI-NEXT: sshr v0.4h, v0.4h, #83371; CHECK-GI-NEXT: saddlv s1, v1.4h3372; CHECK-GI-NEXT: saddlv s0, v0.4h3373; CHECK-GI-NEXT: fmov w8, s13374; CHECK-GI-NEXT: fmov w9, s03375; CHECK-GI-NEXT: sxth x8, w83376; CHECK-GI-NEXT: add x0, x8, w9, sxth3377; CHECK-GI-NEXT: ret3378entry:3379 %xx = sext <4 x i8> %x to <4 x i64>3380 %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)3381 %yy = sext <4 x i8> %y to <4 x i64>3382 %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy)3383 %z = add i64 %z1, %z23384 ret i64 %z3385}3386 3387define i64 @add_pair_v2i8_v2i64_zext(<2 x i8> %x, <2 x i8> %y) {3388; CHECK-SD-LABEL: add_pair_v2i8_v2i64_zext:3389; CHECK-SD: // %bb.0: // %entry3390; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q03391; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q13392; CHECK-SD-NEXT: movi v2.2d, #0x0000ff000000ff3393; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]3394; CHECK-SD-NEXT: and v0.16b, v0.16b, v2.16b3395; CHECK-SD-NEXT: uaddlv d0, v0.4s3396; CHECK-SD-NEXT: fmov x0, d03397; CHECK-SD-NEXT: ret3398;3399; CHECK-GI-LABEL: add_pair_v2i8_v2i64_zext:3400; CHECK-GI: // %bb.0: // %entry3401; CHECK-GI-NEXT: movi v2.2d, #0x000000000000ff3402; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #03403; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #03404; CHECK-GI-NEXT: and v0.16b, v0.16b, v2.16b3405; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b3406; CHECK-GI-NEXT: addp d0, v0.2d3407; CHECK-GI-NEXT: addp d1, v1.2d3408; CHECK-GI-NEXT: fmov x8, d03409; CHECK-GI-NEXT: fmov x9, d13410; CHECK-GI-NEXT: add x0, x8, x93411; CHECK-GI-NEXT: ret3412entry:3413 %xx = zext <2 x i8> %x to <2 x i64>3414 %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)3415 %yy = zext <2 x i8> %y to <2 x i64>3416 %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy)3417 %z = add i64 %z1, %z23418 ret i64 %z3419}3420 3421define i64 @add_pair_v2i8_v2i64_sext(<2 x i8> %x, <2 x i8> %y) {3422; CHECK-SD-LABEL: add_pair_v2i8_v2i64_sext:3423; CHECK-SD: // %bb.0: // %entry3424; CHECK-SD-NEXT: ushll v0.2d, v0.2s, #03425; CHECK-SD-NEXT: ushll v1.2d, v1.2s, #03426; CHECK-SD-NEXT: shl v0.2d, v0.2d, #563427; CHECK-SD-NEXT: shl v1.2d, v1.2d, #563428; CHECK-SD-NEXT: sshr v0.2d, v0.2d, #563429; CHECK-SD-NEXT: ssra v0.2d, v1.2d, #563430; CHECK-SD-NEXT: addp d0, v0.2d3431; CHECK-SD-NEXT: fmov x0, d03432; CHECK-SD-NEXT: ret3433;3434; CHECK-GI-LABEL: add_pair_v2i8_v2i64_sext:3435; CHECK-GI: // %bb.0: // %entry3436; CHECK-GI-NEXT: ushll v0.2d, v0.2s, #03437; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #03438; CHECK-GI-NEXT: shl v0.2d, v0.2d, #563439; CHECK-GI-NEXT: shl v1.2d, v1.2d, #563440; CHECK-GI-NEXT: sshr v0.2d, v0.2d, #563441; CHECK-GI-NEXT: sshr v1.2d, v1.2d, #563442; CHECK-GI-NEXT: addp d0, v0.2d3443; CHECK-GI-NEXT: addp d1, v1.2d3444; CHECK-GI-NEXT: fmov x8, d03445; CHECK-GI-NEXT: fmov x9, d13446; CHECK-GI-NEXT: add x0, x8, x93447; CHECK-GI-NEXT: ret3448entry:3449 %xx = sext <2 x i8> %x to <2 x i64>3450 %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)3451 %yy = sext <2 x i8> %y to <2 x i64>3452 %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy)3453 %z = add i64 %z1, %z23454 ret i64 %z3455}3456 3457define i32 @add_pair_v8i8_v8i32_double_sext_zext(<8 x i8> %ax, <8 x i8> %ay, <8 x i8> %bx, <8 x i8> %by) {3458; CHECK-SD-BASE-LABEL: add_pair_v8i8_v8i32_double_sext_zext:3459; CHECK-SD-BASE: // %bb.0: // %entry3460; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #03461; CHECK-SD-BASE-NEXT: sshll v3.8h, v3.8b, #03462; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #03463; CHECK-SD-BASE-NEXT: sshll v2.8h, v2.8b, #03464; CHECK-SD-BASE-NEXT: uaddlp v1.4s, v1.8h3465; CHECK-SD-BASE-NEXT: saddlp v3.4s, v3.8h3466; CHECK-SD-BASE-NEXT: uadalp v1.4s, v0.8h3467; CHECK-SD-BASE-NEXT: sadalp v3.4s, v2.8h3468; CHECK-SD-BASE-NEXT: add v0.4s, v3.4s, v1.4s3469; CHECK-SD-BASE-NEXT: addv s0, v0.4s3470; CHECK-SD-BASE-NEXT: fmov w0, s03471; CHECK-SD-BASE-NEXT: ret3472;3473; CHECK-SD-DOT-LABEL: add_pair_v8i8_v8i32_double_sext_zext:3474; CHECK-SD-DOT: // %bb.0: // %entry3475; CHECK-SD-DOT-NEXT: movi v4.2d, #00000000000000003476; CHECK-SD-DOT-NEXT: movi v5.8b, #13477; CHECK-SD-DOT-NEXT: movi v6.2d, #00000000000000003478; CHECK-SD-DOT-NEXT: udot v6.2s, v1.8b, v5.8b3479; CHECK-SD-DOT-NEXT: sdot v4.2s, v3.8b, v5.8b3480; CHECK-SD-DOT-NEXT: udot v6.2s, v0.8b, v5.8b3481; CHECK-SD-DOT-NEXT: sdot v4.2s, v2.8b, v5.8b3482; CHECK-SD-DOT-NEXT: add v0.2s, v6.2s, v4.2s3483; CHECK-SD-DOT-NEXT: addp v0.2s, v0.2s, v0.2s3484; CHECK-SD-DOT-NEXT: fmov w0, s03485; CHECK-SD-DOT-NEXT: ret3486;3487; CHECK-GI-BASE-LABEL: add_pair_v8i8_v8i32_double_sext_zext:3488; CHECK-GI-BASE: // %bb.0: // %entry3489; CHECK-GI-BASE-NEXT: saddlv h3, v3.8b3490; CHECK-GI-BASE-NEXT: uaddlv h1, v1.8b3491; CHECK-GI-BASE-NEXT: uaddlv h0, v0.8b3492; CHECK-GI-BASE-NEXT: saddlv h2, v2.8b3493; CHECK-GI-BASE-NEXT: fmov w8, s33494; CHECK-GI-BASE-NEXT: fmov w9, s13495; CHECK-GI-BASE-NEXT: fmov w10, s03496; CHECK-GI-BASE-NEXT: fmov w11, s23497; CHECK-GI-BASE-NEXT: sxth w8, w83498; CHECK-GI-BASE-NEXT: and w9, w9, #0xffff3499; CHECK-GI-BASE-NEXT: add w9, w9, w10, uxth3500; CHECK-GI-BASE-NEXT: add w8, w8, w11, sxth3501; CHECK-GI-BASE-NEXT: add w0, w9, w83502; CHECK-GI-BASE-NEXT: ret3503;3504; CHECK-GI-DOT-LABEL: add_pair_v8i8_v8i32_double_sext_zext:3505; CHECK-GI-DOT: // %bb.0: // %entry3506; CHECK-GI-DOT-NEXT: movi v4.8b, #13507; CHECK-GI-DOT-NEXT: movi v5.2d, #00000000000000003508; CHECK-GI-DOT-NEXT: movi v6.2d, #00000000000000003509; CHECK-GI-DOT-NEXT: movi v7.2d, #00000000000000003510; CHECK-GI-DOT-NEXT: movi v16.2d, #00000000000000003511; CHECK-GI-DOT-NEXT: udot v5.2s, v0.8b, v4.8b3512; CHECK-GI-DOT-NEXT: sdot v6.2s, v3.8b, v4.8b3513; CHECK-GI-DOT-NEXT: udot v7.2s, v1.8b, v4.8b3514; CHECK-GI-DOT-NEXT: sdot v16.2s, v2.8b, v4.8b3515; CHECK-GI-DOT-NEXT: addp v0.2s, v5.2s, v5.2s3516; CHECK-GI-DOT-NEXT: addp v3.2s, v6.2s, v6.2s3517; CHECK-GI-DOT-NEXT: addp v1.2s, v7.2s, v7.2s3518; CHECK-GI-DOT-NEXT: addp v2.2s, v16.2s, v16.2s3519; CHECK-GI-DOT-NEXT: fmov w8, s03520; CHECK-GI-DOT-NEXT: fmov w11, s33521; CHECK-GI-DOT-NEXT: fmov w9, s13522; CHECK-GI-DOT-NEXT: fmov w10, s23523; CHECK-GI-DOT-NEXT: add w8, w8, w93524; CHECK-GI-DOT-NEXT: add w9, w10, w113525; CHECK-GI-DOT-NEXT: add w0, w8, w93526; CHECK-GI-DOT-NEXT: ret3527entry:3528 %axx = zext <8 x i8> %ax to <8 x i32>3529 %az1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %axx)3530 %ayy = zext <8 x i8> %ay to <8 x i32>3531 %az2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %ayy)3532 %az = add i32 %az1, %az23533 %bxx = sext <8 x i8> %bx to <8 x i32>3534 %bz1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %bxx)3535 %byy = sext <8 x i8> %by to <8 x i32>3536 %bz2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %byy)3537 %bz = add i32 %bz1, %bz23538 %z = add i32 %az, %bz3539 ret i32 %z3540}3541 3542define i32 @add_pair_v8i16_v4i32_double_sext_zext_shuffle(<8 x i16> %ax, <8 x i16> %ay, <8 x i16> %bx, <8 x i16> %by) {3543; CHECK-SD-LABEL: add_pair_v8i16_v4i32_double_sext_zext_shuffle:3544; CHECK-SD: // %bb.0: // %entry3545; CHECK-SD-NEXT: uaddlp v1.4s, v1.8h3546; CHECK-SD-NEXT: uaddlp v3.4s, v3.8h3547; CHECK-SD-NEXT: uadalp v1.4s, v0.8h3548; CHECK-SD-NEXT: uadalp v3.4s, v2.8h3549; CHECK-SD-NEXT: add v0.4s, v3.4s, v1.4s3550; CHECK-SD-NEXT: addv s0, v0.4s3551; CHECK-SD-NEXT: fmov w0, s03552; CHECK-SD-NEXT: ret3553;3554; CHECK-GI-LABEL: add_pair_v8i16_v4i32_double_sext_zext_shuffle:3555; CHECK-GI: // %bb.0: // %entry3556; CHECK-GI-NEXT: ushll v4.4s, v0.4h, #03557; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #03558; CHECK-GI-NEXT: ushll v5.4s, v1.4h, #03559; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #03560; CHECK-GI-NEXT: ushll v6.4s, v2.4h, #03561; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #03562; CHECK-GI-NEXT: ushll v7.4s, v3.4h, #03563; CHECK-GI-NEXT: ushll2 v3.4s, v3.8h, #03564; CHECK-GI-NEXT: add v0.4s, v4.4s, v0.4s3565; CHECK-GI-NEXT: add v1.4s, v5.4s, v1.4s3566; CHECK-GI-NEXT: add v2.4s, v6.4s, v2.4s3567; CHECK-GI-NEXT: add v3.4s, v7.4s, v3.4s3568; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s3569; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s3570; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s3571; CHECK-GI-NEXT: addv s0, v0.4s3572; CHECK-GI-NEXT: fmov w0, s03573; CHECK-GI-NEXT: ret3574entry:3575 %axx = zext <8 x i16> %ax to <8 x i32>3576 %s1h = shufflevector <8 x i32> %axx, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>3577 %s1l = shufflevector <8 x i32> %axx, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3578 %axs = add <4 x i32> %s1h, %s1l3579 %ayy = zext <8 x i16> %ay to <8 x i32>3580 %s2h = shufflevector <8 x i32> %ayy, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>3581 %s2l = shufflevector <8 x i32> %ayy, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3582 %ays = add <4 x i32> %s2h, %s2l3583 %az = add <4 x i32> %axs, %ays3584 %bxx = zext <8 x i16> %bx to <8 x i32>3585 %s3h = shufflevector <8 x i32> %bxx, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>3586 %s3l = shufflevector <8 x i32> %bxx, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3587 %bxs = add <4 x i32> %s3h, %s3l3588 %byy = zext <8 x i16> %by to <8 x i32>3589 %s4h = shufflevector <8 x i32> %byy, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>3590 %s4l = shufflevector <8 x i32> %byy, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3591 %bys = add <4 x i32> %s4h, %s4l3592 %bz = add <4 x i32> %bxs, %bys3593 %z = add <4 x i32> %az, %bz3594 %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %z)3595 ret i32 %z23596}3597 3598define i64 @add_pair_v2i64_v2i64(<2 x i64> %x, <2 x i64> %y) {3599; CHECK-SD-LABEL: add_pair_v2i64_v2i64:3600; CHECK-SD: // %bb.0: // %entry3601; CHECK-SD-NEXT: add v0.2d, v0.2d, v1.2d3602; CHECK-SD-NEXT: addp d0, v0.2d3603; CHECK-SD-NEXT: fmov x0, d03604; CHECK-SD-NEXT: ret3605;3606; CHECK-GI-LABEL: add_pair_v2i64_v2i64:3607; CHECK-GI: // %bb.0: // %entry3608; CHECK-GI-NEXT: addp d0, v0.2d3609; CHECK-GI-NEXT: addp d1, v1.2d3610; CHECK-GI-NEXT: fmov x8, d03611; CHECK-GI-NEXT: fmov x9, d13612; CHECK-GI-NEXT: add x0, x8, x93613; CHECK-GI-NEXT: ret3614entry:3615 %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %x)3616 %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %y)3617 %z = add i64 %z1, %z23618 ret i64 %z3619}3620 3621; Irregularly sized vectors3622define i16 @add_v24i8_v24i16_zext(<24 x i8> %x) {3623; CHECK-SD-LABEL: add_v24i8_v24i16_zext:3624; CHECK-SD: // %bb.0: // %entry3625; CHECK-SD-NEXT: fmov s0, w03626; CHECK-SD-NEXT: ldr b1, [sp, #64]3627; CHECK-SD-NEXT: add x8, sp, #723628; CHECK-SD-NEXT: ldr b2, [sp]3629; CHECK-SD-NEXT: add x9, sp, #803630; CHECK-SD-NEXT: ld1 { v1.b }[1], [x8]3631; CHECK-SD-NEXT: add x8, sp, #83632; CHECK-SD-NEXT: mov v0.b[1], w13633; CHECK-SD-NEXT: ld1 { v2.b }[1], [x8]3634; CHECK-SD-NEXT: add x8, sp, #163635; CHECK-SD-NEXT: ld1 { v1.b }[2], [x9]3636; CHECK-SD-NEXT: add x9, sp, #883637; CHECK-SD-NEXT: ld1 { v2.b }[2], [x8]3638; CHECK-SD-NEXT: add x8, sp, #243639; CHECK-SD-NEXT: mov v0.b[2], w23640; CHECK-SD-NEXT: ld1 { v1.b }[3], [x9]3641; CHECK-SD-NEXT: add x9, sp, #963642; CHECK-SD-NEXT: ld1 { v2.b }[3], [x8]3643; CHECK-SD-NEXT: add x8, sp, #323644; CHECK-SD-NEXT: mov v0.b[3], w33645; CHECK-SD-NEXT: ld1 { v1.b }[4], [x9]3646; CHECK-SD-NEXT: add x9, sp, #1043647; CHECK-SD-NEXT: ld1 { v2.b }[4], [x8]3648; CHECK-SD-NEXT: add x8, sp, #403649; CHECK-SD-NEXT: ld1 { v1.b }[5], [x9]3650; CHECK-SD-NEXT: add x9, sp, #1123651; CHECK-SD-NEXT: mov v0.b[4], w43652; CHECK-SD-NEXT: ld1 { v2.b }[5], [x8]3653; CHECK-SD-NEXT: add x8, sp, #483654; CHECK-SD-NEXT: ld1 { v1.b }[6], [x9]3655; CHECK-SD-NEXT: add x9, sp, #1203656; CHECK-SD-NEXT: ld1 { v2.b }[6], [x8]3657; CHECK-SD-NEXT: add x8, sp, #563658; CHECK-SD-NEXT: mov v0.b[5], w53659; CHECK-SD-NEXT: ld1 { v1.b }[7], [x9]3660; CHECK-SD-NEXT: ld1 { v2.b }[7], [x8]3661; CHECK-SD-NEXT: mov v0.b[6], w63662; CHECK-SD-NEXT: mov v0.b[7], w73663; CHECK-SD-NEXT: uaddl v0.8h, v0.8b, v1.8b3664; CHECK-SD-NEXT: uaddw v0.8h, v0.8h, v2.8b3665; CHECK-SD-NEXT: addv h0, v0.8h3666; CHECK-SD-NEXT: fmov w0, s03667; CHECK-SD-NEXT: ret3668;3669; CHECK-GI-LABEL: add_v24i8_v24i16_zext:3670; CHECK-GI: // %bb.0: // %entry3671; CHECK-GI-NEXT: fmov s0, w03672; CHECK-GI-NEXT: ldr w8, [sp]3673; CHECK-GI-NEXT: ldr w9, [sp, #8]3674; CHECK-GI-NEXT: ldr w10, [sp, #72]3675; CHECK-GI-NEXT: mov v0.b[1], w13676; CHECK-GI-NEXT: mov v0.b[2], w23677; CHECK-GI-NEXT: mov v0.b[3], w33678; CHECK-GI-NEXT: mov v0.b[4], w43679; CHECK-GI-NEXT: mov v0.b[5], w53680; CHECK-GI-NEXT: mov v0.b[6], w63681; CHECK-GI-NEXT: mov v0.b[7], w73682; CHECK-GI-NEXT: mov v0.b[8], w83683; CHECK-GI-NEXT: ldr w8, [sp, #64]3684; CHECK-GI-NEXT: fmov s1, w83685; CHECK-GI-NEXT: ldr w8, [sp, #16]3686; CHECK-GI-NEXT: mov v0.b[9], w93687; CHECK-GI-NEXT: ldr w9, [sp, #80]3688; CHECK-GI-NEXT: mov v1.b[1], w103689; CHECK-GI-NEXT: mov v0.b[10], w83690; CHECK-GI-NEXT: ldr w8, [sp, #24]3691; CHECK-GI-NEXT: mov v1.b[2], w93692; CHECK-GI-NEXT: ldr w9, [sp, #88]3693; CHECK-GI-NEXT: mov v0.b[11], w83694; CHECK-GI-NEXT: ldr w8, [sp, #32]3695; CHECK-GI-NEXT: mov v1.b[3], w93696; CHECK-GI-NEXT: ldr w9, [sp, #96]3697; CHECK-GI-NEXT: mov v0.b[12], w83698; CHECK-GI-NEXT: ldr w8, [sp, #40]3699; CHECK-GI-NEXT: mov v1.b[4], w93700; CHECK-GI-NEXT: ldr w9, [sp, #104]3701; CHECK-GI-NEXT: mov v0.b[13], w83702; CHECK-GI-NEXT: ldr w8, [sp, #48]3703; CHECK-GI-NEXT: mov v1.b[5], w93704; CHECK-GI-NEXT: ldr w9, [sp, #112]3705; CHECK-GI-NEXT: mov v0.b[14], w83706; CHECK-GI-NEXT: ldr w8, [sp, #56]3707; CHECK-GI-NEXT: mov v1.b[6], w93708; CHECK-GI-NEXT: ldr w9, [sp, #120]3709; CHECK-GI-NEXT: mov v0.b[15], w83710; CHECK-GI-NEXT: mov v1.b[7], w93711; CHECK-GI-NEXT: uaddlv h0, v0.16b3712; CHECK-GI-NEXT: uaddlv h1, v1.8b3713; CHECK-GI-NEXT: fmov w8, s03714; CHECK-GI-NEXT: fmov w9, s13715; CHECK-GI-NEXT: add w0, w8, w93716; CHECK-GI-NEXT: ret3717entry:3718 %xx = zext <24 x i8> %x to <24 x i16>3719 %z = call i16 @llvm.vector.reduce.add.v24i16(<24 x i16> %xx)3720 ret i16 %z3721}3722 3723define i16 @add_v32i8_v32i16_zext(<32 x i8> %x) {3724; CHECK-SD-LABEL: add_v32i8_v32i16_zext:3725; CHECK-SD: // %bb.0: // %entry3726; CHECK-SD-NEXT: uaddl2 v2.8h, v0.16b, v1.16b3727; CHECK-SD-NEXT: uaddl v0.8h, v0.8b, v1.8b3728; CHECK-SD-NEXT: add v0.8h, v0.8h, v2.8h3729; CHECK-SD-NEXT: addv h0, v0.8h3730; CHECK-SD-NEXT: fmov w0, s03731; CHECK-SD-NEXT: ret3732;3733; CHECK-GI-LABEL: add_v32i8_v32i16_zext:3734; CHECK-GI: // %bb.0: // %entry3735; CHECK-GI-NEXT: uaddlv h0, v0.16b3736; CHECK-GI-NEXT: uaddlv h1, v1.16b3737; CHECK-GI-NEXT: fmov w8, s03738; CHECK-GI-NEXT: fmov w9, s13739; CHECK-GI-NEXT: add w0, w8, w93740; CHECK-GI-NEXT: ret3741entry:3742 %xx = zext <32 x i8> %x to <32 x i16>3743 %z = call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> %xx)3744 ret i16 %z3745}3746 3747define i16 @add_v24i8_v24i16_sext(<24 x i8> %x) {3748; CHECK-SD-LABEL: add_v24i8_v24i16_sext:3749; CHECK-SD: // %bb.0: // %entry3750; CHECK-SD-NEXT: fmov s0, w03751; CHECK-SD-NEXT: ldr b1, [sp, #64]3752; CHECK-SD-NEXT: add x8, sp, #723753; CHECK-SD-NEXT: ldr b2, [sp]3754; CHECK-SD-NEXT: add x9, sp, #803755; CHECK-SD-NEXT: ld1 { v1.b }[1], [x8]3756; CHECK-SD-NEXT: add x8, sp, #83757; CHECK-SD-NEXT: mov v0.b[1], w13758; CHECK-SD-NEXT: ld1 { v2.b }[1], [x8]3759; CHECK-SD-NEXT: add x8, sp, #163760; CHECK-SD-NEXT: ld1 { v1.b }[2], [x9]3761; CHECK-SD-NEXT: add x9, sp, #883762; CHECK-SD-NEXT: ld1 { v2.b }[2], [x8]3763; CHECK-SD-NEXT: add x8, sp, #243764; CHECK-SD-NEXT: mov v0.b[2], w23765; CHECK-SD-NEXT: ld1 { v1.b }[3], [x9]3766; CHECK-SD-NEXT: add x9, sp, #963767; CHECK-SD-NEXT: ld1 { v2.b }[3], [x8]3768; CHECK-SD-NEXT: add x8, sp, #323769; CHECK-SD-NEXT: mov v0.b[3], w33770; CHECK-SD-NEXT: ld1 { v1.b }[4], [x9]3771; CHECK-SD-NEXT: add x9, sp, #1043772; CHECK-SD-NEXT: ld1 { v2.b }[4], [x8]3773; CHECK-SD-NEXT: add x8, sp, #403774; CHECK-SD-NEXT: ld1 { v1.b }[5], [x9]3775; CHECK-SD-NEXT: add x9, sp, #1123776; CHECK-SD-NEXT: mov v0.b[4], w43777; CHECK-SD-NEXT: ld1 { v2.b }[5], [x8]3778; CHECK-SD-NEXT: add x8, sp, #483779; CHECK-SD-NEXT: ld1 { v1.b }[6], [x9]3780; CHECK-SD-NEXT: add x9, sp, #1203781; CHECK-SD-NEXT: ld1 { v2.b }[6], [x8]3782; CHECK-SD-NEXT: add x8, sp, #563783; CHECK-SD-NEXT: mov v0.b[5], w53784; CHECK-SD-NEXT: ld1 { v1.b }[7], [x9]3785; CHECK-SD-NEXT: ld1 { v2.b }[7], [x8]3786; CHECK-SD-NEXT: mov v0.b[6], w63787; CHECK-SD-NEXT: mov v0.b[7], w73788; CHECK-SD-NEXT: saddl v0.8h, v0.8b, v1.8b3789; CHECK-SD-NEXT: saddw v0.8h, v0.8h, v2.8b3790; CHECK-SD-NEXT: addv h0, v0.8h3791; CHECK-SD-NEXT: fmov w0, s03792; CHECK-SD-NEXT: ret3793;3794; CHECK-GI-LABEL: add_v24i8_v24i16_sext:3795; CHECK-GI: // %bb.0: // %entry3796; CHECK-GI-NEXT: fmov s0, w03797; CHECK-GI-NEXT: ldr w8, [sp]3798; CHECK-GI-NEXT: ldr w9, [sp, #8]3799; CHECK-GI-NEXT: ldr w10, [sp, #72]3800; CHECK-GI-NEXT: mov v0.b[1], w13801; CHECK-GI-NEXT: mov v0.b[2], w23802; CHECK-GI-NEXT: mov v0.b[3], w33803; CHECK-GI-NEXT: mov v0.b[4], w43804; CHECK-GI-NEXT: mov v0.b[5], w53805; CHECK-GI-NEXT: mov v0.b[6], w63806; CHECK-GI-NEXT: mov v0.b[7], w73807; CHECK-GI-NEXT: mov v0.b[8], w83808; CHECK-GI-NEXT: ldr w8, [sp, #64]3809; CHECK-GI-NEXT: fmov s1, w83810; CHECK-GI-NEXT: ldr w8, [sp, #16]3811; CHECK-GI-NEXT: mov v0.b[9], w93812; CHECK-GI-NEXT: ldr w9, [sp, #80]3813; CHECK-GI-NEXT: mov v1.b[1], w103814; CHECK-GI-NEXT: mov v0.b[10], w83815; CHECK-GI-NEXT: ldr w8, [sp, #24]3816; CHECK-GI-NEXT: mov v1.b[2], w93817; CHECK-GI-NEXT: ldr w9, [sp, #88]3818; CHECK-GI-NEXT: mov v0.b[11], w83819; CHECK-GI-NEXT: ldr w8, [sp, #32]3820; CHECK-GI-NEXT: mov v1.b[3], w93821; CHECK-GI-NEXT: ldr w9, [sp, #96]3822; CHECK-GI-NEXT: mov v0.b[12], w83823; CHECK-GI-NEXT: ldr w8, [sp, #40]3824; CHECK-GI-NEXT: mov v1.b[4], w93825; CHECK-GI-NEXT: ldr w9, [sp, #104]3826; CHECK-GI-NEXT: mov v0.b[13], w83827; CHECK-GI-NEXT: ldr w8, [sp, #48]3828; CHECK-GI-NEXT: mov v1.b[5], w93829; CHECK-GI-NEXT: ldr w9, [sp, #112]3830; CHECK-GI-NEXT: mov v0.b[14], w83831; CHECK-GI-NEXT: ldr w8, [sp, #56]3832; CHECK-GI-NEXT: mov v1.b[6], w93833; CHECK-GI-NEXT: ldr w9, [sp, #120]3834; CHECK-GI-NEXT: mov v0.b[15], w83835; CHECK-GI-NEXT: mov v1.b[7], w93836; CHECK-GI-NEXT: saddlv h0, v0.16b3837; CHECK-GI-NEXT: saddlv h1, v1.8b3838; CHECK-GI-NEXT: fmov w8, s03839; CHECK-GI-NEXT: fmov w9, s13840; CHECK-GI-NEXT: add w0, w8, w93841; CHECK-GI-NEXT: ret3842entry:3843 %xx = sext <24 x i8> %x to <24 x i16>3844 %z = call i16 @llvm.vector.reduce.add.v24i16(<24 x i16> %xx)3845 ret i16 %z3846}3847 3848define i16 @add_v32i8_v32i16_sext(<32 x i8> %x) {3849; CHECK-SD-LABEL: add_v32i8_v32i16_sext:3850; CHECK-SD: // %bb.0: // %entry3851; CHECK-SD-NEXT: saddl2 v2.8h, v0.16b, v1.16b3852; CHECK-SD-NEXT: saddl v0.8h, v0.8b, v1.8b3853; CHECK-SD-NEXT: add v0.8h, v0.8h, v2.8h3854; CHECK-SD-NEXT: addv h0, v0.8h3855; CHECK-SD-NEXT: fmov w0, s03856; CHECK-SD-NEXT: ret3857;3858; CHECK-GI-LABEL: add_v32i8_v32i16_sext:3859; CHECK-GI: // %bb.0: // %entry3860; CHECK-GI-NEXT: saddlv h0, v0.16b3861; CHECK-GI-NEXT: saddlv h1, v1.16b3862; CHECK-GI-NEXT: fmov w8, s03863; CHECK-GI-NEXT: fmov w9, s13864; CHECK-GI-NEXT: add w0, w8, w93865; CHECK-GI-NEXT: ret3866entry:3867 %xx = sext <32 x i8> %x to <32 x i16>3868 %z = call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> %xx)3869 ret i16 %z3870}3871 3872; Irregularly sized vectors and larger extends3873define i32 @add_v24i8_v24i32_zext(<24 x i8> %x) {3874; CHECK-SD-BASE-LABEL: add_v24i8_v24i32_zext:3875; CHECK-SD-BASE: // %bb.0: // %entry3876; CHECK-SD-BASE-NEXT: fmov s0, w03877; CHECK-SD-BASE-NEXT: ldr b1, [sp, #64]3878; CHECK-SD-BASE-NEXT: add x8, sp, #723879; CHECK-SD-BASE-NEXT: ldr b2, [sp]3880; CHECK-SD-BASE-NEXT: add x9, sp, #803881; CHECK-SD-BASE-NEXT: ld1 { v1.b }[1], [x8]3882; CHECK-SD-BASE-NEXT: add x8, sp, #83883; CHECK-SD-BASE-NEXT: mov v0.b[1], w13884; CHECK-SD-BASE-NEXT: ld1 { v2.b }[1], [x8]3885; CHECK-SD-BASE-NEXT: add x8, sp, #163886; CHECK-SD-BASE-NEXT: ld1 { v1.b }[2], [x9]3887; CHECK-SD-BASE-NEXT: add x9, sp, #883888; CHECK-SD-BASE-NEXT: ld1 { v2.b }[2], [x8]3889; CHECK-SD-BASE-NEXT: add x8, sp, #243890; CHECK-SD-BASE-NEXT: mov v0.b[2], w23891; CHECK-SD-BASE-NEXT: ld1 { v1.b }[3], [x9]3892; CHECK-SD-BASE-NEXT: add x9, sp, #963893; CHECK-SD-BASE-NEXT: ld1 { v2.b }[3], [x8]3894; CHECK-SD-BASE-NEXT: add x8, sp, #323895; CHECK-SD-BASE-NEXT: mov v0.b[3], w33896; CHECK-SD-BASE-NEXT: ld1 { v1.b }[4], [x9]3897; CHECK-SD-BASE-NEXT: add x9, sp, #1043898; CHECK-SD-BASE-NEXT: ld1 { v2.b }[4], [x8]3899; CHECK-SD-BASE-NEXT: add x8, sp, #403900; CHECK-SD-BASE-NEXT: ld1 { v1.b }[5], [x9]3901; CHECK-SD-BASE-NEXT: add x9, sp, #1123902; CHECK-SD-BASE-NEXT: mov v0.b[4], w43903; CHECK-SD-BASE-NEXT: ld1 { v2.b }[5], [x8]3904; CHECK-SD-BASE-NEXT: add x8, sp, #483905; CHECK-SD-BASE-NEXT: ld1 { v1.b }[6], [x9]3906; CHECK-SD-BASE-NEXT: add x9, sp, #1203907; CHECK-SD-BASE-NEXT: ld1 { v2.b }[6], [x8]3908; CHECK-SD-BASE-NEXT: add x8, sp, #563909; CHECK-SD-BASE-NEXT: mov v0.b[5], w53910; CHECK-SD-BASE-NEXT: ld1 { v1.b }[7], [x9]3911; CHECK-SD-BASE-NEXT: ld1 { v2.b }[7], [x8]3912; CHECK-SD-BASE-NEXT: mov v0.b[6], w63913; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #03914; CHECK-SD-BASE-NEXT: ushll v2.8h, v2.8b, #03915; CHECK-SD-BASE-NEXT: mov v0.b[7], w73916; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #03917; CHECK-SD-BASE-NEXT: uaddl2 v3.4s, v0.8h, v1.8h3918; CHECK-SD-BASE-NEXT: uaddl v0.4s, v0.4h, v1.4h3919; CHECK-SD-BASE-NEXT: uaddw2 v1.4s, v3.4s, v2.8h3920; CHECK-SD-BASE-NEXT: uaddw v0.4s, v0.4s, v2.4h3921; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s3922; CHECK-SD-BASE-NEXT: addv s0, v0.4s3923; CHECK-SD-BASE-NEXT: fmov w0, s03924; CHECK-SD-BASE-NEXT: ret3925;3926; CHECK-SD-DOT-LABEL: add_v24i8_v24i32_zext:3927; CHECK-SD-DOT: // %bb.0: // %entry3928; CHECK-SD-DOT-NEXT: fmov s0, w03929; CHECK-SD-DOT-NEXT: mov x8, sp3930; CHECK-SD-DOT-NEXT: ldr b1, [sp, #64]3931; CHECK-SD-DOT-NEXT: add x9, sp, #723932; CHECK-SD-DOT-NEXT: movi v2.16b, #13933; CHECK-SD-DOT-NEXT: movi v3.2d, #00000000000000003934; CHECK-SD-DOT-NEXT: ld1 { v1.b }[1], [x9]3935; CHECK-SD-DOT-NEXT: add x9, sp, #803936; CHECK-SD-DOT-NEXT: movi v4.2d, #00000000000000003937; CHECK-SD-DOT-NEXT: mov v0.b[1], w13938; CHECK-SD-DOT-NEXT: movi v5.8b, #13939; CHECK-SD-DOT-NEXT: ld1 { v1.b }[2], [x9]3940; CHECK-SD-DOT-NEXT: add x9, sp, #883941; CHECK-SD-DOT-NEXT: mov v0.b[2], w23942; CHECK-SD-DOT-NEXT: ld1 { v1.b }[3], [x9]3943; CHECK-SD-DOT-NEXT: add x9, sp, #963944; CHECK-SD-DOT-NEXT: mov v0.b[3], w33945; CHECK-SD-DOT-NEXT: ld1 { v1.b }[4], [x9]3946; CHECK-SD-DOT-NEXT: add x9, sp, #1043947; CHECK-SD-DOT-NEXT: ld1 { v1.b }[5], [x9]3948; CHECK-SD-DOT-NEXT: add x9, sp, #1123949; CHECK-SD-DOT-NEXT: mov v0.b[4], w43950; CHECK-SD-DOT-NEXT: ld1 { v1.b }[6], [x9]3951; CHECK-SD-DOT-NEXT: add x9, sp, #1203952; CHECK-SD-DOT-NEXT: mov v0.b[5], w53953; CHECK-SD-DOT-NEXT: ld1 { v1.b }[7], [x9]3954; CHECK-SD-DOT-NEXT: mov v0.b[6], w63955; CHECK-SD-DOT-NEXT: udot v4.2s, v1.8b, v5.8b3956; CHECK-SD-DOT-NEXT: mov v0.b[7], w73957; CHECK-SD-DOT-NEXT: addp v1.2s, v4.2s, v4.2s3958; CHECK-SD-DOT-NEXT: fmov w9, s13959; CHECK-SD-DOT-NEXT: ld1 { v0.b }[8], [x8]3960; CHECK-SD-DOT-NEXT: add x8, sp, #83961; CHECK-SD-DOT-NEXT: ld1 { v0.b }[9], [x8]3962; CHECK-SD-DOT-NEXT: add x8, sp, #163963; CHECK-SD-DOT-NEXT: ld1 { v0.b }[10], [x8]3964; CHECK-SD-DOT-NEXT: add x8, sp, #243965; CHECK-SD-DOT-NEXT: ld1 { v0.b }[11], [x8]3966; CHECK-SD-DOT-NEXT: add x8, sp, #323967; CHECK-SD-DOT-NEXT: ld1 { v0.b }[12], [x8]3968; CHECK-SD-DOT-NEXT: add x8, sp, #403969; CHECK-SD-DOT-NEXT: ld1 { v0.b }[13], [x8]3970; CHECK-SD-DOT-NEXT: add x8, sp, #483971; CHECK-SD-DOT-NEXT: ld1 { v0.b }[14], [x8]3972; CHECK-SD-DOT-NEXT: add x8, sp, #563973; CHECK-SD-DOT-NEXT: ld1 { v0.b }[15], [x8]3974; CHECK-SD-DOT-NEXT: udot v3.4s, v0.16b, v2.16b3975; CHECK-SD-DOT-NEXT: addv s0, v3.4s3976; CHECK-SD-DOT-NEXT: fmov w8, s03977; CHECK-SD-DOT-NEXT: add w0, w8, w93978; CHECK-SD-DOT-NEXT: ret3979;3980; CHECK-GI-BASE-LABEL: add_v24i8_v24i32_zext:3981; CHECK-GI-BASE: // %bb.0: // %entry3982; CHECK-GI-BASE-NEXT: fmov s0, w03983; CHECK-GI-BASE-NEXT: ldr w8, [sp]3984; CHECK-GI-BASE-NEXT: ldr w9, [sp, #8]3985; CHECK-GI-BASE-NEXT: ldr w10, [sp, #72]3986; CHECK-GI-BASE-NEXT: mov v0.b[1], w13987; CHECK-GI-BASE-NEXT: mov v0.b[2], w23988; CHECK-GI-BASE-NEXT: mov v0.b[3], w33989; CHECK-GI-BASE-NEXT: mov v0.b[4], w43990; CHECK-GI-BASE-NEXT: mov v0.b[5], w53991; CHECK-GI-BASE-NEXT: mov v0.b[6], w63992; CHECK-GI-BASE-NEXT: mov v0.b[7], w73993; CHECK-GI-BASE-NEXT: mov v0.b[8], w83994; CHECK-GI-BASE-NEXT: ldr w8, [sp, #64]3995; CHECK-GI-BASE-NEXT: fmov s1, w83996; CHECK-GI-BASE-NEXT: ldr w8, [sp, #16]3997; CHECK-GI-BASE-NEXT: mov v0.b[9], w93998; CHECK-GI-BASE-NEXT: ldr w9, [sp, #80]3999; CHECK-GI-BASE-NEXT: mov v1.b[1], w104000; CHECK-GI-BASE-NEXT: mov v0.b[10], w84001; CHECK-GI-BASE-NEXT: ldr w8, [sp, #24]4002; CHECK-GI-BASE-NEXT: mov v1.b[2], w94003; CHECK-GI-BASE-NEXT: ldr w9, [sp, #88]4004; CHECK-GI-BASE-NEXT: mov v0.b[11], w84005; CHECK-GI-BASE-NEXT: ldr w8, [sp, #32]4006; CHECK-GI-BASE-NEXT: mov v1.b[3], w94007; CHECK-GI-BASE-NEXT: ldr w9, [sp, #96]4008; CHECK-GI-BASE-NEXT: mov v0.b[12], w84009; CHECK-GI-BASE-NEXT: ldr w8, [sp, #40]4010; CHECK-GI-BASE-NEXT: mov v1.b[4], w94011; CHECK-GI-BASE-NEXT: ldr w9, [sp, #104]4012; CHECK-GI-BASE-NEXT: mov v0.b[13], w84013; CHECK-GI-BASE-NEXT: ldr w8, [sp, #48]4014; CHECK-GI-BASE-NEXT: mov v1.b[5], w94015; CHECK-GI-BASE-NEXT: ldr w9, [sp, #112]4016; CHECK-GI-BASE-NEXT: mov v0.b[14], w84017; CHECK-GI-BASE-NEXT: ldr w8, [sp, #56]4018; CHECK-GI-BASE-NEXT: mov v1.b[6], w94019; CHECK-GI-BASE-NEXT: ldr w9, [sp, #120]4020; CHECK-GI-BASE-NEXT: mov v0.b[15], w84021; CHECK-GI-BASE-NEXT: mov v1.b[7], w94022; CHECK-GI-BASE-NEXT: uaddlv h0, v0.16b4023; CHECK-GI-BASE-NEXT: uaddlv h1, v1.8b4024; CHECK-GI-BASE-NEXT: fmov w8, s04025; CHECK-GI-BASE-NEXT: fmov w9, s14026; CHECK-GI-BASE-NEXT: add w8, w8, w94027; CHECK-GI-BASE-NEXT: and w0, w8, #0xffff4028; CHECK-GI-BASE-NEXT: ret4029;4030; CHECK-GI-DOT-LABEL: add_v24i8_v24i32_zext:4031; CHECK-GI-DOT: // %bb.0: // %entry4032; CHECK-GI-DOT-NEXT: fmov s0, w04033; CHECK-GI-DOT-NEXT: ldr w9, [sp, #64]4034; CHECK-GI-DOT-NEXT: ldr w8, [sp]4035; CHECK-GI-DOT-NEXT: ldr w10, [sp, #72]4036; CHECK-GI-DOT-NEXT: movi v2.8b, #14037; CHECK-GI-DOT-NEXT: movi v3.8b, #14038; CHECK-GI-DOT-NEXT: fmov s1, w94039; CHECK-GI-DOT-NEXT: ldr w9, [sp, #80]4040; CHECK-GI-DOT-NEXT: movi v4.2d, #00000000000000004041; CHECK-GI-DOT-NEXT: mov v0.b[1], w14042; CHECK-GI-DOT-NEXT: movi v5.2d, #00000000000000004043; CHECK-GI-DOT-NEXT: mov v1.b[1], w104044; CHECK-GI-DOT-NEXT: mov v3.d[1], v2.d[0]4045; CHECK-GI-DOT-NEXT: mov v0.b[2], w24046; CHECK-GI-DOT-NEXT: mov v1.b[2], w94047; CHECK-GI-DOT-NEXT: ldr w9, [sp, #88]4048; CHECK-GI-DOT-NEXT: mov v0.b[3], w34049; CHECK-GI-DOT-NEXT: mov v1.b[3], w94050; CHECK-GI-DOT-NEXT: ldr w9, [sp, #96]4051; CHECK-GI-DOT-NEXT: mov v0.b[4], w44052; CHECK-GI-DOT-NEXT: mov v1.b[4], w94053; CHECK-GI-DOT-NEXT: ldr w9, [sp, #104]4054; CHECK-GI-DOT-NEXT: mov v0.b[5], w54055; CHECK-GI-DOT-NEXT: mov v1.b[5], w94056; CHECK-GI-DOT-NEXT: ldr w9, [sp, #112]4057; CHECK-GI-DOT-NEXT: mov v0.b[6], w64058; CHECK-GI-DOT-NEXT: mov v1.b[6], w94059; CHECK-GI-DOT-NEXT: ldr w9, [sp, #120]4060; CHECK-GI-DOT-NEXT: mov v0.b[7], w74061; CHECK-GI-DOT-NEXT: mov v1.b[7], w94062; CHECK-GI-DOT-NEXT: mov v0.b[8], w84063; CHECK-GI-DOT-NEXT: ldr w8, [sp, #8]4064; CHECK-GI-DOT-NEXT: fmov d1, d14065; CHECK-GI-DOT-NEXT: mov v0.b[9], w84066; CHECK-GI-DOT-NEXT: ldr w8, [sp, #16]4067; CHECK-GI-DOT-NEXT: udot v4.4s, v1.16b, v2.16b4068; CHECK-GI-DOT-NEXT: mov v0.b[10], w84069; CHECK-GI-DOT-NEXT: ldr w8, [sp, #24]4070; CHECK-GI-DOT-NEXT: mov v0.b[11], w84071; CHECK-GI-DOT-NEXT: ldr w8, [sp, #32]4072; CHECK-GI-DOT-NEXT: mov v0.b[12], w84073; CHECK-GI-DOT-NEXT: ldr w8, [sp, #40]4074; CHECK-GI-DOT-NEXT: mov v0.b[13], w84075; CHECK-GI-DOT-NEXT: ldr w8, [sp, #48]4076; CHECK-GI-DOT-NEXT: mov v0.b[14], w84077; CHECK-GI-DOT-NEXT: ldr w8, [sp, #56]4078; CHECK-GI-DOT-NEXT: mov v0.b[15], w84079; CHECK-GI-DOT-NEXT: udot v5.4s, v0.16b, v3.16b4080; CHECK-GI-DOT-NEXT: add v0.4s, v5.4s, v4.4s4081; CHECK-GI-DOT-NEXT: addv s0, v0.4s4082; CHECK-GI-DOT-NEXT: fmov w0, s04083; CHECK-GI-DOT-NEXT: ret4084entry:4085 %xx = zext <24 x i8> %x to <24 x i32>4086 %z = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %xx)4087 ret i32 %z4088}4089 4090define i32 @add_v32i8_v32i32_zext(<32 x i8> %x) {4091; CHECK-SD-BASE-LABEL: add_v32i8_v32i32_zext:4092; CHECK-SD-BASE: // %bb.0: // %entry4093; CHECK-SD-BASE-NEXT: ushll2 v2.8h, v1.16b, #04094; CHECK-SD-BASE-NEXT: ushll2 v3.8h, v0.16b, #04095; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #04096; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #04097; CHECK-SD-BASE-NEXT: uaddl2 v4.4s, v3.8h, v2.8h4098; CHECK-SD-BASE-NEXT: uaddl v2.4s, v3.4h, v2.4h4099; CHECK-SD-BASE-NEXT: uaddl2 v5.4s, v0.8h, v1.8h4100; CHECK-SD-BASE-NEXT: uaddl v0.4s, v0.4h, v1.4h4101; CHECK-SD-BASE-NEXT: add v1.4s, v5.4s, v4.4s4102; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v2.4s4103; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s4104; CHECK-SD-BASE-NEXT: addv s0, v0.4s4105; CHECK-SD-BASE-NEXT: fmov w0, s04106; CHECK-SD-BASE-NEXT: ret4107;4108; CHECK-SD-DOT-LABEL: add_v32i8_v32i32_zext:4109; CHECK-SD-DOT: // %bb.0: // %entry4110; CHECK-SD-DOT-NEXT: movi v2.16b, #14111; CHECK-SD-DOT-NEXT: movi v3.2d, #00000000000000004112; CHECK-SD-DOT-NEXT: udot v3.4s, v1.16b, v2.16b4113; CHECK-SD-DOT-NEXT: udot v3.4s, v0.16b, v2.16b4114; CHECK-SD-DOT-NEXT: addv s0, v3.4s4115; CHECK-SD-DOT-NEXT: fmov w0, s04116; CHECK-SD-DOT-NEXT: ret4117;4118; CHECK-GI-BASE-LABEL: add_v32i8_v32i32_zext:4119; CHECK-GI-BASE: // %bb.0: // %entry4120; CHECK-GI-BASE-NEXT: uaddlv h0, v0.16b4121; CHECK-GI-BASE-NEXT: uaddlv h1, v1.16b4122; CHECK-GI-BASE-NEXT: fmov w8, s04123; CHECK-GI-BASE-NEXT: fmov w9, s14124; CHECK-GI-BASE-NEXT: add w8, w8, w94125; CHECK-GI-BASE-NEXT: and w0, w8, #0xffff4126; CHECK-GI-BASE-NEXT: ret4127;4128; CHECK-GI-DOT-LABEL: add_v32i8_v32i32_zext:4129; CHECK-GI-DOT: // %bb.0: // %entry4130; CHECK-GI-DOT-NEXT: movi v2.16b, #14131; CHECK-GI-DOT-NEXT: movi v3.2d, #00000000000000004132; CHECK-GI-DOT-NEXT: movi v4.2d, #00000000000000004133; CHECK-GI-DOT-NEXT: udot v4.4s, v0.16b, v2.16b4134; CHECK-GI-DOT-NEXT: udot v3.4s, v1.16b, v2.16b4135; CHECK-GI-DOT-NEXT: add v0.4s, v4.4s, v3.4s4136; CHECK-GI-DOT-NEXT: addv s0, v0.4s4137; CHECK-GI-DOT-NEXT: fmov w0, s04138; CHECK-GI-DOT-NEXT: ret4139entry:4140 %xx = zext <32 x i8> %x to <32 x i32>4141 %z = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %xx)4142 ret i32 %z4143}4144 4145define i32 @add_v24i8_v24i32_sext(<24 x i8> %x) {4146; CHECK-SD-BASE-LABEL: add_v24i8_v24i32_sext:4147; CHECK-SD-BASE: // %bb.0: // %entry4148; CHECK-SD-BASE-NEXT: fmov s0, w04149; CHECK-SD-BASE-NEXT: ldr b1, [sp, #64]4150; CHECK-SD-BASE-NEXT: add x8, sp, #724151; CHECK-SD-BASE-NEXT: ldr b2, [sp]4152; CHECK-SD-BASE-NEXT: add x9, sp, #804153; CHECK-SD-BASE-NEXT: ld1 { v1.b }[1], [x8]4154; CHECK-SD-BASE-NEXT: add x8, sp, #84155; CHECK-SD-BASE-NEXT: mov v0.b[1], w14156; CHECK-SD-BASE-NEXT: ld1 { v2.b }[1], [x8]4157; CHECK-SD-BASE-NEXT: add x8, sp, #164158; CHECK-SD-BASE-NEXT: ld1 { v1.b }[2], [x9]4159; CHECK-SD-BASE-NEXT: add x9, sp, #884160; CHECK-SD-BASE-NEXT: ld1 { v2.b }[2], [x8]4161; CHECK-SD-BASE-NEXT: add x8, sp, #244162; CHECK-SD-BASE-NEXT: mov v0.b[2], w24163; CHECK-SD-BASE-NEXT: ld1 { v1.b }[3], [x9]4164; CHECK-SD-BASE-NEXT: add x9, sp, #964165; CHECK-SD-BASE-NEXT: ld1 { v2.b }[3], [x8]4166; CHECK-SD-BASE-NEXT: add x8, sp, #324167; CHECK-SD-BASE-NEXT: mov v0.b[3], w34168; CHECK-SD-BASE-NEXT: ld1 { v1.b }[4], [x9]4169; CHECK-SD-BASE-NEXT: add x9, sp, #1044170; CHECK-SD-BASE-NEXT: ld1 { v2.b }[4], [x8]4171; CHECK-SD-BASE-NEXT: add x8, sp, #404172; CHECK-SD-BASE-NEXT: ld1 { v1.b }[5], [x9]4173; CHECK-SD-BASE-NEXT: add x9, sp, #1124174; CHECK-SD-BASE-NEXT: mov v0.b[4], w44175; CHECK-SD-BASE-NEXT: ld1 { v2.b }[5], [x8]4176; CHECK-SD-BASE-NEXT: add x8, sp, #484177; CHECK-SD-BASE-NEXT: ld1 { v1.b }[6], [x9]4178; CHECK-SD-BASE-NEXT: add x9, sp, #1204179; CHECK-SD-BASE-NEXT: ld1 { v2.b }[6], [x8]4180; CHECK-SD-BASE-NEXT: add x8, sp, #564181; CHECK-SD-BASE-NEXT: mov v0.b[5], w54182; CHECK-SD-BASE-NEXT: ld1 { v1.b }[7], [x9]4183; CHECK-SD-BASE-NEXT: ld1 { v2.b }[7], [x8]4184; CHECK-SD-BASE-NEXT: mov v0.b[6], w64185; CHECK-SD-BASE-NEXT: sshll v1.8h, v1.8b, #04186; CHECK-SD-BASE-NEXT: sshll v2.8h, v2.8b, #04187; CHECK-SD-BASE-NEXT: mov v0.b[7], w74188; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #04189; CHECK-SD-BASE-NEXT: saddl2 v3.4s, v0.8h, v1.8h4190; CHECK-SD-BASE-NEXT: saddl v0.4s, v0.4h, v1.4h4191; CHECK-SD-BASE-NEXT: saddw2 v1.4s, v3.4s, v2.8h4192; CHECK-SD-BASE-NEXT: saddw v0.4s, v0.4s, v2.4h4193; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s4194; CHECK-SD-BASE-NEXT: addv s0, v0.4s4195; CHECK-SD-BASE-NEXT: fmov w0, s04196; CHECK-SD-BASE-NEXT: ret4197;4198; CHECK-SD-DOT-LABEL: add_v24i8_v24i32_sext:4199; CHECK-SD-DOT: // %bb.0: // %entry4200; CHECK-SD-DOT-NEXT: fmov s0, w04201; CHECK-SD-DOT-NEXT: mov x8, sp4202; CHECK-SD-DOT-NEXT: ldr b1, [sp, #64]4203; CHECK-SD-DOT-NEXT: add x9, sp, #724204; CHECK-SD-DOT-NEXT: movi v2.16b, #14205; CHECK-SD-DOT-NEXT: movi v3.2d, #00000000000000004206; CHECK-SD-DOT-NEXT: ld1 { v1.b }[1], [x9]4207; CHECK-SD-DOT-NEXT: add x9, sp, #804208; CHECK-SD-DOT-NEXT: movi v4.2d, #00000000000000004209; CHECK-SD-DOT-NEXT: mov v0.b[1], w14210; CHECK-SD-DOT-NEXT: movi v5.8b, #14211; CHECK-SD-DOT-NEXT: ld1 { v1.b }[2], [x9]4212; CHECK-SD-DOT-NEXT: add x9, sp, #884213; CHECK-SD-DOT-NEXT: mov v0.b[2], w24214; CHECK-SD-DOT-NEXT: ld1 { v1.b }[3], [x9]4215; CHECK-SD-DOT-NEXT: add x9, sp, #964216; CHECK-SD-DOT-NEXT: mov v0.b[3], w34217; CHECK-SD-DOT-NEXT: ld1 { v1.b }[4], [x9]4218; CHECK-SD-DOT-NEXT: add x9, sp, #1044219; CHECK-SD-DOT-NEXT: ld1 { v1.b }[5], [x9]4220; CHECK-SD-DOT-NEXT: add x9, sp, #1124221; CHECK-SD-DOT-NEXT: mov v0.b[4], w44222; CHECK-SD-DOT-NEXT: ld1 { v1.b }[6], [x9]4223; CHECK-SD-DOT-NEXT: add x9, sp, #1204224; CHECK-SD-DOT-NEXT: mov v0.b[5], w54225; CHECK-SD-DOT-NEXT: ld1 { v1.b }[7], [x9]4226; CHECK-SD-DOT-NEXT: mov v0.b[6], w64227; CHECK-SD-DOT-NEXT: sdot v4.2s, v1.8b, v5.8b4228; CHECK-SD-DOT-NEXT: mov v0.b[7], w74229; CHECK-SD-DOT-NEXT: addp v1.2s, v4.2s, v4.2s4230; CHECK-SD-DOT-NEXT: fmov w9, s14231; CHECK-SD-DOT-NEXT: ld1 { v0.b }[8], [x8]4232; CHECK-SD-DOT-NEXT: add x8, sp, #84233; CHECK-SD-DOT-NEXT: ld1 { v0.b }[9], [x8]4234; CHECK-SD-DOT-NEXT: add x8, sp, #164235; CHECK-SD-DOT-NEXT: ld1 { v0.b }[10], [x8]4236; CHECK-SD-DOT-NEXT: add x8, sp, #244237; CHECK-SD-DOT-NEXT: ld1 { v0.b }[11], [x8]4238; CHECK-SD-DOT-NEXT: add x8, sp, #324239; CHECK-SD-DOT-NEXT: ld1 { v0.b }[12], [x8]4240; CHECK-SD-DOT-NEXT: add x8, sp, #404241; CHECK-SD-DOT-NEXT: ld1 { v0.b }[13], [x8]4242; CHECK-SD-DOT-NEXT: add x8, sp, #484243; CHECK-SD-DOT-NEXT: ld1 { v0.b }[14], [x8]4244; CHECK-SD-DOT-NEXT: add x8, sp, #564245; CHECK-SD-DOT-NEXT: ld1 { v0.b }[15], [x8]4246; CHECK-SD-DOT-NEXT: sdot v3.4s, v0.16b, v2.16b4247; CHECK-SD-DOT-NEXT: addv s0, v3.4s4248; CHECK-SD-DOT-NEXT: fmov w8, s04249; CHECK-SD-DOT-NEXT: add w0, w8, w94250; CHECK-SD-DOT-NEXT: ret4251;4252; CHECK-GI-BASE-LABEL: add_v24i8_v24i32_sext:4253; CHECK-GI-BASE: // %bb.0: // %entry4254; CHECK-GI-BASE-NEXT: fmov s0, w04255; CHECK-GI-BASE-NEXT: ldr w8, [sp]4256; CHECK-GI-BASE-NEXT: ldr w9, [sp, #8]4257; CHECK-GI-BASE-NEXT: ldr w10, [sp, #72]4258; CHECK-GI-BASE-NEXT: mov v0.b[1], w14259; CHECK-GI-BASE-NEXT: mov v0.b[2], w24260; CHECK-GI-BASE-NEXT: mov v0.b[3], w34261; CHECK-GI-BASE-NEXT: mov v0.b[4], w44262; CHECK-GI-BASE-NEXT: mov v0.b[5], w54263; CHECK-GI-BASE-NEXT: mov v0.b[6], w64264; CHECK-GI-BASE-NEXT: mov v0.b[7], w74265; CHECK-GI-BASE-NEXT: mov v0.b[8], w84266; CHECK-GI-BASE-NEXT: ldr w8, [sp, #64]4267; CHECK-GI-BASE-NEXT: fmov s1, w84268; CHECK-GI-BASE-NEXT: ldr w8, [sp, #16]4269; CHECK-GI-BASE-NEXT: mov v0.b[9], w94270; CHECK-GI-BASE-NEXT: ldr w9, [sp, #80]4271; CHECK-GI-BASE-NEXT: mov v1.b[1], w104272; CHECK-GI-BASE-NEXT: mov v0.b[10], w84273; CHECK-GI-BASE-NEXT: ldr w8, [sp, #24]4274; CHECK-GI-BASE-NEXT: mov v1.b[2], w94275; CHECK-GI-BASE-NEXT: ldr w9, [sp, #88]4276; CHECK-GI-BASE-NEXT: mov v0.b[11], w84277; CHECK-GI-BASE-NEXT: ldr w8, [sp, #32]4278; CHECK-GI-BASE-NEXT: mov v1.b[3], w94279; CHECK-GI-BASE-NEXT: ldr w9, [sp, #96]4280; CHECK-GI-BASE-NEXT: mov v0.b[12], w84281; CHECK-GI-BASE-NEXT: ldr w8, [sp, #40]4282; CHECK-GI-BASE-NEXT: mov v1.b[4], w94283; CHECK-GI-BASE-NEXT: ldr w9, [sp, #104]4284; CHECK-GI-BASE-NEXT: mov v0.b[13], w84285; CHECK-GI-BASE-NEXT: ldr w8, [sp, #48]4286; CHECK-GI-BASE-NEXT: mov v1.b[5], w94287; CHECK-GI-BASE-NEXT: ldr w9, [sp, #112]4288; CHECK-GI-BASE-NEXT: mov v0.b[14], w84289; CHECK-GI-BASE-NEXT: ldr w8, [sp, #56]4290; CHECK-GI-BASE-NEXT: mov v1.b[6], w94291; CHECK-GI-BASE-NEXT: ldr w9, [sp, #120]4292; CHECK-GI-BASE-NEXT: mov v0.b[15], w84293; CHECK-GI-BASE-NEXT: mov v1.b[7], w94294; CHECK-GI-BASE-NEXT: saddlv h0, v0.16b4295; CHECK-GI-BASE-NEXT: saddlv h1, v1.8b4296; CHECK-GI-BASE-NEXT: fmov w8, s04297; CHECK-GI-BASE-NEXT: fmov w9, s14298; CHECK-GI-BASE-NEXT: add w8, w8, w94299; CHECK-GI-BASE-NEXT: sxth w0, w84300; CHECK-GI-BASE-NEXT: ret4301;4302; CHECK-GI-DOT-LABEL: add_v24i8_v24i32_sext:4303; CHECK-GI-DOT: // %bb.0: // %entry4304; CHECK-GI-DOT-NEXT: fmov s0, w04305; CHECK-GI-DOT-NEXT: ldr w9, [sp, #64]4306; CHECK-GI-DOT-NEXT: ldr w8, [sp]4307; CHECK-GI-DOT-NEXT: ldr w10, [sp, #72]4308; CHECK-GI-DOT-NEXT: movi v2.8b, #14309; CHECK-GI-DOT-NEXT: movi v3.8b, #14310; CHECK-GI-DOT-NEXT: fmov s1, w94311; CHECK-GI-DOT-NEXT: ldr w9, [sp, #80]4312; CHECK-GI-DOT-NEXT: movi v4.2d, #00000000000000004313; CHECK-GI-DOT-NEXT: mov v0.b[1], w14314; CHECK-GI-DOT-NEXT: movi v5.2d, #00000000000000004315; CHECK-GI-DOT-NEXT: mov v1.b[1], w104316; CHECK-GI-DOT-NEXT: mov v3.d[1], v2.d[0]4317; CHECK-GI-DOT-NEXT: mov v0.b[2], w24318; CHECK-GI-DOT-NEXT: mov v1.b[2], w94319; CHECK-GI-DOT-NEXT: ldr w9, [sp, #88]4320; CHECK-GI-DOT-NEXT: mov v0.b[3], w34321; CHECK-GI-DOT-NEXT: mov v1.b[3], w94322; CHECK-GI-DOT-NEXT: ldr w9, [sp, #96]4323; CHECK-GI-DOT-NEXT: mov v0.b[4], w44324; CHECK-GI-DOT-NEXT: mov v1.b[4], w94325; CHECK-GI-DOT-NEXT: ldr w9, [sp, #104]4326; CHECK-GI-DOT-NEXT: mov v0.b[5], w54327; CHECK-GI-DOT-NEXT: mov v1.b[5], w94328; CHECK-GI-DOT-NEXT: ldr w9, [sp, #112]4329; CHECK-GI-DOT-NEXT: mov v0.b[6], w64330; CHECK-GI-DOT-NEXT: mov v1.b[6], w94331; CHECK-GI-DOT-NEXT: ldr w9, [sp, #120]4332; CHECK-GI-DOT-NEXT: mov v0.b[7], w74333; CHECK-GI-DOT-NEXT: mov v1.b[7], w94334; CHECK-GI-DOT-NEXT: mov v0.b[8], w84335; CHECK-GI-DOT-NEXT: ldr w8, [sp, #8]4336; CHECK-GI-DOT-NEXT: fmov d1, d14337; CHECK-GI-DOT-NEXT: mov v0.b[9], w84338; CHECK-GI-DOT-NEXT: ldr w8, [sp, #16]4339; CHECK-GI-DOT-NEXT: sdot v4.4s, v1.16b, v2.16b4340; CHECK-GI-DOT-NEXT: mov v0.b[10], w84341; CHECK-GI-DOT-NEXT: ldr w8, [sp, #24]4342; CHECK-GI-DOT-NEXT: mov v0.b[11], w84343; CHECK-GI-DOT-NEXT: ldr w8, [sp, #32]4344; CHECK-GI-DOT-NEXT: mov v0.b[12], w84345; CHECK-GI-DOT-NEXT: ldr w8, [sp, #40]4346; CHECK-GI-DOT-NEXT: mov v0.b[13], w84347; CHECK-GI-DOT-NEXT: ldr w8, [sp, #48]4348; CHECK-GI-DOT-NEXT: mov v0.b[14], w84349; CHECK-GI-DOT-NEXT: ldr w8, [sp, #56]4350; CHECK-GI-DOT-NEXT: mov v0.b[15], w84351; CHECK-GI-DOT-NEXT: sdot v5.4s, v0.16b, v3.16b4352; CHECK-GI-DOT-NEXT: add v0.4s, v5.4s, v4.4s4353; CHECK-GI-DOT-NEXT: addv s0, v0.4s4354; CHECK-GI-DOT-NEXT: fmov w0, s04355; CHECK-GI-DOT-NEXT: ret4356entry:4357 %xx = sext <24 x i8> %x to <24 x i32>4358 %z = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %xx)4359 ret i32 %z4360}4361 4362define i32 @add_v32i8_v32i32_sext(<32 x i8> %x) {4363; CHECK-SD-BASE-LABEL: add_v32i8_v32i32_sext:4364; CHECK-SD-BASE: // %bb.0: // %entry4365; CHECK-SD-BASE-NEXT: sshll2 v2.8h, v1.16b, #04366; CHECK-SD-BASE-NEXT: sshll2 v3.8h, v0.16b, #04367; CHECK-SD-BASE-NEXT: sshll v1.8h, v1.8b, #04368; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #04369; CHECK-SD-BASE-NEXT: saddl2 v4.4s, v3.8h, v2.8h4370; CHECK-SD-BASE-NEXT: saddl v2.4s, v3.4h, v2.4h4371; CHECK-SD-BASE-NEXT: saddl2 v5.4s, v0.8h, v1.8h4372; CHECK-SD-BASE-NEXT: saddl v0.4s, v0.4h, v1.4h4373; CHECK-SD-BASE-NEXT: add v1.4s, v5.4s, v4.4s4374; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v2.4s4375; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s4376; CHECK-SD-BASE-NEXT: addv s0, v0.4s4377; CHECK-SD-BASE-NEXT: fmov w0, s04378; CHECK-SD-BASE-NEXT: ret4379;4380; CHECK-SD-DOT-LABEL: add_v32i8_v32i32_sext:4381; CHECK-SD-DOT: // %bb.0: // %entry4382; CHECK-SD-DOT-NEXT: movi v2.16b, #14383; CHECK-SD-DOT-NEXT: movi v3.2d, #00000000000000004384; CHECK-SD-DOT-NEXT: sdot v3.4s, v1.16b, v2.16b4385; CHECK-SD-DOT-NEXT: sdot v3.4s, v0.16b, v2.16b4386; CHECK-SD-DOT-NEXT: addv s0, v3.4s4387; CHECK-SD-DOT-NEXT: fmov w0, s04388; CHECK-SD-DOT-NEXT: ret4389;4390; CHECK-GI-BASE-LABEL: add_v32i8_v32i32_sext:4391; CHECK-GI-BASE: // %bb.0: // %entry4392; CHECK-GI-BASE-NEXT: saddlv h0, v0.16b4393; CHECK-GI-BASE-NEXT: saddlv h1, v1.16b4394; CHECK-GI-BASE-NEXT: fmov w8, s04395; CHECK-GI-BASE-NEXT: fmov w9, s14396; CHECK-GI-BASE-NEXT: add w8, w8, w94397; CHECK-GI-BASE-NEXT: sxth w0, w84398; CHECK-GI-BASE-NEXT: ret4399;4400; CHECK-GI-DOT-LABEL: add_v32i8_v32i32_sext:4401; CHECK-GI-DOT: // %bb.0: // %entry4402; CHECK-GI-DOT-NEXT: movi v2.16b, #14403; CHECK-GI-DOT-NEXT: movi v3.2d, #00000000000000004404; CHECK-GI-DOT-NEXT: movi v4.2d, #00000000000000004405; CHECK-GI-DOT-NEXT: sdot v4.4s, v0.16b, v2.16b4406; CHECK-GI-DOT-NEXT: sdot v3.4s, v1.16b, v2.16b4407; CHECK-GI-DOT-NEXT: add v0.4s, v4.4s, v3.4s4408; CHECK-GI-DOT-NEXT: addv s0, v0.4s4409; CHECK-GI-DOT-NEXT: fmov w0, s04410; CHECK-GI-DOT-NEXT: ret4411entry:4412 %xx = sext <32 x i8> %x to <32 x i32>4413 %z = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %xx)4414 ret i32 %z4415}4416 4417define i32 @full(ptr %p1, i32 noundef %s1, ptr %p2, i32 noundef %s2) {4418; CHECK-SD-BASE-LABEL: full:4419; CHECK-SD-BASE: // %bb.0: // %entry4420; CHECK-SD-BASE-NEXT: // kill: def $w3 killed $w3 def $x34421; CHECK-SD-BASE-NEXT: // kill: def $w1 killed $w1 def $x14422; CHECK-SD-BASE-NEXT: sxtw x8, w34423; CHECK-SD-BASE-NEXT: sxtw x9, w14424; CHECK-SD-BASE-NEXT: ldr d0, [x0]4425; CHECK-SD-BASE-NEXT: ldr d1, [x2]4426; CHECK-SD-BASE-NEXT: add x10, x0, x94427; CHECK-SD-BASE-NEXT: add x11, x2, x84428; CHECK-SD-BASE-NEXT: uabdl v0.8h, v0.8b, v1.8b4429; CHECK-SD-BASE-NEXT: ldr d1, [x10]4430; CHECK-SD-BASE-NEXT: ldr d2, [x11]4431; CHECK-SD-BASE-NEXT: add x10, x10, x94432; CHECK-SD-BASE-NEXT: add x11, x11, x84433; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b4434; CHECK-SD-BASE-NEXT: ldr d2, [x11]4435; CHECK-SD-BASE-NEXT: add x11, x11, x84436; CHECK-SD-BASE-NEXT: uaddlp v0.4s, v0.8h4437; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h4438; CHECK-SD-BASE-NEXT: ldr d1, [x10]4439; CHECK-SD-BASE-NEXT: add x10, x10, x94440; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b4441; CHECK-SD-BASE-NEXT: ldr d2, [x11]4442; CHECK-SD-BASE-NEXT: add x11, x11, x84443; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h4444; CHECK-SD-BASE-NEXT: ldr d1, [x10]4445; CHECK-SD-BASE-NEXT: add x10, x10, x94446; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b4447; CHECK-SD-BASE-NEXT: ldr d2, [x11]4448; CHECK-SD-BASE-NEXT: add x11, x11, x84449; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h4450; CHECK-SD-BASE-NEXT: ldr d1, [x10]4451; CHECK-SD-BASE-NEXT: add x10, x10, x94452; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b4453; CHECK-SD-BASE-NEXT: ldr d2, [x11]4454; CHECK-SD-BASE-NEXT: add x11, x11, x84455; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h4456; CHECK-SD-BASE-NEXT: ldr d1, [x10]4457; CHECK-SD-BASE-NEXT: add x10, x10, x94458; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b4459; CHECK-SD-BASE-NEXT: ldr d2, [x11]4460; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h4461; CHECK-SD-BASE-NEXT: ldr d1, [x10]4462; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b4463; CHECK-SD-BASE-NEXT: ldr d2, [x11, x8]4464; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h4465; CHECK-SD-BASE-NEXT: ldr d1, [x10, x9]4466; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b4467; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h4468; CHECK-SD-BASE-NEXT: addv s0, v0.4s4469; CHECK-SD-BASE-NEXT: fmov w0, s04470; CHECK-SD-BASE-NEXT: ret4471;4472; CHECK-SD-DOT-LABEL: full:4473; CHECK-SD-DOT: // %bb.0: // %entry4474; CHECK-SD-DOT-NEXT: ldr d0, [x0]4475; CHECK-SD-DOT-NEXT: ldr d1, [x2]4476; CHECK-SD-DOT-NEXT: // kill: def $w3 killed $w3 def $x34477; CHECK-SD-DOT-NEXT: // kill: def $w1 killed $w1 def $x14478; CHECK-SD-DOT-NEXT: sxtw x8, w34479; CHECK-SD-DOT-NEXT: sxtw x9, w14480; CHECK-SD-DOT-NEXT: movi v2.2d, #00000000000000004481; CHECK-SD-DOT-NEXT: movi v3.8b, #14482; CHECK-SD-DOT-NEXT: uabd v0.8b, v0.8b, v1.8b4483; CHECK-SD-DOT-NEXT: add x11, x2, x84484; CHECK-SD-DOT-NEXT: add x10, x0, x94485; CHECK-SD-DOT-NEXT: ldr d4, [x11]4486; CHECK-SD-DOT-NEXT: add x11, x11, x84487; CHECK-SD-DOT-NEXT: ldr d1, [x10]4488; CHECK-SD-DOT-NEXT: add x10, x10, x94489; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b4490; CHECK-SD-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b4491; CHECK-SD-DOT-NEXT: ldr d1, [x10]4492; CHECK-SD-DOT-NEXT: ldr d4, [x11]4493; CHECK-SD-DOT-NEXT: add x10, x10, x94494; CHECK-SD-DOT-NEXT: add x11, x11, x84495; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b4496; CHECK-SD-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b4497; CHECK-SD-DOT-NEXT: ldr d1, [x10]4498; CHECK-SD-DOT-NEXT: ldr d4, [x11]4499; CHECK-SD-DOT-NEXT: add x10, x10, x94500; CHECK-SD-DOT-NEXT: add x11, x11, x84501; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b4502; CHECK-SD-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b4503; CHECK-SD-DOT-NEXT: ldr d1, [x10]4504; CHECK-SD-DOT-NEXT: ldr d4, [x11]4505; CHECK-SD-DOT-NEXT: add x10, x10, x94506; CHECK-SD-DOT-NEXT: add x11, x11, x84507; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b4508; CHECK-SD-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b4509; CHECK-SD-DOT-NEXT: ldr d1, [x10]4510; CHECK-SD-DOT-NEXT: ldr d4, [x11]4511; CHECK-SD-DOT-NEXT: add x10, x10, x94512; CHECK-SD-DOT-NEXT: add x11, x11, x84513; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b4514; CHECK-SD-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b4515; CHECK-SD-DOT-NEXT: ldr d1, [x10]4516; CHECK-SD-DOT-NEXT: ldr d4, [x11]4517; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b4518; CHECK-SD-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b4519; CHECK-SD-DOT-NEXT: ldr d1, [x10, x9]4520; CHECK-SD-DOT-NEXT: ldr d4, [x11, x8]4521; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b4522; CHECK-SD-DOT-NEXT: uabd v0.8b, v1.8b, v4.8b4523; CHECK-SD-DOT-NEXT: udot v2.2s, v0.8b, v3.8b4524; CHECK-SD-DOT-NEXT: addp v0.2s, v2.2s, v2.2s4525; CHECK-SD-DOT-NEXT: fmov w0, s04526; CHECK-SD-DOT-NEXT: ret4527;4528; CHECK-GI-LABEL: full:4529; CHECK-GI: // %bb.0: // %entry4530; CHECK-GI-NEXT: // kill: def $w1 killed $w1 def $x14531; CHECK-GI-NEXT: // kill: def $w3 killed $w3 def $x34532; CHECK-GI-NEXT: sxtw x9, w14533; CHECK-GI-NEXT: sxtw x8, w34534; CHECK-GI-NEXT: ldr d0, [x0]4535; CHECK-GI-NEXT: ldr d1, [x2]4536; CHECK-GI-NEXT: add x10, x0, x94537; CHECK-GI-NEXT: add x11, x2, x84538; CHECK-GI-NEXT: usubl v2.8h, v0.8b, v1.8b4539; CHECK-GI-NEXT: ldr d1, [x10]4540; CHECK-GI-NEXT: ldr d3, [x11]4541; CHECK-GI-NEXT: add x10, x10, x94542; CHECK-GI-NEXT: add x11, x11, x84543; CHECK-GI-NEXT: movi v0.2d, #00000000000000004544; CHECK-GI-NEXT: usubl v3.8h, v1.8b, v3.8b4545; CHECK-GI-NEXT: ldr d4, [x10]4546; CHECK-GI-NEXT: ldr d5, [x11]4547; CHECK-GI-NEXT: add x10, x10, x94548; CHECK-GI-NEXT: add x11, x11, x84549; CHECK-GI-NEXT: sshll v1.4s, v2.4h, #04550; CHECK-GI-NEXT: sshll2 v2.4s, v2.8h, #04551; CHECK-GI-NEXT: ldr d6, [x10]4552; CHECK-GI-NEXT: ldr d7, [x11]4553; CHECK-GI-NEXT: add x10, x10, x94554; CHECK-GI-NEXT: add x11, x11, x84555; CHECK-GI-NEXT: sshll2 v16.4s, v3.8h, #04556; CHECK-GI-NEXT: usubl v4.8h, v4.8b, v5.8b4557; CHECK-GI-NEXT: ldr d5, [x10]4558; CHECK-GI-NEXT: ldr d17, [x11]4559; CHECK-GI-NEXT: add x10, x10, x94560; CHECK-GI-NEXT: add x11, x11, x84561; CHECK-GI-NEXT: usubl v6.8h, v6.8b, v7.8b4562; CHECK-GI-NEXT: ldr d7, [x10]4563; CHECK-GI-NEXT: ldr d19, [x11]4564; CHECK-GI-NEXT: abs v2.4s, v2.4s4565; CHECK-GI-NEXT: sshll v3.4s, v3.4h, #04566; CHECK-GI-NEXT: abs v16.4s, v16.4s4567; CHECK-GI-NEXT: add x10, x10, x94568; CHECK-GI-NEXT: add x11, x11, x84569; CHECK-GI-NEXT: usubl v5.8h, v5.8b, v17.8b4570; CHECK-GI-NEXT: ldr d17, [x10]4571; CHECK-GI-NEXT: ldr d20, [x11]4572; CHECK-GI-NEXT: usubl v7.8h, v7.8b, v19.8b4573; CHECK-GI-NEXT: ldr d19, [x10, x9]4574; CHECK-GI-NEXT: ldr d21, [x11, x8]4575; CHECK-GI-NEXT: sshll2 v18.4s, v4.8h, #04576; CHECK-GI-NEXT: saba v2.4s, v1.4s, v0.4s4577; CHECK-GI-NEXT: sshll2 v1.4s, v6.8h, #04578; CHECK-GI-NEXT: usubl v17.8h, v17.8b, v20.8b4579; CHECK-GI-NEXT: saba v16.4s, v3.4s, v0.4s4580; CHECK-GI-NEXT: sshll2 v3.4s, v5.8h, #04581; CHECK-GI-NEXT: usubl v19.8h, v19.8b, v21.8b4582; CHECK-GI-NEXT: sshll v4.4s, v4.4h, #04583; CHECK-GI-NEXT: abs v18.4s, v18.4s4584; CHECK-GI-NEXT: sshll2 v20.4s, v7.8h, #04585; CHECK-GI-NEXT: sshll v6.4s, v6.4h, #04586; CHECK-GI-NEXT: abs v1.4s, v1.4s4587; CHECK-GI-NEXT: sshll2 v21.4s, v17.8h, #04588; CHECK-GI-NEXT: sshll v5.4s, v5.4h, #04589; CHECK-GI-NEXT: abs v3.4s, v3.4s4590; CHECK-GI-NEXT: sshll2 v22.4s, v19.8h, #04591; CHECK-GI-NEXT: saba v18.4s, v4.4s, v0.4s4592; CHECK-GI-NEXT: sshll v4.4s, v7.4h, #04593; CHECK-GI-NEXT: abs v7.4s, v20.4s4594; CHECK-GI-NEXT: saba v1.4s, v6.4s, v0.4s4595; CHECK-GI-NEXT: sshll v6.4s, v17.4h, #04596; CHECK-GI-NEXT: abs v17.4s, v21.4s4597; CHECK-GI-NEXT: saba v3.4s, v5.4s, v0.4s4598; CHECK-GI-NEXT: sshll v5.4s, v19.4h, #04599; CHECK-GI-NEXT: abs v19.4s, v22.4s4600; CHECK-GI-NEXT: saba v7.4s, v4.4s, v0.4s4601; CHECK-GI-NEXT: saba v17.4s, v6.4s, v0.4s4602; CHECK-GI-NEXT: saba v19.4s, v5.4s, v0.4s4603; CHECK-GI-NEXT: addv s0, v2.4s4604; CHECK-GI-NEXT: addv s2, v16.4s4605; CHECK-GI-NEXT: addv s4, v18.4s4606; CHECK-GI-NEXT: addv s1, v1.4s4607; CHECK-GI-NEXT: fmov w8, s24608; CHECK-GI-NEXT: fmov w9, s04609; CHECK-GI-NEXT: addv s0, v3.4s4610; CHECK-GI-NEXT: fmov w10, s44611; CHECK-GI-NEXT: addv s2, v7.4s4612; CHECK-GI-NEXT: add w8, w8, w94613; CHECK-GI-NEXT: fmov w9, s14614; CHECK-GI-NEXT: addv s1, v17.4s4615; CHECK-GI-NEXT: add w8, w10, w84616; CHECK-GI-NEXT: add w8, w9, w84617; CHECK-GI-NEXT: fmov w9, s04618; CHECK-GI-NEXT: addv s0, v19.4s4619; CHECK-GI-NEXT: add w8, w9, w84620; CHECK-GI-NEXT: fmov w9, s24621; CHECK-GI-NEXT: add w8, w9, w84622; CHECK-GI-NEXT: fmov w9, s14623; CHECK-GI-NEXT: add w8, w9, w84624; CHECK-GI-NEXT: fmov w9, s04625; CHECK-GI-NEXT: add w0, w9, w84626; CHECK-GI-NEXT: ret4627entry:4628 %idx.ext8 = sext i32 %s2 to i644629 %idx.ext = sext i32 %s1 to i644630 %0 = load <8 x i8>, ptr %p1, align 14631 %1 = zext <8 x i8> %0 to <8 x i32>4632 %2 = load <8 x i8>, ptr %p2, align 14633 %3 = zext <8 x i8> %2 to <8 x i32>4634 %4 = sub nsw <8 x i32> %1, %34635 %5 = tail call <8 x i32> @llvm.abs.v8i32(<8 x i32> %4, i1 true)4636 %6 = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %5)4637 %add.ptr = getelementptr inbounds i8, ptr %p1, i64 %idx.ext4638 %add.ptr9 = getelementptr inbounds i8, ptr %p2, i64 %idx.ext84639 %7 = load <8 x i8>, ptr %add.ptr, align 14640 %8 = zext <8 x i8> %7 to <8 x i32>4641 %9 = load <8 x i8>, ptr %add.ptr9, align 14642 %10 = zext <8 x i8> %9 to <8 x i32>4643 %11 = sub nsw <8 x i32> %8, %104644 %12 = tail call <8 x i32> @llvm.abs.v8i32(<8 x i32> %11, i1 true)4645 %13 = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %12)4646 %op.rdx.1 = add i32 %13, %64647 %add.ptr.1 = getelementptr inbounds i8, ptr %add.ptr, i64 %idx.ext4648 %add.ptr9.1 = getelementptr inbounds i8, ptr %add.ptr9, i64 %idx.ext84649 %14 = load <8 x i8>, ptr %add.ptr.1, align 14650 %15 = zext <8 x i8> %14 to <8 x i32>4651 %16 = load <8 x i8>, ptr %add.ptr9.1, align 14652 %17 = zext <8 x i8> %16 to <8 x i32>4653 %18 = sub nsw <8 x i32> %15, %174654 %19 = tail call <8 x i32> @llvm.abs.v8i32(<8 x i32> %18, i1 true)4655 %20 = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %19)4656 %op.rdx.2 = add i32 %20, %op.rdx.14657 %add.ptr.2 = getelementptr inbounds i8, ptr %add.ptr.1, i64 %idx.ext4658 %add.ptr9.2 = getelementptr inbounds i8, ptr %add.ptr9.1, i64 %idx.ext84659 %21 = load <8 x i8>, ptr %add.ptr.2, align 14660 %22 = zext <8 x i8> %21 to <8 x i32>4661 %23 = load <8 x i8>, ptr %add.ptr9.2, align 14662 %24 = zext <8 x i8> %23 to <8 x i32>4663 %25 = sub nsw <8 x i32> %22, %244664 %26 = tail call <8 x i32> @llvm.abs.v8i32(<8 x i32> %25, i1 true)4665 %27 = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %26)4666 %op.rdx.3 = add i32 %27, %op.rdx.24667 %add.ptr.3 = getelementptr inbounds i8, ptr %add.ptr.2, i64 %idx.ext4668 %add.ptr9.3 = getelementptr inbounds i8, ptr %add.ptr9.2, i64 %idx.ext84669 %28 = load <8 x i8>, ptr %add.ptr.3, align 14670 %29 = zext <8 x i8> %28 to <8 x i32>4671 %30 = load <8 x i8>, ptr %add.ptr9.3, align 14672 %31 = zext <8 x i8> %30 to <8 x i32>4673 %32 = sub nsw <8 x i32> %29, %314674 %33 = tail call <8 x i32> @llvm.abs.v8i32(<8 x i32> %32, i1 true)4675 %34 = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %33)4676 %op.rdx.4 = add i32 %34, %op.rdx.34677 %add.ptr.4 = getelementptr inbounds i8, ptr %add.ptr.3, i64 %idx.ext4678 %add.ptr9.4 = getelementptr inbounds i8, ptr %add.ptr9.3, i64 %idx.ext84679 %35 = load <8 x i8>, ptr %add.ptr.4, align 14680 %36 = zext <8 x i8> %35 to <8 x i32>4681 %37 = load <8 x i8>, ptr %add.ptr9.4, align 14682 %38 = zext <8 x i8> %37 to <8 x i32>4683 %39 = sub nsw <8 x i32> %36, %384684 %40 = tail call <8 x i32> @llvm.abs.v8i32(<8 x i32> %39, i1 true)4685 %41 = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %40)4686 %op.rdx.5 = add i32 %41, %op.rdx.44687 %add.ptr.5 = getelementptr inbounds i8, ptr %add.ptr.4, i64 %idx.ext4688 %add.ptr9.5 = getelementptr inbounds i8, ptr %add.ptr9.4, i64 %idx.ext84689 %42 = load <8 x i8>, ptr %add.ptr.5, align 14690 %43 = zext <8 x i8> %42 to <8 x i32>4691 %44 = load <8 x i8>, ptr %add.ptr9.5, align 14692 %45 = zext <8 x i8> %44 to <8 x i32>4693 %46 = sub nsw <8 x i32> %43, %454694 %47 = tail call <8 x i32> @llvm.abs.v8i32(<8 x i32> %46, i1 true)4695 %48 = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %47)4696 %op.rdx.6 = add i32 %48, %op.rdx.54697 %add.ptr.6 = getelementptr inbounds i8, ptr %add.ptr.5, i64 %idx.ext4698 %add.ptr9.6 = getelementptr inbounds i8, ptr %add.ptr9.5, i64 %idx.ext84699 %49 = load <8 x i8>, ptr %add.ptr.6, align 14700 %50 = zext <8 x i8> %49 to <8 x i32>4701 %51 = load <8 x i8>, ptr %add.ptr9.6, align 14702 %52 = zext <8 x i8> %51 to <8 x i32>4703 %53 = sub nsw <8 x i32> %50, %524704 %54 = tail call <8 x i32> @llvm.abs.v8i32(<8 x i32> %53, i1 true)4705 %55 = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %54)4706 %op.rdx.7 = add i32 %55, %op.rdx.64707 ret i32 %op.rdx.74708}4709 4710define i32 @extract_hi_lo(<8 x i16> %a) {4711; CHECK-SD-LABEL: extract_hi_lo:4712; CHECK-SD: // %bb.0: // %entry4713; CHECK-SD-NEXT: uaddlv s0, v0.8h4714; CHECK-SD-NEXT: fmov w0, s04715; CHECK-SD-NEXT: ret4716;4717; CHECK-GI-LABEL: extract_hi_lo:4718; CHECK-GI: // %bb.0: // %entry4719; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #04720; CHECK-GI-NEXT: uaddw2 v0.4s, v1.4s, v0.8h4721; CHECK-GI-NEXT: addv s0, v0.4s4722; CHECK-GI-NEXT: fmov w0, s04723; CHECK-GI-NEXT: ret4724entry:4725 %e1 = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>4726 %e2 = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>4727 %z1 = zext <4 x i16> %e1 to <4 x i32>4728 %z2 = zext <4 x i16> %e2 to <4 x i32>4729 %z4 = add <4 x i32> %z1, %z24730 %z5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %z4)4731 ret i32 %z54732}4733 4734define i32 @extract_hi_hi(<8 x i16> %a) {4735; CHECK-SD-LABEL: extract_hi_hi:4736; CHECK-SD: // %bb.0: // %entry4737; CHECK-SD-NEXT: mov v0.d[0], v0.d[1]4738; CHECK-SD-NEXT: uaddlv s0, v0.8h4739; CHECK-SD-NEXT: fmov w0, s04740; CHECK-SD-NEXT: ret4741;4742; CHECK-GI-LABEL: extract_hi_hi:4743; CHECK-GI: // %bb.0: // %entry4744; CHECK-GI-NEXT: uaddl2 v0.4s, v0.8h, v0.8h4745; CHECK-GI-NEXT: addv s0, v0.4s4746; CHECK-GI-NEXT: fmov w0, s04747; CHECK-GI-NEXT: ret4748entry:4749 %e2 = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>4750 %z2 = zext <4 x i16> %e2 to <4 x i32>4751 %z4 = add <4 x i32> %z2, %z24752 %z5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %z4)4753 ret i32 %z54754}4755 4756define i32 @extract_lo_lo(<8 x i16> %a) {4757; CHECK-SD-LABEL: extract_lo_lo:4758; CHECK-SD: // %bb.0: // %entry4759; CHECK-SD-NEXT: mov v0.d[1], v0.d[0]4760; CHECK-SD-NEXT: uaddlv s0, v0.8h4761; CHECK-SD-NEXT: fmov w0, s04762; CHECK-SD-NEXT: ret4763;4764; CHECK-GI-LABEL: extract_lo_lo:4765; CHECK-GI: // %bb.0: // %entry4766; CHECK-GI-NEXT: uaddl v0.4s, v0.4h, v0.4h4767; CHECK-GI-NEXT: addv s0, v0.4s4768; CHECK-GI-NEXT: fmov w0, s04769; CHECK-GI-NEXT: ret4770entry:4771 %e1 = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>4772 %z1 = zext <4 x i16> %e1 to <4 x i32>4773 %z4 = add <4 x i32> %z1, %z14774 %z5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %z4)4775 ret i32 %z54776}4777 4778define i64 @extract_scalable(<2 x i32> %0) "target-features"="+sve2" {4779; CHECK-SD-LABEL: extract_scalable:4780; CHECK-SD: // %bb.0:4781; CHECK-SD-NEXT: movi v1.2s, #14782; CHECK-SD-NEXT: ptrue p0.s, vl24783; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $z04784; CHECK-SD-NEXT: sdivr z0.s, p0/m, z0.s, z1.s4785; CHECK-SD-NEXT: saddl v0.2d, v0.2s, v0.2s4786; CHECK-SD-NEXT: addp d0, v0.2d4787; CHECK-SD-NEXT: fmov x0, d04788; CHECK-SD-NEXT: ret4789;4790; CHECK-GI-LABEL: extract_scalable:4791; CHECK-GI: // %bb.0:4792; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q04793; CHECK-GI-NEXT: fmov w9, s04794; CHECK-GI-NEXT: mov w8, #1 // =0x14795; CHECK-GI-NEXT: mov w10, v0.s[1]4796; CHECK-GI-NEXT: sdiv w9, w8, w94797; CHECK-GI-NEXT: sdiv w8, w8, w104798; CHECK-GI-NEXT: fmov s0, w94799; CHECK-GI-NEXT: mov v0.s[1], w84800; CHECK-GI-NEXT: saddl v0.2d, v0.2s, v0.2s4801; CHECK-GI-NEXT: addp d0, v0.2d4802; CHECK-GI-NEXT: fmov x0, d04803; CHECK-GI-NEXT: ret4804 %2 = sdiv <2 x i32> splat (i32 1), %04805 %3 = sext <2 x i32> %2 to <2 x i64>4806 %4 = add <2 x i64> %3, %34807 %5 = tail call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %4)4808 ret i64 %54809}4810 4811define i32 @vecreduce_add_from_i21_zero() {4812; CHECK-SD-LABEL: vecreduce_add_from_i21_zero:4813; CHECK-SD: // %bb.0: // %entry4814; CHECK-SD-NEXT: mov w0, wzr4815; CHECK-SD-NEXT: ret4816;4817; CHECK-GI-LABEL: vecreduce_add_from_i21_zero:4818; CHECK-GI: // %bb.0: // %entry4819; CHECK-GI-NEXT: movi v0.2d, #00000000000000004820; CHECK-GI-NEXT: addv s0, v0.4s4821; CHECK-GI-NEXT: fmov w0, s04822; CHECK-GI-NEXT: ret4823entry:4824 %0 = zext <4 x i21> zeroinitializer to <4 x i32>4825 %1 = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %0)4826 ret i32 %14827}4828 4829define i32 @vecreduce_add_from_i21(<4 x i21> %a) {4830; CHECK-LABEL: vecreduce_add_from_i21:4831; CHECK: // %bb.0: // %entry4832; CHECK-NEXT: movi v1.4s, #31, msl #164833; CHECK-NEXT: and v0.16b, v0.16b, v1.16b4834; CHECK-NEXT: addv s0, v0.4s4835; CHECK-NEXT: fmov w0, s04836; CHECK-NEXT: ret4837entry:4838 %0 = zext <4 x i21> %a to <4 x i32>4839 %1 = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %0)4840 ret i32 %14841}4842 4843declare <8 x i32> @llvm.abs.v8i32(<8 x i32>, i1 immarg) #14844declare i16 @llvm.vector.reduce.add.v32i16(<32 x i16>)4845declare i16 @llvm.vector.reduce.add.v24i16(<24 x i16>)4846declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>)4847declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)4848declare i16 @llvm.vector.reduce.add.v4i16(<4 x i16>)4849declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)4850declare i32 @llvm.vector.reduce.add.v2i32(<2 x i32>)4851declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)4852declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)4853declare i32 @llvm.vector.reduce.add.v24i32(<24 x i32>)4854declare i32 @llvm.vector.reduce.add.v32i32(<32 x i32>)4855declare i32 @llvm.vector.reduce.add.v48i32(<48 x i32>)4856declare i64 @llvm.vector.reduce.add.v16i64(<16 x i64>)4857declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>)4858declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)4859declare i64 @llvm.vector.reduce.add.v8i64(<8 x i64>)4860declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)4861declare i8 @llvm.vector.reduce.add.v8i8(<8 x i8>)4862