2230 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s --check-prefixes=CHECK,CHECK-SD3; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI4 5; CHECK-GI: warning: Instruction selection used fallback path for ext_via_i196; CHECK-GI-NEXT: warning: Instruction selection used fallback path for srhadd_v2i32_trunc7; CHECK-GI-NEXT: warning: Instruction selection used fallback path for urhadd_v2i32_trunc8 9define <8 x i8> @shadd8b(ptr nocapture readonly %A, ptr nocapture readonly %B) {10; CHECK-LABEL: shadd8b:11; CHECK: // %bb.0:12; CHECK-NEXT: ldr d0, [x0]13; CHECK-NEXT: ldr d1, [x1]14; CHECK-NEXT: shadd.8b v0, v0, v115; CHECK-NEXT: ret16 %tmp1 = load <8 x i8>, ptr %A, align 817 %tmp2 = load <8 x i8>, ptr %B, align 818 %tmp3 = tail call <8 x i8> @llvm.aarch64.neon.shadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)19 ret <8 x i8> %tmp320}21 22define <16 x i8> @shadd16b(ptr nocapture readonly %A, ptr nocapture readonly %B) {23; CHECK-LABEL: shadd16b:24; CHECK: // %bb.0:25; CHECK-NEXT: ldr q0, [x0]26; CHECK-NEXT: ldr q1, [x1]27; CHECK-NEXT: shadd.16b v0, v0, v128; CHECK-NEXT: ret29 %tmp1 = load <16 x i8>, ptr %A, align 1630 %tmp2 = load <16 x i8>, ptr %B, align 1631 %tmp3 = tail call <16 x i8> @llvm.aarch64.neon.shadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)32 ret <16 x i8> %tmp333}34 35define <4 x i16> @shadd4h(ptr nocapture readonly %A, ptr nocapture readonly %B) {36; CHECK-LABEL: shadd4h:37; CHECK: // %bb.0:38; CHECK-NEXT: ldr d0, [x0]39; CHECK-NEXT: ldr d1, [x1]40; CHECK-NEXT: shadd.4h v0, v0, v141; CHECK-NEXT: ret42 %tmp1 = load <4 x i16>, ptr %A, align 843 %tmp2 = load <4 x i16>, ptr %B, align 844 %tmp3 = tail call <4 x i16> @llvm.aarch64.neon.shadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)45 ret <4 x i16> %tmp346}47 48define <8 x i16> @shadd8h(ptr nocapture readonly %A, ptr nocapture readonly %B) {49; CHECK-LABEL: shadd8h:50; CHECK: // %bb.0:51; CHECK-NEXT: ldr q0, [x0]52; CHECK-NEXT: ldr q1, [x1]53; CHECK-NEXT: shadd.8h v0, v0, v154; CHECK-NEXT: ret55 %tmp1 = load <8 x i16>, ptr %A, align 1656 %tmp2 = load <8 x i16>, ptr %B, align 1657 %tmp3 = tail call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)58 ret <8 x i16> %tmp359}60 61define <2 x i32> @shadd2s(ptr nocapture readonly %A, ptr nocapture readonly %B) {62; CHECK-LABEL: shadd2s:63; CHECK: // %bb.0:64; CHECK-NEXT: ldr d0, [x0]65; CHECK-NEXT: ldr d1, [x1]66; CHECK-NEXT: shadd.2s v0, v0, v167; CHECK-NEXT: ret68 %tmp1 = load <2 x i32>, ptr %A, align 869 %tmp2 = load <2 x i32>, ptr %B, align 870 %tmp3 = tail call <2 x i32> @llvm.aarch64.neon.shadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)71 ret <2 x i32> %tmp372}73 74define <4 x i32> @shadd4s(ptr nocapture readonly %A, ptr nocapture readonly %B) {75; CHECK-LABEL: shadd4s:76; CHECK: // %bb.0:77; CHECK-NEXT: ldr q0, [x0]78; CHECK-NEXT: ldr q1, [x1]79; CHECK-NEXT: shadd.4s v0, v0, v180; CHECK-NEXT: ret81 %tmp1 = load <4 x i32>, ptr %A, align 1682 %tmp2 = load <4 x i32>, ptr %B, align 1683 %tmp3 = tail call <4 x i32> @llvm.aarch64.neon.shadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)84 ret <4 x i32> %tmp385}86 87define <8 x i8> @uhadd8b(ptr nocapture readonly %A, ptr nocapture readonly %B) {88; CHECK-LABEL: uhadd8b:89; CHECK: // %bb.0:90; CHECK-NEXT: ldr d0, [x0]91; CHECK-NEXT: ldr d1, [x1]92; CHECK-NEXT: uhadd.8b v0, v0, v193; CHECK-NEXT: ret94 %tmp1 = load <8 x i8>, ptr %A, align 895 %tmp2 = load <8 x i8>, ptr %B, align 896 %tmp3 = tail call <8 x i8> @llvm.aarch64.neon.uhadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)97 ret <8 x i8> %tmp398}99 100define <16 x i8> @uhadd16b(ptr nocapture readonly %A, ptr nocapture readonly %B) {101; CHECK-LABEL: uhadd16b:102; CHECK: // %bb.0:103; CHECK-NEXT: ldr q0, [x0]104; CHECK-NEXT: ldr q1, [x1]105; CHECK-NEXT: uhadd.16b v0, v0, v1106; CHECK-NEXT: ret107 %tmp1 = load <16 x i8>, ptr %A, align 16108 %tmp2 = load <16 x i8>, ptr %B, align 16109 %tmp3 = tail call <16 x i8> @llvm.aarch64.neon.uhadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)110 ret <16 x i8> %tmp3111}112 113define <4 x i16> @uhadd4h(ptr nocapture readonly %A, ptr nocapture readonly %B) {114; CHECK-LABEL: uhadd4h:115; CHECK: // %bb.0:116; CHECK-NEXT: ldr d0, [x0]117; CHECK-NEXT: ldr d1, [x1]118; CHECK-NEXT: uhadd.4h v0, v0, v1119; CHECK-NEXT: ret120 %tmp1 = load <4 x i16>, ptr %A, align 8121 %tmp2 = load <4 x i16>, ptr %B, align 8122 %tmp3 = tail call <4 x i16> @llvm.aarch64.neon.uhadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)123 ret <4 x i16> %tmp3124}125 126define <8 x i16> @uhadd8h(ptr nocapture readonly %A, ptr nocapture readonly %B) {127; CHECK-LABEL: uhadd8h:128; CHECK: // %bb.0:129; CHECK-NEXT: ldr q0, [x0]130; CHECK-NEXT: ldr q1, [x1]131; CHECK-NEXT: uhadd.8h v0, v0, v1132; CHECK-NEXT: ret133 %tmp1 = load <8 x i16>, ptr %A, align 16134 %tmp2 = load <8 x i16>, ptr %B, align 16135 %tmp3 = tail call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)136 ret <8 x i16> %tmp3137}138 139define <2 x i32> @uhadd2s(ptr nocapture readonly %A, ptr nocapture readonly %B) {140; CHECK-LABEL: uhadd2s:141; CHECK: // %bb.0:142; CHECK-NEXT: ldr d0, [x0]143; CHECK-NEXT: ldr d1, [x1]144; CHECK-NEXT: uhadd.2s v0, v0, v1145; CHECK-NEXT: ret146 %tmp1 = load <2 x i32>, ptr %A, align 8147 %tmp2 = load <2 x i32>, ptr %B, align 8148 %tmp3 = tail call <2 x i32> @llvm.aarch64.neon.uhadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)149 ret <2 x i32> %tmp3150}151 152define <4 x i32> @uhadd4s(ptr nocapture readonly %A, ptr nocapture readonly %B) {153; CHECK-LABEL: uhadd4s:154; CHECK: // %bb.0:155; CHECK-NEXT: ldr q0, [x0]156; CHECK-NEXT: ldr q1, [x1]157; CHECK-NEXT: uhadd.4s v0, v0, v1158; CHECK-NEXT: ret159 %tmp1 = load <4 x i32>, ptr %A, align 16160 %tmp2 = load <4 x i32>, ptr %B, align 16161 %tmp3 = tail call <4 x i32> @llvm.aarch64.neon.uhadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)162 ret <4 x i32> %tmp3163}164 165declare <8 x i8> @llvm.aarch64.neon.shadd.v8i8(<8 x i8>, <8 x i8>)166declare <4 x i16> @llvm.aarch64.neon.shadd.v4i16(<4 x i16>, <4 x i16>)167declare <2 x i32> @llvm.aarch64.neon.shadd.v2i32(<2 x i32>, <2 x i32>)168declare <8 x i8> @llvm.aarch64.neon.uhadd.v8i8(<8 x i8>, <8 x i8>)169declare <4 x i16> @llvm.aarch64.neon.uhadd.v4i16(<4 x i16>, <4 x i16>)170declare <2 x i32> @llvm.aarch64.neon.uhadd.v2i32(<2 x i32>, <2 x i32>)171declare <16 x i8> @llvm.aarch64.neon.shadd.v16i8(<16 x i8>, <16 x i8>)172declare <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16>, <8 x i16>)173declare <4 x i32> @llvm.aarch64.neon.shadd.v4i32(<4 x i32>, <4 x i32>)174declare <16 x i8> @llvm.aarch64.neon.uhadd.v16i8(<16 x i8>, <16 x i8>)175declare <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16>, <8 x i16>)176declare <4 x i32> @llvm.aarch64.neon.uhadd.v4i32(<4 x i32>, <4 x i32>)177 178define <8 x i8> @srhadd8b(ptr nocapture readonly %A, ptr nocapture readonly %B) {179; CHECK-LABEL: srhadd8b:180; CHECK: // %bb.0:181; CHECK-NEXT: ldr d0, [x0]182; CHECK-NEXT: ldr d1, [x1]183; CHECK-NEXT: srhadd.8b v0, v0, v1184; CHECK-NEXT: ret185 %tmp1 = load <8 x i8>, ptr %A, align 8186 %tmp2 = load <8 x i8>, ptr %B, align 8187 %tmp3 = tail call <8 x i8> @llvm.aarch64.neon.srhadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)188 ret <8 x i8> %tmp3189}190 191define <16 x i8> @srhadd16b(ptr nocapture readonly %A, ptr nocapture readonly %B) {192; CHECK-LABEL: srhadd16b:193; CHECK: // %bb.0:194; CHECK-NEXT: ldr q0, [x0]195; CHECK-NEXT: ldr q1, [x1]196; CHECK-NEXT: srhadd.16b v0, v0, v1197; CHECK-NEXT: ret198 %tmp1 = load <16 x i8>, ptr %A, align 16199 %tmp2 = load <16 x i8>, ptr %B, align 16200 %tmp3 = tail call <16 x i8> @llvm.aarch64.neon.srhadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)201 ret <16 x i8> %tmp3202}203 204define <4 x i16> @srhadd4h(ptr nocapture readonly %A, ptr nocapture readonly %B) {205; CHECK-LABEL: srhadd4h:206; CHECK: // %bb.0:207; CHECK-NEXT: ldr d0, [x0]208; CHECK-NEXT: ldr d1, [x1]209; CHECK-NEXT: srhadd.4h v0, v0, v1210; CHECK-NEXT: ret211 %tmp1 = load <4 x i16>, ptr %A, align 8212 %tmp2 = load <4 x i16>, ptr %B, align 8213 %tmp3 = tail call <4 x i16> @llvm.aarch64.neon.srhadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)214 ret <4 x i16> %tmp3215}216 217define <8 x i16> @srhadd8h(ptr nocapture readonly %A, ptr nocapture readonly %B) {218; CHECK-LABEL: srhadd8h:219; CHECK: // %bb.0:220; CHECK-NEXT: ldr q0, [x0]221; CHECK-NEXT: ldr q1, [x1]222; CHECK-NEXT: srhadd.8h v0, v0, v1223; CHECK-NEXT: ret224 %tmp1 = load <8 x i16>, ptr %A, align 16225 %tmp2 = load <8 x i16>, ptr %B, align 16226 %tmp3 = tail call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)227 ret <8 x i16> %tmp3228}229 230define <2 x i32> @srhadd2s(ptr nocapture readonly %A, ptr nocapture readonly %B) {231; CHECK-LABEL: srhadd2s:232; CHECK: // %bb.0:233; CHECK-NEXT: ldr d0, [x0]234; CHECK-NEXT: ldr d1, [x1]235; CHECK-NEXT: srhadd.2s v0, v0, v1236; CHECK-NEXT: ret237 %tmp1 = load <2 x i32>, ptr %A, align 8238 %tmp2 = load <2 x i32>, ptr %B, align 8239 %tmp3 = tail call <2 x i32> @llvm.aarch64.neon.srhadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)240 ret <2 x i32> %tmp3241}242 243define <4 x i32> @srhadd4s(ptr nocapture readonly %A, ptr nocapture readonly %B) {244; CHECK-LABEL: srhadd4s:245; CHECK: // %bb.0:246; CHECK-NEXT: ldr q0, [x0]247; CHECK-NEXT: ldr q1, [x1]248; CHECK-NEXT: srhadd.4s v0, v0, v1249; CHECK-NEXT: ret250 %tmp1 = load <4 x i32>, ptr %A, align 16251 %tmp2 = load <4 x i32>, ptr %B, align 16252 %tmp3 = tail call <4 x i32> @llvm.aarch64.neon.srhadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)253 ret <4 x i32> %tmp3254}255 256define <8 x i8> @urhadd8b(ptr nocapture readonly %A, ptr nocapture readonly %B) {257; CHECK-LABEL: urhadd8b:258; CHECK: // %bb.0:259; CHECK-NEXT: ldr d0, [x0]260; CHECK-NEXT: ldr d1, [x1]261; CHECK-NEXT: urhadd.8b v0, v0, v1262; CHECK-NEXT: ret263 %tmp1 = load <8 x i8>, ptr %A, align 8264 %tmp2 = load <8 x i8>, ptr %B, align 8265 %tmp3 = tail call <8 x i8> @llvm.aarch64.neon.urhadd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)266 ret <8 x i8> %tmp3267}268 269define <16 x i8> @urhadd16b(ptr nocapture readonly %A, ptr nocapture readonly %B) {270; CHECK-LABEL: urhadd16b:271; CHECK: // %bb.0:272; CHECK-NEXT: ldr q0, [x0]273; CHECK-NEXT: ldr q1, [x1]274; CHECK-NEXT: urhadd.16b v0, v0, v1275; CHECK-NEXT: ret276 %tmp1 = load <16 x i8>, ptr %A, align 16277 %tmp2 = load <16 x i8>, ptr %B, align 16278 %tmp3 = tail call <16 x i8> @llvm.aarch64.neon.urhadd.v16i8(<16 x i8> %tmp1, <16 x i8> %tmp2)279 ret <16 x i8> %tmp3280}281 282define <4 x i16> @urhadd4h(ptr nocapture readonly %A, ptr nocapture readonly %B) {283; CHECK-LABEL: urhadd4h:284; CHECK: // %bb.0:285; CHECK-NEXT: ldr d0, [x0]286; CHECK-NEXT: ldr d1, [x1]287; CHECK-NEXT: urhadd.4h v0, v0, v1288; CHECK-NEXT: ret289 %tmp1 = load <4 x i16>, ptr %A, align 8290 %tmp2 = load <4 x i16>, ptr %B, align 8291 %tmp3 = tail call <4 x i16> @llvm.aarch64.neon.urhadd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)292 ret <4 x i16> %tmp3293}294 295define <8 x i16> @urhadd8h(ptr nocapture readonly %A, ptr nocapture readonly %B) {296; CHECK-LABEL: urhadd8h:297; CHECK: // %bb.0:298; CHECK-NEXT: ldr q0, [x0]299; CHECK-NEXT: ldr q1, [x1]300; CHECK-NEXT: urhadd.8h v0, v0, v1301; CHECK-NEXT: ret302 %tmp1 = load <8 x i16>, ptr %A, align 16303 %tmp2 = load <8 x i16>, ptr %B, align 16304 %tmp3 = tail call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)305 ret <8 x i16> %tmp3306}307 308define <2 x i32> @urhadd2s(ptr nocapture readonly %A, ptr nocapture readonly %B) {309; CHECK-LABEL: urhadd2s:310; CHECK: // %bb.0:311; CHECK-NEXT: ldr d0, [x0]312; CHECK-NEXT: ldr d1, [x1]313; CHECK-NEXT: urhadd.2s v0, v0, v1314; CHECK-NEXT: ret315 %tmp1 = load <2 x i32>, ptr %A, align 8316 %tmp2 = load <2 x i32>, ptr %B, align 8317 %tmp3 = tail call <2 x i32> @llvm.aarch64.neon.urhadd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)318 ret <2 x i32> %tmp3319}320 321define <4 x i32> @urhadd4s(ptr nocapture readonly %A, ptr nocapture readonly %B) {322; CHECK-LABEL: urhadd4s:323; CHECK: // %bb.0:324; CHECK-NEXT: ldr q0, [x0]325; CHECK-NEXT: ldr q1, [x1]326; CHECK-NEXT: urhadd.4s v0, v0, v1327; CHECK-NEXT: ret328 %tmp1 = load <4 x i32>, ptr %A, align 16329 %tmp2 = load <4 x i32>, ptr %B, align 16330 %tmp3 = tail call <4 x i32> @llvm.aarch64.neon.urhadd.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)331 ret <4 x i32> %tmp3332}333 334define void @testLowerToSRHADD8b(<8 x i8> %src1, <8 x i8> %src2, ptr nocapture writeonly %dest) {335; CHECK-SD-LABEL: testLowerToSRHADD8b:336; CHECK-SD: // %bb.0:337; CHECK-SD-NEXT: srhadd.8b v0, v0, v1338; CHECK-SD-NEXT: str d0, [x0]339; CHECK-SD-NEXT: ret340;341; CHECK-GI-LABEL: testLowerToSRHADD8b:342; CHECK-GI: // %bb.0:343; CHECK-GI-NEXT: movi.8h v2, #1344; CHECK-GI-NEXT: saddl.8h v0, v0, v1345; CHECK-GI-NEXT: add.8h v0, v0, v2346; CHECK-GI-NEXT: shrn.8b v0, v0, #1347; CHECK-GI-NEXT: str d0, [x0]348; CHECK-GI-NEXT: ret349 %sextsrc1 = sext <8 x i8> %src1 to <8 x i16>350 %sextsrc2 = sext <8 x i8> %src2 to <8 x i16>351 %add1 = add nsw <8 x i16> %sextsrc1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>352 %add2 = add nsw <8 x i16> %add1, %sextsrc2353 %resulti16 = lshr <8 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>354 %result = trunc <8 x i16> %resulti16 to <8 x i8>355 store <8 x i8> %result, ptr %dest, align 8356 ret void357}358 359define void @testLowerToSRHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture writeonly %dest) {360; CHECK-SD-LABEL: testLowerToSRHADD4h:361; CHECK-SD: // %bb.0:362; CHECK-SD-NEXT: srhadd.4h v0, v0, v1363; CHECK-SD-NEXT: str d0, [x0]364; CHECK-SD-NEXT: ret365;366; CHECK-GI-LABEL: testLowerToSRHADD4h:367; CHECK-GI: // %bb.0:368; CHECK-GI-NEXT: movi.4s v2, #1369; CHECK-GI-NEXT: saddl.4s v0, v0, v1370; CHECK-GI-NEXT: add.4s v0, v0, v2371; CHECK-GI-NEXT: shrn.4h v0, v0, #1372; CHECK-GI-NEXT: str d0, [x0]373; CHECK-GI-NEXT: ret374 %sextsrc1 = sext <4 x i16> %src1 to <4 x i32>375 %sextsrc2 = sext <4 x i16> %src2 to <4 x i32>376 %add1 = add nsw <4 x i32> %sextsrc1, <i32 1, i32 1, i32 1, i32 1>377 %add2 = add nsw <4 x i32> %add1, %sextsrc2378 %resulti16 = lshr <4 x i32> %add2, <i32 1, i32 1, i32 1, i32 1>379 %result = trunc <4 x i32> %resulti16 to <4 x i16>380 store <4 x i16> %result, ptr %dest, align 8381 ret void382}383 384define void @testLowerToSRHADD2s(<2 x i32> %src1, <2 x i32> %src2, ptr nocapture writeonly %dest) {385; CHECK-SD-LABEL: testLowerToSRHADD2s:386; CHECK-SD: // %bb.0:387; CHECK-SD-NEXT: srhadd.2s v0, v0, v1388; CHECK-SD-NEXT: str d0, [x0]389; CHECK-SD-NEXT: ret390;391; CHECK-GI-LABEL: testLowerToSRHADD2s:392; CHECK-GI: // %bb.0:393; CHECK-GI-NEXT: adrp x8, .LCPI26_0394; CHECK-GI-NEXT: saddl.2d v0, v0, v1395; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI26_0]396; CHECK-GI-NEXT: add.2d v0, v0, v1397; CHECK-GI-NEXT: shrn.2s v0, v0, #1398; CHECK-GI-NEXT: str d0, [x0]399; CHECK-GI-NEXT: ret400 %sextsrc1 = sext <2 x i32> %src1 to <2 x i64>401 %sextsrc2 = sext <2 x i32> %src2 to <2 x i64>402 %add1 = add nsw <2 x i64> %sextsrc1, <i64 1, i64 1>403 %add2 = add nsw <2 x i64> %add1, %sextsrc2404 %resulti16 = lshr <2 x i64> %add2, <i64 1, i64 1>405 %result = trunc <2 x i64> %resulti16 to <2 x i32>406 store <2 x i32> %result, ptr %dest, align 8407 ret void408}409 410define void @testLowerToSRHADD16b(<16 x i8> %src1, <16 x i8> %src2, ptr nocapture writeonly %dest) {411; CHECK-SD-LABEL: testLowerToSRHADD16b:412; CHECK-SD: // %bb.0:413; CHECK-SD-NEXT: srhadd.16b v0, v0, v1414; CHECK-SD-NEXT: str q0, [x0]415; CHECK-SD-NEXT: ret416;417; CHECK-GI-LABEL: testLowerToSRHADD16b:418; CHECK-GI: // %bb.0:419; CHECK-GI-NEXT: movi.8h v2, #1420; CHECK-GI-NEXT: saddl.8h v3, v0, v1421; CHECK-GI-NEXT: saddl2.8h v0, v0, v1422; CHECK-GI-NEXT: add.8h v1, v3, v2423; CHECK-GI-NEXT: add.8h v0, v0, v2424; CHECK-GI-NEXT: shrn.8b v1, v1, #1425; CHECK-GI-NEXT: shrn2.16b v1, v0, #1426; CHECK-GI-NEXT: str q1, [x0]427; CHECK-GI-NEXT: ret428 %sextsrc1 = sext <16 x i8> %src1 to <16 x i16>429 %sextsrc2 = sext <16 x i8> %src2 to <16 x i16>430 %add1 = add nsw <16 x i16> %sextsrc1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>431 %add2 = add nsw <16 x i16> %add1, %sextsrc2432 %resulti16 = lshr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>433 %result = trunc <16 x i16> %resulti16 to <16 x i8>434 store <16 x i8> %result, ptr %dest, align 16435 ret void436}437 438define void @testLowerToSRHADD8h(<8 x i16> %src1, <8 x i16> %src2, ptr nocapture writeonly %dest) {439; CHECK-SD-LABEL: testLowerToSRHADD8h:440; CHECK-SD: // %bb.0:441; CHECK-SD-NEXT: srhadd.8h v0, v0, v1442; CHECK-SD-NEXT: str q0, [x0]443; CHECK-SD-NEXT: ret444;445; CHECK-GI-LABEL: testLowerToSRHADD8h:446; CHECK-GI: // %bb.0:447; CHECK-GI-NEXT: movi.4s v2, #1448; CHECK-GI-NEXT: saddl.4s v3, v0, v1449; CHECK-GI-NEXT: saddl2.4s v0, v0, v1450; CHECK-GI-NEXT: add.4s v1, v3, v2451; CHECK-GI-NEXT: add.4s v0, v0, v2452; CHECK-GI-NEXT: shrn.4h v1, v1, #1453; CHECK-GI-NEXT: shrn2.8h v1, v0, #1454; CHECK-GI-NEXT: str q1, [x0]455; CHECK-GI-NEXT: ret456 %sextsrc1 = sext <8 x i16> %src1 to <8 x i32>457 %sextsrc2 = sext <8 x i16> %src2 to <8 x i32>458 %add1 = add nsw <8 x i32> %sextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>459 %add2 = add nsw <8 x i32> %add1, %sextsrc2460 %resulti16 = lshr <8 x i32> %add2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>461 %result = trunc <8 x i32> %resulti16 to <8 x i16>462 store <8 x i16> %result, ptr %dest, align 16463 ret void464}465 466define void @testLowerToSRHADD4s(<4 x i32> %src1, <4 x i32> %src2, ptr nocapture writeonly %dest) {467; CHECK-SD-LABEL: testLowerToSRHADD4s:468; CHECK-SD: // %bb.0:469; CHECK-SD-NEXT: srhadd.4s v0, v0, v1470; CHECK-SD-NEXT: str q0, [x0]471; CHECK-SD-NEXT: ret472;473; CHECK-GI-LABEL: testLowerToSRHADD4s:474; CHECK-GI: // %bb.0:475; CHECK-GI-NEXT: adrp x8, .LCPI29_0476; CHECK-GI-NEXT: saddl.2d v2, v0, v1477; CHECK-GI-NEXT: saddl2.2d v0, v0, v1478; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI29_0]479; CHECK-GI-NEXT: add.2d v1, v2, v3480; CHECK-GI-NEXT: add.2d v0, v0, v3481; CHECK-GI-NEXT: shrn.2s v1, v1, #1482; CHECK-GI-NEXT: shrn2.4s v1, v0, #1483; CHECK-GI-NEXT: str q1, [x0]484; CHECK-GI-NEXT: ret485 %sextsrc1 = sext <4 x i32> %src1 to <4 x i64>486 %sextsrc2 = sext <4 x i32> %src2 to <4 x i64>487 %add1 = add nsw <4 x i64> %sextsrc1, <i64 1, i64 1, i64 1, i64 1>488 %add2 = add nsw <4 x i64> %add1, %sextsrc2489 %resulti16 = lshr <4 x i64> %add2, <i64 1, i64 1, i64 1, i64 1>490 %result = trunc <4 x i64> %resulti16 to <4 x i32>491 store <4 x i32> %result, ptr %dest, align 16492 ret void493}494 495define void @testLowerToSHADD8b(<8 x i8> %src1, <8 x i8> %src2, ptr nocapture writeonly %dest) {496; CHECK-SD-LABEL: testLowerToSHADD8b:497; CHECK-SD: // %bb.0:498; CHECK-SD-NEXT: shadd.8b v0, v0, v1499; CHECK-SD-NEXT: str d0, [x0]500; CHECK-SD-NEXT: ret501;502; CHECK-GI-LABEL: testLowerToSHADD8b:503; CHECK-GI: // %bb.0:504; CHECK-GI-NEXT: saddl.8h v0, v0, v1505; CHECK-GI-NEXT: shrn.8b v0, v0, #1506; CHECK-GI-NEXT: str d0, [x0]507; CHECK-GI-NEXT: ret508 %sextsrc1 = sext <8 x i8> %src1 to <8 x i16>509 %sextsrc2 = sext <8 x i8> %src2 to <8 x i16>510 %add = add nsw <8 x i16> %sextsrc1, %sextsrc2511 %resulti16 = lshr <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>512 %result = trunc <8 x i16> %resulti16 to <8 x i8>513 store <8 x i8> %result, ptr %dest, align 8514 ret void515}516 517define void @testLowerToSHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture writeonly %dest) {518; CHECK-SD-LABEL: testLowerToSHADD4h:519; CHECK-SD: // %bb.0:520; CHECK-SD-NEXT: shadd.4h v0, v0, v1521; CHECK-SD-NEXT: str d0, [x0]522; CHECK-SD-NEXT: ret523;524; CHECK-GI-LABEL: testLowerToSHADD4h:525; CHECK-GI: // %bb.0:526; CHECK-GI-NEXT: saddl.4s v0, v0, v1527; CHECK-GI-NEXT: shrn.4h v0, v0, #1528; CHECK-GI-NEXT: str d0, [x0]529; CHECK-GI-NEXT: ret530 %sextsrc1 = sext <4 x i16> %src1 to <4 x i32>531 %sextsrc2 = sext <4 x i16> %src2 to <4 x i32>532 %add = add nsw <4 x i32> %sextsrc1, %sextsrc2533 %resulti16 = lshr <4 x i32> %add, <i32 1, i32 1, i32 1, i32 1>534 %result = trunc <4 x i32> %resulti16 to <4 x i16>535 store <4 x i16> %result, ptr %dest, align 8536 ret void537}538 539define void @testLowerToSHADD2s(<2 x i32> %src1, <2 x i32> %src2, ptr nocapture writeonly %dest) {540; CHECK-SD-LABEL: testLowerToSHADD2s:541; CHECK-SD: // %bb.0:542; CHECK-SD-NEXT: shadd.2s v0, v0, v1543; CHECK-SD-NEXT: str d0, [x0]544; CHECK-SD-NEXT: ret545;546; CHECK-GI-LABEL: testLowerToSHADD2s:547; CHECK-GI: // %bb.0:548; CHECK-GI-NEXT: saddl.2d v0, v0, v1549; CHECK-GI-NEXT: shrn.2s v0, v0, #1550; CHECK-GI-NEXT: str d0, [x0]551; CHECK-GI-NEXT: ret552 %sextsrc1 = sext <2 x i32> %src1 to <2 x i64>553 %sextsrc2 = sext <2 x i32> %src2 to <2 x i64>554 %add = add nsw <2 x i64> %sextsrc1, %sextsrc2555 %resulti16 = lshr <2 x i64> %add, <i64 1, i64 1>556 %result = trunc <2 x i64> %resulti16 to <2 x i32>557 store <2 x i32> %result, ptr %dest, align 8558 ret void559}560 561define void @testLowerToSHADD16b(<16 x i8> %src1, <16 x i8> %src2, ptr nocapture writeonly %dest) {562; CHECK-SD-LABEL: testLowerToSHADD16b:563; CHECK-SD: // %bb.0:564; CHECK-SD-NEXT: shadd.16b v0, v0, v1565; CHECK-SD-NEXT: str q0, [x0]566; CHECK-SD-NEXT: ret567;568; CHECK-GI-LABEL: testLowerToSHADD16b:569; CHECK-GI: // %bb.0:570; CHECK-GI-NEXT: saddl.8h v2, v0, v1571; CHECK-GI-NEXT: saddl2.8h v0, v0, v1572; CHECK-GI-NEXT: shrn.8b v1, v2, #1573; CHECK-GI-NEXT: shrn2.16b v1, v0, #1574; CHECK-GI-NEXT: str q1, [x0]575; CHECK-GI-NEXT: ret576 %sextsrc1 = sext <16 x i8> %src1 to <16 x i16>577 %sextsrc2 = sext <16 x i8> %src2 to <16 x i16>578 %add = add nsw <16 x i16> %sextsrc1, %sextsrc2579 %resulti16 = lshr <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>580 %result = trunc <16 x i16> %resulti16 to <16 x i8>581 store <16 x i8> %result, ptr %dest, align 16582 ret void583}584 585define void @testLowerToSHADD8h(<8 x i16> %src1, <8 x i16> %src2, ptr nocapture writeonly %dest) {586; CHECK-SD-LABEL: testLowerToSHADD8h:587; CHECK-SD: // %bb.0:588; CHECK-SD-NEXT: shadd.8h v0, v0, v1589; CHECK-SD-NEXT: str q0, [x0]590; CHECK-SD-NEXT: ret591;592; CHECK-GI-LABEL: testLowerToSHADD8h:593; CHECK-GI: // %bb.0:594; CHECK-GI-NEXT: saddl.4s v2, v0, v1595; CHECK-GI-NEXT: saddl2.4s v0, v0, v1596; CHECK-GI-NEXT: shrn.4h v1, v2, #1597; CHECK-GI-NEXT: shrn2.8h v1, v0, #1598; CHECK-GI-NEXT: str q1, [x0]599; CHECK-GI-NEXT: ret600 %sextsrc1 = sext <8 x i16> %src1 to <8 x i32>601 %sextsrc2 = sext <8 x i16> %src2 to <8 x i32>602 %add = add nsw <8 x i32> %sextsrc1, %sextsrc2603 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>604 %result = trunc <8 x i32> %resulti16 to <8 x i16>605 store <8 x i16> %result, ptr %dest, align 16606 ret void607}608 609define void @testLowerToSHADD4s(<4 x i32> %src1, <4 x i32> %src2, ptr nocapture writeonly %dest) {610; CHECK-SD-LABEL: testLowerToSHADD4s:611; CHECK-SD: // %bb.0:612; CHECK-SD-NEXT: shadd.4s v0, v0, v1613; CHECK-SD-NEXT: str q0, [x0]614; CHECK-SD-NEXT: ret615;616; CHECK-GI-LABEL: testLowerToSHADD4s:617; CHECK-GI: // %bb.0:618; CHECK-GI-NEXT: saddl.2d v2, v0, v1619; CHECK-GI-NEXT: saddl2.2d v0, v0, v1620; CHECK-GI-NEXT: shrn.2s v1, v2, #1621; CHECK-GI-NEXT: shrn2.4s v1, v0, #1622; CHECK-GI-NEXT: str q1, [x0]623; CHECK-GI-NEXT: ret624 %sextsrc1 = sext <4 x i32> %src1 to <4 x i64>625 %sextsrc2 = sext <4 x i32> %src2 to <4 x i64>626 %add = add nsw <4 x i64> %sextsrc1, %sextsrc2627 %resulti16 = lshr <4 x i64> %add, <i64 1, i64 1, i64 1, i64 1>628 %result = trunc <4 x i64> %resulti16 to <4 x i32>629 store <4 x i32> %result, ptr %dest, align 16630 ret void631}632 633define void @testLowerToURHADD8b(<8 x i8> %src1, <8 x i8> %src2, ptr nocapture writeonly %dest) {634; CHECK-SD-LABEL: testLowerToURHADD8b:635; CHECK-SD: // %bb.0:636; CHECK-SD-NEXT: urhadd.8b v0, v0, v1637; CHECK-SD-NEXT: str d0, [x0]638; CHECK-SD-NEXT: ret639;640; CHECK-GI-LABEL: testLowerToURHADD8b:641; CHECK-GI: // %bb.0:642; CHECK-GI-NEXT: movi.8h v2, #1643; CHECK-GI-NEXT: uaddl.8h v0, v0, v1644; CHECK-GI-NEXT: add.8h v0, v0, v2645; CHECK-GI-NEXT: shrn.8b v0, v0, #1646; CHECK-GI-NEXT: str d0, [x0]647; CHECK-GI-NEXT: ret648 %zextsrc1 = zext <8 x i8> %src1 to <8 x i16>649 %zextsrc2 = zext <8 x i8> %src2 to <8 x i16>650 %add1 = add nuw nsw <8 x i16> %zextsrc1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>651 %add2 = add nuw nsw <8 x i16> %add1, %zextsrc2652 %resulti16 = lshr <8 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>653 %result = trunc <8 x i16> %resulti16 to <8 x i8>654 store <8 x i8> %result, ptr %dest, align 8655 ret void656}657 658define void @testLowerToURHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture writeonly %dest) {659; CHECK-SD-LABEL: testLowerToURHADD4h:660; CHECK-SD: // %bb.0:661; CHECK-SD-NEXT: urhadd.4h v0, v0, v1662; CHECK-SD-NEXT: str d0, [x0]663; CHECK-SD-NEXT: ret664;665; CHECK-GI-LABEL: testLowerToURHADD4h:666; CHECK-GI: // %bb.0:667; CHECK-GI-NEXT: movi.4s v2, #1668; CHECK-GI-NEXT: uaddl.4s v0, v0, v1669; CHECK-GI-NEXT: add.4s v0, v0, v2670; CHECK-GI-NEXT: shrn.4h v0, v0, #1671; CHECK-GI-NEXT: str d0, [x0]672; CHECK-GI-NEXT: ret673 %zextsrc1 = zext <4 x i16> %src1 to <4 x i32>674 %zextsrc2 = zext <4 x i16> %src2 to <4 x i32>675 %add1 = add nuw nsw <4 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1>676 %add2 = add nuw nsw <4 x i32> %add1, %zextsrc2677 %resulti16 = lshr <4 x i32> %add2, <i32 1, i32 1, i32 1, i32 1>678 %result = trunc <4 x i32> %resulti16 to <4 x i16>679 store <4 x i16> %result, ptr %dest, align 8680 ret void681}682 683define void @testLowerToURHADD2s(<2 x i32> %src1, <2 x i32> %src2, ptr nocapture writeonly %dest) {684; CHECK-SD-LABEL: testLowerToURHADD2s:685; CHECK-SD: // %bb.0:686; CHECK-SD-NEXT: urhadd.2s v0, v0, v1687; CHECK-SD-NEXT: str d0, [x0]688; CHECK-SD-NEXT: ret689;690; CHECK-GI-LABEL: testLowerToURHADD2s:691; CHECK-GI: // %bb.0:692; CHECK-GI-NEXT: adrp x8, .LCPI38_0693; CHECK-GI-NEXT: uaddl.2d v0, v0, v1694; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI38_0]695; CHECK-GI-NEXT: add.2d v0, v0, v1696; CHECK-GI-NEXT: shrn.2s v0, v0, #1697; CHECK-GI-NEXT: str d0, [x0]698; CHECK-GI-NEXT: ret699 %zextsrc1 = zext <2 x i32> %src1 to <2 x i64>700 %zextsrc2 = zext <2 x i32> %src2 to <2 x i64>701 %add1 = add nuw nsw <2 x i64> %zextsrc1, <i64 1, i64 1>702 %add2 = add nuw nsw <2 x i64> %add1, %zextsrc2703 %resulti16 = lshr <2 x i64> %add2, <i64 1, i64 1>704 %result = trunc <2 x i64> %resulti16 to <2 x i32>705 store <2 x i32> %result, ptr %dest, align 8706 ret void707}708 709define void @testLowerToURHADD16b(<16 x i8> %src1, <16 x i8> %src2, ptr nocapture writeonly %dest) {710; CHECK-SD-LABEL: testLowerToURHADD16b:711; CHECK-SD: // %bb.0:712; CHECK-SD-NEXT: urhadd.16b v0, v0, v1713; CHECK-SD-NEXT: str q0, [x0]714; CHECK-SD-NEXT: ret715;716; CHECK-GI-LABEL: testLowerToURHADD16b:717; CHECK-GI: // %bb.0:718; CHECK-GI-NEXT: movi.8h v2, #1719; CHECK-GI-NEXT: uaddl.8h v3, v0, v1720; CHECK-GI-NEXT: uaddl2.8h v0, v0, v1721; CHECK-GI-NEXT: add.8h v1, v3, v2722; CHECK-GI-NEXT: add.8h v0, v0, v2723; CHECK-GI-NEXT: shrn.8b v1, v1, #1724; CHECK-GI-NEXT: shrn2.16b v1, v0, #1725; CHECK-GI-NEXT: str q1, [x0]726; CHECK-GI-NEXT: ret727 %zextsrc1 = zext <16 x i8> %src1 to <16 x i16>728 %zextsrc2 = zext <16 x i8> %src2 to <16 x i16>729 %add1 = add nuw nsw <16 x i16> %zextsrc1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>730 %add2 = add nuw nsw <16 x i16> %add1, %zextsrc2731 %resulti16 = lshr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>732 %result = trunc <16 x i16> %resulti16 to <16 x i8>733 store <16 x i8> %result, ptr %dest, align 16734 ret void735}736 737define void @testLowerToURHADD8h(<8 x i16> %src1, <8 x i16> %src2, ptr nocapture writeonly %dest) {738; CHECK-SD-LABEL: testLowerToURHADD8h:739; CHECK-SD: // %bb.0:740; CHECK-SD-NEXT: urhadd.8h v0, v0, v1741; CHECK-SD-NEXT: str q0, [x0]742; CHECK-SD-NEXT: ret743;744; CHECK-GI-LABEL: testLowerToURHADD8h:745; CHECK-GI: // %bb.0:746; CHECK-GI-NEXT: movi.4s v2, #1747; CHECK-GI-NEXT: uaddl.4s v3, v0, v1748; CHECK-GI-NEXT: uaddl2.4s v0, v0, v1749; CHECK-GI-NEXT: add.4s v1, v3, v2750; CHECK-GI-NEXT: add.4s v0, v0, v2751; CHECK-GI-NEXT: shrn.4h v1, v1, #1752; CHECK-GI-NEXT: shrn2.8h v1, v0, #1753; CHECK-GI-NEXT: str q1, [x0]754; CHECK-GI-NEXT: ret755 %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>756 %zextsrc2 = zext <8 x i16> %src2 to <8 x i32>757 %add1 = add nuw nsw <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>758 %add2 = add nuw nsw <8 x i32> %add1, %zextsrc2759 %resulti16 = lshr <8 x i32> %add2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>760 %result = trunc <8 x i32> %resulti16 to <8 x i16>761 store <8 x i16> %result, ptr %dest, align 16762 ret void763}764 765define void @testLowerToURHADD4s(<4 x i32> %src1, <4 x i32> %src2, ptr nocapture writeonly %dest) {766; CHECK-SD-LABEL: testLowerToURHADD4s:767; CHECK-SD: // %bb.0:768; CHECK-SD-NEXT: urhadd.4s v0, v0, v1769; CHECK-SD-NEXT: str q0, [x0]770; CHECK-SD-NEXT: ret771;772; CHECK-GI-LABEL: testLowerToURHADD4s:773; CHECK-GI: // %bb.0:774; CHECK-GI-NEXT: adrp x8, .LCPI41_0775; CHECK-GI-NEXT: uaddl.2d v2, v0, v1776; CHECK-GI-NEXT: uaddl2.2d v0, v0, v1777; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI41_0]778; CHECK-GI-NEXT: add.2d v1, v2, v3779; CHECK-GI-NEXT: add.2d v0, v0, v3780; CHECK-GI-NEXT: shrn.2s v1, v1, #1781; CHECK-GI-NEXT: shrn2.4s v1, v0, #1782; CHECK-GI-NEXT: str q1, [x0]783; CHECK-GI-NEXT: ret784 %zextsrc1 = zext <4 x i32> %src1 to <4 x i64>785 %zextsrc2 = zext <4 x i32> %src2 to <4 x i64>786 %add1 = add nuw nsw <4 x i64> %zextsrc1, <i64 1, i64 1, i64 1, i64 1>787 %add2 = add nuw nsw <4 x i64> %add1, %zextsrc2788 %resulti16 = lshr <4 x i64> %add2, <i64 1, i64 1, i64 1, i64 1>789 %result = trunc <4 x i64> %resulti16 to <4 x i32>790 store <4 x i32> %result, ptr %dest, align 16791 ret void792}793 794define void @testLowerToUHADD8b(<8 x i8> %src1, <8 x i8> %src2, ptr nocapture writeonly %dest) {795; CHECK-SD-LABEL: testLowerToUHADD8b:796; CHECK-SD: // %bb.0:797; CHECK-SD-NEXT: uhadd.8b v0, v0, v1798; CHECK-SD-NEXT: str d0, [x0]799; CHECK-SD-NEXT: ret800;801; CHECK-GI-LABEL: testLowerToUHADD8b:802; CHECK-GI: // %bb.0:803; CHECK-GI-NEXT: uaddl.8h v0, v0, v1804; CHECK-GI-NEXT: shrn.8b v0, v0, #1805; CHECK-GI-NEXT: str d0, [x0]806; CHECK-GI-NEXT: ret807 %zextsrc1 = zext <8 x i8> %src1 to <8 x i16>808 %zextsrc2 = zext <8 x i8> %src2 to <8 x i16>809 %add = add nuw nsw <8 x i16> %zextsrc1, %zextsrc2810 %resulti16 = lshr <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>811 %result = trunc <8 x i16> %resulti16 to <8 x i8>812 store <8 x i8> %result, ptr %dest, align 8813 ret void814}815 816define void @testLowerToUHADD4h(<4 x i16> %src1, <4 x i16> %src2, ptr nocapture writeonly %dest) {817; CHECK-SD-LABEL: testLowerToUHADD4h:818; CHECK-SD: // %bb.0:819; CHECK-SD-NEXT: uhadd.4h v0, v0, v1820; CHECK-SD-NEXT: str d0, [x0]821; CHECK-SD-NEXT: ret822;823; CHECK-GI-LABEL: testLowerToUHADD4h:824; CHECK-GI: // %bb.0:825; CHECK-GI-NEXT: uaddl.4s v0, v0, v1826; CHECK-GI-NEXT: shrn.4h v0, v0, #1827; CHECK-GI-NEXT: str d0, [x0]828; CHECK-GI-NEXT: ret829 %zextsrc1 = zext <4 x i16> %src1 to <4 x i32>830 %zextsrc2 = zext <4 x i16> %src2 to <4 x i32>831 %add = add nuw nsw <4 x i32> %zextsrc1, %zextsrc2832 %resulti16 = lshr <4 x i32> %add, <i32 1, i32 1, i32 1, i32 1>833 %result = trunc <4 x i32> %resulti16 to <4 x i16>834 store <4 x i16> %result, ptr %dest, align 8835 ret void836}837 838define void @testLowerToUHADD2s(<2 x i32> %src1, <2 x i32> %src2, ptr nocapture writeonly %dest) {839; CHECK-SD-LABEL: testLowerToUHADD2s:840; CHECK-SD: // %bb.0:841; CHECK-SD-NEXT: uhadd.2s v0, v0, v1842; CHECK-SD-NEXT: str d0, [x0]843; CHECK-SD-NEXT: ret844;845; CHECK-GI-LABEL: testLowerToUHADD2s:846; CHECK-GI: // %bb.0:847; CHECK-GI-NEXT: uaddl.2d v0, v0, v1848; CHECK-GI-NEXT: shrn.2s v0, v0, #1849; CHECK-GI-NEXT: str d0, [x0]850; CHECK-GI-NEXT: ret851 %zextsrc1 = zext <2 x i32> %src1 to <2 x i64>852 %zextsrc2 = zext <2 x i32> %src2 to <2 x i64>853 %add = add nuw nsw <2 x i64> %zextsrc1, %zextsrc2854 %resulti16 = lshr <2 x i64> %add, <i64 1, i64 1>855 %result = trunc <2 x i64> %resulti16 to <2 x i32>856 store <2 x i32> %result, ptr %dest, align 8857 ret void858}859 860define void @testLowerToUHADD16b(<16 x i8> %src1, <16 x i8> %src2, ptr nocapture writeonly %dest) {861; CHECK-SD-LABEL: testLowerToUHADD16b:862; CHECK-SD: // %bb.0:863; CHECK-SD-NEXT: uhadd.16b v0, v0, v1864; CHECK-SD-NEXT: str q0, [x0]865; CHECK-SD-NEXT: ret866;867; CHECK-GI-LABEL: testLowerToUHADD16b:868; CHECK-GI: // %bb.0:869; CHECK-GI-NEXT: uaddl.8h v2, v0, v1870; CHECK-GI-NEXT: uaddl2.8h v0, v0, v1871; CHECK-GI-NEXT: shrn.8b v1, v2, #1872; CHECK-GI-NEXT: shrn2.16b v1, v0, #1873; CHECK-GI-NEXT: str q1, [x0]874; CHECK-GI-NEXT: ret875 %zextsrc1 = zext <16 x i8> %src1 to <16 x i16>876 %zextsrc2 = zext <16 x i8> %src2 to <16 x i16>877 %add = add nuw nsw <16 x i16> %zextsrc1, %zextsrc2878 %resulti16 = lshr <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>879 %result = trunc <16 x i16> %resulti16 to <16 x i8>880 store <16 x i8> %result, ptr %dest, align 16881 ret void882}883 884define void @testLowerToUHADD8h(<8 x i16> %src1, <8 x i16> %src2, ptr nocapture writeonly %dest) {885; CHECK-SD-LABEL: testLowerToUHADD8h:886; CHECK-SD: // %bb.0:887; CHECK-SD-NEXT: uhadd.8h v0, v0, v1888; CHECK-SD-NEXT: str q0, [x0]889; CHECK-SD-NEXT: ret890;891; CHECK-GI-LABEL: testLowerToUHADD8h:892; CHECK-GI: // %bb.0:893; CHECK-GI-NEXT: uaddl.4s v2, v0, v1894; CHECK-GI-NEXT: uaddl2.4s v0, v0, v1895; CHECK-GI-NEXT: shrn.4h v1, v2, #1896; CHECK-GI-NEXT: shrn2.8h v1, v0, #1897; CHECK-GI-NEXT: str q1, [x0]898; CHECK-GI-NEXT: ret899 %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>900 %zextsrc2 = zext <8 x i16> %src2 to <8 x i32>901 %add = add nuw nsw <8 x i32> %zextsrc1, %zextsrc2902 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>903 %result = trunc <8 x i32> %resulti16 to <8 x i16>904 store <8 x i16> %result, ptr %dest, align 16905 ret void906}907 908define void @testLowerToUHADD4s(<4 x i32> %src1, <4 x i32> %src2, ptr nocapture writeonly %dest) {909; CHECK-SD-LABEL: testLowerToUHADD4s:910; CHECK-SD: // %bb.0:911; CHECK-SD-NEXT: uhadd.4s v0, v0, v1912; CHECK-SD-NEXT: str q0, [x0]913; CHECK-SD-NEXT: ret914;915; CHECK-GI-LABEL: testLowerToUHADD4s:916; CHECK-GI: // %bb.0:917; CHECK-GI-NEXT: uaddl.2d v2, v0, v1918; CHECK-GI-NEXT: uaddl2.2d v0, v0, v1919; CHECK-GI-NEXT: shrn.2s v1, v2, #1920; CHECK-GI-NEXT: shrn2.4s v1, v0, #1921; CHECK-GI-NEXT: str q1, [x0]922; CHECK-GI-NEXT: ret923 %zextsrc1 = zext <4 x i32> %src1 to <4 x i64>924 %zextsrc2 = zext <4 x i32> %src2 to <4 x i64>925 %add = add nuw nsw <4 x i64> %zextsrc1, %zextsrc2926 %resulti16 = lshr <4 x i64> %add, <i64 1, i64 1, i64 1, i64 1>927 %result = trunc <4 x i64> %resulti16 to <4 x i32>928 store <4 x i32> %result, ptr %dest, align 16929 ret void930}931 932define <4 x i32> @hadd16_sext_asr(<4 x i16> %src1, <4 x i16> %src2) {933; CHECK-SD-LABEL: hadd16_sext_asr:934; CHECK-SD: // %bb.0:935; CHECK-SD-NEXT: shadd.4h v0, v0, v1936; CHECK-SD-NEXT: sshll.4s v0, v0, #0937; CHECK-SD-NEXT: ret938;939; CHECK-GI-LABEL: hadd16_sext_asr:940; CHECK-GI: // %bb.0:941; CHECK-GI-NEXT: saddl.4s v0, v0, v1942; CHECK-GI-NEXT: sshr.4s v0, v0, #1943; CHECK-GI-NEXT: ret944 %zextsrc1 = sext <4 x i16> %src1 to <4 x i32>945 %zextsrc2 = sext <4 x i16> %src2 to <4 x i32>946 %add = add nsw <4 x i32> %zextsrc1, %zextsrc2947 %resulti16 = ashr <4 x i32> %add, <i32 1, i32 1, i32 1, i32 1>948 ret <4 x i32> %resulti16949}950 951define <4 x i32> @hadd16_zext_asr(<4 x i16> %src1, <4 x i16> %src2) {952; CHECK-SD-LABEL: hadd16_zext_asr:953; CHECK-SD: // %bb.0:954; CHECK-SD-NEXT: uhadd.4h v0, v0, v1955; CHECK-SD-NEXT: ushll.4s v0, v0, #0956; CHECK-SD-NEXT: ret957;958; CHECK-GI-LABEL: hadd16_zext_asr:959; CHECK-GI: // %bb.0:960; CHECK-GI-NEXT: uaddl.4s v0, v0, v1961; CHECK-GI-NEXT: ushr.4s v0, v0, #1962; CHECK-GI-NEXT: ret963 %zextsrc1 = zext <4 x i16> %src1 to <4 x i32>964 %zextsrc2 = zext <4 x i16> %src2 to <4 x i32>965 %add = add nuw nsw <4 x i32> %zextsrc1, %zextsrc2966 %resulti16 = lshr <4 x i32> %add, <i32 1, i32 1, i32 1, i32 1>967 ret <4 x i32> %resulti16968}969 970define <4 x i32> @hadd16_sext_lsr(<4 x i16> %src1, <4 x i16> %src2) {971; CHECK-LABEL: hadd16_sext_lsr:972; CHECK: // %bb.0:973; CHECK-NEXT: saddl.4s v0, v0, v1974; CHECK-NEXT: ushr.4s v0, v0, #1975; CHECK-NEXT: ret976 %zextsrc1 = sext <4 x i16> %src1 to <4 x i32>977 %zextsrc2 = sext <4 x i16> %src2 to <4 x i32>978 %add = add nsw <4 x i32> %zextsrc1, %zextsrc2979 %resulti16 = lshr <4 x i32> %add, <i32 1, i32 1, i32 1, i32 1>980 ret <4 x i32> %resulti16981}982 983define <4 x i32> @hadd16_zext_lsr(<4 x i16> %src1, <4 x i16> %src2) {984; CHECK-SD-LABEL: hadd16_zext_lsr:985; CHECK-SD: // %bb.0:986; CHECK-SD-NEXT: uhadd.4h v0, v0, v1987; CHECK-SD-NEXT: ushll.4s v0, v0, #0988; CHECK-SD-NEXT: ret989;990; CHECK-GI-LABEL: hadd16_zext_lsr:991; CHECK-GI: // %bb.0:992; CHECK-GI-NEXT: uaddl.4s v0, v0, v1993; CHECK-GI-NEXT: ushr.4s v0, v0, #1994; CHECK-GI-NEXT: ret995 %zextsrc1 = zext <4 x i16> %src1 to <4 x i32>996 %zextsrc2 = zext <4 x i16> %src2 to <4 x i32>997 %add = add nuw nsw <4 x i32> %zextsrc1, %zextsrc2998 %resulti16 = lshr <4 x i32> %add, <i32 1, i32 1, i32 1, i32 1>999 ret <4 x i32> %resulti161000}1001 1002define <4 x i64> @hadd32_sext_asr(<4 x i32> %src1, <4 x i32> %src2) {1003; CHECK-SD-LABEL: hadd32_sext_asr:1004; CHECK-SD: // %bb.0:1005; CHECK-SD-NEXT: shadd.4s v0, v0, v11006; CHECK-SD-NEXT: sshll2.2d v1, v0, #01007; CHECK-SD-NEXT: sshll.2d v0, v0, #01008; CHECK-SD-NEXT: ret1009;1010; CHECK-GI-LABEL: hadd32_sext_asr:1011; CHECK-GI: // %bb.0:1012; CHECK-GI-NEXT: saddl.2d v2, v0, v11013; CHECK-GI-NEXT: saddl2.2d v1, v0, v11014; CHECK-GI-NEXT: sshr.2d v0, v2, #11015; CHECK-GI-NEXT: sshr.2d v1, v1, #11016; CHECK-GI-NEXT: ret1017 %zextsrc1 = sext <4 x i32> %src1 to <4 x i64>1018 %zextsrc2 = sext <4 x i32> %src2 to <4 x i64>1019 %add = add nsw <4 x i64> %zextsrc1, %zextsrc21020 %resulti32 = ashr <4 x i64> %add, <i64 1, i64 1, i64 1, i64 1>1021 ret <4 x i64> %resulti321022}1023 1024define <4 x i64> @hadd32_zext_asr(<4 x i32> %src1, <4 x i32> %src2) {1025; CHECK-SD-LABEL: hadd32_zext_asr:1026; CHECK-SD: // %bb.0:1027; CHECK-SD-NEXT: uhadd.4s v0, v0, v11028; CHECK-SD-NEXT: ushll2.2d v1, v0, #01029; CHECK-SD-NEXT: ushll.2d v0, v0, #01030; CHECK-SD-NEXT: ret1031;1032; CHECK-GI-LABEL: hadd32_zext_asr:1033; CHECK-GI: // %bb.0:1034; CHECK-GI-NEXT: uaddl.2d v2, v0, v11035; CHECK-GI-NEXT: uaddl2.2d v1, v0, v11036; CHECK-GI-NEXT: ushr.2d v0, v2, #11037; CHECK-GI-NEXT: ushr.2d v1, v1, #11038; CHECK-GI-NEXT: ret1039 %zextsrc1 = zext <4 x i32> %src1 to <4 x i64>1040 %zextsrc2 = zext <4 x i32> %src2 to <4 x i64>1041 %add = add nuw nsw <4 x i64> %zextsrc1, %zextsrc21042 %resulti32 = lshr <4 x i64> %add, <i64 1, i64 1, i64 1, i64 1>1043 ret <4 x i64> %resulti321044}1045 1046define <4 x i64> @hadd32_sext_lsr(<4 x i32> %src1, <4 x i32> %src2) {1047; CHECK-SD-LABEL: hadd32_sext_lsr:1048; CHECK-SD: // %bb.0:1049; CHECK-SD-NEXT: saddl.2d v2, v0, v11050; CHECK-SD-NEXT: saddl2.2d v0, v0, v11051; CHECK-SD-NEXT: ushr.2d v1, v0, #11052; CHECK-SD-NEXT: ushr.2d v0, v2, #11053; CHECK-SD-NEXT: ret1054;1055; CHECK-GI-LABEL: hadd32_sext_lsr:1056; CHECK-GI: // %bb.0:1057; CHECK-GI-NEXT: saddl.2d v2, v0, v11058; CHECK-GI-NEXT: saddl2.2d v1, v0, v11059; CHECK-GI-NEXT: ushr.2d v0, v2, #11060; CHECK-GI-NEXT: ushr.2d v1, v1, #11061; CHECK-GI-NEXT: ret1062 %zextsrc1 = sext <4 x i32> %src1 to <4 x i64>1063 %zextsrc2 = sext <4 x i32> %src2 to <4 x i64>1064 %add = add nsw <4 x i64> %zextsrc1, %zextsrc21065 %resulti32 = lshr <4 x i64> %add, <i64 1, i64 1, i64 1, i64 1>1066 ret <4 x i64> %resulti321067}1068 1069define <4 x i64> @hadd32_zext_lsr(<4 x i32> %src1, <4 x i32> %src2) {1070; CHECK-SD-LABEL: hadd32_zext_lsr:1071; CHECK-SD: // %bb.0:1072; CHECK-SD-NEXT: uhadd.4s v0, v0, v11073; CHECK-SD-NEXT: ushll2.2d v1, v0, #01074; CHECK-SD-NEXT: ushll.2d v0, v0, #01075; CHECK-SD-NEXT: ret1076;1077; CHECK-GI-LABEL: hadd32_zext_lsr:1078; CHECK-GI: // %bb.0:1079; CHECK-GI-NEXT: uaddl.2d v2, v0, v11080; CHECK-GI-NEXT: uaddl2.2d v1, v0, v11081; CHECK-GI-NEXT: ushr.2d v0, v2, #11082; CHECK-GI-NEXT: ushr.2d v1, v1, #11083; CHECK-GI-NEXT: ret1084 %zextsrc1 = zext <4 x i32> %src1 to <4 x i64>1085 %zextsrc2 = zext <4 x i32> %src2 to <4 x i64>1086 %add = add nuw nsw <4 x i64> %zextsrc1, %zextsrc21087 %resulti32 = lshr <4 x i64> %add, <i64 1, i64 1, i64 1, i64 1>1088 ret <4 x i64> %resulti321089}1090 1091define <4 x i16> @hadd8_sext_asr(<4 x i8> %src1, <4 x i8> %src2) {1092; CHECK-SD-LABEL: hadd8_sext_asr:1093; CHECK-SD: // %bb.0:1094; CHECK-SD-NEXT: shl.4h v1, v1, #81095; CHECK-SD-NEXT: shl.4h v0, v0, #81096; CHECK-SD-NEXT: sshr.4h v1, v1, #81097; CHECK-SD-NEXT: sshr.4h v0, v0, #81098; CHECK-SD-NEXT: shadd.4h v0, v0, v11099; CHECK-SD-NEXT: ret1100;1101; CHECK-GI-LABEL: hadd8_sext_asr:1102; CHECK-GI: // %bb.0:1103; CHECK-GI-NEXT: shl.4h v1, v1, #81104; CHECK-GI-NEXT: shl.4h v0, v0, #81105; CHECK-GI-NEXT: sshr.4h v1, v1, #81106; CHECK-GI-NEXT: ssra.4h v1, v0, #81107; CHECK-GI-NEXT: sshr.4h v0, v1, #11108; CHECK-GI-NEXT: ret1109 %zextsrc1 = sext <4 x i8> %src1 to <4 x i16>1110 %zextsrc2 = sext <4 x i8> %src2 to <4 x i16>1111 %add = add nsw <4 x i16> %zextsrc1, %zextsrc21112 %resulti8 = ashr <4 x i16> %add, <i16 1, i16 1, i16 1, i16 1>1113 ret <4 x i16> %resulti81114}1115 1116define <4 x i16> @hadd8_zext_asr(<4 x i8> %src1, <4 x i8> %src2) {1117; CHECK-SD-LABEL: hadd8_zext_asr:1118; CHECK-SD: // %bb.0:1119; CHECK-SD-NEXT: bic.4h v1, #255, lsl #81120; CHECK-SD-NEXT: bic.4h v0, #255, lsl #81121; CHECK-SD-NEXT: uhadd.4h v0, v0, v11122; CHECK-SD-NEXT: ret1123;1124; CHECK-GI-LABEL: hadd8_zext_asr:1125; CHECK-GI: // %bb.0:1126; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff1127; CHECK-GI-NEXT: and.8b v0, v0, v21128; CHECK-GI-NEXT: and.8b v1, v1, v21129; CHECK-GI-NEXT: add.4h v0, v0, v11130; CHECK-GI-NEXT: ushr.4h v0, v0, #11131; CHECK-GI-NEXT: ret1132 %zextsrc1 = zext <4 x i8> %src1 to <4 x i16>1133 %zextsrc2 = zext <4 x i8> %src2 to <4 x i16>1134 %add = add nuw nsw <4 x i16> %zextsrc1, %zextsrc21135 %resulti8 = lshr <4 x i16> %add, <i16 1, i16 1, i16 1, i16 1>1136 ret <4 x i16> %resulti81137}1138 1139define <4 x i16> @hadd8_sext_lsr(<4 x i8> %src1, <4 x i8> %src2) {1140; CHECK-SD-LABEL: hadd8_sext_lsr:1141; CHECK-SD: // %bb.0:1142; CHECK-SD-NEXT: shl.4h v0, v0, #81143; CHECK-SD-NEXT: shl.4h v1, v1, #81144; CHECK-SD-NEXT: sshr.4h v0, v0, #81145; CHECK-SD-NEXT: ssra.4h v0, v1, #81146; CHECK-SD-NEXT: ushr.4h v0, v0, #11147; CHECK-SD-NEXT: ret1148;1149; CHECK-GI-LABEL: hadd8_sext_lsr:1150; CHECK-GI: // %bb.0:1151; CHECK-GI-NEXT: shl.4h v1, v1, #81152; CHECK-GI-NEXT: shl.4h v0, v0, #81153; CHECK-GI-NEXT: sshr.4h v1, v1, #81154; CHECK-GI-NEXT: ssra.4h v1, v0, #81155; CHECK-GI-NEXT: ushr.4h v0, v1, #11156; CHECK-GI-NEXT: ret1157 %zextsrc1 = sext <4 x i8> %src1 to <4 x i16>1158 %zextsrc2 = sext <4 x i8> %src2 to <4 x i16>1159 %add = add nsw <4 x i16> %zextsrc1, %zextsrc21160 %resulti8 = lshr <4 x i16> %add, <i16 1, i16 1, i16 1, i16 1>1161 ret <4 x i16> %resulti81162}1163 1164define <4 x i16> @hadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {1165; CHECK-SD-LABEL: hadd8_zext_lsr:1166; CHECK-SD: // %bb.0:1167; CHECK-SD-NEXT: bic.4h v1, #255, lsl #81168; CHECK-SD-NEXT: bic.4h v0, #255, lsl #81169; CHECK-SD-NEXT: uhadd.4h v0, v0, v11170; CHECK-SD-NEXT: ret1171;1172; CHECK-GI-LABEL: hadd8_zext_lsr:1173; CHECK-GI: // %bb.0:1174; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff1175; CHECK-GI-NEXT: and.8b v0, v0, v21176; CHECK-GI-NEXT: and.8b v1, v1, v21177; CHECK-GI-NEXT: add.4h v0, v0, v11178; CHECK-GI-NEXT: ushr.4h v0, v0, #11179; CHECK-GI-NEXT: ret1180 %zextsrc1 = zext <4 x i8> %src1 to <4 x i16>1181 %zextsrc2 = zext <4 x i8> %src2 to <4 x i16>1182 %add = add nuw nsw <4 x i16> %zextsrc1, %zextsrc21183 %resulti8 = lshr <4 x i16> %add, <i16 1, i16 1, i16 1, i16 1>1184 ret <4 x i16> %resulti81185}1186 1187define <2 x i16> @hadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {1188; CHECK-SD-LABEL: hadd8x2_sext_asr:1189; CHECK-SD: // %bb.0:1190; CHECK-SD-NEXT: shl.2s v1, v1, #241191; CHECK-SD-NEXT: shl.2s v0, v0, #241192; CHECK-SD-NEXT: sshr.2s v1, v1, #241193; CHECK-SD-NEXT: sshr.2s v0, v0, #241194; CHECK-SD-NEXT: shadd.2s v0, v0, v11195; CHECK-SD-NEXT: ret1196;1197; CHECK-GI-LABEL: hadd8x2_sext_asr:1198; CHECK-GI: // %bb.0:1199; CHECK-GI-NEXT: shl.2s v1, v1, #241200; CHECK-GI-NEXT: mov w8, #1 // =0x11201; CHECK-GI-NEXT: shl.2s v0, v0, #241202; CHECK-GI-NEXT: fmov s2, w81203; CHECK-GI-NEXT: sshr.2s v1, v1, #241204; CHECK-GI-NEXT: mov.h v2[1], w81205; CHECK-GI-NEXT: ssra.2s v1, v0, #241206; CHECK-GI-NEXT: uzp1.4h v0, v1, v01207; CHECK-GI-NEXT: neg.4h v1, v21208; CHECK-GI-NEXT: sshl.4h v0, v0, v11209; CHECK-GI-NEXT: ushll.4s v0, v0, #01210; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q01211; CHECK-GI-NEXT: ret1212 %zextsrc1 = sext <2 x i8> %src1 to <2 x i16>1213 %zextsrc2 = sext <2 x i8> %src2 to <2 x i16>1214 %add = add nsw <2 x i16> %zextsrc1, %zextsrc21215 %resulti8 = ashr <2 x i16> %add, <i16 1, i16 1>1216 ret <2 x i16> %resulti81217}1218 1219define <2 x i16> @hadd8x2_zext_asr(<2 x i8> %src1, <2 x i8> %src2) {1220; CHECK-SD-LABEL: hadd8x2_zext_asr:1221; CHECK-SD: // %bb.0:1222; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff1223; CHECK-SD-NEXT: and.8b v1, v1, v21224; CHECK-SD-NEXT: and.8b v0, v0, v21225; CHECK-SD-NEXT: uhadd.2s v0, v0, v11226; CHECK-SD-NEXT: ret1227;1228; CHECK-GI-LABEL: hadd8x2_zext_asr:1229; CHECK-GI: // %bb.0:1230; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff1231; CHECK-GI-NEXT: mov w8, #1 // =0x11232; CHECK-GI-NEXT: and.8b v0, v0, v21233; CHECK-GI-NEXT: and.8b v1, v1, v21234; CHECK-GI-NEXT: fmov s2, w81235; CHECK-GI-NEXT: add.2s v0, v0, v11236; CHECK-GI-NEXT: mov.h v2[1], w81237; CHECK-GI-NEXT: uzp1.4h v0, v0, v01238; CHECK-GI-NEXT: neg.4h v1, v21239; CHECK-GI-NEXT: ushl.4h v0, v0, v11240; CHECK-GI-NEXT: ushll.4s v0, v0, #01241; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q01242; CHECK-GI-NEXT: ret1243 %zextsrc1 = zext <2 x i8> %src1 to <2 x i16>1244 %zextsrc2 = zext <2 x i8> %src2 to <2 x i16>1245 %add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc21246 %resulti8 = lshr <2 x i16> %add, <i16 1, i16 1>1247 ret <2 x i16> %resulti81248}1249 1250define <2 x i16> @hadd8x2_sext_lsr(<2 x i8> %src1, <2 x i8> %src2) {1251; CHECK-SD-LABEL: hadd8x2_sext_lsr:1252; CHECK-SD: // %bb.0:1253; CHECK-SD-NEXT: shl.2s v0, v0, #241254; CHECK-SD-NEXT: shl.2s v1, v1, #241255; CHECK-SD-NEXT: movi d2, #0x00ffff0000ffff1256; CHECK-SD-NEXT: sshr.2s v0, v0, #241257; CHECK-SD-NEXT: ssra.2s v0, v1, #241258; CHECK-SD-NEXT: and.8b v0, v0, v21259; CHECK-SD-NEXT: ushr.2s v0, v0, #11260; CHECK-SD-NEXT: ret1261;1262; CHECK-GI-LABEL: hadd8x2_sext_lsr:1263; CHECK-GI: // %bb.0:1264; CHECK-GI-NEXT: shl.2s v1, v1, #241265; CHECK-GI-NEXT: mov w8, #1 // =0x11266; CHECK-GI-NEXT: shl.2s v0, v0, #241267; CHECK-GI-NEXT: fmov s2, w81268; CHECK-GI-NEXT: sshr.2s v1, v1, #241269; CHECK-GI-NEXT: mov.h v2[1], w81270; CHECK-GI-NEXT: ssra.2s v1, v0, #241271; CHECK-GI-NEXT: uzp1.4h v0, v1, v01272; CHECK-GI-NEXT: neg.4h v1, v21273; CHECK-GI-NEXT: ushl.4h v0, v0, v11274; CHECK-GI-NEXT: ushll.4s v0, v0, #01275; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q01276; CHECK-GI-NEXT: ret1277 %zextsrc1 = sext <2 x i8> %src1 to <2 x i16>1278 %zextsrc2 = sext <2 x i8> %src2 to <2 x i16>1279 %add = add nsw <2 x i16> %zextsrc1, %zextsrc21280 %resulti8 = lshr <2 x i16> %add, <i16 1, i16 1>1281 ret <2 x i16> %resulti81282}1283 1284define <2 x i16> @hadd8x2_zext_lsr(<2 x i8> %src1, <2 x i8> %src2) {1285; CHECK-SD-LABEL: hadd8x2_zext_lsr:1286; CHECK-SD: // %bb.0:1287; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff1288; CHECK-SD-NEXT: and.8b v1, v1, v21289; CHECK-SD-NEXT: and.8b v0, v0, v21290; CHECK-SD-NEXT: uhadd.2s v0, v0, v11291; CHECK-SD-NEXT: ret1292;1293; CHECK-GI-LABEL: hadd8x2_zext_lsr:1294; CHECK-GI: // %bb.0:1295; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff1296; CHECK-GI-NEXT: mov w8, #1 // =0x11297; CHECK-GI-NEXT: and.8b v0, v0, v21298; CHECK-GI-NEXT: and.8b v1, v1, v21299; CHECK-GI-NEXT: fmov s2, w81300; CHECK-GI-NEXT: add.2s v0, v0, v11301; CHECK-GI-NEXT: mov.h v2[1], w81302; CHECK-GI-NEXT: uzp1.4h v0, v0, v01303; CHECK-GI-NEXT: neg.4h v1, v21304; CHECK-GI-NEXT: ushl.4h v0, v0, v11305; CHECK-GI-NEXT: ushll.4s v0, v0, #01306; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q01307; CHECK-GI-NEXT: ret1308 %zextsrc1 = zext <2 x i8> %src1 to <2 x i16>1309 %zextsrc2 = zext <2 x i8> %src2 to <2 x i16>1310 %add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc21311 %resulti8 = lshr <2 x i16> %add, <i16 1, i16 1>1312 ret <2 x i16> %resulti81313}1314 1315define <4 x i16> @rhadd8_sext_asr(<4 x i8> %src1, <4 x i8> %src2) {1316; CHECK-SD-LABEL: rhadd8_sext_asr:1317; CHECK-SD: // %bb.0:1318; CHECK-SD-NEXT: shl.4h v1, v1, #81319; CHECK-SD-NEXT: shl.4h v0, v0, #81320; CHECK-SD-NEXT: sshr.4h v1, v1, #81321; CHECK-SD-NEXT: sshr.4h v0, v0, #81322; CHECK-SD-NEXT: srhadd.4h v0, v0, v11323; CHECK-SD-NEXT: ret1324;1325; CHECK-GI-LABEL: rhadd8_sext_asr:1326; CHECK-GI: // %bb.0:1327; CHECK-GI-NEXT: shl.4h v1, v1, #81328; CHECK-GI-NEXT: shl.4h v0, v0, #81329; CHECK-GI-NEXT: movi.4h v2, #11330; CHECK-GI-NEXT: sshr.4h v1, v1, #81331; CHECK-GI-NEXT: ssra.4h v1, v0, #81332; CHECK-GI-NEXT: add.4h v0, v1, v21333; CHECK-GI-NEXT: sshr.4h v0, v0, #11334; CHECK-GI-NEXT: ret1335 %zextsrc1 = sext <4 x i8> %src1 to <4 x i16>1336 %zextsrc2 = sext <4 x i8> %src2 to <4 x i16>1337 %add = add nsw <4 x i16> %zextsrc1, %zextsrc21338 %add2 = add nsw <4 x i16> %add, <i16 1, i16 1, i16 1, i16 1>1339 %resulti8 = ashr <4 x i16> %add2, <i16 1, i16 1, i16 1, i16 1>1340 ret <4 x i16> %resulti81341}1342 1343define <4 x i16> @rhadd8_zext_asr(<4 x i8> %src1, <4 x i8> %src2) {1344; CHECK-SD-LABEL: rhadd8_zext_asr:1345; CHECK-SD: // %bb.0:1346; CHECK-SD-NEXT: bic.4h v1, #255, lsl #81347; CHECK-SD-NEXT: bic.4h v0, #255, lsl #81348; CHECK-SD-NEXT: urhadd.4h v0, v0, v11349; CHECK-SD-NEXT: ret1350;1351; CHECK-GI-LABEL: rhadd8_zext_asr:1352; CHECK-GI: // %bb.0:1353; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff1354; CHECK-GI-NEXT: and.8b v0, v0, v21355; CHECK-GI-NEXT: and.8b v1, v1, v21356; CHECK-GI-NEXT: movi.4h v2, #11357; CHECK-GI-NEXT: add.4h v0, v0, v11358; CHECK-GI-NEXT: add.4h v0, v0, v21359; CHECK-GI-NEXT: ushr.4h v0, v0, #11360; CHECK-GI-NEXT: ret1361 %zextsrc1 = zext <4 x i8> %src1 to <4 x i16>1362 %zextsrc2 = zext <4 x i8> %src2 to <4 x i16>1363 %add = add nuw nsw <4 x i16> %zextsrc1, %zextsrc21364 %add2 = add nuw nsw <4 x i16> %add, <i16 1, i16 1, i16 1, i16 1>1365 %resulti8 = lshr <4 x i16> %add2, <i16 1, i16 1, i16 1, i16 1>1366 ret <4 x i16> %resulti81367}1368 1369define <4 x i16> @rhadd8_sext_lsr(<4 x i8> %src1, <4 x i8> %src2) {1370; CHECK-SD-LABEL: rhadd8_sext_lsr:1371; CHECK-SD: // %bb.0:1372; CHECK-SD-NEXT: shl.4h v0, v0, #81373; CHECK-SD-NEXT: shl.4h v1, v1, #81374; CHECK-SD-NEXT: movi.4h v2, #11375; CHECK-SD-NEXT: sshr.4h v0, v0, #81376; CHECK-SD-NEXT: ssra.4h v0, v1, #81377; CHECK-SD-NEXT: add.4h v0, v0, v21378; CHECK-SD-NEXT: ushr.4h v0, v0, #11379; CHECK-SD-NEXT: ret1380;1381; CHECK-GI-LABEL: rhadd8_sext_lsr:1382; CHECK-GI: // %bb.0:1383; CHECK-GI-NEXT: shl.4h v1, v1, #81384; CHECK-GI-NEXT: shl.4h v0, v0, #81385; CHECK-GI-NEXT: movi.4h v2, #11386; CHECK-GI-NEXT: sshr.4h v1, v1, #81387; CHECK-GI-NEXT: ssra.4h v1, v0, #81388; CHECK-GI-NEXT: add.4h v0, v1, v21389; CHECK-GI-NEXT: ushr.4h v0, v0, #11390; CHECK-GI-NEXT: ret1391 %zextsrc1 = sext <4 x i8> %src1 to <4 x i16>1392 %zextsrc2 = sext <4 x i8> %src2 to <4 x i16>1393 %add = add nsw <4 x i16> %zextsrc1, %zextsrc21394 %add2 = add nsw <4 x i16> %add, <i16 1, i16 1, i16 1, i16 1>1395 %resulti8 = lshr <4 x i16> %add2, <i16 1, i16 1, i16 1, i16 1>1396 ret <4 x i16> %resulti81397}1398 1399define <4 x i16> @rhadd8_zext_lsr(<4 x i8> %src1, <4 x i8> %src2) {1400; CHECK-SD-LABEL: rhadd8_zext_lsr:1401; CHECK-SD: // %bb.0:1402; CHECK-SD-NEXT: bic.4h v1, #255, lsl #81403; CHECK-SD-NEXT: bic.4h v0, #255, lsl #81404; CHECK-SD-NEXT: urhadd.4h v0, v0, v11405; CHECK-SD-NEXT: ret1406;1407; CHECK-GI-LABEL: rhadd8_zext_lsr:1408; CHECK-GI: // %bb.0:1409; CHECK-GI-NEXT: movi d2, #0xff00ff00ff00ff1410; CHECK-GI-NEXT: and.8b v0, v0, v21411; CHECK-GI-NEXT: and.8b v1, v1, v21412; CHECK-GI-NEXT: movi.4h v2, #11413; CHECK-GI-NEXT: add.4h v0, v0, v11414; CHECK-GI-NEXT: add.4h v0, v0, v21415; CHECK-GI-NEXT: ushr.4h v0, v0, #11416; CHECK-GI-NEXT: ret1417 %zextsrc1 = zext <4 x i8> %src1 to <4 x i16>1418 %zextsrc2 = zext <4 x i8> %src2 to <4 x i16>1419 %add = add nuw nsw <4 x i16> %zextsrc1, %zextsrc21420 %add2 = add nuw nsw <4 x i16> %add, <i16 1, i16 1, i16 1, i16 1>1421 %resulti8 = lshr <4 x i16> %add2, <i16 1, i16 1, i16 1, i16 1>1422 ret <4 x i16> %resulti81423}1424 1425define <2 x i16> @rhadd8x2_sext_asr(<2 x i8> %src1, <2 x i8> %src2) {1426; CHECK-SD-LABEL: rhadd8x2_sext_asr:1427; CHECK-SD: // %bb.0:1428; CHECK-SD-NEXT: shl.2s v1, v1, #241429; CHECK-SD-NEXT: shl.2s v0, v0, #241430; CHECK-SD-NEXT: sshr.2s v1, v1, #241431; CHECK-SD-NEXT: sshr.2s v0, v0, #241432; CHECK-SD-NEXT: srhadd.2s v0, v0, v11433; CHECK-SD-NEXT: ret1434;1435; CHECK-GI-LABEL: rhadd8x2_sext_asr:1436; CHECK-GI: // %bb.0:1437; CHECK-GI-NEXT: shl.2s v1, v1, #241438; CHECK-GI-NEXT: shl.2s v0, v0, #241439; CHECK-GI-NEXT: mov w8, #1 // =0x11440; CHECK-GI-NEXT: dup.2s v2, w81441; CHECK-GI-NEXT: sshr.2s v1, v1, #241442; CHECK-GI-NEXT: ssra.2s v1, v0, #241443; CHECK-GI-NEXT: fmov s0, w81444; CHECK-GI-NEXT: mov.h v0[1], w81445; CHECK-GI-NEXT: add.2s v1, v1, v21446; CHECK-GI-NEXT: uzp1.4h v1, v1, v01447; CHECK-GI-NEXT: neg.4h v0, v01448; CHECK-GI-NEXT: sshl.4h v0, v1, v01449; CHECK-GI-NEXT: ushll.4s v0, v0, #01450; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q01451; CHECK-GI-NEXT: ret1452 %zextsrc1 = sext <2 x i8> %src1 to <2 x i16>1453 %zextsrc2 = sext <2 x i8> %src2 to <2 x i16>1454 %add = add nsw <2 x i16> %zextsrc1, %zextsrc21455 %add2 = add nsw <2 x i16> %add, <i16 1, i16 1>1456 %resulti8 = ashr <2 x i16> %add2, <i16 1, i16 1>1457 ret <2 x i16> %resulti81458}1459 1460define <2 x i16> @rhadd8x2_zext_asr(<2 x i8> %src1, <2 x i8> %src2) {1461; CHECK-SD-LABEL: rhadd8x2_zext_asr:1462; CHECK-SD: // %bb.0:1463; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff1464; CHECK-SD-NEXT: and.8b v1, v1, v21465; CHECK-SD-NEXT: and.8b v0, v0, v21466; CHECK-SD-NEXT: urhadd.2s v0, v0, v11467; CHECK-SD-NEXT: ret1468;1469; CHECK-GI-LABEL: rhadd8x2_zext_asr:1470; CHECK-GI: // %bb.0:1471; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff1472; CHECK-GI-NEXT: mov w8, #1 // =0x11473; CHECK-GI-NEXT: and.8b v0, v0, v21474; CHECK-GI-NEXT: and.8b v1, v1, v21475; CHECK-GI-NEXT: dup.2s v2, w81476; CHECK-GI-NEXT: add.2s v0, v0, v11477; CHECK-GI-NEXT: fmov s1, w81478; CHECK-GI-NEXT: add.2s v0, v0, v21479; CHECK-GI-NEXT: mov.h v1[1], w81480; CHECK-GI-NEXT: uzp1.4h v0, v0, v01481; CHECK-GI-NEXT: neg.4h v1, v11482; CHECK-GI-NEXT: ushl.4h v0, v0, v11483; CHECK-GI-NEXT: ushll.4s v0, v0, #01484; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q01485; CHECK-GI-NEXT: ret1486 %zextsrc1 = zext <2 x i8> %src1 to <2 x i16>1487 %zextsrc2 = zext <2 x i8> %src2 to <2 x i16>1488 %add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc21489 %add2 = add nuw nsw <2 x i16> %add, <i16 1, i16 1>1490 %resulti8 = lshr <2 x i16> %add2, <i16 1, i16 1>1491 ret <2 x i16> %resulti81492}1493 1494define <2 x i16> @rhadd8x2_sext_lsr(<2 x i8> %src1, <2 x i8> %src2) {1495; CHECK-SD-LABEL: rhadd8x2_sext_lsr:1496; CHECK-SD: // %bb.0:1497; CHECK-SD-NEXT: shl.2s v0, v0, #241498; CHECK-SD-NEXT: shl.2s v1, v1, #241499; CHECK-SD-NEXT: movi d2, #0x00ffff0000ffff1500; CHECK-SD-NEXT: sshr.2s v0, v0, #241501; CHECK-SD-NEXT: sshr.2s v1, v1, #241502; CHECK-SD-NEXT: mvn.8b v0, v01503; CHECK-SD-NEXT: sub.2s v0, v1, v01504; CHECK-SD-NEXT: and.8b v0, v0, v21505; CHECK-SD-NEXT: ushr.2s v0, v0, #11506; CHECK-SD-NEXT: ret1507;1508; CHECK-GI-LABEL: rhadd8x2_sext_lsr:1509; CHECK-GI: // %bb.0:1510; CHECK-GI-NEXT: shl.2s v1, v1, #241511; CHECK-GI-NEXT: shl.2s v0, v0, #241512; CHECK-GI-NEXT: mov w8, #1 // =0x11513; CHECK-GI-NEXT: dup.2s v2, w81514; CHECK-GI-NEXT: sshr.2s v1, v1, #241515; CHECK-GI-NEXT: ssra.2s v1, v0, #241516; CHECK-GI-NEXT: fmov s0, w81517; CHECK-GI-NEXT: mov.h v0[1], w81518; CHECK-GI-NEXT: add.2s v1, v1, v21519; CHECK-GI-NEXT: uzp1.4h v1, v1, v01520; CHECK-GI-NEXT: neg.4h v0, v01521; CHECK-GI-NEXT: ushl.4h v0, v1, v01522; CHECK-GI-NEXT: ushll.4s v0, v0, #01523; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q01524; CHECK-GI-NEXT: ret1525 %zextsrc1 = sext <2 x i8> %src1 to <2 x i16>1526 %zextsrc2 = sext <2 x i8> %src2 to <2 x i16>1527 %add = add nsw <2 x i16> %zextsrc1, %zextsrc21528 %add2 = add nsw <2 x i16> %add, <i16 1, i16 1>1529 %resulti8 = lshr <2 x i16> %add2, <i16 1, i16 1>1530 ret <2 x i16> %resulti81531}1532 1533define <2 x i16> @rhadd8x2_zext_lsr(<2 x i8> %src1, <2 x i8> %src2) {1534; CHECK-SD-LABEL: rhadd8x2_zext_lsr:1535; CHECK-SD: // %bb.0:1536; CHECK-SD-NEXT: movi d2, #0x0000ff000000ff1537; CHECK-SD-NEXT: and.8b v1, v1, v21538; CHECK-SD-NEXT: and.8b v0, v0, v21539; CHECK-SD-NEXT: urhadd.2s v0, v0, v11540; CHECK-SD-NEXT: ret1541;1542; CHECK-GI-LABEL: rhadd8x2_zext_lsr:1543; CHECK-GI: // %bb.0:1544; CHECK-GI-NEXT: movi d2, #0x0000ff000000ff1545; CHECK-GI-NEXT: mov w8, #1 // =0x11546; CHECK-GI-NEXT: and.8b v0, v0, v21547; CHECK-GI-NEXT: and.8b v1, v1, v21548; CHECK-GI-NEXT: dup.2s v2, w81549; CHECK-GI-NEXT: add.2s v0, v0, v11550; CHECK-GI-NEXT: fmov s1, w81551; CHECK-GI-NEXT: add.2s v0, v0, v21552; CHECK-GI-NEXT: mov.h v1[1], w81553; CHECK-GI-NEXT: uzp1.4h v0, v0, v01554; CHECK-GI-NEXT: neg.4h v1, v11555; CHECK-GI-NEXT: ushl.4h v0, v0, v11556; CHECK-GI-NEXT: ushll.4s v0, v0, #01557; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q01558; CHECK-GI-NEXT: ret1559 %zextsrc1 = zext <2 x i8> %src1 to <2 x i16>1560 %zextsrc2 = zext <2 x i8> %src2 to <2 x i16>1561 %add = add nuw nsw <2 x i16> %zextsrc1, %zextsrc21562 %add2 = add nuw nsw <2 x i16> %add, <i16 1, i16 1>1563 %resulti8 = lshr <2 x i16> %add2, <i16 1, i16 1>1564 ret <2 x i16> %resulti81565}1566 1567 1568define void @testLowerToSHADD8b_c(<8 x i8> %src1, ptr nocapture writeonly %dest) {1569; CHECK-SD-LABEL: testLowerToSHADD8b_c:1570; CHECK-SD: // %bb.0:1571; CHECK-SD-NEXT: movi.8b v1, #101572; CHECK-SD-NEXT: shadd.8b v0, v0, v11573; CHECK-SD-NEXT: str d0, [x0]1574; CHECK-SD-NEXT: ret1575;1576; CHECK-GI-LABEL: testLowerToSHADD8b_c:1577; CHECK-GI: // %bb.0:1578; CHECK-GI-NEXT: movi.8h v1, #101579; CHECK-GI-NEXT: saddw.8h v0, v1, v01580; CHECK-GI-NEXT: shrn.8b v0, v0, #11581; CHECK-GI-NEXT: str d0, [x0]1582; CHECK-GI-NEXT: ret1583 %sextsrc1 = sext <8 x i8> %src1 to <8 x i16>1584 %add = add nsw <8 x i16> %sextsrc1, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>1585 %resulti16 = lshr <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>1586 %result = trunc <8 x i16> %resulti16 to <8 x i8>1587 store <8 x i8> %result, ptr %dest, align 81588 ret void1589}1590 1591define void @testLowerToSHADD4h_c(<4 x i16> %src1, ptr nocapture writeonly %dest) {1592; CHECK-SD-LABEL: testLowerToSHADD4h_c:1593; CHECK-SD: // %bb.0:1594; CHECK-SD-NEXT: movi.4h v1, #101595; CHECK-SD-NEXT: shadd.4h v0, v0, v11596; CHECK-SD-NEXT: str d0, [x0]1597; CHECK-SD-NEXT: ret1598;1599; CHECK-GI-LABEL: testLowerToSHADD4h_c:1600; CHECK-GI: // %bb.0:1601; CHECK-GI-NEXT: movi.4s v1, #101602; CHECK-GI-NEXT: saddw.4s v0, v1, v01603; CHECK-GI-NEXT: shrn.4h v0, v0, #11604; CHECK-GI-NEXT: str d0, [x0]1605; CHECK-GI-NEXT: ret1606 %sextsrc1 = sext <4 x i16> %src1 to <4 x i32>1607 %add = add nsw <4 x i32> %sextsrc1, <i32 10, i32 10, i32 10, i32 10>1608 %resulti16 = lshr <4 x i32> %add, <i32 1, i32 1, i32 1, i32 1>1609 %result = trunc <4 x i32> %resulti16 to <4 x i16>1610 store <4 x i16> %result, ptr %dest, align 81611 ret void1612}1613 1614define void @testLowerToSHADD2s_c(<2 x i32> %src1, ptr nocapture writeonly %dest) {1615; CHECK-SD-LABEL: testLowerToSHADD2s_c:1616; CHECK-SD: // %bb.0:1617; CHECK-SD-NEXT: movi.2s v1, #101618; CHECK-SD-NEXT: shadd.2s v0, v0, v11619; CHECK-SD-NEXT: str d0, [x0]1620; CHECK-SD-NEXT: ret1621;1622; CHECK-GI-LABEL: testLowerToSHADD2s_c:1623; CHECK-GI: // %bb.0:1624; CHECK-GI-NEXT: adrp x8, .LCPI74_01625; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI74_0]1626; CHECK-GI-NEXT: saddw.2d v0, v1, v01627; CHECK-GI-NEXT: shrn.2s v0, v0, #11628; CHECK-GI-NEXT: str d0, [x0]1629; CHECK-GI-NEXT: ret1630 %sextsrc1 = sext <2 x i32> %src1 to <2 x i64>1631 %add = add nsw <2 x i64> %sextsrc1, <i64 10, i64 10>1632 %resulti16 = lshr <2 x i64> %add, <i64 1, i64 1>1633 %result = trunc <2 x i64> %resulti16 to <2 x i32>1634 store <2 x i32> %result, ptr %dest, align 81635 ret void1636}1637 1638define void @testLowerToSHADD16b_c(<16 x i8> %src1, ptr nocapture writeonly %dest) {1639; CHECK-SD-LABEL: testLowerToSHADD16b_c:1640; CHECK-SD: // %bb.0:1641; CHECK-SD-NEXT: movi.16b v1, #101642; CHECK-SD-NEXT: shadd.16b v0, v0, v11643; CHECK-SD-NEXT: str q0, [x0]1644; CHECK-SD-NEXT: ret1645;1646; CHECK-GI-LABEL: testLowerToSHADD16b_c:1647; CHECK-GI: // %bb.0:1648; CHECK-GI-NEXT: movi.8h v1, #101649; CHECK-GI-NEXT: saddw.8h v2, v1, v01650; CHECK-GI-NEXT: saddw2.8h v0, v1, v01651; CHECK-GI-NEXT: shrn.8b v1, v2, #11652; CHECK-GI-NEXT: shrn2.16b v1, v0, #11653; CHECK-GI-NEXT: str q1, [x0]1654; CHECK-GI-NEXT: ret1655 %sextsrc1 = sext <16 x i8> %src1 to <16 x i16>1656 %add = add nsw <16 x i16> %sextsrc1, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>1657 %resulti16 = lshr <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>1658 %result = trunc <16 x i16> %resulti16 to <16 x i8>1659 store <16 x i8> %result, ptr %dest, align 161660 ret void1661}1662 1663define void @testLowerToSHADD8h_c(<8 x i16> %src1, ptr nocapture writeonly %dest) {1664; CHECK-SD-LABEL: testLowerToSHADD8h_c:1665; CHECK-SD: // %bb.0:1666; CHECK-SD-NEXT: movi.8h v1, #101667; CHECK-SD-NEXT: shadd.8h v0, v0, v11668; CHECK-SD-NEXT: str q0, [x0]1669; CHECK-SD-NEXT: ret1670;1671; CHECK-GI-LABEL: testLowerToSHADD8h_c:1672; CHECK-GI: // %bb.0:1673; CHECK-GI-NEXT: movi.4s v1, #101674; CHECK-GI-NEXT: saddw.4s v2, v1, v01675; CHECK-GI-NEXT: saddw2.4s v0, v1, v01676; CHECK-GI-NEXT: shrn.4h v1, v2, #11677; CHECK-GI-NEXT: shrn2.8h v1, v0, #11678; CHECK-GI-NEXT: str q1, [x0]1679; CHECK-GI-NEXT: ret1680 %sextsrc1 = sext <8 x i16> %src1 to <8 x i32>1681 %add = add nsw <8 x i32> %sextsrc1, <i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>1682 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>1683 %result = trunc <8 x i32> %resulti16 to <8 x i16>1684 store <8 x i16> %result, ptr %dest, align 161685 ret void1686}1687 1688define void @testLowerToSHADD4s_c(<4 x i32> %src1, ptr nocapture writeonly %dest) {1689; CHECK-SD-LABEL: testLowerToSHADD4s_c:1690; CHECK-SD: // %bb.0:1691; CHECK-SD-NEXT: movi.4s v1, #101692; CHECK-SD-NEXT: shadd.4s v0, v0, v11693; CHECK-SD-NEXT: str q0, [x0]1694; CHECK-SD-NEXT: ret1695;1696; CHECK-GI-LABEL: testLowerToSHADD4s_c:1697; CHECK-GI: // %bb.0:1698; CHECK-GI-NEXT: adrp x8, .LCPI77_01699; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI77_0]1700; CHECK-GI-NEXT: saddw.2d v2, v1, v01701; CHECK-GI-NEXT: saddw2.2d v0, v1, v01702; CHECK-GI-NEXT: shrn.2s v1, v2, #11703; CHECK-GI-NEXT: shrn2.4s v1, v0, #11704; CHECK-GI-NEXT: str q1, [x0]1705; CHECK-GI-NEXT: ret1706 %sextsrc1 = sext <4 x i32> %src1 to <4 x i64>1707 %add = add nsw <4 x i64> %sextsrc1, <i64 10, i64 10, i64 10, i64 10>1708 %resulti16 = lshr <4 x i64> %add, <i64 1, i64 1, i64 1, i64 1>1709 %result = trunc <4 x i64> %resulti16 to <4 x i32>1710 store <4 x i32> %result, ptr %dest, align 161711 ret void1712}1713 1714define void @testLowerToUHADD8b_c(<8 x i8> %src1, ptr nocapture writeonly %dest) {1715; CHECK-SD-LABEL: testLowerToUHADD8b_c:1716; CHECK-SD: // %bb.0:1717; CHECK-SD-NEXT: movi.8b v1, #101718; CHECK-SD-NEXT: uhadd.8b v0, v0, v11719; CHECK-SD-NEXT: str d0, [x0]1720; CHECK-SD-NEXT: ret1721;1722; CHECK-GI-LABEL: testLowerToUHADD8b_c:1723; CHECK-GI: // %bb.0:1724; CHECK-GI-NEXT: movi.8h v1, #101725; CHECK-GI-NEXT: uaddw.8h v0, v1, v01726; CHECK-GI-NEXT: shrn.8b v0, v0, #11727; CHECK-GI-NEXT: str d0, [x0]1728; CHECK-GI-NEXT: ret1729 %zextsrc1 = zext <8 x i8> %src1 to <8 x i16>1730 %add = add nuw nsw <8 x i16> %zextsrc1, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>1731 %resulti16 = lshr <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>1732 %result = trunc <8 x i16> %resulti16 to <8 x i8>1733 store <8 x i8> %result, ptr %dest, align 81734 ret void1735}1736 1737define void @testLowerToUHADD4h_c(<4 x i16> %src1, ptr nocapture writeonly %dest) {1738; CHECK-SD-LABEL: testLowerToUHADD4h_c:1739; CHECK-SD: // %bb.0:1740; CHECK-SD-NEXT: movi.4h v1, #101741; CHECK-SD-NEXT: uhadd.4h v0, v0, v11742; CHECK-SD-NEXT: str d0, [x0]1743; CHECK-SD-NEXT: ret1744;1745; CHECK-GI-LABEL: testLowerToUHADD4h_c:1746; CHECK-GI: // %bb.0:1747; CHECK-GI-NEXT: movi.4s v1, #101748; CHECK-GI-NEXT: uaddw.4s v0, v1, v01749; CHECK-GI-NEXT: shrn.4h v0, v0, #11750; CHECK-GI-NEXT: str d0, [x0]1751; CHECK-GI-NEXT: ret1752 %zextsrc1 = zext <4 x i16> %src1 to <4 x i32>1753 %add = add nuw nsw <4 x i32> %zextsrc1, <i32 10, i32 10, i32 10, i32 10>1754 %resulti16 = lshr <4 x i32> %add, <i32 1, i32 1, i32 1, i32 1>1755 %result = trunc <4 x i32> %resulti16 to <4 x i16>1756 store <4 x i16> %result, ptr %dest, align 81757 ret void1758}1759 1760define void @testLowerToUHADD2s_c(<2 x i32> %src1, ptr nocapture writeonly %dest) {1761; CHECK-SD-LABEL: testLowerToUHADD2s_c:1762; CHECK-SD: // %bb.0:1763; CHECK-SD-NEXT: movi.2s v1, #101764; CHECK-SD-NEXT: uhadd.2s v0, v0, v11765; CHECK-SD-NEXT: str d0, [x0]1766; CHECK-SD-NEXT: ret1767;1768; CHECK-GI-LABEL: testLowerToUHADD2s_c:1769; CHECK-GI: // %bb.0:1770; CHECK-GI-NEXT: adrp x8, .LCPI80_01771; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI80_0]1772; CHECK-GI-NEXT: uaddw.2d v0, v1, v01773; CHECK-GI-NEXT: shrn.2s v0, v0, #11774; CHECK-GI-NEXT: str d0, [x0]1775; CHECK-GI-NEXT: ret1776 %zextsrc1 = zext <2 x i32> %src1 to <2 x i64>1777 %add = add nuw nsw <2 x i64> %zextsrc1, <i64 10, i64 10>1778 %resulti16 = lshr <2 x i64> %add, <i64 1, i64 1>1779 %result = trunc <2 x i64> %resulti16 to <2 x i32>1780 store <2 x i32> %result, ptr %dest, align 81781 ret void1782}1783 1784define void @testLowerToUHADD16b_c(<16 x i8> %src1, ptr nocapture writeonly %dest) {1785; CHECK-SD-LABEL: testLowerToUHADD16b_c:1786; CHECK-SD: // %bb.0:1787; CHECK-SD-NEXT: movi.16b v1, #101788; CHECK-SD-NEXT: uhadd.16b v0, v0, v11789; CHECK-SD-NEXT: str q0, [x0]1790; CHECK-SD-NEXT: ret1791;1792; CHECK-GI-LABEL: testLowerToUHADD16b_c:1793; CHECK-GI: // %bb.0:1794; CHECK-GI-NEXT: movi.8h v1, #101795; CHECK-GI-NEXT: uaddw.8h v2, v1, v01796; CHECK-GI-NEXT: uaddw2.8h v0, v1, v01797; CHECK-GI-NEXT: shrn.8b v1, v2, #11798; CHECK-GI-NEXT: shrn2.16b v1, v0, #11799; CHECK-GI-NEXT: str q1, [x0]1800; CHECK-GI-NEXT: ret1801 %zextsrc1 = zext <16 x i8> %src1 to <16 x i16>1802 %add = add nuw nsw <16 x i16> %zextsrc1, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>1803 %resulti16 = lshr <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>1804 %result = trunc <16 x i16> %resulti16 to <16 x i8>1805 store <16 x i8> %result, ptr %dest, align 161806 ret void1807}1808 1809define void @testLowerToUHADD8h_c(<8 x i16> %src1, ptr nocapture writeonly %dest) {1810; CHECK-SD-LABEL: testLowerToUHADD8h_c:1811; CHECK-SD: // %bb.0:1812; CHECK-SD-NEXT: movi.8h v1, #101813; CHECK-SD-NEXT: uhadd.8h v0, v0, v11814; CHECK-SD-NEXT: str q0, [x0]1815; CHECK-SD-NEXT: ret1816;1817; CHECK-GI-LABEL: testLowerToUHADD8h_c:1818; CHECK-GI: // %bb.0:1819; CHECK-GI-NEXT: movi.4s v1, #101820; CHECK-GI-NEXT: uaddw.4s v2, v1, v01821; CHECK-GI-NEXT: uaddw2.4s v0, v1, v01822; CHECK-GI-NEXT: shrn.4h v1, v2, #11823; CHECK-GI-NEXT: shrn2.8h v1, v0, #11824; CHECK-GI-NEXT: str q1, [x0]1825; CHECK-GI-NEXT: ret1826 %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>1827 %add = add nuw nsw <8 x i32> %zextsrc1, <i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>1828 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>1829 %result = trunc <8 x i32> %resulti16 to <8 x i16>1830 store <8 x i16> %result, ptr %dest, align 161831 ret void1832}1833 1834define void @testLowerToUHADD4s_c(<4 x i32> %src1, ptr nocapture writeonly %dest) {1835; CHECK-SD-LABEL: testLowerToUHADD4s_c:1836; CHECK-SD: // %bb.0:1837; CHECK-SD-NEXT: movi.4s v1, #101838; CHECK-SD-NEXT: uhadd.4s v0, v0, v11839; CHECK-SD-NEXT: str q0, [x0]1840; CHECK-SD-NEXT: ret1841;1842; CHECK-GI-LABEL: testLowerToUHADD4s_c:1843; CHECK-GI: // %bb.0:1844; CHECK-GI-NEXT: adrp x8, .LCPI83_01845; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI83_0]1846; CHECK-GI-NEXT: uaddw.2d v2, v1, v01847; CHECK-GI-NEXT: uaddw2.2d v0, v1, v01848; CHECK-GI-NEXT: shrn.2s v1, v2, #11849; CHECK-GI-NEXT: shrn2.4s v1, v0, #11850; CHECK-GI-NEXT: str q1, [x0]1851; CHECK-GI-NEXT: ret1852 %zextsrc1 = zext <4 x i32> %src1 to <4 x i64>1853 %add = add nuw nsw <4 x i64> %zextsrc1, <i64 10, i64 10, i64 10, i64 10>1854 %resulti16 = lshr <4 x i64> %add, <i64 1, i64 1, i64 1, i64 1>1855 %result = trunc <4 x i64> %resulti16 to <4 x i32>1856 store <4 x i32> %result, ptr %dest, align 161857 ret void1858}1859 1860define <8 x i8> @andmaskv8i8(<8 x i16> %src1, <8 x i8> %src2) {1861; CHECK-SD-LABEL: andmaskv8i8:1862; CHECK-SD: // %bb.0:1863; CHECK-SD-NEXT: movi.8b v2, #71864; CHECK-SD-NEXT: xtn.8b v0, v01865; CHECK-SD-NEXT: and.8b v0, v0, v21866; CHECK-SD-NEXT: uhadd.8b v0, v0, v11867; CHECK-SD-NEXT: ret1868;1869; CHECK-GI-LABEL: andmaskv8i8:1870; CHECK-GI: // %bb.0:1871; CHECK-GI-NEXT: movi.8h v2, #71872; CHECK-GI-NEXT: and.16b v0, v0, v21873; CHECK-GI-NEXT: uaddw.8h v0, v0, v11874; CHECK-GI-NEXT: shrn.8b v0, v0, #11875; CHECK-GI-NEXT: ret1876 %zextsrc1 = and <8 x i16> %src1, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>1877 %zextsrc2 = zext <8 x i8> %src2 to <8 x i16>1878 %add = add nuw nsw <8 x i16> %zextsrc1, %zextsrc21879 %resulti16 = lshr <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>1880 %result = trunc <8 x i16> %resulti16 to <8 x i8>1881 ret <8 x i8> %result1882}1883 1884define <16 x i8> @andmaskv16i8(<16 x i16> %src1, <16 x i8> %src2) {1885; CHECK-SD-LABEL: andmaskv16i8:1886; CHECK-SD: // %bb.0:1887; CHECK-SD-NEXT: movi.16b v3, #71888; CHECK-SD-NEXT: uzp1.16b v0, v0, v11889; CHECK-SD-NEXT: and.16b v0, v0, v31890; CHECK-SD-NEXT: uhadd.16b v0, v0, v21891; CHECK-SD-NEXT: ret1892;1893; CHECK-GI-LABEL: andmaskv16i8:1894; CHECK-GI: // %bb.0:1895; CHECK-GI-NEXT: movi.8h v3, #71896; CHECK-GI-NEXT: and.16b v0, v0, v31897; CHECK-GI-NEXT: and.16b v1, v1, v31898; CHECK-GI-NEXT: uaddw.8h v0, v0, v21899; CHECK-GI-NEXT: uaddw2.8h v1, v1, v21900; CHECK-GI-NEXT: shrn.8b v0, v0, #11901; CHECK-GI-NEXT: shrn2.16b v0, v1, #11902; CHECK-GI-NEXT: ret1903 %zextsrc1 = and <16 x i16> %src1, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>1904 %zextsrc2 = zext <16 x i8> %src2 to <16 x i16>1905 %add = add nuw nsw <16 x i16> %zextsrc1, %zextsrc21906 %resulti16 = lshr <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>1907 %result = trunc <16 x i16> %resulti16 to <16 x i8>1908 ret <16 x i8> %result1909}1910 1911define <16 x i8> @andmask2v16i8(<16 x i16> %src1, <16 x i16> %src2) {1912; CHECK-SD-LABEL: andmask2v16i8:1913; CHECK-SD: // %bb.0:1914; CHECK-SD-NEXT: uzp1.16b v2, v2, v31915; CHECK-SD-NEXT: movi.16b v3, #31916; CHECK-SD-NEXT: uzp1.16b v0, v0, v11917; CHECK-SD-NEXT: movi.16b v1, #71918; CHECK-SD-NEXT: and.16b v2, v2, v31919; CHECK-SD-NEXT: and.16b v0, v0, v11920; CHECK-SD-NEXT: uhadd.16b v0, v0, v21921; CHECK-SD-NEXT: ret1922;1923; CHECK-GI-LABEL: andmask2v16i8:1924; CHECK-GI: // %bb.0:1925; CHECK-GI-NEXT: movi.8h v4, #71926; CHECK-GI-NEXT: movi.8h v5, #31927; CHECK-GI-NEXT: and.16b v0, v0, v41928; CHECK-GI-NEXT: and.16b v2, v2, v51929; CHECK-GI-NEXT: and.16b v1, v1, v41930; CHECK-GI-NEXT: and.16b v3, v3, v51931; CHECK-GI-NEXT: add.8h v0, v0, v21932; CHECK-GI-NEXT: add.8h v1, v1, v31933; CHECK-GI-NEXT: shrn.8b v0, v0, #11934; CHECK-GI-NEXT: shrn2.16b v0, v1, #11935; CHECK-GI-NEXT: ret1936 %zextsrc1 = and <16 x i16> %src1, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>1937 %zextsrc2 = and <16 x i16> %src2, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>1938 %add = add nuw nsw <16 x i16> %zextsrc1, %zextsrc21939 %resulti16 = lshr <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>1940 %result = trunc <16 x i16> %resulti16 to <16 x i8>1941 ret <16 x i8> %result1942}1943 1944define <8 x i8> @andmask2v8i8(<8 x i16> %src1, <8 x i16> %src2) {1945; CHECK-SD-LABEL: andmask2v8i8:1946; CHECK-SD: // %bb.0:1947; CHECK-SD-NEXT: movi.8b v2, #71948; CHECK-SD-NEXT: xtn.8b v0, v01949; CHECK-SD-NEXT: xtn.8b v1, v11950; CHECK-SD-NEXT: and.8b v0, v0, v21951; CHECK-SD-NEXT: uhadd.8b v0, v0, v11952; CHECK-SD-NEXT: ret1953;1954; CHECK-GI-LABEL: andmask2v8i8:1955; CHECK-GI: // %bb.0:1956; CHECK-GI-NEXT: movi.8h v2, #71957; CHECK-GI-NEXT: movi.2d v3, #0xff00ff00ff00ff1958; CHECK-GI-NEXT: and.16b v0, v0, v21959; CHECK-GI-NEXT: and.16b v1, v1, v31960; CHECK-GI-NEXT: add.8h v0, v0, v11961; CHECK-GI-NEXT: shrn.8b v0, v0, #11962; CHECK-GI-NEXT: ret1963 %zextsrc1 = and <8 x i16> %src1, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>1964 %zextsrc2 = and <8 x i16> %src2, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>1965 %add = add nuw nsw <8 x i16> %zextsrc1, %zextsrc21966 %resulti16 = lshr <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>1967 %result = trunc <8 x i16> %resulti16 to <8 x i8>1968 ret <8 x i8> %result1969}1970 1971define <8 x i16> @andmask3v8i8(<8 x i16> %src1, <8 x i16> %src2) {1972; CHECK-SD-LABEL: andmask3v8i8:1973; CHECK-SD: // %bb.0:1974; CHECK-SD-NEXT: movi.8h v2, #71975; CHECK-SD-NEXT: bic.8h v1, #254, lsl #81976; CHECK-SD-NEXT: and.16b v0, v0, v21977; CHECK-SD-NEXT: uhadd.8h v0, v0, v11978; CHECK-SD-NEXT: ret1979;1980; CHECK-GI-LABEL: andmask3v8i8:1981; CHECK-GI: // %bb.0:1982; CHECK-GI-NEXT: movi.8h v2, #71983; CHECK-GI-NEXT: mvni.8h v3, #254, lsl #81984; CHECK-GI-NEXT: and.16b v1, v1, v31985; CHECK-GI-NEXT: and.16b v0, v0, v21986; CHECK-GI-NEXT: add.8h v0, v0, v11987; CHECK-GI-NEXT: ushr.8h v0, v0, #11988; CHECK-GI-NEXT: ret1989 %zextsrc1 = and <8 x i16> %src1, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>1990 %zextsrc2 = and <8 x i16> %src2, <i16 511, i16 511, i16 511, i16 511, i16 511, i16 511, i16 511, i16 511>1991 %add = add nuw nsw <8 x i16> %zextsrc1, %zextsrc21992 %resulti16 = lshr <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>1993 ret <8 x i16> %resulti161994}1995 1996define <16 x i8> @sextmaskv16i8(<16 x i16> %src1, <16 x i8> %src2) {1997; CHECK-SD-LABEL: sextmaskv16i8:1998; CHECK-SD: // %bb.0:1999; CHECK-SD-NEXT: sshr.8h v1, v1, #112000; CHECK-SD-NEXT: sshr.8h v0, v0, #112001; CHECK-SD-NEXT: uzp1.16b v0, v0, v12002; CHECK-SD-NEXT: shadd.16b v0, v0, v22003; CHECK-SD-NEXT: ret2004;2005; CHECK-GI-LABEL: sextmaskv16i8:2006; CHECK-GI: // %bb.0:2007; CHECK-GI-NEXT: sshll.8h v3, v2, #02008; CHECK-GI-NEXT: sshr.8h v1, v1, #112009; CHECK-GI-NEXT: ssra.8h v3, v0, #112010; CHECK-GI-NEXT: saddw2.8h v1, v1, v22011; CHECK-GI-NEXT: shrn.8b v0, v3, #12012; CHECK-GI-NEXT: shrn2.16b v0, v1, #12013; CHECK-GI-NEXT: ret2014 %sextsrc1 = ashr <16 x i16> %src1, <i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11>2015 %sextsrc2 = sext <16 x i8> %src2 to <16 x i16>2016 %add = add nsw <16 x i16> %sextsrc1, %sextsrc22017 %1 = ashr <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>2018 %result = trunc <16 x i16> %1 to <16 x i8>2019 ret <16 x i8> %result2020}2021 2022define <8 x i8> @sextmaskv8i8(<8 x i16> %src1, <8 x i8> %src2) {2023; CHECK-SD-LABEL: sextmaskv8i8:2024; CHECK-SD: // %bb.0:2025; CHECK-SD-NEXT: sshr.8h v0, v0, #112026; CHECK-SD-NEXT: xtn.8b v0, v02027; CHECK-SD-NEXT: shadd.8b v0, v0, v12028; CHECK-SD-NEXT: ret2029;2030; CHECK-GI-LABEL: sextmaskv8i8:2031; CHECK-GI: // %bb.0:2032; CHECK-GI-NEXT: sshll.8h v1, v1, #02033; CHECK-GI-NEXT: ssra.8h v1, v0, #112034; CHECK-GI-NEXT: shrn.8b v0, v1, #12035; CHECK-GI-NEXT: ret2036 %sextsrc1 = ashr <8 x i16> %src1, <i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11>2037 %sextsrc2 = sext <8 x i8> %src2 to <8 x i16>2038 %add = add nsw <8 x i16> %sextsrc1, %sextsrc22039 %1 = ashr <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>2040 %result = trunc <8 x i16> %1 to <8 x i8>2041 ret <8 x i8> %result2042}2043 2044define <8 x i8> @sextmask2v8i8(<8 x i16> %src1, <8 x i8> %src2) {2045; CHECK-SD-LABEL: sextmask2v8i8:2046; CHECK-SD: // %bb.0:2047; CHECK-SD-NEXT: shrn.8b v0, v0, #82048; CHECK-SD-NEXT: shadd.8b v0, v0, v12049; CHECK-SD-NEXT: ret2050;2051; CHECK-GI-LABEL: sextmask2v8i8:2052; CHECK-GI: // %bb.0:2053; CHECK-GI-NEXT: sshll.8h v1, v1, #02054; CHECK-GI-NEXT: ssra.8h v1, v0, #82055; CHECK-GI-NEXT: shrn.8b v0, v1, #12056; CHECK-GI-NEXT: ret2057 %sextsrc1 = ashr <8 x i16> %src1, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>2058 %sextsrc2 = sext <8 x i8> %src2 to <8 x i16>2059 %add = add nsw <8 x i16> %sextsrc1, %sextsrc22060 %1 = ashr <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>2061 %result = trunc <8 x i16> %1 to <8 x i8>2062 ret <8 x i8> %result2063}2064 2065define <8 x i8> @sextmask3v8i8(<8 x i16> %src1, <8 x i8> %src2) {2066; CHECK-SD-LABEL: sextmask3v8i8:2067; CHECK-SD: // %bb.0:2068; CHECK-SD-NEXT: ushr.8h v0, v0, #72069; CHECK-SD-NEXT: sshll.8h v1, v1, #02070; CHECK-SD-NEXT: shadd.8h v0, v0, v12071; CHECK-SD-NEXT: xtn.8b v0, v02072; CHECK-SD-NEXT: ret2073;2074; CHECK-GI-LABEL: sextmask3v8i8:2075; CHECK-GI: // %bb.0:2076; CHECK-GI-NEXT: sshll.8h v1, v1, #02077; CHECK-GI-NEXT: ssra.8h v1, v0, #72078; CHECK-GI-NEXT: shrn.8b v0, v1, #12079; CHECK-GI-NEXT: ret2080 %1 = ashr <8 x i16> %src1, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>2081 %sextsrc2 = sext <8 x i8> %src2 to <8 x i16>2082 %add = add nsw <8 x i16> %1, %sextsrc22083 %2 = ashr <8 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>2084 %result = trunc <8 x i16> %2 to <8 x i8>2085 ret <8 x i8> %result2086}2087 2088define <4 x i16> @ext_via_i19(<4 x i16> %a) {2089; CHECK-LABEL: ext_via_i19:2090; CHECK: // %bb.0:2091; CHECK-NEXT: movi.4h v1, #12092; CHECK-NEXT: urhadd.4h v0, v0, v12093; CHECK-NEXT: ret2094 %t3 = zext <4 x i16> %a to <4 x i32>2095 %t4 = add <4 x i32> %t3, <i32 1, i32 1, i32 1, i32 1>2096 %t5 = trunc <4 x i32> %t4 to <4 x i19>2097 %new0 = add <4 x i19> %t5, <i19 1, i19 1, i19 1, i19 1>2098 %new1 = lshr <4 x i19> %new0, <i19 1, i19 1, i19 1, i19 1>2099 %last = zext <4 x i19> %new1 to <4 x i32>2100 %t6 = trunc <4 x i32> %last to <4 x i16>2101 ret <4 x i16> %t62102}2103 2104define <8 x i8> @srhadd_v8i8_trunc(<8 x i8> %s0, <8 x i8> %s1) {2105; CHECK-SD-LABEL: srhadd_v8i8_trunc:2106; CHECK-SD: // %bb.0:2107; CHECK-SD-NEXT: srhadd.8b v0, v0, v12108; CHECK-SD-NEXT: ret2109;2110; CHECK-GI-LABEL: srhadd_v8i8_trunc:2111; CHECK-GI: // %bb.0:2112; CHECK-GI-NEXT: sshll.8h v0, v0, #02113; CHECK-GI-NEXT: sshll.8h v1, v1, #02114; CHECK-GI-NEXT: urhadd.8h v0, v0, v12115; CHECK-GI-NEXT: xtn.8b v0, v02116; CHECK-GI-NEXT: ret2117 %s0s = sext <8 x i8> %s0 to <8 x i16>2118 %s1s = sext <8 x i8> %s1 to <8 x i16>2119 %s = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> %s0s, <8 x i16> %s1s)2120 %s2 = trunc <8 x i16> %s to <8 x i8>2121 ret <8 x i8> %s22122}2123 2124define <4 x i16> @srhadd_v4i16_trunc(<4 x i16> %s0, <4 x i16> %s1) {2125; CHECK-SD-LABEL: srhadd_v4i16_trunc:2126; CHECK-SD: // %bb.0:2127; CHECK-SD-NEXT: srhadd.4h v0, v0, v12128; CHECK-SD-NEXT: ret2129;2130; CHECK-GI-LABEL: srhadd_v4i16_trunc:2131; CHECK-GI: // %bb.0:2132; CHECK-GI-NEXT: sshll.4s v0, v0, #02133; CHECK-GI-NEXT: sshll.4s v1, v1, #02134; CHECK-GI-NEXT: urhadd.4s v0, v0, v12135; CHECK-GI-NEXT: xtn.4h v0, v02136; CHECK-GI-NEXT: ret2137 %s0s = sext <4 x i16> %s0 to <4 x i32>2138 %s1s = sext <4 x i16> %s1 to <4 x i32>2139 %s = call <4 x i32> @llvm.aarch64.neon.urhadd.v4i32(<4 x i32> %s0s, <4 x i32> %s1s)2140 %s2 = trunc <4 x i32> %s to <4 x i16>2141 ret <4 x i16> %s22142}2143 2144define <2 x i32> @srhadd_v2i32_trunc(<2 x i32> %s0, <2 x i32> %s1) {2145; CHECK-LABEL: srhadd_v2i32_trunc:2146; CHECK: // %bb.0:2147; CHECK-NEXT: sshll.2d v0, v0, #02148; CHECK-NEXT: sshll.2d v1, v1, #02149; CHECK-NEXT: eor.16b v2, v0, v12150; CHECK-NEXT: orr.16b v0, v0, v12151; CHECK-NEXT: ushr.2d v1, v2, #12152; CHECK-NEXT: sub.2d v0, v0, v12153; CHECK-NEXT: xtn.2s v0, v02154; CHECK-NEXT: ret2155 %s0s = sext <2 x i32> %s0 to <2 x i64>2156 %s1s = sext <2 x i32> %s1 to <2 x i64>2157 %s = call <2 x i64> @llvm.aarch64.neon.urhadd.v2i64(<2 x i64> %s0s, <2 x i64> %s1s)2158 %s2 = trunc <2 x i64> %s to <2 x i32>2159 ret <2 x i32> %s22160}2161 2162define <8 x i8> @urhadd_v8i8_trunc(<8 x i8> %s0, <8 x i8> %s1) {2163; CHECK-SD-LABEL: urhadd_v8i8_trunc:2164; CHECK-SD: // %bb.0:2165; CHECK-SD-NEXT: urhadd.8b v0, v0, v12166; CHECK-SD-NEXT: ret2167;2168; CHECK-GI-LABEL: urhadd_v8i8_trunc:2169; CHECK-GI: // %bb.0:2170; CHECK-GI-NEXT: ushll.8h v0, v0, #02171; CHECK-GI-NEXT: ushll.8h v1, v1, #02172; CHECK-GI-NEXT: srhadd.8h v0, v0, v12173; CHECK-GI-NEXT: xtn.8b v0, v02174; CHECK-GI-NEXT: ret2175 %s0s = zext <8 x i8> %s0 to <8 x i16>2176 %s1s = zext <8 x i8> %s1 to <8 x i16>2177 %s = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> %s0s, <8 x i16> %s1s)2178 %s2 = trunc <8 x i16> %s to <8 x i8>2179 ret <8 x i8> %s22180}2181 2182define <4 x i16> @urhadd_v4i16_trunc(<4 x i16> %s0, <4 x i16> %s1) {2183; CHECK-SD-LABEL: urhadd_v4i16_trunc:2184; CHECK-SD: // %bb.0:2185; CHECK-SD-NEXT: urhadd.4h v0, v0, v12186; CHECK-SD-NEXT: ret2187;2188; CHECK-GI-LABEL: urhadd_v4i16_trunc:2189; CHECK-GI: // %bb.0:2190; CHECK-GI-NEXT: ushll.4s v0, v0, #02191; CHECK-GI-NEXT: ushll.4s v1, v1, #02192; CHECK-GI-NEXT: srhadd.4s v0, v0, v12193; CHECK-GI-NEXT: xtn.4h v0, v02194; CHECK-GI-NEXT: ret2195 %s0s = zext <4 x i16> %s0 to <4 x i32>2196 %s1s = zext <4 x i16> %s1 to <4 x i32>2197 %s = call <4 x i32> @llvm.aarch64.neon.srhadd.v4i32(<4 x i32> %s0s, <4 x i32> %s1s)2198 %s2 = trunc <4 x i32> %s to <4 x i16>2199 ret <4 x i16> %s22200}2201 2202define <2 x i32> @urhadd_v2i32_trunc(<2 x i32> %s0, <2 x i32> %s1) {2203; CHECK-LABEL: urhadd_v2i32_trunc:2204; CHECK: // %bb.0:2205; CHECK-NEXT: mov w8, #1 // =0x12206; CHECK-NEXT: uaddl.2d v0, v0, v12207; CHECK-NEXT: dup.2d v1, x82208; CHECK-NEXT: add.2d v0, v0, v12209; CHECK-NEXT: shrn.2s v0, v0, #12210; CHECK-NEXT: ret2211 %s0s = zext <2 x i32> %s0 to <2 x i64>2212 %s1s = zext <2 x i32> %s1 to <2 x i64>2213 %s = call <2 x i64> @llvm.aarch64.neon.srhadd.v2i64(<2 x i64> %s0s, <2 x i64> %s1s)2214 %s2 = trunc <2 x i64> %s to <2 x i32>2215 ret <2 x i32> %s22216}2217 2218declare <8 x i8> @llvm.aarch64.neon.srhadd.v8i8(<8 x i8>, <8 x i8>)2219declare <4 x i16> @llvm.aarch64.neon.srhadd.v4i16(<4 x i16>, <4 x i16>)2220declare <2 x i32> @llvm.aarch64.neon.srhadd.v2i32(<2 x i32>, <2 x i32>)2221declare <8 x i8> @llvm.aarch64.neon.urhadd.v8i8(<8 x i8>, <8 x i8>)2222declare <4 x i16> @llvm.aarch64.neon.urhadd.v4i16(<4 x i16>, <4 x i16>)2223declare <2 x i32> @llvm.aarch64.neon.urhadd.v2i32(<2 x i32>, <2 x i32>)2224declare <16 x i8> @llvm.aarch64.neon.srhadd.v16i8(<16 x i8>, <16 x i8>)2225declare <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16>, <8 x i16>)2226declare <4 x i32> @llvm.aarch64.neon.srhadd.v4i32(<4 x i32>, <4 x i32>)2227declare <16 x i8> @llvm.aarch64.neon.urhadd.v16i8(<16 x i8>, <16 x i8>)2228declare <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16>, <8 x i16>)2229declare <4 x i32> @llvm.aarch64.neon.urhadd.v4i32(<4 x i32>, <4 x i32>)2230