1061 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=aarch64 | FileCheck %s3 4define <8 x i16> @haddu_base(<8 x i16> %src1, <8 x i16> %src2) {5; CHECK-LABEL: haddu_base:6; CHECK: // %bb.0:7; CHECK-NEXT: uhadd v0.8h, v0.8h, v1.8h8; CHECK-NEXT: ret9 %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>10 %zextsrc2 = zext <8 x i16> %src2 to <8 x i32>11 %add = add <8 x i32> %zextsrc1, %zextsrc212 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>13 %result = trunc <8 x i32> %resulti16 to <8 x i16>14 ret <8 x i16> %result15}16 17define <8 x i16> @haddu_const(<8 x i16> %src1) {18; CHECK-LABEL: haddu_const:19; CHECK: // %bb.0:20; CHECK-NEXT: movi v1.8h, #121; CHECK-NEXT: uhadd v0.8h, v0.8h, v1.8h22; CHECK-NEXT: ret23 %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>24 %add = add <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>25 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>26 %result = trunc <8 x i32> %resulti16 to <8 x i16>27 ret <8 x i16> %result28}29 30define <8 x i16> @haddu_const_lhs(<8 x i16> %src1) {31; CHECK-LABEL: haddu_const_lhs:32; CHECK: // %bb.0:33; CHECK-NEXT: movi v1.8h, #134; CHECK-NEXT: uhadd v0.8h, v0.8h, v1.8h35; CHECK-NEXT: ret36 %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>37 %add = add <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %zextsrc138 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>39 %result = trunc <8 x i32> %resulti16 to <8 x i16>40 ret <8 x i16> %result41}42 43define <8 x i16> @haddu_const_zero(<8 x i16> %src1) {44; CHECK-LABEL: haddu_const_zero:45; CHECK: // %bb.0:46; CHECK-NEXT: ushll v1.4s, v0.4h, #047; CHECK-NEXT: ushll2 v2.4s, v0.8h, #048; CHECK-NEXT: shrn v0.4h, v1.4s, #149; CHECK-NEXT: shrn2 v0.8h, v2.4s, #150; CHECK-NEXT: ret51 %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>52 %add = add <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, %zextsrc153 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>54 %result = trunc <8 x i32> %resulti16 to <8 x i16>55 ret <8 x i16> %result56}57 58define <8 x i16> @haddu_const_both() {59; CHECK-LABEL: haddu_const_both:60; CHECK: // %bb.0:61; CHECK-NEXT: movi v0.8h, #262; CHECK-NEXT: ret63 %add = add <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>64 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>65 %result = trunc <8 x i32> %resulti16 to <8 x i16>66 ret <8 x i16> %result67}68 69define <8 x i16> @haddu_const_bothhigh() {70; CHECK-LABEL: haddu_const_bothhigh:71; CHECK: // %bb.0:72; CHECK-NEXT: mvni v0.8h, #173; CHECK-NEXT: ret74 %ext1 = zext <8 x i16> <i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534> to <8 x i32>75 %ext2 = zext <8 x i16> <i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535> to <8 x i32>76 %add = add <8 x i32> %ext1, %ext277 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>78 %result = trunc <8 x i32> %resulti16 to <8 x i16>79 ret <8 x i16> %result80}81 82define <8 x i16> @haddu_undef(<8 x i16> %src1) {83; CHECK-LABEL: haddu_undef:84; CHECK: // %bb.0:85; CHECK-NEXT: ushll v1.4s, v0.4h, #086; CHECK-NEXT: ushll2 v2.4s, v0.8h, #087; CHECK-NEXT: shrn v0.4h, v1.4s, #188; CHECK-NEXT: shrn2 v0.8h, v2.4s, #189; CHECK-NEXT: ret90 %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>91 %zextsrc2 = zext <8 x i16> undef to <8 x i32>92 %add = add <8 x i32> %zextsrc2, %zextsrc193 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>94 %result = trunc <8 x i32> %resulti16 to <8 x i16>95 ret <8 x i16> %result96}97 98 99 100define <8 x i16> @haddu_i_base(<8 x i16> %src1, <8 x i16> %src2) {101; CHECK-LABEL: haddu_i_base:102; CHECK: // %bb.0:103; CHECK-NEXT: uhadd v0.8h, v0.8h, v1.8h104; CHECK-NEXT: ret105 %result = call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> %src1, <8 x i16> %src2)106 ret <8 x i16> %result107}108 109define <8 x i16> @haddu_i_const(<8 x i16> %src1) {110; CHECK-LABEL: haddu_i_const:111; CHECK: // %bb.0:112; CHECK-NEXT: movi v1.8h, #1113; CHECK-NEXT: uhadd v0.8h, v0.8h, v1.8h114; CHECK-NEXT: ret115 %result = call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> %src1, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)116 ret <8 x i16> %result117}118 119define <8 x i16> @haddu_i_const_lhs(<8 x i16> %src1) {120; CHECK-LABEL: haddu_i_const_lhs:121; CHECK: // %bb.0:122; CHECK-NEXT: movi v1.8h, #1123; CHECK-NEXT: uhadd v0.8h, v0.8h, v1.8h124; CHECK-NEXT: ret125 %result = call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>, <8 x i16> %src1)126 ret <8 x i16> %result127}128 129define <8 x i16> @haddu_i_const_zero(<8 x i16> %src1) {130; CHECK-LABEL: haddu_i_const_zero:131; CHECK: // %bb.0:132; CHECK-NEXT: ushr v0.8h, v0.8h, #1133; CHECK-NEXT: ret134 %result = call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, <8 x i16> %src1)135 ret <8 x i16> %result136}137 138define <8 x i16> @haddu_i_const_both() {139; CHECK-LABEL: haddu_i_const_both:140; CHECK: // %bb.0:141; CHECK-NEXT: movi v0.8h, #2142; CHECK-NEXT: ret143 %result = call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)144 ret <8 x i16> %result145}146 147define <8 x i16> @haddu_i_const_bothhigh() {148; CHECK-LABEL: haddu_i_const_bothhigh:149; CHECK: // %bb.0:150; CHECK-NEXT: mvni v0.8h, #1151; CHECK-NEXT: ret152 %result = call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> <i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534>, <8 x i16> <i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535>)153 ret <8 x i16> %result154}155 156define <8 x i16> @haddu_i_undef(<8 x i16> %t, <8 x i16> %src1) {157; CHECK-LABEL: haddu_i_undef:158; CHECK: // %bb.0:159; CHECK-NEXT: mov v0.16b, v1.16b160; CHECK-NEXT: ret161 %result = call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> undef, <8 x i16> %src1)162 ret <8 x i16> %result163}164 165 166 167 168 169define <8 x i16> @hadds_base(<8 x i16> %src1, <8 x i16> %src2) {170; CHECK-LABEL: hadds_base:171; CHECK: // %bb.0:172; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h173; CHECK-NEXT: ret174 %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>175 %zextsrc2 = sext <8 x i16> %src2 to <8 x i32>176 %add = add <8 x i32> %zextsrc1, %zextsrc2177 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>178 %result = trunc <8 x i32> %resulti16 to <8 x i16>179 ret <8 x i16> %result180}181 182define <8 x i16> @hadds_const(<8 x i16> %src1) {183; CHECK-LABEL: hadds_const:184; CHECK: // %bb.0:185; CHECK-NEXT: movi v1.8h, #1186; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h187; CHECK-NEXT: ret188 %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>189 %add = add <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>190 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>191 %result = trunc <8 x i32> %resulti16 to <8 x i16>192 ret <8 x i16> %result193}194 195define <8 x i16> @hadds_const_lhs(<8 x i16> %src1) {196; CHECK-LABEL: hadds_const_lhs:197; CHECK: // %bb.0:198; CHECK-NEXT: movi v1.8h, #1199; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h200; CHECK-NEXT: ret201 %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>202 %add = add <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %zextsrc1203 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>204 %result = trunc <8 x i32> %resulti16 to <8 x i16>205 ret <8 x i16> %result206}207 208define <8 x i16> @hadds_const_zero(<8 x i16> %src1) {209; CHECK-LABEL: hadds_const_zero:210; CHECK: // %bb.0:211; CHECK-NEXT: sshll v1.4s, v0.4h, #0212; CHECK-NEXT: sshll2 v2.4s, v0.8h, #0213; CHECK-NEXT: shrn v0.4h, v1.4s, #1214; CHECK-NEXT: shrn2 v0.8h, v2.4s, #1215; CHECK-NEXT: ret216 %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>217 %add = add <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, %zextsrc1218 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>219 %result = trunc <8 x i32> %resulti16 to <8 x i16>220 ret <8 x i16> %result221}222 223define <8 x i16> @hadds_const_both() {224; CHECK-LABEL: hadds_const_both:225; CHECK: // %bb.0:226; CHECK-NEXT: movi v0.8h, #2227; CHECK-NEXT: ret228 %add = add <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>229 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>230 %result = trunc <8 x i32> %resulti16 to <8 x i16>231 ret <8 x i16> %result232}233 234define <8 x i16> @hadds_const_bothhigh() {235; CHECK-LABEL: hadds_const_bothhigh:236; CHECK: // %bb.0:237; CHECK-NEXT: mov w8, #32766 // =0x7ffe238; CHECK-NEXT: dup v0.8h, w8239; CHECK-NEXT: ret240 %ext1 = sext <8 x i16> <i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766> to <8 x i32>241 %ext2 = sext <8 x i16> <i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767> to <8 x i32>242 %add = add <8 x i32> %ext1, %ext2243 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>244 %result = trunc <8 x i32> %resulti16 to <8 x i16>245 ret <8 x i16> %result246}247 248define <8 x i16> @hadds_undef(<8 x i16> %src1) {249; CHECK-LABEL: hadds_undef:250; CHECK: // %bb.0:251; CHECK-NEXT: sshll v1.4s, v0.4h, #0252; CHECK-NEXT: sshll2 v2.4s, v0.8h, #0253; CHECK-NEXT: shrn v0.4h, v1.4s, #1254; CHECK-NEXT: shrn2 v0.8h, v2.4s, #1255; CHECK-NEXT: ret256 %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>257 %zextsrc2 = sext <8 x i16> undef to <8 x i32>258 %add = add <8 x i32> %zextsrc2, %zextsrc1259 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>260 %result = trunc <8 x i32> %resulti16 to <8 x i16>261 ret <8 x i16> %result262}263 264 265 266define <8 x i16> @hadds_i_base(<8 x i16> %src1, <8 x i16> %src2) {267; CHECK-LABEL: hadds_i_base:268; CHECK: // %bb.0:269; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h270; CHECK-NEXT: ret271 %result = call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> %src1, <8 x i16> %src2)272 ret <8 x i16> %result273}274 275define <8 x i16> @hadds_i_const(<8 x i16> %src1) {276; CHECK-LABEL: hadds_i_const:277; CHECK: // %bb.0:278; CHECK-NEXT: movi v1.8h, #1279; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h280; CHECK-NEXT: ret281 %result = call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> %src1, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)282 ret <8 x i16> %result283}284 285define <8 x i16> @hadds_i_const_lhs(<8 x i16> %src1) {286; CHECK-LABEL: hadds_i_const_lhs:287; CHECK: // %bb.0:288; CHECK-NEXT: movi v1.8h, #1289; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h290; CHECK-NEXT: ret291 %result = call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>, <8 x i16> %src1)292 ret <8 x i16> %result293}294 295define <8 x i16> @hadds_i_const_zero(<8 x i16> %src1) {296; CHECK-LABEL: hadds_i_const_zero:297; CHECK: // %bb.0:298; CHECK-NEXT: sshr v0.8h, v0.8h, #1299; CHECK-NEXT: ret300 %result = call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, <8 x i16> %src1)301 ret <8 x i16> %result302}303 304define <8 x i16> @hadds_i_const_both() {305; CHECK-LABEL: hadds_i_const_both:306; CHECK: // %bb.0:307; CHECK-NEXT: movi v0.8h, #2308; CHECK-NEXT: ret309 %result = call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)310 ret <8 x i16> %result311}312 313define <8 x i16> @hadds_i_const_bothhigh() {314; CHECK-LABEL: hadds_i_const_bothhigh:315; CHECK: // %bb.0:316; CHECK-NEXT: mov w8, #32766 // =0x7ffe317; CHECK-NEXT: dup v0.8h, w8318; CHECK-NEXT: ret319 %result = call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> <i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766>, <8 x i16> <i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767>)320 ret <8 x i16> %result321}322 323define <8 x i16> @hadds_i_undef(<8 x i16> %t, <8 x i16> %src1) {324; CHECK-LABEL: hadds_i_undef:325; CHECK: // %bb.0:326; CHECK-NEXT: mov v0.16b, v1.16b327; CHECK-NEXT: ret328 %result = call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> undef, <8 x i16> %src1)329 ret <8 x i16> %result330}331 332define <8 x i16> @sub_fixedwidth_v4i32(<8 x i16> %a0, <8 x i16> %a1) {333; CHECK-LABEL: sub_fixedwidth_v4i32:334; CHECK: // %bb.0:335; CHECK-NEXT: urhadd v0.8h, v0.8h, v1.8h336; CHECK-NEXT: ret337 %or = or <8 x i16> %a0, %a1338 %xor = xor <8 x i16> %a0, %a1339 %srl = lshr <8 x i16> %xor, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>340 %res = sub <8 x i16> %or, %srl341 ret <8 x i16> %res342}343 344define <8 x i16> @srhadd_fixedwidth_v8i16(<8 x i16> %a0, <8 x i16> %a1) {345; CHECK-LABEL: srhadd_fixedwidth_v8i16:346; CHECK: // %bb.0:347; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h348; CHECK-NEXT: ret349 %or = or <8 x i16> %a0, %a1350 %xor = xor <8 x i16> %a0, %a1351 %srl = ashr <8 x i16> %xor, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>352 %res = sub <8 x i16> %or, %srl353 ret <8 x i16> %res354}355 356define <8 x i16> @rhaddu_base(<8 x i16> %src1, <8 x i16> %src2) {357; CHECK-LABEL: rhaddu_base:358; CHECK: // %bb.0:359; CHECK-NEXT: urhadd v0.8h, v0.8h, v1.8h360; CHECK-NEXT: ret361 %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>362 %zextsrc2 = zext <8 x i16> %src2 to <8 x i32>363 %add1 = add <8 x i32> %zextsrc1, %zextsrc2364 %add = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>365 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>366 %result = trunc <8 x i32> %resulti16 to <8 x i16>367 ret <8 x i16> %result368}369 370define <8 x i16> @rhaddu_const(<8 x i16> %src1) {371; CHECK-LABEL: rhaddu_const:372; CHECK: // %bb.0:373; CHECK-NEXT: movi v1.8h, #1374; CHECK-NEXT: urhadd v0.8h, v0.8h, v1.8h375; CHECK-NEXT: ret376 %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>377 %add1 = add <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>378 %add = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>379 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>380 %result = trunc <8 x i32> %resulti16 to <8 x i16>381 ret <8 x i16> %result382}383 384define <8 x i16> @rhaddu_const_lhs(<8 x i16> %src1) {385; CHECK-LABEL: rhaddu_const_lhs:386; CHECK: // %bb.0:387; CHECK-NEXT: movi v1.8h, #1388; CHECK-NEXT: urhadd v0.8h, v0.8h, v1.8h389; CHECK-NEXT: ret390 %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>391 %add1 = add <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %zextsrc1392 %add = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>393 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>394 %result = trunc <8 x i32> %resulti16 to <8 x i16>395 ret <8 x i16> %result396}397 398define <8 x i16> @rhaddu_const_zero(<8 x i16> %src1) {399; CHECK-LABEL: rhaddu_const_zero:400; CHECK: // %bb.0:401; CHECK-NEXT: movi v1.8h, #1402; CHECK-NEXT: uhadd v0.8h, v0.8h, v1.8h403; CHECK-NEXT: ret404 %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>405 %add1 = add <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, %zextsrc1406 %add = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>407 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>408 %result = trunc <8 x i32> %resulti16 to <8 x i16>409 ret <8 x i16> %result410}411 412define <8 x i16> @rhaddu_const_both() {413; CHECK-LABEL: rhaddu_const_both:414; CHECK: // %bb.0:415; CHECK-NEXT: movi v0.8h, #2416; CHECK-NEXT: ret417 %add1 = add <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>418 %add = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>419 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>420 %result = trunc <8 x i32> %resulti16 to <8 x i16>421 ret <8 x i16> %result422}423 424define <8 x i16> @rhaddu_const_bothhigh() {425; CHECK-LABEL: rhaddu_const_bothhigh:426; CHECK: // %bb.0:427; CHECK-NEXT: movi v0.2d, #0xffffffffffffffff428; CHECK-NEXT: ret429 %ext1 = zext <8 x i16> <i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534> to <8 x i32>430 %ext2 = zext <8 x i16> <i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535> to <8 x i32>431 %add1 = add <8 x i32> %ext1, %ext2432 %add = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>433 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>434 %result = trunc <8 x i32> %resulti16 to <8 x i16>435 ret <8 x i16> %result436}437 438define <8 x i16> @rhaddu_undef(<8 x i16> %src1) {439; CHECK-LABEL: rhaddu_undef:440; CHECK: // %bb.0:441; CHECK-NEXT: movi v1.8h, #1442; CHECK-NEXT: uhadd v0.8h, v0.8h, v1.8h443; CHECK-NEXT: ret444 %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>445 %zextsrc2 = zext <8 x i16> undef to <8 x i32>446 %add1 = add <8 x i32> %zextsrc2, %zextsrc1447 %add = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>448 %resulti16 = lshr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>449 %result = trunc <8 x i32> %resulti16 to <8 x i16>450 ret <8 x i16> %result451}452 453 454 455define <8 x i16> @rhaddu_i_base(<8 x i16> %src1, <8 x i16> %src2) {456; CHECK-LABEL: rhaddu_i_base:457; CHECK: // %bb.0:458; CHECK-NEXT: urhadd v0.8h, v0.8h, v1.8h459; CHECK-NEXT: ret460 %result = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> %src1, <8 x i16> %src2)461 ret <8 x i16> %result462}463 464define <8 x i16> @rhaddu_i_const(<8 x i16> %src1) {465; CHECK-LABEL: rhaddu_i_const:466; CHECK: // %bb.0:467; CHECK-NEXT: movi v1.8h, #1468; CHECK-NEXT: urhadd v0.8h, v0.8h, v1.8h469; CHECK-NEXT: ret470 %result = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> %src1, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)471 ret <8 x i16> %result472}473 474define <8 x i16> @rhaddu_i_const_lhs(<8 x i16> %src1) {475; CHECK-LABEL: rhaddu_i_const_lhs:476; CHECK: // %bb.0:477; CHECK-NEXT: movi v1.8h, #1478; CHECK-NEXT: urhadd v0.8h, v0.8h, v1.8h479; CHECK-NEXT: ret480 %result = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>, <8 x i16> %src1)481 ret <8 x i16> %result482}483 484define <8 x i16> @rhaddu_i_const_zero(<8 x i16> %src1) {485; CHECK-LABEL: rhaddu_i_const_zero:486; CHECK: // %bb.0:487; CHECK-NEXT: movi v1.2d, #0000000000000000488; CHECK-NEXT: urhadd v0.8h, v0.8h, v1.8h489; CHECK-NEXT: ret490 %result = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, <8 x i16> %src1)491 ret <8 x i16> %result492}493 494define <8 x i16> @rhaddu_i_const_both() {495; CHECK-LABEL: rhaddu_i_const_both:496; CHECK: // %bb.0:497; CHECK-NEXT: movi v0.8h, #2498; CHECK-NEXT: ret499 %result = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)500 ret <8 x i16> %result501}502 503define <8 x i16> @rhaddu_i_const_bothhigh() {504; CHECK-LABEL: rhaddu_i_const_bothhigh:505; CHECK: // %bb.0:506; CHECK-NEXT: movi v0.2d, #0xffffffffffffffff507; CHECK-NEXT: ret508 %result = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> <i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534>, <8 x i16> <i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535>)509 ret <8 x i16> %result510}511 512define <8 x i16> @rhaddu_i_undef(<8 x i16> %t, <8 x i16> %src1) {513; CHECK-LABEL: rhaddu_i_undef:514; CHECK: // %bb.0:515; CHECK-NEXT: mov v0.16b, v1.16b516; CHECK-NEXT: ret517 %result = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> undef, <8 x i16> %src1)518 ret <8 x i16> %result519}520 521 522 523 524 525define <8 x i16> @rhadds_base(<8 x i16> %src1, <8 x i16> %src2) {526; CHECK-LABEL: rhadds_base:527; CHECK: // %bb.0:528; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h529; CHECK-NEXT: ret530 %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>531 %zextsrc2 = sext <8 x i16> %src2 to <8 x i32>532 %add1 = add <8 x i32> %zextsrc1, %zextsrc2533 %add = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>534 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>535 %result = trunc <8 x i32> %resulti16 to <8 x i16>536 ret <8 x i16> %result537}538 539define <8 x i16> @rhadds_const(<8 x i16> %src1) {540; CHECK-LABEL: rhadds_const:541; CHECK: // %bb.0:542; CHECK-NEXT: movi v1.8h, #1543; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h544; CHECK-NEXT: ret545 %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>546 %add1 = add <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>547 %add = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>548 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>549 %result = trunc <8 x i32> %resulti16 to <8 x i16>550 ret <8 x i16> %result551}552 553define <8 x i16> @rhadds_const_lhs(<8 x i16> %src1) {554; CHECK-LABEL: rhadds_const_lhs:555; CHECK: // %bb.0:556; CHECK-NEXT: movi v1.8h, #1557; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h558; CHECK-NEXT: ret559 %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>560 %add1 = add <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %zextsrc1561 %add = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>562 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>563 %result = trunc <8 x i32> %resulti16 to <8 x i16>564 ret <8 x i16> %result565}566 567define <8 x i16> @rhadds_const_zero(<8 x i16> %src1) {568; CHECK-LABEL: rhadds_const_zero:569; CHECK: // %bb.0:570; CHECK-NEXT: movi v1.8h, #1571; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h572; CHECK-NEXT: ret573 %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>574 %add1 = add <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, %zextsrc1575 %add = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>576 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>577 %result = trunc <8 x i32> %resulti16 to <8 x i16>578 ret <8 x i16> %result579}580 581define <8 x i16> @rhadds_const_both() {582; CHECK-LABEL: rhadds_const_both:583; CHECK: // %bb.0:584; CHECK-NEXT: movi v0.8h, #2585; CHECK-NEXT: ret586 %add1 = add <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>587 %add = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>588 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>589 %result = trunc <8 x i32> %resulti16 to <8 x i16>590 ret <8 x i16> %result591}592 593define <8 x i16> @rhadds_const_bothhigh() {594; CHECK-LABEL: rhadds_const_bothhigh:595; CHECK: // %bb.0:596; CHECK-NEXT: mvni v0.8h, #128, lsl #8597; CHECK-NEXT: ret598 %ext1 = sext <8 x i16> <i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766> to <8 x i32>599 %ext2 = sext <8 x i16> <i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767> to <8 x i32>600 %add1 = add <8 x i32> %ext1, %ext2601 %add = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>602 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>603 %result = trunc <8 x i32> %resulti16 to <8 x i16>604 ret <8 x i16> %result605}606 607define <8 x i16> @rhadds_undef(<8 x i16> %src1) {608; CHECK-LABEL: rhadds_undef:609; CHECK: // %bb.0:610; CHECK-NEXT: movi v1.8h, #1611; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h612; CHECK-NEXT: ret613 %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>614 %zextsrc2 = sext <8 x i16> undef to <8 x i32>615 %add1 = add <8 x i32> %zextsrc2, %zextsrc1616 %add = add <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>617 %resulti16 = ashr <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>618 %result = trunc <8 x i32> %resulti16 to <8 x i16>619 ret <8 x i16> %result620}621 622 623 624define <8 x i16> @rhadds_i_base(<8 x i16> %src1, <8 x i16> %src2) {625; CHECK-LABEL: rhadds_i_base:626; CHECK: // %bb.0:627; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h628; CHECK-NEXT: ret629 %result = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> %src1, <8 x i16> %src2)630 ret <8 x i16> %result631}632 633define <8 x i16> @rhadds_i_const(<8 x i16> %src1) {634; CHECK-LABEL: rhadds_i_const:635; CHECK: // %bb.0:636; CHECK-NEXT: movi v1.8h, #1637; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h638; CHECK-NEXT: ret639 %result = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> %src1, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)640 ret <8 x i16> %result641}642 643define <8 x i16> @rhadds_i_const_lhs(<8 x i16> %src1) {644; CHECK-LABEL: rhadds_i_const_lhs:645; CHECK: // %bb.0:646; CHECK-NEXT: movi v1.8h, #1647; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h648; CHECK-NEXT: ret649 %result = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>, <8 x i16> %src1)650 ret <8 x i16> %result651}652 653define <8 x i16> @rhadds_i_const_zero(<8 x i16> %src1) {654; CHECK-LABEL: rhadds_i_const_zero:655; CHECK: // %bb.0:656; CHECK-NEXT: movi v1.2d, #0000000000000000657; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h658; CHECK-NEXT: ret659 %result = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, <8 x i16> %src1)660 ret <8 x i16> %result661}662 663define <8 x i16> @rhadds_i_const_both() {664; CHECK-LABEL: rhadds_i_const_both:665; CHECK: // %bb.0:666; CHECK-NEXT: movi v0.8h, #2667; CHECK-NEXT: ret668 %result = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)669 ret <8 x i16> %result670}671 672define <8 x i16> @rhadds_i_const_bothhigh() {673; CHECK-LABEL: rhadds_i_const_bothhigh:674; CHECK: // %bb.0:675; CHECK-NEXT: mvni v0.8h, #128, lsl #8676; CHECK-NEXT: ret677 %result = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> <i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766>, <8 x i16> <i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767>)678 ret <8 x i16> %result679}680 681define <8 x i16> @rhadds_i_undef(<8 x i16> %t, <8 x i16> %src1) {682; CHECK-LABEL: rhadds_i_undef:683; CHECK: // %bb.0:684; CHECK-NEXT: mov v0.16b, v1.16b685; CHECK-NEXT: ret686 %result = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> undef, <8 x i16> %src1)687 ret <8 x i16> %result688}689 690 691define <8 x i8> @shadd_v8i8(<8 x i8> %x) {692; CHECK-LABEL: shadd_v8i8:693; CHECK: // %bb.0:694; CHECK-NEXT: ret695 %r = tail call <8 x i8> @llvm.aarch64.neon.shadd.v8i8(<8 x i8> %x, <8 x i8> %x)696 ret <8 x i8> %r697}698 699define <4 x i16> @shadd_v4i16(<4 x i16> %x) {700; CHECK-LABEL: shadd_v4i16:701; CHECK: // %bb.0:702; CHECK-NEXT: ret703 %r = tail call <4 x i16> @llvm.aarch64.neon.shadd.v4i16(<4 x i16> %x, <4 x i16> %x)704 ret <4 x i16> %r705}706 707define <2 x i32> @shadd_v2i32(<2 x i32> %x) {708; CHECK-LABEL: shadd_v2i32:709; CHECK: // %bb.0:710; CHECK-NEXT: ret711 %r = tail call <2 x i32> @llvm.aarch64.neon.shadd.v2i32(<2 x i32> %x, <2 x i32> %x)712 ret <2 x i32> %r713}714 715define <16 x i8> @shadd_v16i8(<16 x i8> %x) {716; CHECK-LABEL: shadd_v16i8:717; CHECK: // %bb.0:718; CHECK-NEXT: ret719 %r = tail call <16 x i8> @llvm.aarch64.neon.shadd.v16i8(<16 x i8> %x, <16 x i8> %x)720 ret <16 x i8> %r721}722 723define <8 x i16> @shadd_v8i16(<8 x i16> %x) {724; CHECK-LABEL: shadd_v8i16:725; CHECK: // %bb.0:726; CHECK-NEXT: ret727 %r = tail call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> %x, <8 x i16> %x)728 ret <8 x i16> %r729}730 731define <4 x i32> @shadd_v4i32(<4 x i32> %x) {732; CHECK-LABEL: shadd_v4i32:733; CHECK: // %bb.0:734; CHECK-NEXT: ret735 %r = tail call <4 x i32> @llvm.aarch64.neon.shadd.v4i32(<4 x i32> %x, <4 x i32> %x)736 ret <4 x i32> %r737}738 739define <8 x i8> @uhadd_v8i8(<8 x i8> %x) {740; CHECK-LABEL: uhadd_v8i8:741; CHECK: // %bb.0:742; CHECK-NEXT: ret743 %r = tail call <8 x i8> @llvm.aarch64.neon.uhadd.v8i8(<8 x i8> %x, <8 x i8> %x)744 ret <8 x i8> %r745}746 747define <4 x i16> @uhadd_v4i16(<4 x i16> %x) {748; CHECK-LABEL: uhadd_v4i16:749; CHECK: // %bb.0:750; CHECK-NEXT: ret751 %r = tail call <4 x i16> @llvm.aarch64.neon.uhadd.v4i16(<4 x i16> %x, <4 x i16> %x)752 ret <4 x i16> %r753}754 755define <2 x i32> @uhadd_v2i32(<2 x i32> %x) {756; CHECK-LABEL: uhadd_v2i32:757; CHECK: // %bb.0:758; CHECK-NEXT: ret759 %r = tail call <2 x i32> @llvm.aarch64.neon.uhadd.v2i32(<2 x i32> %x, <2 x i32> %x)760 ret <2 x i32> %r761}762 763define <16 x i8> @uhadd_v16i8(<16 x i8> %x) {764; CHECK-LABEL: uhadd_v16i8:765; CHECK: // %bb.0:766; CHECK-NEXT: ret767 %r = tail call <16 x i8> @llvm.aarch64.neon.uhadd.v16i8(<16 x i8> %x, <16 x i8> %x)768 ret <16 x i8> %r769}770 771define <8 x i16> @uhadd_v8i16(<8 x i16> %x) {772; CHECK-LABEL: uhadd_v8i16:773; CHECK: // %bb.0:774; CHECK-NEXT: ret775 %r = tail call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> %x, <8 x i16> %x)776 ret <8 x i16> %r777}778 779define <4 x i32> @uhadd_v4i32(<4 x i32> %x) {780; CHECK-LABEL: uhadd_v4i32:781; CHECK: // %bb.0:782; CHECK-NEXT: ret783 %r = tail call <4 x i32> @llvm.aarch64.neon.uhadd.v4i32(<4 x i32> %x, <4 x i32> %x)784 ret <4 x i32> %r785}786define <8 x i8> @srhadd_v8i8(<8 x i8> %x) {787; CHECK-LABEL: srhadd_v8i8:788; CHECK: // %bb.0:789; CHECK-NEXT: ret790 %r = tail call <8 x i8> @llvm.aarch64.neon.srhadd.v8i8(<8 x i8> %x, <8 x i8> %x)791 ret <8 x i8> %r792}793 794define <4 x i16> @srhadd_v4i16(<4 x i16> %x) {795; CHECK-LABEL: srhadd_v4i16:796; CHECK: // %bb.0:797; CHECK-NEXT: ret798 %r = tail call <4 x i16> @llvm.aarch64.neon.srhadd.v4i16(<4 x i16> %x, <4 x i16> %x)799 ret <4 x i16> %r800}801 802define <2 x i32> @srhadd_v2i32(<2 x i32> %x) {803; CHECK-LABEL: srhadd_v2i32:804; CHECK: // %bb.0:805; CHECK-NEXT: ret806 %r = tail call <2 x i32> @llvm.aarch64.neon.srhadd.v2i32(<2 x i32> %x, <2 x i32> %x)807 ret <2 x i32> %r808}809 810define <16 x i8> @srhadd_v16i8(<16 x i8> %x) {811; CHECK-LABEL: srhadd_v16i8:812; CHECK: // %bb.0:813; CHECK-NEXT: ret814 %r = tail call <16 x i8> @llvm.aarch64.neon.srhadd.v16i8(<16 x i8> %x, <16 x i8> %x)815 ret <16 x i8> %r816}817 818define <8 x i16> @srhadd_v8i16(<8 x i16> %x) {819; CHECK-LABEL: srhadd_v8i16:820; CHECK: // %bb.0:821; CHECK-NEXT: ret822 %r = tail call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> %x, <8 x i16> %x)823 ret <8 x i16> %r824}825 826define <4 x i32> @srhadd_v4i32(<4 x i32> %x) {827; CHECK-LABEL: srhadd_v4i32:828; CHECK: // %bb.0:829; CHECK-NEXT: ret830 %r = tail call <4 x i32> @llvm.aarch64.neon.srhadd.v4i32(<4 x i32> %x, <4 x i32> %x)831 ret <4 x i32> %r832}833 834define <8 x i8> @urhadd_v8i8(<8 x i8> %x) {835; CHECK-LABEL: urhadd_v8i8:836; CHECK: // %bb.0:837; CHECK-NEXT: ret838 %r = tail call <8 x i8> @llvm.aarch64.neon.urhadd.v8i8(<8 x i8> %x, <8 x i8> %x)839 ret <8 x i8> %r840}841 842define <4 x i16> @urhadd_v4i16(<4 x i16> %x) {843; CHECK-LABEL: urhadd_v4i16:844; CHECK: // %bb.0:845; CHECK-NEXT: ret846 %r = tail call <4 x i16> @llvm.aarch64.neon.urhadd.v4i16(<4 x i16> %x, <4 x i16> %x)847 ret <4 x i16> %r848}849 850define <2 x i32> @urhadd_v2i32(<2 x i32> %x) {851; CHECK-LABEL: urhadd_v2i32:852; CHECK: // %bb.0:853; CHECK-NEXT: ret854 %r = tail call <2 x i32> @llvm.aarch64.neon.urhadd.v2i32(<2 x i32> %x, <2 x i32> %x)855 ret <2 x i32> %r856}857 858define <16 x i8> @urhadd_v16i8(<16 x i8> %x) {859; CHECK-LABEL: urhadd_v16i8:860; CHECK: // %bb.0:861; CHECK-NEXT: ret862 %r = tail call <16 x i8> @llvm.aarch64.neon.urhadd.v16i8(<16 x i8> %x, <16 x i8> %x)863 ret <16 x i8> %r864}865 866define <8 x i16> @urhadd_v8i16(<8 x i16> %x) {867; CHECK-LABEL: urhadd_v8i16:868; CHECK: // %bb.0:869; CHECK-NEXT: ret870 %r = tail call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> %x, <8 x i16> %x)871 ret <8 x i16> %r872}873 874define <4 x i32> @urhadd_v4i32(<4 x i32> %x) {875; CHECK-LABEL: urhadd_v4i32:876; CHECK: // %bb.0:877; CHECK-NEXT: ret878 %r = tail call <4 x i32> @llvm.aarch64.neon.urhadd.v4i32(<4 x i32> %x, <4 x i32> %x)879 ret <4 x i32> %r880}881 882define <8 x i16> @uhadd_fixedwidth_v4i32(<8 x i16> %a0, <8 x i16> %a1) {883; CHECK-LABEL: uhadd_fixedwidth_v4i32:884; CHECK: // %bb.0:885; CHECK-NEXT: uhadd v0.8h, v0.8h, v1.8h886; CHECK-NEXT: ret887 %and = and <8 x i16> %a0, %a1888 %xor = xor <8 x i16> %a0, %a1889 %srl = lshr <8 x i16> %xor, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>890 %res = add <8 x i16> %and, %srl891 ret <8 x i16> %res892}893 894define <8 x i16> @shadd_fixedwidth_v8i16(<8 x i16> %a0, <8 x i16> %a1) {895; CHECK-LABEL: shadd_fixedwidth_v8i16:896; CHECK: // %bb.0:897; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h898; CHECK-NEXT: ret899 %and = and <8 x i16> %a0, %a1900 %xor = xor <8 x i16> %a0, %a1901 %srl = ashr <8 x i16> %xor, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>902 %res = add <8 x i16> %and, %srl903 ret <8 x i16> %res904}905 906define <8 x i16> @shadd_demandedelts(<8 x i16> %a0, <8 x i16> %a1) {907; CHECK-LABEL: shadd_demandedelts:908; CHECK: // %bb.0:909; CHECK-NEXT: dup v0.8h, v0.h[0]910; CHECK-NEXT: shadd v0.8h, v0.8h, v1.8h911; CHECK-NEXT: dup v0.8h, v0.h[0]912; CHECK-NEXT: ret913 %s0 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> zeroinitializer914 %op = call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> %s0, <8 x i16> %a1)915 %r0 = shufflevector <8 x i16> %op, <8 x i16> undef, <8 x i32> zeroinitializer916 ret <8 x i16> %r0917}918 919define <8 x i16> @srhadd_demandedelts(<8 x i16> %a0, <8 x i16> %a1) {920; CHECK-LABEL: srhadd_demandedelts:921; CHECK: // %bb.0:922; CHECK-NEXT: dup v0.8h, v0.h[0]923; CHECK-NEXT: srhadd v0.8h, v0.8h, v1.8h924; CHECK-NEXT: dup v0.8h, v0.h[0]925; CHECK-NEXT: ret926 %s0 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> zeroinitializer927 %op = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> %s0, <8 x i16> %a1)928 %r0 = shufflevector <8 x i16> %op, <8 x i16> undef, <8 x i32> zeroinitializer929 ret <8 x i16> %r0930}931 932define <8 x i16> @uhadd_demandedelts(<8 x i16> %a0, <8 x i16> %a1) {933; CHECK-LABEL: uhadd_demandedelts:934; CHECK: // %bb.0:935; CHECK-NEXT: dup v0.8h, v0.h[0]936; CHECK-NEXT: uhadd v0.8h, v0.8h, v1.8h937; CHECK-NEXT: dup v0.8h, v0.h[0]938; CHECK-NEXT: ret939 %s0 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> zeroinitializer940 %op = call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> %s0, <8 x i16> %a1)941 %r0 = shufflevector <8 x i16> %op, <8 x i16> undef, <8 x i32> zeroinitializer942 ret <8 x i16> %r0943}944 945define <8 x i16> @urhadd_demandedelts(<8 x i16> %a0, <8 x i16> %a1) {946; CHECK-LABEL: urhadd_demandedelts:947; CHECK: // %bb.0:948; CHECK-NEXT: dup v0.8h, v0.h[0]949; CHECK-NEXT: urhadd v0.8h, v0.8h, v1.8h950; CHECK-NEXT: dup v0.8h, v0.h[0]951; CHECK-NEXT: ret952 %s0 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> zeroinitializer953 %op = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> %s0, <8 x i16> %a1)954 %r0 = shufflevector <8 x i16> %op, <8 x i16> undef, <8 x i32> zeroinitializer955 ret <8 x i16> %r0956}957 958; Remove unnecessary sign_extend_inreg after shadd959define <2 x i32> @shadd_signbits_v2i32(<2 x i32> %a0, <2 x i32> %a1, ptr %p2) {960; CHECK-LABEL: shadd_signbits_v2i32:961; CHECK: // %bb.0:962; CHECK-NEXT: sshr v0.2s, v0.2s, #17963; CHECK-NEXT: sshr v1.2s, v1.2s, #17964; CHECK-NEXT: shadd v0.2s, v0.2s, v1.2s965; CHECK-NEXT: str d0, [x0]966; CHECK-NEXT: ret967 %x0 = ashr <2 x i32> %a0, <i32 17, i32 17>968 %x1 = ashr <2 x i32> %a1, <i32 17, i32 17>969 %m = and <2 x i32> %x0, %x1970 %s = xor <2 x i32> %x0, %x1971 %x = ashr <2 x i32> %s, <i32 1, i32 1>972 %avg = add <2 x i32> %m, %x973 %avg1 = shl <2 x i32> %avg, <i32 17, i32 17>974 %avg2 = ashr <2 x i32> %avg1, <i32 17, i32 17>975 store <2 x i32> %avg, ptr %p2 ; extra use976 ret <2 x i32> %avg2977}978 979; Remove unnecessary sign_extend_inreg after srhadd980define <2 x i32> @srhadd_signbits_v2i32(<2 x i32> %a0, <2 x i32> %a1, ptr %p2) {981; CHECK-LABEL: srhadd_signbits_v2i32:982; CHECK: // %bb.0:983; CHECK-NEXT: sshr v0.2s, v0.2s, #17984; CHECK-NEXT: sshr v1.2s, v1.2s, #17985; CHECK-NEXT: srhadd v0.2s, v0.2s, v1.2s986; CHECK-NEXT: str d0, [x0]987; CHECK-NEXT: ret988 %x0 = ashr <2 x i32> %a0, <i32 17, i32 17>989 %x1 = ashr <2 x i32> %a1, <i32 17, i32 17>990 %m = or <2 x i32> %x0, %x1991 %s = xor <2 x i32> %x0, %x1992 %x = ashr <2 x i32> %s, <i32 1, i32 1>993 %avg = sub <2 x i32> %m, %x994 %avg1 = shl <2 x i32> %avg, <i32 17, i32 17>995 %avg2 = ashr <2 x i32> %avg1, <i32 17, i32 17>996 store <2 x i32> %avg, ptr %p2 ; extra use997 ret <2 x i32> %avg2998}999 1000; negative test - not enough signbits to remove sign_extend_inreg after srhadd1001define <2 x i32> @srhadd_signbits_v2i32_negative(<2 x i32> %a0, <2 x i32> %a1, ptr %p2) {1002; CHECK-LABEL: srhadd_signbits_v2i32_negative:1003; CHECK: // %bb.0:1004; CHECK-NEXT: sshr v0.2s, v0.2s, #171005; CHECK-NEXT: sshr v1.2s, v1.2s, #171006; CHECK-NEXT: srhadd v1.2s, v0.2s, v1.2s1007; CHECK-NEXT: shl v0.2s, v1.2s, #221008; CHECK-NEXT: str d1, [x0]1009; CHECK-NEXT: sshr v0.2s, v0.2s, #221010; CHECK-NEXT: ret1011 %x0 = ashr <2 x i32> %a0, <i32 17, i32 17>1012 %x1 = ashr <2 x i32> %a1, <i32 17, i32 17>1013 %m = or <2 x i32> %x0, %x11014 %s = xor <2 x i32> %x0, %x11015 %x = ashr <2 x i32> %s, <i32 1, i32 1>1016 %avg = sub <2 x i32> %m, %x1017 %avg1 = shl <2 x i32> %avg, <i32 22, i32 22>1018 %avg2 = ashr <2 x i32> %avg1, <i32 22, i32 22>1019 store <2 x i32> %avg, ptr %p2 ; extra use1020 ret <2 x i32> %avg21021}1022 1023define <8 x i8> @dontcrashonnvcasts() {1024; CHECK-LABEL: dontcrashonnvcasts:1025; CHECK: // %bb.0:1026; CHECK-NEXT: movi v0.2s, #128, lsl #241027; CHECK-NEXT: movi v1.2d, #00000000000000001028; CHECK-NEXT: fneg d0, d01029; CHECK-NEXT: uzp1 v0.8b, v0.8b, v1.8b1030; CHECK-NEXT: ret1031 %vrhadd_v.i = tail call <8 x i8> @llvm.aarch64.neon.urhadd.v8i8(<8 x i8> <i8 0, i8 0, i8 0, i8 -1, i8 0, i8 0, i8 0, i8 0>, <8 x i8> zeroinitializer)1032 %shuffle.i = shufflevector <8 x i8> %vrhadd_v.i, <8 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 0, i8 poison, i8 0, i8 poison>, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>1033 ret <8 x i8> %shuffle.i1034}1035 1036declare <8 x i8> @llvm.aarch64.neon.shadd.v8i8(<8 x i8>, <8 x i8>)1037declare <4 x i16> @llvm.aarch64.neon.shadd.v4i16(<4 x i16>, <4 x i16>)1038declare <2 x i32> @llvm.aarch64.neon.shadd.v2i32(<2 x i32>, <2 x i32>)1039declare <8 x i8> @llvm.aarch64.neon.uhadd.v8i8(<8 x i8>, <8 x i8>)1040declare <4 x i16> @llvm.aarch64.neon.uhadd.v4i16(<4 x i16>, <4 x i16>)1041declare <2 x i32> @llvm.aarch64.neon.uhadd.v2i32(<2 x i32>, <2 x i32>)1042declare <16 x i8> @llvm.aarch64.neon.shadd.v16i8(<16 x i8>, <16 x i8>)1043declare <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16>, <8 x i16>)1044declare <4 x i32> @llvm.aarch64.neon.shadd.v4i32(<4 x i32>, <4 x i32>)1045declare <16 x i8> @llvm.aarch64.neon.uhadd.v16i8(<16 x i8>, <16 x i8>)1046declare <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16>, <8 x i16>)1047declare <4 x i32> @llvm.aarch64.neon.uhadd.v4i32(<4 x i32>, <4 x i32>)1048 1049declare <8 x i8> @llvm.aarch64.neon.srhadd.v8i8(<8 x i8>, <8 x i8>)1050declare <4 x i16> @llvm.aarch64.neon.srhadd.v4i16(<4 x i16>, <4 x i16>)1051declare <2 x i32> @llvm.aarch64.neon.srhadd.v2i32(<2 x i32>, <2 x i32>)1052declare <8 x i8> @llvm.aarch64.neon.urhadd.v8i8(<8 x i8>, <8 x i8>)1053declare <4 x i16> @llvm.aarch64.neon.urhadd.v4i16(<4 x i16>, <4 x i16>)1054declare <2 x i32> @llvm.aarch64.neon.urhadd.v2i32(<2 x i32>, <2 x i32>)1055declare <16 x i8> @llvm.aarch64.neon.srhadd.v16i8(<16 x i8>, <16 x i8>)1056declare <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16>, <8 x i16>)1057declare <4 x i32> @llvm.aarch64.neon.srhadd.v4i32(<4 x i32>, <4 x i32>)1058declare <16 x i8> @llvm.aarch64.neon.urhadd.v16i8(<16 x i8>, <16 x i8>)1059declare <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16>, <8 x i16>)1060declare <4 x i32> @llvm.aarch64.neon.urhadd.v4i32(<4 x i32>, <4 x i32>)1061