1405 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+sve | FileCheck %s -check-prefixes=CHECK,SVE3; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+sve2 | FileCheck %s -check-prefixes=CHECK,SVE24 5define <vscale x 2 x i64> @hadds_v2i64(<vscale x 2 x i64> %s0, <vscale x 2 x i64> %s1) {6; SVE-LABEL: hadds_v2i64:7; SVE: // %bb.0: // %entry8; SVE-NEXT: eor z2.d, z0.d, z1.d9; SVE-NEXT: and z0.d, z0.d, z1.d10; SVE-NEXT: asr z1.d, z2.d, #111; SVE-NEXT: add z0.d, z0.d, z1.d12; SVE-NEXT: ret13;14; SVE2-LABEL: hadds_v2i64:15; SVE2: // %bb.0: // %entry16; SVE2-NEXT: ptrue p0.d17; SVE2-NEXT: shadd z0.d, p0/m, z0.d, z1.d18; SVE2-NEXT: ret19entry:20 %s0s = sext <vscale x 2 x i64> %s0 to <vscale x 2 x i128>21 %s1s = sext <vscale x 2 x i64> %s1 to <vscale x 2 x i128>22 %m = add nsw <vscale x 2 x i128> %s0s, %s1s23 %s = ashr <vscale x 2 x i128> %m, splat (i128 1)24 %s2 = trunc <vscale x 2 x i128> %s to <vscale x 2 x i64>25 ret <vscale x 2 x i64> %s226}27 28define <vscale x 2 x i64> @hadds_v2i64_lsh(<vscale x 2 x i64> %s0, <vscale x 2 x i64> %s1) {29; SVE-LABEL: hadds_v2i64_lsh:30; SVE: // %bb.0: // %entry31; SVE-NEXT: eor z2.d, z0.d, z1.d32; SVE-NEXT: and z0.d, z0.d, z1.d33; SVE-NEXT: asr z1.d, z2.d, #134; SVE-NEXT: add z0.d, z0.d, z1.d35; SVE-NEXT: ret36;37; SVE2-LABEL: hadds_v2i64_lsh:38; SVE2: // %bb.0: // %entry39; SVE2-NEXT: ptrue p0.d40; SVE2-NEXT: shadd z0.d, p0/m, z0.d, z1.d41; SVE2-NEXT: ret42entry:43 %s0s = sext <vscale x 2 x i64> %s0 to <vscale x 2 x i128>44 %s1s = sext <vscale x 2 x i64> %s1 to <vscale x 2 x i128>45 %m = add nsw <vscale x 2 x i128> %s0s, %s1s46 %s = lshr <vscale x 2 x i128> %m, splat (i128 1)47 %s2 = trunc <vscale x 2 x i128> %s to <vscale x 2 x i64>48 ret <vscale x 2 x i64> %s249}50 51define <vscale x 2 x i64> @haddu_v2i64(<vscale x 2 x i64> %s0, <vscale x 2 x i64> %s1) {52; SVE-LABEL: haddu_v2i64:53; SVE: // %bb.0: // %entry54; SVE-NEXT: eor z2.d, z0.d, z1.d55; SVE-NEXT: and z0.d, z0.d, z1.d56; SVE-NEXT: lsr z1.d, z2.d, #157; SVE-NEXT: add z0.d, z0.d, z1.d58; SVE-NEXT: ret59;60; SVE2-LABEL: haddu_v2i64:61; SVE2: // %bb.0: // %entry62; SVE2-NEXT: ptrue p0.d63; SVE2-NEXT: uhadd z0.d, p0/m, z0.d, z1.d64; SVE2-NEXT: ret65entry:66 %s0s = zext <vscale x 2 x i64> %s0 to <vscale x 2 x i128>67 %s1s = zext <vscale x 2 x i64> %s1 to <vscale x 2 x i128>68 %m = add nuw nsw <vscale x 2 x i128> %s0s, %s1s69 %s = lshr <vscale x 2 x i128> %m, splat (i128 1)70 %s2 = trunc <vscale x 2 x i128> %s to <vscale x 2 x i64>71 ret <vscale x 2 x i64> %s272}73 74define <vscale x 2 x i32> @hadds_v2i32(<vscale x 2 x i32> %s0, <vscale x 2 x i32> %s1) {75; SVE-LABEL: hadds_v2i32:76; SVE: // %bb.0: // %entry77; SVE-NEXT: ptrue p0.d78; SVE-NEXT: sxtw z0.d, p0/m, z0.d79; SVE-NEXT: adr z0.d, [z0.d, z1.d, sxtw]80; SVE-NEXT: asr z0.d, z0.d, #181; SVE-NEXT: ret82;83; SVE2-LABEL: hadds_v2i32:84; SVE2: // %bb.0: // %entry85; SVE2-NEXT: ptrue p0.d86; SVE2-NEXT: sxtw z1.d, p0/m, z1.d87; SVE2-NEXT: sxtw z0.d, p0/m, z0.d88; SVE2-NEXT: shadd z0.d, p0/m, z0.d, z1.d89; SVE2-NEXT: ret90entry:91 %s0s = sext <vscale x 2 x i32> %s0 to <vscale x 2 x i64>92 %s1s = sext <vscale x 2 x i32> %s1 to <vscale x 2 x i64>93 %m = add nsw <vscale x 2 x i64> %s0s, %s1s94 %s = ashr <vscale x 2 x i64> %m, splat (i64 1)95 %s2 = trunc <vscale x 2 x i64> %s to <vscale x 2 x i32>96 ret <vscale x 2 x i32> %s297}98 99define <vscale x 2 x i32> @hadds_v2i32_lsh(<vscale x 2 x i32> %s0, <vscale x 2 x i32> %s1) {100; CHECK-LABEL: hadds_v2i32_lsh:101; CHECK: // %bb.0: // %entry102; CHECK-NEXT: ptrue p0.d103; CHECK-NEXT: sxtw z0.d, p0/m, z0.d104; CHECK-NEXT: adr z0.d, [z0.d, z1.d, sxtw]105; CHECK-NEXT: lsr z0.d, z0.d, #1106; CHECK-NEXT: ret107entry:108 %s0s = sext <vscale x 2 x i32> %s0 to <vscale x 2 x i64>109 %s1s = sext <vscale x 2 x i32> %s1 to <vscale x 2 x i64>110 %m = add nsw <vscale x 2 x i64> %s0s, %s1s111 %s = lshr <vscale x 2 x i64> %m, splat (i64 1)112 %s2 = trunc <vscale x 2 x i64> %s to <vscale x 2 x i32>113 ret <vscale x 2 x i32> %s2114}115 116define <vscale x 2 x i32> @haddu_v2i32(<vscale x 2 x i32> %s0, <vscale x 2 x i32> %s1) {117; SVE-LABEL: haddu_v2i32:118; SVE: // %bb.0: // %entry119; SVE-NEXT: and z0.d, z0.d, #0xffffffff120; SVE-NEXT: adr z0.d, [z0.d, z1.d, uxtw]121; SVE-NEXT: lsr z0.d, z0.d, #1122; SVE-NEXT: ret123;124; SVE2-LABEL: haddu_v2i32:125; SVE2: // %bb.0: // %entry126; SVE2-NEXT: and z1.d, z1.d, #0xffffffff127; SVE2-NEXT: and z0.d, z0.d, #0xffffffff128; SVE2-NEXT: ptrue p0.d129; SVE2-NEXT: uhadd z0.d, p0/m, z0.d, z1.d130; SVE2-NEXT: ret131entry:132 %s0s = zext <vscale x 2 x i32> %s0 to <vscale x 2 x i64>133 %s1s = zext <vscale x 2 x i32> %s1 to <vscale x 2 x i64>134 %m = add nuw nsw <vscale x 2 x i64> %s0s, %s1s135 %s = lshr <vscale x 2 x i64> %m, splat (i64 1)136 %s2 = trunc <vscale x 2 x i64> %s to <vscale x 2 x i32>137 ret <vscale x 2 x i32> %s2138}139 140define <vscale x 4 x i32> @hadds_v4i32(<vscale x 4 x i32> %s0, <vscale x 4 x i32> %s1) {141; SVE-LABEL: hadds_v4i32:142; SVE: // %bb.0: // %entry143; SVE-NEXT: eor z2.d, z0.d, z1.d144; SVE-NEXT: and z0.d, z0.d, z1.d145; SVE-NEXT: asr z1.s, z2.s, #1146; SVE-NEXT: add z0.s, z0.s, z1.s147; SVE-NEXT: ret148;149; SVE2-LABEL: hadds_v4i32:150; SVE2: // %bb.0: // %entry151; SVE2-NEXT: ptrue p0.s152; SVE2-NEXT: shadd z0.s, p0/m, z0.s, z1.s153; SVE2-NEXT: ret154entry:155 %s0s = sext <vscale x 4 x i32> %s0 to <vscale x 4 x i64>156 %s1s = sext <vscale x 4 x i32> %s1 to <vscale x 4 x i64>157 %m = add nsw <vscale x 4 x i64> %s0s, %s1s158 %s = ashr <vscale x 4 x i64> %m, splat (i64 1)159 %s2 = trunc <vscale x 4 x i64> %s to <vscale x 4 x i32>160 ret <vscale x 4 x i32> %s2161}162 163define <vscale x 4 x i32> @hadds_v4i32_lsh(<vscale x 4 x i32> %s0, <vscale x 4 x i32> %s1) {164; SVE-LABEL: hadds_v4i32_lsh:165; SVE: // %bb.0: // %entry166; SVE-NEXT: eor z2.d, z0.d, z1.d167; SVE-NEXT: and z0.d, z0.d, z1.d168; SVE-NEXT: asr z1.s, z2.s, #1169; SVE-NEXT: add z0.s, z0.s, z1.s170; SVE-NEXT: ret171;172; SVE2-LABEL: hadds_v4i32_lsh:173; SVE2: // %bb.0: // %entry174; SVE2-NEXT: ptrue p0.s175; SVE2-NEXT: shadd z0.s, p0/m, z0.s, z1.s176; SVE2-NEXT: ret177entry:178 %s0s = sext <vscale x 4 x i32> %s0 to <vscale x 4 x i64>179 %s1s = sext <vscale x 4 x i32> %s1 to <vscale x 4 x i64>180 %m = add nsw <vscale x 4 x i64> %s0s, %s1s181 %s = lshr <vscale x 4 x i64> %m, splat (i64 1)182 %s2 = trunc <vscale x 4 x i64> %s to <vscale x 4 x i32>183 ret <vscale x 4 x i32> %s2184}185 186define <vscale x 4 x i32> @haddu_v4i32(<vscale x 4 x i32> %s0, <vscale x 4 x i32> %s1) {187; SVE-LABEL: haddu_v4i32:188; SVE: // %bb.0: // %entry189; SVE-NEXT: eor z2.d, z0.d, z1.d190; SVE-NEXT: and z0.d, z0.d, z1.d191; SVE-NEXT: lsr z1.s, z2.s, #1192; SVE-NEXT: add z0.s, z0.s, z1.s193; SVE-NEXT: ret194;195; SVE2-LABEL: haddu_v4i32:196; SVE2: // %bb.0: // %entry197; SVE2-NEXT: ptrue p0.s198; SVE2-NEXT: uhadd z0.s, p0/m, z0.s, z1.s199; SVE2-NEXT: ret200entry:201 %s0s = zext <vscale x 4 x i32> %s0 to <vscale x 4 x i64>202 %s1s = zext <vscale x 4 x i32> %s1 to <vscale x 4 x i64>203 %m = add nuw nsw <vscale x 4 x i64> %s0s, %s1s204 %s = lshr <vscale x 4 x i64> %m, splat (i64 1)205 %s2 = trunc <vscale x 4 x i64> %s to <vscale x 4 x i32>206 ret <vscale x 4 x i32> %s2207}208 209define <vscale x 2 x i16> @hadds_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {210; SVE-LABEL: hadds_v2i16:211; SVE: // %bb.0: // %entry212; SVE-NEXT: ptrue p0.d213; SVE-NEXT: sxth z1.d, p0/m, z1.d214; SVE-NEXT: sxth z0.d, p0/m, z0.d215; SVE-NEXT: add z0.d, z0.d, z1.d216; SVE-NEXT: asr z0.d, z0.d, #1217; SVE-NEXT: ret218;219; SVE2-LABEL: hadds_v2i16:220; SVE2: // %bb.0: // %entry221; SVE2-NEXT: ptrue p0.d222; SVE2-NEXT: sxth z1.d, p0/m, z1.d223; SVE2-NEXT: sxth z0.d, p0/m, z0.d224; SVE2-NEXT: shadd z0.d, p0/m, z0.d, z1.d225; SVE2-NEXT: ret226entry:227 %s0s = sext <vscale x 2 x i16> %s0 to <vscale x 2 x i32>228 %s1s = sext <vscale x 2 x i16> %s1 to <vscale x 2 x i32>229 %m = add nsw <vscale x 2 x i32> %s0s, %s1s230 %s = ashr <vscale x 2 x i32> %m, splat (i32 1)231 %s2 = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>232 ret <vscale x 2 x i16> %s2233}234 235define <vscale x 2 x i16> @hadds_v2i16_lsh(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {236; CHECK-LABEL: hadds_v2i16_lsh:237; CHECK: // %bb.0: // %entry238; CHECK-NEXT: ptrue p0.d239; CHECK-NEXT: sxth z0.d, p0/m, z0.d240; CHECK-NEXT: sxth z1.d, p0/m, z1.d241; CHECK-NEXT: add z0.d, z0.d, z1.d242; CHECK-NEXT: and z0.d, z0.d, #0xffffffff243; CHECK-NEXT: lsr z0.d, z0.d, #1244; CHECK-NEXT: ret245entry:246 %s0s = sext <vscale x 2 x i16> %s0 to <vscale x 2 x i32>247 %s1s = sext <vscale x 2 x i16> %s1 to <vscale x 2 x i32>248 %m = add nsw <vscale x 2 x i32> %s0s, %s1s249 %s = lshr <vscale x 2 x i32> %m, splat (i32 1)250 %s2 = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>251 ret <vscale x 2 x i16> %s2252}253 254define <vscale x 2 x i16> @haddu_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {255; SVE-LABEL: haddu_v2i16:256; SVE: // %bb.0: // %entry257; SVE-NEXT: and z1.d, z1.d, #0xffff258; SVE-NEXT: and z0.d, z0.d, #0xffff259; SVE-NEXT: add z0.d, z0.d, z1.d260; SVE-NEXT: lsr z0.d, z0.d, #1261; SVE-NEXT: ret262;263; SVE2-LABEL: haddu_v2i16:264; SVE2: // %bb.0: // %entry265; SVE2-NEXT: and z1.d, z1.d, #0xffff266; SVE2-NEXT: and z0.d, z0.d, #0xffff267; SVE2-NEXT: ptrue p0.d268; SVE2-NEXT: uhadd z0.d, p0/m, z0.d, z1.d269; SVE2-NEXT: ret270entry:271 %s0s = zext <vscale x 2 x i16> %s0 to <vscale x 2 x i32>272 %s1s = zext <vscale x 2 x i16> %s1 to <vscale x 2 x i32>273 %m = add nuw nsw <vscale x 2 x i32> %s0s, %s1s274 %s = lshr <vscale x 2 x i32> %m, splat (i32 1)275 %s2 = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>276 ret <vscale x 2 x i16> %s2277}278 279define <vscale x 4 x i16> @hadds_v4i16(<vscale x 4 x i16> %s0, <vscale x 4 x i16> %s1) {280; SVE-LABEL: hadds_v4i16:281; SVE: // %bb.0: // %entry282; SVE-NEXT: ptrue p0.s283; SVE-NEXT: sxth z1.s, p0/m, z1.s284; SVE-NEXT: sxth z0.s, p0/m, z0.s285; SVE-NEXT: add z0.s, z0.s, z1.s286; SVE-NEXT: asr z0.s, z0.s, #1287; SVE-NEXT: ret288;289; SVE2-LABEL: hadds_v4i16:290; SVE2: // %bb.0: // %entry291; SVE2-NEXT: ptrue p0.s292; SVE2-NEXT: sxth z1.s, p0/m, z1.s293; SVE2-NEXT: sxth z0.s, p0/m, z0.s294; SVE2-NEXT: shadd z0.s, p0/m, z0.s, z1.s295; SVE2-NEXT: ret296entry:297 %s0s = sext <vscale x 4 x i16> %s0 to <vscale x 4 x i32>298 %s1s = sext <vscale x 4 x i16> %s1 to <vscale x 4 x i32>299 %m = add nsw <vscale x 4 x i32> %s0s, %s1s300 %s = ashr <vscale x 4 x i32> %m, splat (i32 1)301 %s2 = trunc <vscale x 4 x i32> %s to <vscale x 4 x i16>302 ret <vscale x 4 x i16> %s2303}304 305define <vscale x 4 x i16> @hadds_v4i16_lsh(<vscale x 4 x i16> %s0, <vscale x 4 x i16> %s1) {306; CHECK-LABEL: hadds_v4i16_lsh:307; CHECK: // %bb.0: // %entry308; CHECK-NEXT: ptrue p0.s309; CHECK-NEXT: sxth z0.s, p0/m, z0.s310; CHECK-NEXT: sxth z1.s, p0/m, z1.s311; CHECK-NEXT: add z0.s, z0.s, z1.s312; CHECK-NEXT: lsr z0.s, z0.s, #1313; CHECK-NEXT: ret314entry:315 %s0s = sext <vscale x 4 x i16> %s0 to <vscale x 4 x i32>316 %s1s = sext <vscale x 4 x i16> %s1 to <vscale x 4 x i32>317 %m = add nsw <vscale x 4 x i32> %s0s, %s1s318 %s = lshr <vscale x 4 x i32> %m, splat (i32 1)319 %s2 = trunc <vscale x 4 x i32> %s to <vscale x 4 x i16>320 ret <vscale x 4 x i16> %s2321}322 323define <vscale x 4 x i16> @haddu_v4i16(<vscale x 4 x i16> %s0, <vscale x 4 x i16> %s1) {324; SVE-LABEL: haddu_v4i16:325; SVE: // %bb.0: // %entry326; SVE-NEXT: and z1.s, z1.s, #0xffff327; SVE-NEXT: and z0.s, z0.s, #0xffff328; SVE-NEXT: add z0.s, z0.s, z1.s329; SVE-NEXT: lsr z0.s, z0.s, #1330; SVE-NEXT: ret331;332; SVE2-LABEL: haddu_v4i16:333; SVE2: // %bb.0: // %entry334; SVE2-NEXT: and z1.s, z1.s, #0xffff335; SVE2-NEXT: and z0.s, z0.s, #0xffff336; SVE2-NEXT: ptrue p0.s337; SVE2-NEXT: uhadd z0.s, p0/m, z0.s, z1.s338; SVE2-NEXT: ret339entry:340 %s0s = zext <vscale x 4 x i16> %s0 to <vscale x 4 x i32>341 %s1s = zext <vscale x 4 x i16> %s1 to <vscale x 4 x i32>342 %m = add nuw nsw <vscale x 4 x i32> %s0s, %s1s343 %s = lshr <vscale x 4 x i32> %m, splat (i32 1)344 %s2 = trunc <vscale x 4 x i32> %s to <vscale x 4 x i16>345 ret <vscale x 4 x i16> %s2346}347 348define <vscale x 8 x i16> @hadds_v8i16(<vscale x 8 x i16> %s0, <vscale x 8 x i16> %s1) {349; SVE-LABEL: hadds_v8i16:350; SVE: // %bb.0: // %entry351; SVE-NEXT: eor z2.d, z0.d, z1.d352; SVE-NEXT: and z0.d, z0.d, z1.d353; SVE-NEXT: asr z1.h, z2.h, #1354; SVE-NEXT: add z0.h, z0.h, z1.h355; SVE-NEXT: ret356;357; SVE2-LABEL: hadds_v8i16:358; SVE2: // %bb.0: // %entry359; SVE2-NEXT: ptrue p0.h360; SVE2-NEXT: shadd z0.h, p0/m, z0.h, z1.h361; SVE2-NEXT: ret362entry:363 %s0s = sext <vscale x 8 x i16> %s0 to <vscale x 8 x i32>364 %s1s = sext <vscale x 8 x i16> %s1 to <vscale x 8 x i32>365 %m = add nsw <vscale x 8 x i32> %s0s, %s1s366 %s = ashr <vscale x 8 x i32> %m, splat (i32 1)367 %s2 = trunc <vscale x 8 x i32> %s to <vscale x 8 x i16>368 ret <vscale x 8 x i16> %s2369}370 371define <vscale x 8 x i16> @hadds_v8i16_lsh(<vscale x 8 x i16> %s0, <vscale x 8 x i16> %s1) {372; SVE-LABEL: hadds_v8i16_lsh:373; SVE: // %bb.0: // %entry374; SVE-NEXT: eor z2.d, z0.d, z1.d375; SVE-NEXT: and z0.d, z0.d, z1.d376; SVE-NEXT: asr z1.h, z2.h, #1377; SVE-NEXT: add z0.h, z0.h, z1.h378; SVE-NEXT: ret379;380; SVE2-LABEL: hadds_v8i16_lsh:381; SVE2: // %bb.0: // %entry382; SVE2-NEXT: ptrue p0.h383; SVE2-NEXT: shadd z0.h, p0/m, z0.h, z1.h384; SVE2-NEXT: ret385entry:386 %s0s = sext <vscale x 8 x i16> %s0 to <vscale x 8 x i32>387 %s1s = sext <vscale x 8 x i16> %s1 to <vscale x 8 x i32>388 %m = add nsw <vscale x 8 x i32> %s0s, %s1s389 %s = lshr <vscale x 8 x i32> %m, splat (i32 1)390 %s2 = trunc <vscale x 8 x i32> %s to <vscale x 8 x i16>391 ret <vscale x 8 x i16> %s2392}393 394define <vscale x 8 x i16> @haddu_v8i16(<vscale x 8 x i16> %s0, <vscale x 8 x i16> %s1) {395; SVE-LABEL: haddu_v8i16:396; SVE: // %bb.0: // %entry397; SVE-NEXT: eor z2.d, z0.d, z1.d398; SVE-NEXT: and z0.d, z0.d, z1.d399; SVE-NEXT: lsr z1.h, z2.h, #1400; SVE-NEXT: add z0.h, z0.h, z1.h401; SVE-NEXT: ret402;403; SVE2-LABEL: haddu_v8i16:404; SVE2: // %bb.0: // %entry405; SVE2-NEXT: ptrue p0.h406; SVE2-NEXT: uhadd z0.h, p0/m, z0.h, z1.h407; SVE2-NEXT: ret408entry:409 %s0s = zext <vscale x 8 x i16> %s0 to <vscale x 8 x i32>410 %s1s = zext <vscale x 8 x i16> %s1 to <vscale x 8 x i32>411 %m = add nuw nsw <vscale x 8 x i32> %s0s, %s1s412 %s = lshr <vscale x 8 x i32> %m, splat (i32 1)413 %s2 = trunc <vscale x 8 x i32> %s to <vscale x 8 x i16>414 ret <vscale x 8 x i16> %s2415}416 417define <vscale x 4 x i8> @hadds_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {418; SVE-LABEL: hadds_v4i8:419; SVE: // %bb.0: // %entry420; SVE-NEXT: ptrue p0.s421; SVE-NEXT: sxtb z1.s, p0/m, z1.s422; SVE-NEXT: sxtb z0.s, p0/m, z0.s423; SVE-NEXT: add z0.s, z0.s, z1.s424; SVE-NEXT: asr z0.s, z0.s, #1425; SVE-NEXT: ret426;427; SVE2-LABEL: hadds_v4i8:428; SVE2: // %bb.0: // %entry429; SVE2-NEXT: ptrue p0.s430; SVE2-NEXT: sxtb z1.s, p0/m, z1.s431; SVE2-NEXT: sxtb z0.s, p0/m, z0.s432; SVE2-NEXT: shadd z0.s, p0/m, z0.s, z1.s433; SVE2-NEXT: ret434entry:435 %s0s = sext <vscale x 4 x i8> %s0 to <vscale x 4 x i16>436 %s1s = sext <vscale x 4 x i8> %s1 to <vscale x 4 x i16>437 %m = add nsw <vscale x 4 x i16> %s0s, %s1s438 %s = ashr <vscale x 4 x i16> %m, splat (i16 1)439 %s2 = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>440 ret <vscale x 4 x i8> %s2441}442 443define <vscale x 4 x i8> @hadds_v4i8_lsh(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {444; CHECK-LABEL: hadds_v4i8_lsh:445; CHECK: // %bb.0: // %entry446; CHECK-NEXT: ptrue p0.s447; CHECK-NEXT: sxtb z0.s, p0/m, z0.s448; CHECK-NEXT: sxtb z1.s, p0/m, z1.s449; CHECK-NEXT: add z0.s, z0.s, z1.s450; CHECK-NEXT: and z0.s, z0.s, #0xffff451; CHECK-NEXT: lsr z0.s, z0.s, #1452; CHECK-NEXT: ret453entry:454 %s0s = sext <vscale x 4 x i8> %s0 to <vscale x 4 x i16>455 %s1s = sext <vscale x 4 x i8> %s1 to <vscale x 4 x i16>456 %m = add nsw <vscale x 4 x i16> %s0s, %s1s457 %s = lshr <vscale x 4 x i16> %m, splat (i16 1)458 %s2 = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>459 ret <vscale x 4 x i8> %s2460}461 462define <vscale x 4 x i8> @haddu_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {463; SVE-LABEL: haddu_v4i8:464; SVE: // %bb.0: // %entry465; SVE-NEXT: and z1.s, z1.s, #0xff466; SVE-NEXT: and z0.s, z0.s, #0xff467; SVE-NEXT: add z0.s, z0.s, z1.s468; SVE-NEXT: lsr z0.s, z0.s, #1469; SVE-NEXT: ret470;471; SVE2-LABEL: haddu_v4i8:472; SVE2: // %bb.0: // %entry473; SVE2-NEXT: and z1.s, z1.s, #0xff474; SVE2-NEXT: and z0.s, z0.s, #0xff475; SVE2-NEXT: ptrue p0.s476; SVE2-NEXT: uhadd z0.s, p0/m, z0.s, z1.s477; SVE2-NEXT: ret478entry:479 %s0s = zext <vscale x 4 x i8> %s0 to <vscale x 4 x i16>480 %s1s = zext <vscale x 4 x i8> %s1 to <vscale x 4 x i16>481 %m = add nuw nsw <vscale x 4 x i16> %s0s, %s1s482 %s = lshr <vscale x 4 x i16> %m, splat (i16 1)483 %s2 = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>484 ret <vscale x 4 x i8> %s2485}486 487define <vscale x 8 x i8> @hadds_v8i8(<vscale x 8 x i8> %s0, <vscale x 8 x i8> %s1) {488; SVE-LABEL: hadds_v8i8:489; SVE: // %bb.0: // %entry490; SVE-NEXT: ptrue p0.h491; SVE-NEXT: sxtb z1.h, p0/m, z1.h492; SVE-NEXT: sxtb z0.h, p0/m, z0.h493; SVE-NEXT: add z0.h, z0.h, z1.h494; SVE-NEXT: asr z0.h, z0.h, #1495; SVE-NEXT: ret496;497; SVE2-LABEL: hadds_v8i8:498; SVE2: // %bb.0: // %entry499; SVE2-NEXT: ptrue p0.h500; SVE2-NEXT: sxtb z1.h, p0/m, z1.h501; SVE2-NEXT: sxtb z0.h, p0/m, z0.h502; SVE2-NEXT: shadd z0.h, p0/m, z0.h, z1.h503; SVE2-NEXT: ret504entry:505 %s0s = sext <vscale x 8 x i8> %s0 to <vscale x 8 x i16>506 %s1s = sext <vscale x 8 x i8> %s1 to <vscale x 8 x i16>507 %m = add nsw <vscale x 8 x i16> %s0s, %s1s508 %s = ashr <vscale x 8 x i16> %m, splat (i16 1)509 %s2 = trunc <vscale x 8 x i16> %s to <vscale x 8 x i8>510 ret <vscale x 8 x i8> %s2511}512 513define <vscale x 8 x i8> @hadds_v8i8_lsh(<vscale x 8 x i8> %s0, <vscale x 8 x i8> %s1) {514; CHECK-LABEL: hadds_v8i8_lsh:515; CHECK: // %bb.0: // %entry516; CHECK-NEXT: ptrue p0.h517; CHECK-NEXT: sxtb z0.h, p0/m, z0.h518; CHECK-NEXT: sxtb z1.h, p0/m, z1.h519; CHECK-NEXT: add z0.h, z0.h, z1.h520; CHECK-NEXT: lsr z0.h, z0.h, #1521; CHECK-NEXT: ret522entry:523 %s0s = sext <vscale x 8 x i8> %s0 to <vscale x 8 x i16>524 %s1s = sext <vscale x 8 x i8> %s1 to <vscale x 8 x i16>525 %m = add nsw <vscale x 8 x i16> %s0s, %s1s526 %s = lshr <vscale x 8 x i16> %m, splat (i16 1)527 %s2 = trunc <vscale x 8 x i16> %s to <vscale x 8 x i8>528 ret <vscale x 8 x i8> %s2529}530 531define <vscale x 8 x i8> @haddu_v8i8(<vscale x 8 x i8> %s0, <vscale x 8 x i8> %s1) {532; SVE-LABEL: haddu_v8i8:533; SVE: // %bb.0: // %entry534; SVE-NEXT: and z1.h, z1.h, #0xff535; SVE-NEXT: and z0.h, z0.h, #0xff536; SVE-NEXT: add z0.h, z0.h, z1.h537; SVE-NEXT: lsr z0.h, z0.h, #1538; SVE-NEXT: ret539;540; SVE2-LABEL: haddu_v8i8:541; SVE2: // %bb.0: // %entry542; SVE2-NEXT: and z1.h, z1.h, #0xff543; SVE2-NEXT: and z0.h, z0.h, #0xff544; SVE2-NEXT: ptrue p0.h545; SVE2-NEXT: uhadd z0.h, p0/m, z0.h, z1.h546; SVE2-NEXT: ret547entry:548 %s0s = zext <vscale x 8 x i8> %s0 to <vscale x 8 x i16>549 %s1s = zext <vscale x 8 x i8> %s1 to <vscale x 8 x i16>550 %m = add nuw nsw <vscale x 8 x i16> %s0s, %s1s551 %s = lshr <vscale x 8 x i16> %m, splat (i16 1)552 %s2 = trunc <vscale x 8 x i16> %s to <vscale x 8 x i8>553 ret <vscale x 8 x i8> %s2554}555 556define <vscale x 16 x i8> @hadds_v16i8(<vscale x 16 x i8> %s0, <vscale x 16 x i8> %s1) {557; SVE-LABEL: hadds_v16i8:558; SVE: // %bb.0: // %entry559; SVE-NEXT: eor z2.d, z0.d, z1.d560; SVE-NEXT: and z0.d, z0.d, z1.d561; SVE-NEXT: asr z1.b, z2.b, #1562; SVE-NEXT: add z0.b, z0.b, z1.b563; SVE-NEXT: ret564;565; SVE2-LABEL: hadds_v16i8:566; SVE2: // %bb.0: // %entry567; SVE2-NEXT: ptrue p0.b568; SVE2-NEXT: shadd z0.b, p0/m, z0.b, z1.b569; SVE2-NEXT: ret570entry:571 %s0s = sext <vscale x 16 x i8> %s0 to <vscale x 16 x i16>572 %s1s = sext <vscale x 16 x i8> %s1 to <vscale x 16 x i16>573 %m = add nsw <vscale x 16 x i16> %s0s, %s1s574 %s = ashr <vscale x 16 x i16> %m, splat (i16 1)575 %s2 = trunc <vscale x 16 x i16> %s to <vscale x 16 x i8>576 ret <vscale x 16 x i8> %s2577}578 579define <vscale x 16 x i8> @hadds_v16i8_lsh(<vscale x 16 x i8> %s0, <vscale x 16 x i8> %s1) {580; SVE-LABEL: hadds_v16i8_lsh:581; SVE: // %bb.0: // %entry582; SVE-NEXT: eor z2.d, z0.d, z1.d583; SVE-NEXT: and z0.d, z0.d, z1.d584; SVE-NEXT: asr z1.b, z2.b, #1585; SVE-NEXT: add z0.b, z0.b, z1.b586; SVE-NEXT: ret587;588; SVE2-LABEL: hadds_v16i8_lsh:589; SVE2: // %bb.0: // %entry590; SVE2-NEXT: ptrue p0.b591; SVE2-NEXT: shadd z0.b, p0/m, z0.b, z1.b592; SVE2-NEXT: ret593entry:594 %s0s = sext <vscale x 16 x i8> %s0 to <vscale x 16 x i16>595 %s1s = sext <vscale x 16 x i8> %s1 to <vscale x 16 x i16>596 %m = add nsw <vscale x 16 x i16> %s0s, %s1s597 %s = lshr <vscale x 16 x i16> %m, splat (i16 1)598 %s2 = trunc <vscale x 16 x i16> %s to <vscale x 16 x i8>599 ret <vscale x 16 x i8> %s2600}601 602define <vscale x 16 x i8> @haddu_v16i8(<vscale x 16 x i8> %s0, <vscale x 16 x i8> %s1) {603; SVE-LABEL: haddu_v16i8:604; SVE: // %bb.0: // %entry605; SVE-NEXT: eor z2.d, z0.d, z1.d606; SVE-NEXT: and z0.d, z0.d, z1.d607; SVE-NEXT: lsr z1.b, z2.b, #1608; SVE-NEXT: add z0.b, z0.b, z1.b609; SVE-NEXT: ret610;611; SVE2-LABEL: haddu_v16i8:612; SVE2: // %bb.0: // %entry613; SVE2-NEXT: ptrue p0.b614; SVE2-NEXT: uhadd z0.b, p0/m, z0.b, z1.b615; SVE2-NEXT: ret616entry:617 %s0s = zext <vscale x 16 x i8> %s0 to <vscale x 16 x i16>618 %s1s = zext <vscale x 16 x i8> %s1 to <vscale x 16 x i16>619 %m = add nuw nsw <vscale x 16 x i16> %s0s, %s1s620 %s = lshr <vscale x 16 x i16> %m, splat (i16 1)621 %s2 = trunc <vscale x 16 x i16> %s to <vscale x 16 x i8>622 ret <vscale x 16 x i8> %s2623}624 625define <vscale x 2 x i64> @rhadds_v2i64(<vscale x 2 x i64> %s0, <vscale x 2 x i64> %s1) {626; SVE-LABEL: rhadds_v2i64:627; SVE: // %bb.0: // %entry628; SVE-NEXT: eor z2.d, z0.d, z1.d629; SVE-NEXT: orr z0.d, z0.d, z1.d630; SVE-NEXT: asr z1.d, z2.d, #1631; SVE-NEXT: sub z0.d, z0.d, z1.d632; SVE-NEXT: ret633;634; SVE2-LABEL: rhadds_v2i64:635; SVE2: // %bb.0: // %entry636; SVE2-NEXT: ptrue p0.d637; SVE2-NEXT: srhadd z0.d, p0/m, z0.d, z1.d638; SVE2-NEXT: ret639entry:640 %s0s = sext <vscale x 2 x i64> %s0 to <vscale x 2 x i128>641 %s1s = sext <vscale x 2 x i64> %s1 to <vscale x 2 x i128>642 %add = add <vscale x 2 x i128> %s0s, splat (i128 1)643 %add2 = add <vscale x 2 x i128> %add, %s1s644 %s = ashr <vscale x 2 x i128> %add2, splat (i128 1)645 %result = trunc <vscale x 2 x i128> %s to <vscale x 2 x i64>646 ret <vscale x 2 x i64> %result647}648 649define <vscale x 2 x i64> @rhadds_v2i64_lsh(<vscale x 2 x i64> %s0, <vscale x 2 x i64> %s1) {650; SVE-LABEL: rhadds_v2i64_lsh:651; SVE: // %bb.0: // %entry652; SVE-NEXT: eor z2.d, z0.d, z1.d653; SVE-NEXT: orr z0.d, z0.d, z1.d654; SVE-NEXT: asr z1.d, z2.d, #1655; SVE-NEXT: sub z0.d, z0.d, z1.d656; SVE-NEXT: ret657;658; SVE2-LABEL: rhadds_v2i64_lsh:659; SVE2: // %bb.0: // %entry660; SVE2-NEXT: ptrue p0.d661; SVE2-NEXT: srhadd z0.d, p0/m, z0.d, z1.d662; SVE2-NEXT: ret663entry:664 %s0s = sext <vscale x 2 x i64> %s0 to <vscale x 2 x i128>665 %s1s = sext <vscale x 2 x i64> %s1 to <vscale x 2 x i128>666 %add = add <vscale x 2 x i128> %s0s, splat (i128 1)667 %add2 = add <vscale x 2 x i128> %add, %s1s668 %s = lshr <vscale x 2 x i128> %add2, splat (i128 1)669 %result = trunc <vscale x 2 x i128> %s to <vscale x 2 x i64>670 ret <vscale x 2 x i64> %result671}672 673define <vscale x 2 x i64> @rhaddu_v2i64(<vscale x 2 x i64> %s0, <vscale x 2 x i64> %s1) {674; SVE-LABEL: rhaddu_v2i64:675; SVE: // %bb.0: // %entry676; SVE-NEXT: eor z2.d, z0.d, z1.d677; SVE-NEXT: orr z0.d, z0.d, z1.d678; SVE-NEXT: lsr z1.d, z2.d, #1679; SVE-NEXT: sub z0.d, z0.d, z1.d680; SVE-NEXT: ret681;682; SVE2-LABEL: rhaddu_v2i64:683; SVE2: // %bb.0: // %entry684; SVE2-NEXT: ptrue p0.d685; SVE2-NEXT: urhadd z0.d, p0/m, z0.d, z1.d686; SVE2-NEXT: ret687entry:688 %s0s = zext <vscale x 2 x i64> %s0 to <vscale x 2 x i128>689 %s1s = zext <vscale x 2 x i64> %s1 to <vscale x 2 x i128>690 %add = add nuw nsw <vscale x 2 x i128> %s0s, splat (i128 1)691 %add2 = add nuw nsw <vscale x 2 x i128> %add, %s1s692 %s = lshr <vscale x 2 x i128> %add2, splat (i128 1)693 %result = trunc <vscale x 2 x i128> %s to <vscale x 2 x i64>694 ret <vscale x 2 x i64> %result695}696 697define <vscale x 2 x i32> @rhadds_v2i32(<vscale x 2 x i32> %s0, <vscale x 2 x i32> %s1) {698; SVE-LABEL: rhadds_v2i32:699; SVE: // %bb.0: // %entry700; SVE-NEXT: ptrue p0.d701; SVE-NEXT: mov z2.d, #-1 // =0xffffffffffffffff702; SVE-NEXT: sxtw z0.d, p0/m, z0.d703; SVE-NEXT: sxtw z1.d, p0/m, z1.d704; SVE-NEXT: eor z0.d, z0.d, z2.d705; SVE-NEXT: sub z0.d, z1.d, z0.d706; SVE-NEXT: asr z0.d, z0.d, #1707; SVE-NEXT: ret708;709; SVE2-LABEL: rhadds_v2i32:710; SVE2: // %bb.0: // %entry711; SVE2-NEXT: ptrue p0.d712; SVE2-NEXT: sxtw z1.d, p0/m, z1.d713; SVE2-NEXT: sxtw z0.d, p0/m, z0.d714; SVE2-NEXT: srhadd z0.d, p0/m, z0.d, z1.d715; SVE2-NEXT: ret716entry:717 %s0s = sext <vscale x 2 x i32> %s0 to <vscale x 2 x i64>718 %s1s = sext <vscale x 2 x i32> %s1 to <vscale x 2 x i64>719 %add = add <vscale x 2 x i64> %s0s, splat (i64 1)720 %add2 = add <vscale x 2 x i64> %add, %s1s721 %s = ashr <vscale x 2 x i64> %add2, splat (i64 1)722 %result = trunc <vscale x 2 x i64> %s to <vscale x 2 x i32>723 ret <vscale x 2 x i32> %result724}725 726define <vscale x 2 x i32> @rhadds_v2i32_lsh(<vscale x 2 x i32> %s0, <vscale x 2 x i32> %s1) {727; CHECK-LABEL: rhadds_v2i32_lsh:728; CHECK: // %bb.0: // %entry729; CHECK-NEXT: ptrue p0.d730; CHECK-NEXT: mov z2.d, #-1 // =0xffffffffffffffff731; CHECK-NEXT: sxtw z0.d, p0/m, z0.d732; CHECK-NEXT: sxtw z1.d, p0/m, z1.d733; CHECK-NEXT: eor z0.d, z0.d, z2.d734; CHECK-NEXT: sub z0.d, z1.d, z0.d735; CHECK-NEXT: lsr z0.d, z0.d, #1736; CHECK-NEXT: ret737entry:738 %s0s = sext <vscale x 2 x i32> %s0 to <vscale x 2 x i64>739 %s1s = sext <vscale x 2 x i32> %s1 to <vscale x 2 x i64>740 %add = add <vscale x 2 x i64> %s0s, splat (i64 1)741 %add2 = add <vscale x 2 x i64> %add, %s1s742 %s = lshr <vscale x 2 x i64> %add2, splat (i64 1)743 %result = trunc <vscale x 2 x i64> %s to <vscale x 2 x i32>744 ret <vscale x 2 x i32> %result745}746 747define <vscale x 2 x i32> @rhaddu_v2i32(<vscale x 2 x i32> %s0, <vscale x 2 x i32> %s1) {748; SVE-LABEL: rhaddu_v2i32:749; SVE: // %bb.0: // %entry750; SVE-NEXT: mov z2.d, #-1 // =0xffffffffffffffff751; SVE-NEXT: and z0.d, z0.d, #0xffffffff752; SVE-NEXT: and z1.d, z1.d, #0xffffffff753; SVE-NEXT: eor z0.d, z0.d, z2.d754; SVE-NEXT: sub z0.d, z1.d, z0.d755; SVE-NEXT: lsr z0.d, z0.d, #1756; SVE-NEXT: ret757;758; SVE2-LABEL: rhaddu_v2i32:759; SVE2: // %bb.0: // %entry760; SVE2-NEXT: and z1.d, z1.d, #0xffffffff761; SVE2-NEXT: and z0.d, z0.d, #0xffffffff762; SVE2-NEXT: ptrue p0.d763; SVE2-NEXT: urhadd z0.d, p0/m, z0.d, z1.d764; SVE2-NEXT: ret765entry:766 %s0s = zext <vscale x 2 x i32> %s0 to <vscale x 2 x i64>767 %s1s = zext <vscale x 2 x i32> %s1 to <vscale x 2 x i64>768 %add = add nuw nsw <vscale x 2 x i64> %s0s, splat (i64 1)769 %add2 = add nuw nsw <vscale x 2 x i64> %add, %s1s770 %s = lshr <vscale x 2 x i64> %add2, splat (i64 1)771 %result = trunc <vscale x 2 x i64> %s to <vscale x 2 x i32>772 ret <vscale x 2 x i32> %result773}774 775define <vscale x 4 x i32> @rhadds_v4i32(<vscale x 4 x i32> %s0, <vscale x 4 x i32> %s1) {776; SVE-LABEL: rhadds_v4i32:777; SVE: // %bb.0: // %entry778; SVE-NEXT: eor z2.d, z0.d, z1.d779; SVE-NEXT: orr z0.d, z0.d, z1.d780; SVE-NEXT: asr z1.s, z2.s, #1781; SVE-NEXT: sub z0.s, z0.s, z1.s782; SVE-NEXT: ret783;784; SVE2-LABEL: rhadds_v4i32:785; SVE2: // %bb.0: // %entry786; SVE2-NEXT: ptrue p0.s787; SVE2-NEXT: srhadd z0.s, p0/m, z0.s, z1.s788; SVE2-NEXT: ret789entry:790 %s0s = sext <vscale x 4 x i32> %s0 to <vscale x 4 x i64>791 %s1s = sext <vscale x 4 x i32> %s1 to <vscale x 4 x i64>792 %add = add <vscale x 4 x i64> %s0s, splat (i64 1)793 %add2 = add <vscale x 4 x i64> %add, %s1s794 %s = ashr <vscale x 4 x i64> %add2, splat (i64 1)795 %result = trunc <vscale x 4 x i64> %s to <vscale x 4 x i32>796 ret <vscale x 4 x i32> %result797}798 799define <vscale x 4 x i32> @rhadds_v4i32_lsh(<vscale x 4 x i32> %s0, <vscale x 4 x i32> %s1) {800; SVE-LABEL: rhadds_v4i32_lsh:801; SVE: // %bb.0: // %entry802; SVE-NEXT: eor z2.d, z0.d, z1.d803; SVE-NEXT: orr z0.d, z0.d, z1.d804; SVE-NEXT: asr z1.s, z2.s, #1805; SVE-NEXT: sub z0.s, z0.s, z1.s806; SVE-NEXT: ret807;808; SVE2-LABEL: rhadds_v4i32_lsh:809; SVE2: // %bb.0: // %entry810; SVE2-NEXT: ptrue p0.s811; SVE2-NEXT: srhadd z0.s, p0/m, z0.s, z1.s812; SVE2-NEXT: ret813entry:814 %s0s = sext <vscale x 4 x i32> %s0 to <vscale x 4 x i64>815 %s1s = sext <vscale x 4 x i32> %s1 to <vscale x 4 x i64>816 %add = add <vscale x 4 x i64> %s0s, splat (i64 1)817 %add2 = add <vscale x 4 x i64> %add, %s1s818 %s = lshr <vscale x 4 x i64> %add2, splat (i64 1)819 %result = trunc <vscale x 4 x i64> %s to <vscale x 4 x i32>820 ret <vscale x 4 x i32> %result821}822 823define <vscale x 4 x i32> @rhaddu_v4i32(<vscale x 4 x i32> %s0, <vscale x 4 x i32> %s1) {824; SVE-LABEL: rhaddu_v4i32:825; SVE: // %bb.0: // %entry826; SVE-NEXT: eor z2.d, z0.d, z1.d827; SVE-NEXT: orr z0.d, z0.d, z1.d828; SVE-NEXT: lsr z1.s, z2.s, #1829; SVE-NEXT: sub z0.s, z0.s, z1.s830; SVE-NEXT: ret831;832; SVE2-LABEL: rhaddu_v4i32:833; SVE2: // %bb.0: // %entry834; SVE2-NEXT: ptrue p0.s835; SVE2-NEXT: urhadd z0.s, p0/m, z0.s, z1.s836; SVE2-NEXT: ret837entry:838 %s0s = zext <vscale x 4 x i32> %s0 to <vscale x 4 x i64>839 %s1s = zext <vscale x 4 x i32> %s1 to <vscale x 4 x i64>840 %add = add nuw nsw <vscale x 4 x i64> %s0s, splat (i64 1)841 %add2 = add nuw nsw <vscale x 4 x i64> %add, %s1s842 %s = lshr <vscale x 4 x i64> %add2, splat (i64 1)843 %result = trunc <vscale x 4 x i64> %s to <vscale x 4 x i32>844 ret <vscale x 4 x i32> %result845}846 847define <vscale x 2 x i16> @rhadds_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {848; SVE-LABEL: rhadds_v2i16:849; SVE: // %bb.0: // %entry850; SVE-NEXT: ptrue p0.d851; SVE-NEXT: mov z2.d, #-1 // =0xffffffffffffffff852; SVE-NEXT: sxth z0.d, p0/m, z0.d853; SVE-NEXT: sxth z1.d, p0/m, z1.d854; SVE-NEXT: eor z0.d, z0.d, z2.d855; SVE-NEXT: sub z0.d, z1.d, z0.d856; SVE-NEXT: asr z0.d, z0.d, #1857; SVE-NEXT: ret858;859; SVE2-LABEL: rhadds_v2i16:860; SVE2: // %bb.0: // %entry861; SVE2-NEXT: ptrue p0.d862; SVE2-NEXT: sxth z1.d, p0/m, z1.d863; SVE2-NEXT: sxth z0.d, p0/m, z0.d864; SVE2-NEXT: srhadd z0.d, p0/m, z0.d, z1.d865; SVE2-NEXT: ret866entry:867 %s0s = sext <vscale x 2 x i16> %s0 to <vscale x 2 x i32>868 %s1s = sext <vscale x 2 x i16> %s1 to <vscale x 2 x i32>869 %add = add <vscale x 2 x i32> %s0s, splat (i32 1)870 %add2 = add <vscale x 2 x i32> %add, %s1s871 %s = ashr <vscale x 2 x i32> %add2, splat (i32 1)872 %result = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>873 ret <vscale x 2 x i16> %result874}875 876define <vscale x 2 x i16> @rhadds_v2i16_lsh(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {877; CHECK-LABEL: rhadds_v2i16_lsh:878; CHECK: // %bb.0: // %entry879; CHECK-NEXT: ptrue p0.d880; CHECK-NEXT: mov z2.d, #-1 // =0xffffffffffffffff881; CHECK-NEXT: sxth z0.d, p0/m, z0.d882; CHECK-NEXT: sxth z1.d, p0/m, z1.d883; CHECK-NEXT: eor z0.d, z0.d, z2.d884; CHECK-NEXT: sub z0.d, z1.d, z0.d885; CHECK-NEXT: and z0.d, z0.d, #0xffffffff886; CHECK-NEXT: lsr z0.d, z0.d, #1887; CHECK-NEXT: ret888entry:889 %s0s = sext <vscale x 2 x i16> %s0 to <vscale x 2 x i32>890 %s1s = sext <vscale x 2 x i16> %s1 to <vscale x 2 x i32>891 %add = add <vscale x 2 x i32> %s0s, splat (i32 1)892 %add2 = add <vscale x 2 x i32> %add, %s1s893 %s = lshr <vscale x 2 x i32> %add2, splat (i32 1)894 %result = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>895 ret <vscale x 2 x i16> %result896}897 898define <vscale x 2 x i16> @rhaddu_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {899; SVE-LABEL: rhaddu_v2i16:900; SVE: // %bb.0: // %entry901; SVE-NEXT: mov z2.d, #-1 // =0xffffffffffffffff902; SVE-NEXT: and z0.d, z0.d, #0xffff903; SVE-NEXT: and z1.d, z1.d, #0xffff904; SVE-NEXT: eor z0.d, z0.d, z2.d905; SVE-NEXT: sub z0.d, z1.d, z0.d906; SVE-NEXT: lsr z0.d, z0.d, #1907; SVE-NEXT: ret908;909; SVE2-LABEL: rhaddu_v2i16:910; SVE2: // %bb.0: // %entry911; SVE2-NEXT: and z1.d, z1.d, #0xffff912; SVE2-NEXT: and z0.d, z0.d, #0xffff913; SVE2-NEXT: ptrue p0.d914; SVE2-NEXT: urhadd z0.d, p0/m, z0.d, z1.d915; SVE2-NEXT: ret916entry:917 %s0s = zext <vscale x 2 x i16> %s0 to <vscale x 2 x i32>918 %s1s = zext <vscale x 2 x i16> %s1 to <vscale x 2 x i32>919 %add = add nuw nsw <vscale x 2 x i32> %s0s, splat (i32 1)920 %add2 = add nuw nsw <vscale x 2 x i32> %add, %s1s921 %s = lshr <vscale x 2 x i32> %add2, splat (i32 1)922 %result = trunc <vscale x 2 x i32> %s to <vscale x 2 x i16>923 ret <vscale x 2 x i16> %result924}925 926define <vscale x 4 x i16> @rhadds_v4i16(<vscale x 4 x i16> %s0, <vscale x 4 x i16> %s1) {927; SVE-LABEL: rhadds_v4i16:928; SVE: // %bb.0: // %entry929; SVE-NEXT: ptrue p0.s930; SVE-NEXT: mov z2.s, #-1 // =0xffffffffffffffff931; SVE-NEXT: sxth z0.s, p0/m, z0.s932; SVE-NEXT: sxth z1.s, p0/m, z1.s933; SVE-NEXT: eor z0.d, z0.d, z2.d934; SVE-NEXT: sub z0.s, z1.s, z0.s935; SVE-NEXT: asr z0.s, z0.s, #1936; SVE-NEXT: ret937;938; SVE2-LABEL: rhadds_v4i16:939; SVE2: // %bb.0: // %entry940; SVE2-NEXT: ptrue p0.s941; SVE2-NEXT: sxth z1.s, p0/m, z1.s942; SVE2-NEXT: sxth z0.s, p0/m, z0.s943; SVE2-NEXT: srhadd z0.s, p0/m, z0.s, z1.s944; SVE2-NEXT: ret945entry:946 %s0s = sext <vscale x 4 x i16> %s0 to <vscale x 4 x i32>947 %s1s = sext <vscale x 4 x i16> %s1 to <vscale x 4 x i32>948 %add = add <vscale x 4 x i32> %s0s, splat (i32 1)949 %add2 = add <vscale x 4 x i32> %add, %s1s950 %s = ashr <vscale x 4 x i32> %add2, splat (i32 1)951 %result = trunc <vscale x 4 x i32> %s to <vscale x 4 x i16>952 ret <vscale x 4 x i16> %result953}954 955define <vscale x 4 x i16> @rhadds_v4i16_lsh(<vscale x 4 x i16> %s0, <vscale x 4 x i16> %s1) {956; CHECK-LABEL: rhadds_v4i16_lsh:957; CHECK: // %bb.0: // %entry958; CHECK-NEXT: ptrue p0.s959; CHECK-NEXT: mov z2.s, #-1 // =0xffffffffffffffff960; CHECK-NEXT: sxth z0.s, p0/m, z0.s961; CHECK-NEXT: sxth z1.s, p0/m, z1.s962; CHECK-NEXT: eor z0.d, z0.d, z2.d963; CHECK-NEXT: sub z0.s, z1.s, z0.s964; CHECK-NEXT: lsr z0.s, z0.s, #1965; CHECK-NEXT: ret966entry:967 %s0s = sext <vscale x 4 x i16> %s0 to <vscale x 4 x i32>968 %s1s = sext <vscale x 4 x i16> %s1 to <vscale x 4 x i32>969 %add = add <vscale x 4 x i32> %s0s, splat (i32 1)970 %add2 = add <vscale x 4 x i32> %add, %s1s971 %s = lshr <vscale x 4 x i32> %add2, splat (i32 1)972 %result = trunc <vscale x 4 x i32> %s to <vscale x 4 x i16>973 ret <vscale x 4 x i16> %result974}975 976define <vscale x 4 x i16> @rhaddu_v4i16(<vscale x 4 x i16> %s0, <vscale x 4 x i16> %s1) {977; SVE-LABEL: rhaddu_v4i16:978; SVE: // %bb.0: // %entry979; SVE-NEXT: mov z2.s, #-1 // =0xffffffffffffffff980; SVE-NEXT: and z0.s, z0.s, #0xffff981; SVE-NEXT: and z1.s, z1.s, #0xffff982; SVE-NEXT: eor z0.d, z0.d, z2.d983; SVE-NEXT: sub z0.s, z1.s, z0.s984; SVE-NEXT: lsr z0.s, z0.s, #1985; SVE-NEXT: ret986;987; SVE2-LABEL: rhaddu_v4i16:988; SVE2: // %bb.0: // %entry989; SVE2-NEXT: and z1.s, z1.s, #0xffff990; SVE2-NEXT: and z0.s, z0.s, #0xffff991; SVE2-NEXT: ptrue p0.s992; SVE2-NEXT: urhadd z0.s, p0/m, z0.s, z1.s993; SVE2-NEXT: ret994entry:995 %s0s = zext <vscale x 4 x i16> %s0 to <vscale x 4 x i32>996 %s1s = zext <vscale x 4 x i16> %s1 to <vscale x 4 x i32>997 %add = add nuw nsw <vscale x 4 x i32> %s0s, splat (i32 1)998 %add2 = add nuw nsw <vscale x 4 x i32> %add, %s1s999 %s = lshr <vscale x 4 x i32> %add2, splat (i32 1)1000 %result = trunc <vscale x 4 x i32> %s to <vscale x 4 x i16>1001 ret <vscale x 4 x i16> %result1002}1003 1004define <vscale x 8 x i16> @rhadds_v8i16(<vscale x 8 x i16> %s0, <vscale x 8 x i16> %s1) {1005; SVE-LABEL: rhadds_v8i16:1006; SVE: // %bb.0: // %entry1007; SVE-NEXT: eor z2.d, z0.d, z1.d1008; SVE-NEXT: orr z0.d, z0.d, z1.d1009; SVE-NEXT: asr z1.h, z2.h, #11010; SVE-NEXT: sub z0.h, z0.h, z1.h1011; SVE-NEXT: ret1012;1013; SVE2-LABEL: rhadds_v8i16:1014; SVE2: // %bb.0: // %entry1015; SVE2-NEXT: ptrue p0.h1016; SVE2-NEXT: srhadd z0.h, p0/m, z0.h, z1.h1017; SVE2-NEXT: ret1018entry:1019 %s0s = sext <vscale x 8 x i16> %s0 to <vscale x 8 x i32>1020 %s1s = sext <vscale x 8 x i16> %s1 to <vscale x 8 x i32>1021 %add = add <vscale x 8 x i32> %s0s, splat (i32 1)1022 %add2 = add <vscale x 8 x i32> %add, %s1s1023 %s = ashr <vscale x 8 x i32> %add2, splat (i32 1)1024 %result = trunc <vscale x 8 x i32> %s to <vscale x 8 x i16>1025 ret <vscale x 8 x i16> %result1026}1027 1028define <vscale x 8 x i16> @rhadds_v8i16_lsh(<vscale x 8 x i16> %s0, <vscale x 8 x i16> %s1) {1029; SVE-LABEL: rhadds_v8i16_lsh:1030; SVE: // %bb.0: // %entry1031; SVE-NEXT: eor z2.d, z0.d, z1.d1032; SVE-NEXT: orr z0.d, z0.d, z1.d1033; SVE-NEXT: asr z1.h, z2.h, #11034; SVE-NEXT: sub z0.h, z0.h, z1.h1035; SVE-NEXT: ret1036;1037; SVE2-LABEL: rhadds_v8i16_lsh:1038; SVE2: // %bb.0: // %entry1039; SVE2-NEXT: ptrue p0.h1040; SVE2-NEXT: srhadd z0.h, p0/m, z0.h, z1.h1041; SVE2-NEXT: ret1042entry:1043 %s0s = sext <vscale x 8 x i16> %s0 to <vscale x 8 x i32>1044 %s1s = sext <vscale x 8 x i16> %s1 to <vscale x 8 x i32>1045 %add = add <vscale x 8 x i32> %s0s, splat (i32 1)1046 %add2 = add <vscale x 8 x i32> %add, %s1s1047 %s = lshr <vscale x 8 x i32> %add2, splat (i32 1)1048 %result = trunc <vscale x 8 x i32> %s to <vscale x 8 x i16>1049 ret <vscale x 8 x i16> %result1050}1051 1052define <vscale x 8 x i16> @rhaddu_v8i16(<vscale x 8 x i16> %s0, <vscale x 8 x i16> %s1) {1053; SVE-LABEL: rhaddu_v8i16:1054; SVE: // %bb.0: // %entry1055; SVE-NEXT: eor z2.d, z0.d, z1.d1056; SVE-NEXT: orr z0.d, z0.d, z1.d1057; SVE-NEXT: lsr z1.h, z2.h, #11058; SVE-NEXT: sub z0.h, z0.h, z1.h1059; SVE-NEXT: ret1060;1061; SVE2-LABEL: rhaddu_v8i16:1062; SVE2: // %bb.0: // %entry1063; SVE2-NEXT: ptrue p0.h1064; SVE2-NEXT: urhadd z0.h, p0/m, z0.h, z1.h1065; SVE2-NEXT: ret1066entry:1067 %s0s = zext <vscale x 8 x i16> %s0 to <vscale x 8 x i32>1068 %s1s = zext <vscale x 8 x i16> %s1 to <vscale x 8 x i32>1069 %add = add nuw nsw <vscale x 8 x i32> %s0s, splat (i32 1)1070 %add2 = add nuw nsw <vscale x 8 x i32> %add, %s1s1071 %s = lshr <vscale x 8 x i32> %add2, splat (i32 1)1072 %result = trunc <vscale x 8 x i32> %s to <vscale x 8 x i16>1073 ret <vscale x 8 x i16> %result1074}1075 1076define <vscale x 4 x i8> @rhadds_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {1077; SVE-LABEL: rhadds_v4i8:1078; SVE: // %bb.0: // %entry1079; SVE-NEXT: ptrue p0.s1080; SVE-NEXT: mov z2.s, #-1 // =0xffffffffffffffff1081; SVE-NEXT: sxtb z0.s, p0/m, z0.s1082; SVE-NEXT: sxtb z1.s, p0/m, z1.s1083; SVE-NEXT: eor z0.d, z0.d, z2.d1084; SVE-NEXT: sub z0.s, z1.s, z0.s1085; SVE-NEXT: asr z0.s, z0.s, #11086; SVE-NEXT: ret1087;1088; SVE2-LABEL: rhadds_v4i8:1089; SVE2: // %bb.0: // %entry1090; SVE2-NEXT: ptrue p0.s1091; SVE2-NEXT: sxtb z1.s, p0/m, z1.s1092; SVE2-NEXT: sxtb z0.s, p0/m, z0.s1093; SVE2-NEXT: srhadd z0.s, p0/m, z0.s, z1.s1094; SVE2-NEXT: ret1095entry:1096 %s0s = sext <vscale x 4 x i8> %s0 to <vscale x 4 x i16>1097 %s1s = sext <vscale x 4 x i8> %s1 to <vscale x 4 x i16>1098 %add = add <vscale x 4 x i16> %s0s, splat (i16 1)1099 %add2 = add <vscale x 4 x i16> %add, %s1s1100 %s = ashr <vscale x 4 x i16> %add2, splat (i16 1)1101 %result = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>1102 ret <vscale x 4 x i8> %result1103}1104 1105define <vscale x 4 x i8> @rhadds_v4i8_lsh(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {1106; CHECK-LABEL: rhadds_v4i8_lsh:1107; CHECK: // %bb.0: // %entry1108; CHECK-NEXT: ptrue p0.s1109; CHECK-NEXT: mov z2.s, #-1 // =0xffffffffffffffff1110; CHECK-NEXT: sxtb z0.s, p0/m, z0.s1111; CHECK-NEXT: sxtb z1.s, p0/m, z1.s1112; CHECK-NEXT: eor z0.d, z0.d, z2.d1113; CHECK-NEXT: sub z0.s, z1.s, z0.s1114; CHECK-NEXT: and z0.s, z0.s, #0xffff1115; CHECK-NEXT: lsr z0.s, z0.s, #11116; CHECK-NEXT: ret1117entry:1118 %s0s = sext <vscale x 4 x i8> %s0 to <vscale x 4 x i16>1119 %s1s = sext <vscale x 4 x i8> %s1 to <vscale x 4 x i16>1120 %add = add <vscale x 4 x i16> %s0s, splat (i16 1)1121 %add2 = add <vscale x 4 x i16> %add, %s1s1122 %s = lshr <vscale x 4 x i16> %add2, splat (i16 1)1123 %result = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>1124 ret <vscale x 4 x i8> %result1125}1126 1127define <vscale x 4 x i8> @rhaddu_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {1128; SVE-LABEL: rhaddu_v4i8:1129; SVE: // %bb.0: // %entry1130; SVE-NEXT: mov z2.s, #-1 // =0xffffffffffffffff1131; SVE-NEXT: and z0.s, z0.s, #0xff1132; SVE-NEXT: and z1.s, z1.s, #0xff1133; SVE-NEXT: eor z0.d, z0.d, z2.d1134; SVE-NEXT: sub z0.s, z1.s, z0.s1135; SVE-NEXT: lsr z0.s, z0.s, #11136; SVE-NEXT: ret1137;1138; SVE2-LABEL: rhaddu_v4i8:1139; SVE2: // %bb.0: // %entry1140; SVE2-NEXT: and z1.s, z1.s, #0xff1141; SVE2-NEXT: and z0.s, z0.s, #0xff1142; SVE2-NEXT: ptrue p0.s1143; SVE2-NEXT: urhadd z0.s, p0/m, z0.s, z1.s1144; SVE2-NEXT: ret1145entry:1146 %s0s = zext <vscale x 4 x i8> %s0 to <vscale x 4 x i16>1147 %s1s = zext <vscale x 4 x i8> %s1 to <vscale x 4 x i16>1148 %add = add nuw nsw <vscale x 4 x i16> %s0s, splat (i16 1)1149 %add2 = add nuw nsw <vscale x 4 x i16> %add, %s1s1150 %s = lshr <vscale x 4 x i16> %add2, splat (i16 1)1151 %result = trunc <vscale x 4 x i16> %s to <vscale x 4 x i8>1152 ret <vscale x 4 x i8> %result1153}1154 1155define <vscale x 8 x i8> @rhadds_v8i8(<vscale x 8 x i8> %s0, <vscale x 8 x i8> %s1) {1156; SVE-LABEL: rhadds_v8i8:1157; SVE: // %bb.0: // %entry1158; SVE-NEXT: ptrue p0.h1159; SVE-NEXT: mov z2.h, #-1 // =0xffffffffffffffff1160; SVE-NEXT: sxtb z0.h, p0/m, z0.h1161; SVE-NEXT: sxtb z1.h, p0/m, z1.h1162; SVE-NEXT: eor z0.d, z0.d, z2.d1163; SVE-NEXT: sub z0.h, z1.h, z0.h1164; SVE-NEXT: asr z0.h, z0.h, #11165; SVE-NEXT: ret1166;1167; SVE2-LABEL: rhadds_v8i8:1168; SVE2: // %bb.0: // %entry1169; SVE2-NEXT: ptrue p0.h1170; SVE2-NEXT: sxtb z1.h, p0/m, z1.h1171; SVE2-NEXT: sxtb z0.h, p0/m, z0.h1172; SVE2-NEXT: srhadd z0.h, p0/m, z0.h, z1.h1173; SVE2-NEXT: ret1174entry:1175 %s0s = sext <vscale x 8 x i8> %s0 to <vscale x 8 x i16>1176 %s1s = sext <vscale x 8 x i8> %s1 to <vscale x 8 x i16>1177 %add = add <vscale x 8 x i16> %s0s, splat (i16 1)1178 %add2 = add <vscale x 8 x i16> %add, %s1s1179 %s = ashr <vscale x 8 x i16> %add2, splat (i16 1)1180 %result = trunc <vscale x 8 x i16> %s to <vscale x 8 x i8>1181 ret <vscale x 8 x i8> %result1182}1183 1184define <vscale x 8 x i8> @rhadds_v8i8_lsh(<vscale x 8 x i8> %s0, <vscale x 8 x i8> %s1) {1185; CHECK-LABEL: rhadds_v8i8_lsh:1186; CHECK: // %bb.0: // %entry1187; CHECK-NEXT: ptrue p0.h1188; CHECK-NEXT: mov z2.h, #-1 // =0xffffffffffffffff1189; CHECK-NEXT: sxtb z0.h, p0/m, z0.h1190; CHECK-NEXT: sxtb z1.h, p0/m, z1.h1191; CHECK-NEXT: eor z0.d, z0.d, z2.d1192; CHECK-NEXT: sub z0.h, z1.h, z0.h1193; CHECK-NEXT: lsr z0.h, z0.h, #11194; CHECK-NEXT: ret1195entry:1196 %s0s = sext <vscale x 8 x i8> %s0 to <vscale x 8 x i16>1197 %s1s = sext <vscale x 8 x i8> %s1 to <vscale x 8 x i16>1198 %add = add <vscale x 8 x i16> %s0s, splat (i16 1)1199 %add2 = add <vscale x 8 x i16> %add, %s1s1200 %s = lshr <vscale x 8 x i16> %add2, splat (i16 1)1201 %result = trunc <vscale x 8 x i16> %s to <vscale x 8 x i8>1202 ret <vscale x 8 x i8> %result1203}1204 1205define <vscale x 8 x i8> @rhaddu_v8i8(<vscale x 8 x i8> %s0, <vscale x 8 x i8> %s1) {1206; SVE-LABEL: rhaddu_v8i8:1207; SVE: // %bb.0: // %entry1208; SVE-NEXT: mov z2.h, #-1 // =0xffffffffffffffff1209; SVE-NEXT: and z0.h, z0.h, #0xff1210; SVE-NEXT: and z1.h, z1.h, #0xff1211; SVE-NEXT: eor z0.d, z0.d, z2.d1212; SVE-NEXT: sub z0.h, z1.h, z0.h1213; SVE-NEXT: lsr z0.h, z0.h, #11214; SVE-NEXT: ret1215;1216; SVE2-LABEL: rhaddu_v8i8:1217; SVE2: // %bb.0: // %entry1218; SVE2-NEXT: and z1.h, z1.h, #0xff1219; SVE2-NEXT: and z0.h, z0.h, #0xff1220; SVE2-NEXT: ptrue p0.h1221; SVE2-NEXT: urhadd z0.h, p0/m, z0.h, z1.h1222; SVE2-NEXT: ret1223entry:1224 %s0s = zext <vscale x 8 x i8> %s0 to <vscale x 8 x i16>1225 %s1s = zext <vscale x 8 x i8> %s1 to <vscale x 8 x i16>1226 %add = add nuw nsw <vscale x 8 x i16> %s0s, splat (i16 1)1227 %add2 = add nuw nsw <vscale x 8 x i16> %add, %s1s1228 %s = lshr <vscale x 8 x i16> %add2, splat (i16 1)1229 %result = trunc <vscale x 8 x i16> %s to <vscale x 8 x i8>1230 ret <vscale x 8 x i8> %result1231}1232 1233define <vscale x 16 x i8> @rhadds_v16i8(<vscale x 16 x i8> %s0, <vscale x 16 x i8> %s1) {1234; SVE-LABEL: rhadds_v16i8:1235; SVE: // %bb.0: // %entry1236; SVE-NEXT: eor z2.d, z0.d, z1.d1237; SVE-NEXT: orr z0.d, z0.d, z1.d1238; SVE-NEXT: asr z1.b, z2.b, #11239; SVE-NEXT: sub z0.b, z0.b, z1.b1240; SVE-NEXT: ret1241;1242; SVE2-LABEL: rhadds_v16i8:1243; SVE2: // %bb.0: // %entry1244; SVE2-NEXT: ptrue p0.b1245; SVE2-NEXT: srhadd z0.b, p0/m, z0.b, z1.b1246; SVE2-NEXT: ret1247entry:1248 %s0s = sext <vscale x 16 x i8> %s0 to <vscale x 16 x i16>1249 %s1s = sext <vscale x 16 x i8> %s1 to <vscale x 16 x i16>1250 %add = add <vscale x 16 x i16> %s0s, splat (i16 1)1251 %add2 = add <vscale x 16 x i16> %add, %s1s1252 %s = ashr <vscale x 16 x i16> %add2, splat (i16 1)1253 %result = trunc <vscale x 16 x i16> %s to <vscale x 16 x i8>1254 ret <vscale x 16 x i8> %result1255}1256 1257define <vscale x 16 x i8> @rhadds_v16i8_lsh(<vscale x 16 x i8> %s0, <vscale x 16 x i8> %s1) {1258; SVE-LABEL: rhadds_v16i8_lsh:1259; SVE: // %bb.0: // %entry1260; SVE-NEXT: eor z2.d, z0.d, z1.d1261; SVE-NEXT: orr z0.d, z0.d, z1.d1262; SVE-NEXT: asr z1.b, z2.b, #11263; SVE-NEXT: sub z0.b, z0.b, z1.b1264; SVE-NEXT: ret1265;1266; SVE2-LABEL: rhadds_v16i8_lsh:1267; SVE2: // %bb.0: // %entry1268; SVE2-NEXT: ptrue p0.b1269; SVE2-NEXT: srhadd z0.b, p0/m, z0.b, z1.b1270; SVE2-NEXT: ret1271entry:1272 %s0s = sext <vscale x 16 x i8> %s0 to <vscale x 16 x i16>1273 %s1s = sext <vscale x 16 x i8> %s1 to <vscale x 16 x i16>1274 %add = add <vscale x 16 x i16> %s0s, splat (i16 1)1275 %add2 = add <vscale x 16 x i16> %add, %s1s1276 %s = lshr <vscale x 16 x i16> %add2, splat (i16 1)1277 %result = trunc <vscale x 16 x i16> %s to <vscale x 16 x i8>1278 ret <vscale x 16 x i8> %result1279}1280 1281define <vscale x 16 x i8> @rhaddu_v16i8(<vscale x 16 x i8> %s0, <vscale x 16 x i8> %s1) {1282; SVE-LABEL: rhaddu_v16i8:1283; SVE: // %bb.0: // %entry1284; SVE-NEXT: eor z2.d, z0.d, z1.d1285; SVE-NEXT: orr z0.d, z0.d, z1.d1286; SVE-NEXT: lsr z1.b, z2.b, #11287; SVE-NEXT: sub z0.b, z0.b, z1.b1288; SVE-NEXT: ret1289;1290; SVE2-LABEL: rhaddu_v16i8:1291; SVE2: // %bb.0: // %entry1292; SVE2-NEXT: ptrue p0.b1293; SVE2-NEXT: urhadd z0.b, p0/m, z0.b, z1.b1294; SVE2-NEXT: ret1295entry:1296 %s0s = zext <vscale x 16 x i8> %s0 to <vscale x 16 x i16>1297 %s1s = zext <vscale x 16 x i8> %s1 to <vscale x 16 x i16>1298 %add = add nuw nsw <vscale x 16 x i16> %s0s, splat (i16 1)1299 %add2 = add nuw nsw <vscale x 16 x i16> %add, %s1s1300 %s = lshr <vscale x 16 x i16> %add2, splat (i16 1)1301 %result = trunc <vscale x 16 x i16> %s to <vscale x 16 x i8>1302 ret <vscale x 16 x i8> %result1303}1304 1305define <vscale x 2 x i64> @haddu_v2i64_add(<vscale x 2 x i64> %s0, <vscale x 2 x i64> %s1) {1306; SVE-LABEL: haddu_v2i64_add:1307; SVE: // %bb.0: // %entry1308; SVE-NEXT: eor z2.d, z0.d, z1.d1309; SVE-NEXT: and z0.d, z0.d, z1.d1310; SVE-NEXT: lsr z1.d, z2.d, #11311; SVE-NEXT: add z0.d, z0.d, z1.d1312; SVE-NEXT: ret1313;1314; SVE2-LABEL: haddu_v2i64_add:1315; SVE2: // %bb.0: // %entry1316; SVE2-NEXT: ptrue p0.d1317; SVE2-NEXT: uhadd z0.d, p0/m, z0.d, z1.d1318; SVE2-NEXT: ret1319entry:1320 %add = add nuw nsw <vscale x 2 x i64> %s0, %s11321 %avg = lshr <vscale x 2 x i64> %add, splat (i64 1)1322 ret <vscale x 2 x i64> %avg1323}1324 1325define <vscale x 2 x i64> @hadds_v2i64_add(<vscale x 2 x i64> %s0, <vscale x 2 x i64> %s1) {1326; SVE-LABEL: hadds_v2i64_add:1327; SVE: // %bb.0: // %entry1328; SVE-NEXT: eor z2.d, z0.d, z1.d1329; SVE-NEXT: and z0.d, z0.d, z1.d1330; SVE-NEXT: asr z1.d, z2.d, #11331; SVE-NEXT: add z0.d, z0.d, z1.d1332; SVE-NEXT: ret1333;1334; SVE2-LABEL: hadds_v2i64_add:1335; SVE2: // %bb.0: // %entry1336; SVE2-NEXT: ptrue p0.d1337; SVE2-NEXT: shadd z0.d, p0/m, z0.d, z1.d1338; SVE2-NEXT: ret1339entry:1340 %add = add nuw nsw <vscale x 2 x i64> %s0, %s11341 %avg = ashr <vscale x 2 x i64> %add, splat (i64 1)1342 ret <vscale x 2 x i64> %avg1343}1344 1345define void @zext_mload_avgflooru(ptr %p1, ptr %p2, <vscale x 8 x i1> %mask) {1346; SVE-LABEL: zext_mload_avgflooru:1347; SVE: // %bb.0:1348; SVE-NEXT: ld1b { z0.h }, p0/z, [x0]1349; SVE-NEXT: ld1b { z1.h }, p0/z, [x1]1350; SVE-NEXT: add z0.h, z0.h, z1.h1351; SVE-NEXT: lsr z0.h, z0.h, #11352; SVE-NEXT: st1h { z0.h }, p0, [x0]1353; SVE-NEXT: ret1354;1355; SVE2-LABEL: zext_mload_avgflooru:1356; SVE2: // %bb.0:1357; SVE2-NEXT: ld1b { z0.h }, p0/z, [x0]1358; SVE2-NEXT: ld1b { z1.h }, p0/z, [x1]1359; SVE2-NEXT: ptrue p1.h1360; SVE2-NEXT: uhadd z0.h, p1/m, z0.h, z1.h1361; SVE2-NEXT: st1h { z0.h }, p0, [x0]1362; SVE2-NEXT: ret1363 %ld1 = call <vscale x 8 x i8> @llvm.masked.load(ptr %p1, i32 16, <vscale x 8 x i1> %mask, <vscale x 8 x i8> zeroinitializer)1364 %ld2 = call <vscale x 8 x i8> @llvm.masked.load(ptr %p2, i32 16, <vscale x 8 x i1> %mask, <vscale x 8 x i8> zeroinitializer)1365 %and = and <vscale x 8 x i8> %ld1, %ld21366 %xor = xor <vscale x 8 x i8> %ld1, %ld21367 %shift = lshr <vscale x 8 x i8> %xor, splat(i8 1)1368 %avg = add <vscale x 8 x i8> %and, %shift1369 %avgext = zext <vscale x 8 x i8> %avg to <vscale x 8 x i16>1370 call void @llvm.masked.store.nxv8i16(<vscale x 8 x i16> %avgext, ptr %p1, i32 16, <vscale x 8 x i1> %mask)1371 ret void1372}1373 1374define void @zext_mload_avgceilu(ptr %p1, ptr %p2, <vscale x 8 x i1> %mask) {1375; SVE-LABEL: zext_mload_avgceilu:1376; SVE: // %bb.0:1377; SVE-NEXT: ld1b { z0.h }, p0/z, [x0]1378; SVE-NEXT: mov z1.h, #-1 // =0xffffffffffffffff1379; SVE-NEXT: ld1b { z2.h }, p0/z, [x1]1380; SVE-NEXT: eor z0.d, z0.d, z1.d1381; SVE-NEXT: sub z0.h, z2.h, z0.h1382; SVE-NEXT: lsr z0.h, z0.h, #11383; SVE-NEXT: st1b { z0.h }, p0, [x0]1384; SVE-NEXT: ret1385;1386; SVE2-LABEL: zext_mload_avgceilu:1387; SVE2: // %bb.0:1388; SVE2-NEXT: ld1b { z0.h }, p0/z, [x0]1389; SVE2-NEXT: ld1b { z1.h }, p0/z, [x1]1390; SVE2-NEXT: ptrue p1.h1391; SVE2-NEXT: urhadd z0.h, p1/m, z0.h, z1.h1392; SVE2-NEXT: st1b { z0.h }, p0, [x0]1393; SVE2-NEXT: ret1394 %ld1 = call <vscale x 8 x i8> @llvm.masked.load(ptr %p1, i32 16, <vscale x 8 x i1> %mask, <vscale x 8 x i8> zeroinitializer)1395 %ld2 = call <vscale x 8 x i8> @llvm.masked.load(ptr %p2, i32 16, <vscale x 8 x i1> %mask, <vscale x 8 x i8> zeroinitializer)1396 %zext1 = zext <vscale x 8 x i8> %ld1 to <vscale x 8 x i16>1397 %zext2 = zext <vscale x 8 x i8> %ld2 to <vscale x 8 x i16>1398 %add1 = add nuw nsw <vscale x 8 x i16> %zext1, splat(i16 1)1399 %add2 = add nuw nsw <vscale x 8 x i16> %add1, %zext21400 %shift = lshr <vscale x 8 x i16> %add2, splat(i16 1)1401 %trunc = trunc <vscale x 8 x i16> %shift to <vscale x 8 x i8>1402 call void @llvm.masked.store.nxv8i8(<vscale x 8 x i8> %trunc, ptr %p1, i32 16, <vscale x 8 x i1> %mask)1403 ret void1404}1405