934 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s3 4define arm_aapcs_vfpcc <4 x i32> @vhadds_v4i32(<4 x i32> %s0, <4 x i32> %s1) {5; CHECK-LABEL: vhadds_v4i32:6; CHECK: @ %bb.0: @ %entry7; CHECK-NEXT: vhadd.s32 q0, q0, q18; CHECK-NEXT: bx lr9entry:10 %s0s = sext <4 x i32> %s0 to <4 x i64>11 %s1s = sext <4 x i32> %s1 to <4 x i64>12 %m = add nsw <4 x i64> %s0s, %s1s13 %s = lshr <4 x i64> %m, <i64 1, i64 1, i64 1, i64 1>14 %s2 = trunc <4 x i64> %s to <4 x i32>15 ret <4 x i32> %s216}17 18define arm_aapcs_vfpcc <4 x i32> @vhaddu_v4i32(<4 x i32> %s0, <4 x i32> %s1) {19; CHECK-LABEL: vhaddu_v4i32:20; CHECK: @ %bb.0: @ %entry21; CHECK-NEXT: vhadd.u32 q0, q0, q122; CHECK-NEXT: bx lr23entry:24 %s0s = zext <4 x i32> %s0 to <4 x i64>25 %s1s = zext <4 x i32> %s1 to <4 x i64>26 %m = add nuw nsw <4 x i64> %s0s, %s1s27 %s = lshr <4 x i64> %m, <i64 1, i64 1, i64 1, i64 1>28 %s2 = trunc <4 x i64> %s to <4 x i32>29 ret <4 x i32> %s230}31 32define arm_aapcs_vfpcc <4 x i16> @vhadds_v4i16(<4 x i16> %s0, <4 x i16> %s1) {33; CHECK-LABEL: vhadds_v4i16:34; CHECK: @ %bb.0: @ %entry35; CHECK-NEXT: vmovlb.s16 q1, q136; CHECK-NEXT: vmovlb.s16 q0, q037; CHECK-NEXT: vadd.i32 q0, q0, q138; CHECK-NEXT: vshr.u32 q0, q0, #139; CHECK-NEXT: bx lr40entry:41 %s0s = sext <4 x i16> %s0 to <4 x i32>42 %s1s = sext <4 x i16> %s1 to <4 x i32>43 %m = add nsw <4 x i32> %s0s, %s1s44 %s = lshr <4 x i32> %m, <i32 1, i32 1, i32 1, i32 1>45 %s2 = trunc <4 x i32> %s to <4 x i16>46 ret <4 x i16> %s247}48 49define arm_aapcs_vfpcc <4 x i16> @vhaddu_v4i16(<4 x i16> %s0, <4 x i16> %s1) {50; CHECK-LABEL: vhaddu_v4i16:51; CHECK: @ %bb.0: @ %entry52; CHECK-NEXT: vmovlb.u16 q1, q153; CHECK-NEXT: vmovlb.u16 q0, q054; CHECK-NEXT: vhadd.u32 q0, q0, q155; CHECK-NEXT: bx lr56entry:57 %s0s = zext <4 x i16> %s0 to <4 x i32>58 %s1s = zext <4 x i16> %s1 to <4 x i32>59 %m = add nuw nsw <4 x i32> %s0s, %s1s60 %s = lshr <4 x i32> %m, <i32 1, i32 1, i32 1, i32 1>61 %s2 = trunc <4 x i32> %s to <4 x i16>62 ret <4 x i16> %s263}64 65define arm_aapcs_vfpcc <8 x i16> @vhadds_v8i16(<8 x i16> %s0, <8 x i16> %s1) {66; CHECK-LABEL: vhadds_v8i16:67; CHECK: @ %bb.0: @ %entry68; CHECK-NEXT: vhadd.s16 q0, q0, q169; CHECK-NEXT: bx lr70entry:71 %s0s = sext <8 x i16> %s0 to <8 x i32>72 %s1s = sext <8 x i16> %s1 to <8 x i32>73 %m = add nsw <8 x i32> %s0s, %s1s74 %s = lshr <8 x i32> %m, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>75 %s2 = trunc <8 x i32> %s to <8 x i16>76 ret <8 x i16> %s277}78 79define arm_aapcs_vfpcc <8 x i16> @vhaddu_v8i16(<8 x i16> %s0, <8 x i16> %s1) {80; CHECK-LABEL: vhaddu_v8i16:81; CHECK: @ %bb.0: @ %entry82; CHECK-NEXT: vhadd.u16 q0, q0, q183; CHECK-NEXT: bx lr84entry:85 %s0s = zext <8 x i16> %s0 to <8 x i32>86 %s1s = zext <8 x i16> %s1 to <8 x i32>87 %m = add nuw nsw <8 x i32> %s0s, %s1s88 %s = lshr <8 x i32> %m, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>89 %s2 = trunc <8 x i32> %s to <8 x i16>90 ret <8 x i16> %s291}92 93define arm_aapcs_vfpcc <4 x i8> @vhadds_v4i8(<4 x i8> %s0, <4 x i8> %s1) {94; CHECK-LABEL: vhadds_v4i8:95; CHECK: @ %bb.0: @ %entry96; CHECK-NEXT: vmovlb.s8 q1, q197; CHECK-NEXT: vmovlb.s8 q0, q098; CHECK-NEXT: vmovlb.s16 q1, q199; CHECK-NEXT: vmovlb.s16 q0, q0100; CHECK-NEXT: vadd.i32 q0, q0, q1101; CHECK-NEXT: vmovlb.u16 q0, q0102; CHECK-NEXT: vshr.u32 q0, q0, #1103; CHECK-NEXT: bx lr104entry:105 %s0s = sext <4 x i8> %s0 to <4 x i16>106 %s1s = sext <4 x i8> %s1 to <4 x i16>107 %m = add nsw <4 x i16> %s0s, %s1s108 %s = lshr <4 x i16> %m, <i16 1, i16 1, i16 1, i16 1>109 %s2 = trunc <4 x i16> %s to <4 x i8>110 ret <4 x i8> %s2111}112 113define arm_aapcs_vfpcc <4 x i8> @vhaddu_v4i8(<4 x i8> %s0, <4 x i8> %s1) {114; CHECK-LABEL: vhaddu_v4i8:115; CHECK: @ %bb.0: @ %entry116; CHECK-NEXT: vmov.i32 q2, #0xff117; CHECK-NEXT: vand q1, q1, q2118; CHECK-NEXT: vand q0, q0, q2119; CHECK-NEXT: vhadd.u32 q0, q0, q1120; CHECK-NEXT: bx lr121entry:122 %s0s = zext <4 x i8> %s0 to <4 x i16>123 %s1s = zext <4 x i8> %s1 to <4 x i16>124 %m = add nuw nsw <4 x i16> %s0s, %s1s125 %s = lshr <4 x i16> %m, <i16 1, i16 1, i16 1, i16 1>126 %s2 = trunc <4 x i16> %s to <4 x i8>127 ret <4 x i8> %s2128}129 130define arm_aapcs_vfpcc <8 x i8> @vhadds_v8i8(<8 x i8> %s0, <8 x i8> %s1) {131; CHECK-LABEL: vhadds_v8i8:132; CHECK: @ %bb.0: @ %entry133; CHECK-NEXT: vmovlb.s8 q1, q1134; CHECK-NEXT: vmovlb.s8 q0, q0135; CHECK-NEXT: vadd.i16 q0, q0, q1136; CHECK-NEXT: vshr.u16 q0, q0, #1137; CHECK-NEXT: bx lr138entry:139 %s0s = sext <8 x i8> %s0 to <8 x i16>140 %s1s = sext <8 x i8> %s1 to <8 x i16>141 %m = add nsw <8 x i16> %s0s, %s1s142 %s = lshr <8 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>143 %s2 = trunc <8 x i16> %s to <8 x i8>144 ret <8 x i8> %s2145}146 147define arm_aapcs_vfpcc <8 x i8> @vhaddu_v8i8(<8 x i8> %s0, <8 x i8> %s1) {148; CHECK-LABEL: vhaddu_v8i8:149; CHECK: @ %bb.0: @ %entry150; CHECK-NEXT: vmovlb.u8 q1, q1151; CHECK-NEXT: vmovlb.u8 q0, q0152; CHECK-NEXT: vhadd.u16 q0, q0, q1153; CHECK-NEXT: bx lr154entry:155 %s0s = zext <8 x i8> %s0 to <8 x i16>156 %s1s = zext <8 x i8> %s1 to <8 x i16>157 %m = add nuw nsw <8 x i16> %s0s, %s1s158 %s = lshr <8 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>159 %s2 = trunc <8 x i16> %s to <8 x i8>160 ret <8 x i8> %s2161}162 163define arm_aapcs_vfpcc <16 x i8> @vhadds_v16i8(<16 x i8> %s0, <16 x i8> %s1) {164; CHECK-LABEL: vhadds_v16i8:165; CHECK: @ %bb.0: @ %entry166; CHECK-NEXT: vhadd.s8 q0, q0, q1167; CHECK-NEXT: bx lr168entry:169 %s0s = sext <16 x i8> %s0 to <16 x i16>170 %s1s = sext <16 x i8> %s1 to <16 x i16>171 %m = add nsw <16 x i16> %s0s, %s1s172 %s = lshr <16 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>173 %s2 = trunc <16 x i16> %s to <16 x i8>174 ret <16 x i8> %s2175}176 177define arm_aapcs_vfpcc <16 x i8> @vhaddu_v16i8(<16 x i8> %s0, <16 x i8> %s1) {178; CHECK-LABEL: vhaddu_v16i8:179; CHECK: @ %bb.0: @ %entry180; CHECK-NEXT: vhadd.u8 q0, q0, q1181; CHECK-NEXT: bx lr182entry:183 %s0s = zext <16 x i8> %s0 to <16 x i16>184 %s1s = zext <16 x i8> %s1 to <16 x i16>185 %m = add nuw nsw <16 x i16> %s0s, %s1s186 %s = lshr <16 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>187 %s2 = trunc <16 x i16> %s to <16 x i8>188 ret <16 x i8> %s2189}190 191define arm_aapcs_vfpcc <4 x i32> @vrhadds_v4i32(<4 x i32> %s0, <4 x i32> %s1) {192; CHECK-LABEL: vrhadds_v4i32:193; CHECK: @ %bb.0: @ %entry194; CHECK-NEXT: vrhadd.s32 q0, q0, q1195; CHECK-NEXT: bx lr196entry:197 %s0s = sext <4 x i32> %s0 to <4 x i64>198 %s1s = sext <4 x i32> %s1 to <4 x i64>199 %add = add nsw <4 x i64> %s0s, <i64 1, i64 1, i64 1, i64 1>200 %add2 = add nsw <4 x i64> %add, %s1s201 %s = lshr <4 x i64> %add2, <i64 1, i64 1, i64 1, i64 1>202 %result = trunc <4 x i64> %s to <4 x i32>203 ret <4 x i32> %result204}205 206define arm_aapcs_vfpcc <4 x i32> @vrhaddu_v4i32(<4 x i32> %s0, <4 x i32> %s1) {207; CHECK-LABEL: vrhaddu_v4i32:208; CHECK: @ %bb.0: @ %entry209; CHECK-NEXT: vrhadd.u32 q0, q0, q1210; CHECK-NEXT: bx lr211entry:212 %s0s = zext <4 x i32> %s0 to <4 x i64>213 %s1s = zext <4 x i32> %s1 to <4 x i64>214 %add = add nuw nsw <4 x i64> %s0s, <i64 1, i64 1, i64 1, i64 1>215 %add2 = add nuw nsw <4 x i64> %add, %s1s216 %s = lshr <4 x i64> %add2, <i64 1, i64 1, i64 1, i64 1>217 %result = trunc <4 x i64> %s to <4 x i32>218 ret <4 x i32> %result219}220 221define arm_aapcs_vfpcc <4 x i16> @vrhadds_v4i16(<4 x i16> %s0, <4 x i16> %s1) {222; CHECK-LABEL: vrhadds_v4i16:223; CHECK: @ %bb.0: @ %entry224; CHECK-NEXT: vmovlb.s16 q1, q1225; CHECK-NEXT: vmovlb.s16 q0, q0226; CHECK-NEXT: vadd.i32 q0, q0, q1227; CHECK-NEXT: movs r0, #1228; CHECK-NEXT: vadd.i32 q0, q0, r0229; CHECK-NEXT: vshr.u32 q0, q0, #1230; CHECK-NEXT: bx lr231entry:232 %s0s = sext <4 x i16> %s0 to <4 x i32>233 %s1s = sext <4 x i16> %s1 to <4 x i32>234 %add = add nsw <4 x i32> %s0s, <i32 1, i32 1, i32 1, i32 1>235 %add2 = add nsw <4 x i32> %add, %s1s236 %s = lshr <4 x i32> %add2, <i32 1, i32 1, i32 1, i32 1>237 %result = trunc <4 x i32> %s to <4 x i16>238 ret <4 x i16> %result239}240 241define arm_aapcs_vfpcc <4 x i16> @vrhaddu_v4i16(<4 x i16> %s0, <4 x i16> %s1) {242; CHECK-LABEL: vrhaddu_v4i16:243; CHECK: @ %bb.0: @ %entry244; CHECK-NEXT: vmovlb.u16 q1, q1245; CHECK-NEXT: vmovlb.u16 q0, q0246; CHECK-NEXT: vrhadd.u32 q0, q0, q1247; CHECK-NEXT: bx lr248entry:249 %s0s = zext <4 x i16> %s0 to <4 x i32>250 %s1s = zext <4 x i16> %s1 to <4 x i32>251 %add = add nuw nsw <4 x i32> %s0s, <i32 1, i32 1, i32 1, i32 1>252 %add2 = add nuw nsw <4 x i32> %add, %s1s253 %s = lshr <4 x i32> %add2, <i32 1, i32 1, i32 1, i32 1>254 %result = trunc <4 x i32> %s to <4 x i16>255 ret <4 x i16> %result256}257 258define arm_aapcs_vfpcc <8 x i16> @vrhadds_v8i16(<8 x i16> %s0, <8 x i16> %s1) {259; CHECK-LABEL: vrhadds_v8i16:260; CHECK: @ %bb.0: @ %entry261; CHECK-NEXT: vrhadd.s16 q0, q0, q1262; CHECK-NEXT: bx lr263entry:264 %s0s = sext <8 x i16> %s0 to <8 x i32>265 %s1s = sext <8 x i16> %s1 to <8 x i32>266 %add = add nsw <8 x i32> %s0s, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>267 %add2 = add nsw <8 x i32> %add, %s1s268 %s = lshr <8 x i32> %add2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>269 %result = trunc <8 x i32> %s to <8 x i16>270 ret <8 x i16> %result271}272 273define arm_aapcs_vfpcc <8 x i16> @vrhaddu_v8i16(<8 x i16> %s0, <8 x i16> %s1) {274; CHECK-LABEL: vrhaddu_v8i16:275; CHECK: @ %bb.0: @ %entry276; CHECK-NEXT: vrhadd.u16 q0, q0, q1277; CHECK-NEXT: bx lr278entry:279 %s0s = zext <8 x i16> %s0 to <8 x i32>280 %s1s = zext <8 x i16> %s1 to <8 x i32>281 %add = add nuw nsw <8 x i32> %s0s, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>282 %add2 = add nuw nsw <8 x i32> %add, %s1s283 %s = lshr <8 x i32> %add2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>284 %result = trunc <8 x i32> %s to <8 x i16>285 ret <8 x i16> %result286}287 288define arm_aapcs_vfpcc <4 x i8> @vrhadds_v4i8(<4 x i8> %s0, <4 x i8> %s1) {289; CHECK-LABEL: vrhadds_v4i8:290; CHECK: @ %bb.0: @ %entry291; CHECK-NEXT: vmovlb.s8 q1, q1292; CHECK-NEXT: vmovlb.s8 q0, q0293; CHECK-NEXT: vmovlb.s16 q1, q1294; CHECK-NEXT: vmovlb.s16 q0, q0295; CHECK-NEXT: vadd.i32 q0, q0, q1296; CHECK-NEXT: movs r0, #1297; CHECK-NEXT: vadd.i32 q0, q0, r0298; CHECK-NEXT: vmovlb.u16 q0, q0299; CHECK-NEXT: vshr.u32 q0, q0, #1300; CHECK-NEXT: bx lr301entry:302 %s0s = sext <4 x i8> %s0 to <4 x i16>303 %s1s = sext <4 x i8> %s1 to <4 x i16>304 %add = add nsw <4 x i16> %s0s, <i16 1, i16 1, i16 1, i16 1>305 %add2 = add nsw <4 x i16> %add, %s1s306 %s = lshr <4 x i16> %add2, <i16 1, i16 1, i16 1, i16 1>307 %result = trunc <4 x i16> %s to <4 x i8>308 ret <4 x i8> %result309}310 311define arm_aapcs_vfpcc <4 x i8> @vrhaddu_v4i8(<4 x i8> %s0, <4 x i8> %s1) {312; CHECK-LABEL: vrhaddu_v4i8:313; CHECK: @ %bb.0: @ %entry314; CHECK-NEXT: vmov.i32 q2, #0xff315; CHECK-NEXT: vand q1, q1, q2316; CHECK-NEXT: vand q0, q0, q2317; CHECK-NEXT: vrhadd.u32 q0, q0, q1318; CHECK-NEXT: bx lr319entry:320 %s0s = zext <4 x i8> %s0 to <4 x i16>321 %s1s = zext <4 x i8> %s1 to <4 x i16>322 %add = add nuw nsw <4 x i16> %s0s, <i16 1, i16 1, i16 1, i16 1>323 %add2 = add nuw nsw <4 x i16> %add, %s1s324 %s = lshr <4 x i16> %add2, <i16 1, i16 1, i16 1, i16 1>325 %result = trunc <4 x i16> %s to <4 x i8>326 ret <4 x i8> %result327}328 329define arm_aapcs_vfpcc <8 x i8> @vrhadds_v8i8(<8 x i8> %s0, <8 x i8> %s1) {330; CHECK-LABEL: vrhadds_v8i8:331; CHECK: @ %bb.0: @ %entry332; CHECK-NEXT: vmovlb.s8 q1, q1333; CHECK-NEXT: vmovlb.s8 q0, q0334; CHECK-NEXT: vadd.i16 q0, q0, q1335; CHECK-NEXT: movs r0, #1336; CHECK-NEXT: vadd.i16 q0, q0, r0337; CHECK-NEXT: vshr.u16 q0, q0, #1338; CHECK-NEXT: bx lr339entry:340 %s0s = sext <8 x i8> %s0 to <8 x i16>341 %s1s = sext <8 x i8> %s1 to <8 x i16>342 %add = add nsw <8 x i16> %s0s, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>343 %add2 = add nsw <8 x i16> %add, %s1s344 %s = lshr <8 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>345 %result = trunc <8 x i16> %s to <8 x i8>346 ret <8 x i8> %result347}348 349define arm_aapcs_vfpcc <8 x i8> @vrhaddu_v8i8(<8 x i8> %s0, <8 x i8> %s1) {350; CHECK-LABEL: vrhaddu_v8i8:351; CHECK: @ %bb.0: @ %entry352; CHECK-NEXT: vmovlb.u8 q1, q1353; CHECK-NEXT: vmovlb.u8 q0, q0354; CHECK-NEXT: vrhadd.u16 q0, q0, q1355; CHECK-NEXT: bx lr356entry:357 %s0s = zext <8 x i8> %s0 to <8 x i16>358 %s1s = zext <8 x i8> %s1 to <8 x i16>359 %add = add nuw nsw <8 x i16> %s0s, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>360 %add2 = add nuw nsw <8 x i16> %add, %s1s361 %s = lshr <8 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>362 %result = trunc <8 x i16> %s to <8 x i8>363 ret <8 x i8> %result364}365 366define arm_aapcs_vfpcc <16 x i8> @vrhadds_v16i8(<16 x i8> %s0, <16 x i8> %s1) {367; CHECK-LABEL: vrhadds_v16i8:368; CHECK: @ %bb.0: @ %entry369; CHECK-NEXT: vrhadd.s8 q0, q0, q1370; CHECK-NEXT: bx lr371entry:372 %s0s = sext <16 x i8> %s0 to <16 x i16>373 %s1s = sext <16 x i8> %s1 to <16 x i16>374 %add = add nsw <16 x i16> %s0s, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>375 %add2 = add nsw <16 x i16> %add, %s1s376 %s = lshr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>377 %result = trunc <16 x i16> %s to <16 x i8>378 ret <16 x i8> %result379}380 381define arm_aapcs_vfpcc <16 x i8> @vrhaddu_v16i8(<16 x i8> %s0, <16 x i8> %s1) {382; CHECK-LABEL: vrhaddu_v16i8:383; CHECK: @ %bb.0: @ %entry384; CHECK-NEXT: vrhadd.u8 q0, q0, q1385; CHECK-NEXT: bx lr386entry:387 %s0s = zext <16 x i8> %s0 to <16 x i16>388 %s1s = zext <16 x i8> %s1 to <16 x i16>389 %add = add nuw nsw <16 x i16> %s0s, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>390 %add2 = add nuw nsw <16 x i16> %add, %s1s391 %s = lshr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>392 %result = trunc <16 x i16> %s to <16 x i8>393 ret <16 x i8> %result394}395 396define void @vhadd_loop_s8(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {397; CHECK-LABEL: vhadd_loop_s8:398; CHECK: @ %bb.0: @ %entry399; CHECK-NEXT: .save {r7, lr}400; CHECK-NEXT: push {r7, lr}401; CHECK-NEXT: mov.w lr, #64402; CHECK-NEXT: .LBB24_1: @ %vector.body403; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1404; CHECK-NEXT: vldrb.u8 q0, [r0], #16405; CHECK-NEXT: vldrb.u8 q1, [r1], #16406; CHECK-NEXT: vhadd.s8 q0, q1, q0407; CHECK-NEXT: vstrb.8 q0, [r2], #16408; CHECK-NEXT: le lr, .LBB24_1409; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup410; CHECK-NEXT: pop {r7, pc}411entry:412 br label %vector.body413 414vector.body: ; preds = %vector.body, %entry415 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]416 %0 = getelementptr inbounds i8, ptr %x, i32 %index417 %wide.load = load <16 x i8>, ptr %0, align 1418 %1 = sext <16 x i8> %wide.load to <16 x i16>419 %2 = getelementptr inbounds i8, ptr %y, i32 %index420 %wide.load16 = load <16 x i8>, ptr %2, align 1421 %3 = sext <16 x i8> %wide.load16 to <16 x i16>422 %4 = add nsw <16 x i16> %3, %1423 %5 = lshr <16 x i16> %4, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>424 %6 = trunc <16 x i16> %5 to <16 x i8>425 %7 = getelementptr inbounds i8, ptr %z, i32 %index426 store <16 x i8> %6, ptr %7, align 1427 %index.next = add i32 %index, 16428 %8 = icmp eq i32 %index.next, 1024429 br i1 %8, label %for.cond.cleanup, label %vector.body430 431for.cond.cleanup: ; preds = %vector.body432 ret void433}434 435define void @vhadd_loop_s16(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {436; CHECK-LABEL: vhadd_loop_s16:437; CHECK: @ %bb.0: @ %entry438; CHECK-NEXT: .save {r7, lr}439; CHECK-NEXT: push {r7, lr}440; CHECK-NEXT: mov.w lr, #128441; CHECK-NEXT: .LBB25_1: @ %vector.body442; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1443; CHECK-NEXT: vldrh.u16 q0, [r0], #16444; CHECK-NEXT: vldrh.u16 q1, [r1], #16445; CHECK-NEXT: vhadd.s16 q0, q1, q0446; CHECK-NEXT: vstrb.8 q0, [r2], #16447; CHECK-NEXT: le lr, .LBB25_1448; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup449; CHECK-NEXT: pop {r7, pc}450entry:451 br label %vector.body452 453vector.body: ; preds = %vector.body, %entry454 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]455 %0 = getelementptr inbounds i16, ptr %x, i32 %index456 %wide.load = load <8 x i16>, ptr %0, align 2457 %1 = sext <8 x i16> %wide.load to <8 x i32>458 %2 = getelementptr inbounds i16, ptr %y, i32 %index459 %wide.load16 = load <8 x i16>, ptr %2, align 2460 %3 = sext <8 x i16> %wide.load16 to <8 x i32>461 %4 = add nsw <8 x i32> %3, %1462 %5 = lshr <8 x i32> %4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>463 %6 = trunc <8 x i32> %5 to <8 x i16>464 %7 = getelementptr inbounds i16, ptr %z, i32 %index465 store <8 x i16> %6, ptr %7, align 2466 %index.next = add i32 %index, 8467 %8 = icmp eq i32 %index.next, 1024468 br i1 %8, label %for.cond.cleanup, label %vector.body469 470for.cond.cleanup: ; preds = %vector.body471 ret void472}473 474define void @vhadd_loop_s32(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {475; CHECK-LABEL: vhadd_loop_s32:476; CHECK: @ %bb.0: @ %entry477; CHECK-NEXT: .save {r7, lr}478; CHECK-NEXT: push {r7, lr}479; CHECK-NEXT: mov.w lr, #256480; CHECK-NEXT: .LBB26_1: @ %vector.body481; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1482; CHECK-NEXT: vldrw.u32 q0, [r0], #16483; CHECK-NEXT: vldrw.u32 q1, [r1], #16484; CHECK-NEXT: vhadd.s32 q0, q1, q0485; CHECK-NEXT: vstrb.8 q0, [r2], #16486; CHECK-NEXT: le lr, .LBB26_1487; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup488; CHECK-NEXT: pop {r7, pc}489entry:490 br label %vector.body491 492vector.body: ; preds = %vector.body, %entry493 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]494 %0 = getelementptr inbounds i32, ptr %x, i32 %index495 %wide.load = load <4 x i32>, ptr %0, align 4496 %1 = sext <4 x i32> %wide.load to <4 x i64>497 %2 = getelementptr inbounds i32, ptr %y, i32 %index498 %wide.load16 = load <4 x i32>, ptr %2, align 4499 %3 = sext <4 x i32> %wide.load16 to <4 x i64>500 %4 = add nsw <4 x i64> %3, %1501 %5 = lshr <4 x i64> %4, <i64 1, i64 1, i64 1, i64 1>502 %6 = trunc <4 x i64> %5 to <4 x i32>503 %7 = getelementptr inbounds i32, ptr %z, i32 %index504 store <4 x i32> %6, ptr %7, align 4505 %index.next = add i32 %index, 4506 %8 = icmp eq i32 %index.next, 1024507 br i1 %8, label %for.cond.cleanup, label %vector.body508 509for.cond.cleanup: ; preds = %vector.body510 ret void511}512 513define void @vhadd_loop_u8(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {514; CHECK-LABEL: vhadd_loop_u8:515; CHECK: @ %bb.0: @ %entry516; CHECK-NEXT: .save {r7, lr}517; CHECK-NEXT: push {r7, lr}518; CHECK-NEXT: mov.w lr, #64519; CHECK-NEXT: .LBB27_1: @ %vector.body520; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1521; CHECK-NEXT: vldrb.u8 q0, [r0], #16522; CHECK-NEXT: vldrb.u8 q1, [r1], #16523; CHECK-NEXT: vhadd.u8 q0, q1, q0524; CHECK-NEXT: vstrb.8 q0, [r2], #16525; CHECK-NEXT: le lr, .LBB27_1526; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup527; CHECK-NEXT: pop {r7, pc}528entry:529 br label %vector.body530 531vector.body: ; preds = %vector.body, %entry532 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]533 %0 = getelementptr inbounds i8, ptr %x, i32 %index534 %wide.load = load <16 x i8>, ptr %0, align 1535 %1 = zext <16 x i8> %wide.load to <16 x i16>536 %2 = getelementptr inbounds i8, ptr %y, i32 %index537 %wide.load16 = load <16 x i8>, ptr %2, align 1538 %3 = zext <16 x i8> %wide.load16 to <16 x i16>539 %4 = add nuw nsw <16 x i16> %3, %1540 %5 = lshr <16 x i16> %4, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>541 %6 = trunc <16 x i16> %5 to <16 x i8>542 %7 = getelementptr inbounds i8, ptr %z, i32 %index543 store <16 x i8> %6, ptr %7, align 1544 %index.next = add i32 %index, 16545 %8 = icmp eq i32 %index.next, 1024546 br i1 %8, label %for.cond.cleanup, label %vector.body547 548for.cond.cleanup: ; preds = %vector.body549 ret void550}551 552define void @vhadd_loop_u16(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {553; CHECK-LABEL: vhadd_loop_u16:554; CHECK: @ %bb.0: @ %entry555; CHECK-NEXT: .save {r7, lr}556; CHECK-NEXT: push {r7, lr}557; CHECK-NEXT: mov.w lr, #128558; CHECK-NEXT: .LBB28_1: @ %vector.body559; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1560; CHECK-NEXT: vldrh.u16 q0, [r0], #16561; CHECK-NEXT: vldrh.u16 q1, [r1], #16562; CHECK-NEXT: vhadd.u16 q0, q1, q0563; CHECK-NEXT: vstrb.8 q0, [r2], #16564; CHECK-NEXT: le lr, .LBB28_1565; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup566; CHECK-NEXT: pop {r7, pc}567entry:568 br label %vector.body569 570vector.body: ; preds = %vector.body, %entry571 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]572 %0 = getelementptr inbounds i16, ptr %x, i32 %index573 %wide.load = load <8 x i16>, ptr %0, align 2574 %1 = zext <8 x i16> %wide.load to <8 x i32>575 %2 = getelementptr inbounds i16, ptr %y, i32 %index576 %wide.load16 = load <8 x i16>, ptr %2, align 2577 %3 = zext <8 x i16> %wide.load16 to <8 x i32>578 %4 = add nuw nsw <8 x i32> %3, %1579 %5 = lshr <8 x i32> %4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>580 %6 = trunc <8 x i32> %5 to <8 x i16>581 %7 = getelementptr inbounds i16, ptr %z, i32 %index582 store <8 x i16> %6, ptr %7, align 2583 %index.next = add i32 %index, 8584 %8 = icmp eq i32 %index.next, 1024585 br i1 %8, label %for.cond.cleanup, label %vector.body586 587for.cond.cleanup: ; preds = %vector.body588 ret void589}590 591define void @vhadd_loop_u32(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {592; CHECK-LABEL: vhadd_loop_u32:593; CHECK: @ %bb.0: @ %entry594; CHECK-NEXT: .save {r7, lr}595; CHECK-NEXT: push {r7, lr}596; CHECK-NEXT: mov.w lr, #256597; CHECK-NEXT: .LBB29_1: @ %vector.body598; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1599; CHECK-NEXT: vldrw.u32 q0, [r0], #16600; CHECK-NEXT: vldrw.u32 q1, [r1], #16601; CHECK-NEXT: vhadd.u32 q0, q1, q0602; CHECK-NEXT: vstrb.8 q0, [r2], #16603; CHECK-NEXT: le lr, .LBB29_1604; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup605; CHECK-NEXT: pop {r7, pc}606entry:607 br label %vector.body608 609vector.body: ; preds = %vector.body, %entry610 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]611 %0 = getelementptr inbounds i32, ptr %x, i32 %index612 %wide.load = load <4 x i32>, ptr %0, align 4613 %1 = zext <4 x i32> %wide.load to <4 x i64>614 %2 = getelementptr inbounds i32, ptr %y, i32 %index615 %wide.load16 = load <4 x i32>, ptr %2, align 4616 %3 = zext <4 x i32> %wide.load16 to <4 x i64>617 %4 = add nuw nsw <4 x i64> %3, %1618 %5 = lshr <4 x i64> %4, <i64 1, i64 1, i64 1, i64 1>619 %6 = trunc <4 x i64> %5 to <4 x i32>620 %7 = getelementptr inbounds i32, ptr %z, i32 %index621 store <4 x i32> %6, ptr %7, align 4622 %index.next = add i32 %index, 4623 %8 = icmp eq i32 %index.next, 1024624 br i1 %8, label %for.cond.cleanup, label %vector.body625 626for.cond.cleanup: ; preds = %vector.body627 ret void628}629 630define void @vrhadd_loop_s8(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {631; CHECK-LABEL: vrhadd_loop_s8:632; CHECK: @ %bb.0: @ %entry633; CHECK-NEXT: .save {r7, lr}634; CHECK-NEXT: push {r7, lr}635; CHECK-NEXT: mov.w lr, #64636; CHECK-NEXT: .LBB30_1: @ %vector.body637; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1638; CHECK-NEXT: vldrb.u8 q0, [r1], #16639; CHECK-NEXT: vldrb.u8 q1, [r0], #16640; CHECK-NEXT: vrhadd.u8 q0, q1, q0641; CHECK-NEXT: vstrb.8 q0, [r2], #16642; CHECK-NEXT: le lr, .LBB30_1643; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup644; CHECK-NEXT: pop {r7, pc}645entry:646 br label %vector.body647 648vector.body: ; preds = %vector.body, %entry649 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]650 %0 = getelementptr inbounds i8, ptr %x, i32 %index651 %wide.load = load <16 x i8>, ptr %0, align 1652 %1 = zext <16 x i8> %wide.load to <16 x i16>653 %2 = getelementptr inbounds i8, ptr %y, i32 %index654 %wide.load16 = load <16 x i8>, ptr %2, align 1655 %3 = zext <16 x i8> %wide.load16 to <16 x i16>656 %4 = add nuw nsw <16 x i16> %1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>657 %5 = add nuw nsw <16 x i16> %4, %3658 %6 = lshr <16 x i16> %5, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>659 %7 = trunc <16 x i16> %6 to <16 x i8>660 %8 = getelementptr inbounds i8, ptr %z, i32 %index661 store <16 x i8> %7, ptr %8, align 1662 %index.next = add i32 %index, 16663 %9 = icmp eq i32 %index.next, 1024664 br i1 %9, label %for.cond.cleanup, label %vector.body665 666for.cond.cleanup: ; preds = %vector.body667 ret void668}669 670define void @vrhadd_loop_s16(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {671; CHECK-LABEL: vrhadd_loop_s16:672; CHECK: @ %bb.0: @ %entry673; CHECK-NEXT: .save {r7, lr}674; CHECK-NEXT: push {r7, lr}675; CHECK-NEXT: mov.w lr, #128676; CHECK-NEXT: .LBB31_1: @ %vector.body677; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1678; CHECK-NEXT: vldrh.u16 q0, [r1], #16679; CHECK-NEXT: vldrh.u16 q1, [r0], #16680; CHECK-NEXT: vrhadd.u16 q0, q1, q0681; CHECK-NEXT: vstrb.8 q0, [r2], #16682; CHECK-NEXT: le lr, .LBB31_1683; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup684; CHECK-NEXT: pop {r7, pc}685entry:686 br label %vector.body687 688vector.body: ; preds = %vector.body, %entry689 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]690 %0 = getelementptr inbounds i16, ptr %x, i32 %index691 %wide.load = load <8 x i16>, ptr %0, align 2692 %1 = zext <8 x i16> %wide.load to <8 x i32>693 %2 = getelementptr inbounds i16, ptr %y, i32 %index694 %wide.load16 = load <8 x i16>, ptr %2, align 2695 %3 = zext <8 x i16> %wide.load16 to <8 x i32>696 %4 = add nuw nsw <8 x i32> %1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>697 %5 = add nuw nsw <8 x i32> %4, %3698 %6 = lshr <8 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>699 %7 = trunc <8 x i32> %6 to <8 x i16>700 %8 = getelementptr inbounds i16, ptr %z, i32 %index701 store <8 x i16> %7, ptr %8, align 2702 %index.next = add i32 %index, 8703 %9 = icmp eq i32 %index.next, 1024704 br i1 %9, label %for.cond.cleanup, label %vector.body705 706for.cond.cleanup: ; preds = %vector.body707 ret void708}709 710define void @vrhadd_loop_s32(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {711; CHECK-LABEL: vrhadd_loop_s32:712; CHECK: @ %bb.0: @ %entry713; CHECK-NEXT: .save {r7, lr}714; CHECK-NEXT: push {r7, lr}715; CHECK-NEXT: mov.w lr, #256716; CHECK-NEXT: .LBB32_1: @ %vector.body717; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1718; CHECK-NEXT: vldrw.u32 q0, [r1], #16719; CHECK-NEXT: vldrw.u32 q1, [r0], #16720; CHECK-NEXT: vrhadd.u32 q0, q1, q0721; CHECK-NEXT: vstrb.8 q0, [r2], #16722; CHECK-NEXT: le lr, .LBB32_1723; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup724; CHECK-NEXT: pop {r7, pc}725entry:726 br label %vector.body727 728vector.body: ; preds = %vector.body, %entry729 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]730 %0 = getelementptr inbounds i32, ptr %x, i32 %index731 %wide.load = load <4 x i32>, ptr %0, align 4732 %1 = zext <4 x i32> %wide.load to <4 x i64>733 %2 = getelementptr inbounds i32, ptr %y, i32 %index734 %wide.load16 = load <4 x i32>, ptr %2, align 4735 %3 = zext <4 x i32> %wide.load16 to <4 x i64>736 %4 = add nuw nsw <4 x i64> %1, <i64 1, i64 1, i64 1, i64 1>737 %5 = add nuw nsw <4 x i64> %4, %3738 %6 = lshr <4 x i64> %5, <i64 1, i64 1, i64 1, i64 1>739 %7 = trunc <4 x i64> %6 to <4 x i32>740 %8 = getelementptr inbounds i32, ptr %z, i32 %index741 store <4 x i32> %7, ptr %8, align 4742 %index.next = add i32 %index, 4743 %9 = icmp eq i32 %index.next, 1024744 br i1 %9, label %for.cond.cleanup, label %vector.body745 746for.cond.cleanup: ; preds = %vector.body747 ret void748}749 750define void @vrhadd_loop_u8(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {751; CHECK-LABEL: vrhadd_loop_u8:752; CHECK: @ %bb.0: @ %entry753; CHECK-NEXT: .save {r7, lr}754; CHECK-NEXT: push {r7, lr}755; CHECK-NEXT: mov.w lr, #64756; CHECK-NEXT: .LBB33_1: @ %vector.body757; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1758; CHECK-NEXT: vldrb.u8 q0, [r1], #16759; CHECK-NEXT: vldrb.u8 q1, [r0], #16760; CHECK-NEXT: vrhadd.u8 q0, q1, q0761; CHECK-NEXT: vstrb.8 q0, [r2], #16762; CHECK-NEXT: le lr, .LBB33_1763; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup764; CHECK-NEXT: pop {r7, pc}765entry:766 br label %vector.body767 768vector.body: ; preds = %vector.body, %entry769 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]770 %0 = getelementptr inbounds i8, ptr %x, i32 %index771 %wide.load = load <16 x i8>, ptr %0, align 1772 %1 = zext <16 x i8> %wide.load to <16 x i16>773 %2 = getelementptr inbounds i8, ptr %y, i32 %index774 %wide.load16 = load <16 x i8>, ptr %2, align 1775 %3 = zext <16 x i8> %wide.load16 to <16 x i16>776 %4 = add nuw nsw <16 x i16> %1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>777 %5 = add nuw nsw <16 x i16> %4, %3778 %6 = lshr <16 x i16> %5, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>779 %7 = trunc <16 x i16> %6 to <16 x i8>780 %8 = getelementptr inbounds i8, ptr %z, i32 %index781 store <16 x i8> %7, ptr %8, align 1782 %index.next = add i32 %index, 16783 %9 = icmp eq i32 %index.next, 1024784 br i1 %9, label %for.cond.cleanup, label %vector.body785 786for.cond.cleanup: ; preds = %vector.body787 ret void788}789 790define void @vrhadd_loop_u16(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {791; CHECK-LABEL: vrhadd_loop_u16:792; CHECK: @ %bb.0: @ %entry793; CHECK-NEXT: .save {r7, lr}794; CHECK-NEXT: push {r7, lr}795; CHECK-NEXT: mov.w lr, #128796; CHECK-NEXT: .LBB34_1: @ %vector.body797; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1798; CHECK-NEXT: vldrh.u16 q0, [r1], #16799; CHECK-NEXT: vldrh.u16 q1, [r0], #16800; CHECK-NEXT: vrhadd.u16 q0, q1, q0801; CHECK-NEXT: vstrb.8 q0, [r2], #16802; CHECK-NEXT: le lr, .LBB34_1803; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup804; CHECK-NEXT: pop {r7, pc}805entry:806 br label %vector.body807 808vector.body: ; preds = %vector.body, %entry809 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]810 %0 = getelementptr inbounds i16, ptr %x, i32 %index811 %wide.load = load <8 x i16>, ptr %0, align 2812 %1 = zext <8 x i16> %wide.load to <8 x i32>813 %2 = getelementptr inbounds i16, ptr %y, i32 %index814 %wide.load16 = load <8 x i16>, ptr %2, align 2815 %3 = zext <8 x i16> %wide.load16 to <8 x i32>816 %4 = add nuw nsw <8 x i32> %1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>817 %5 = add nuw nsw <8 x i32> %4, %3818 %6 = lshr <8 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>819 %7 = trunc <8 x i32> %6 to <8 x i16>820 %8 = getelementptr inbounds i16, ptr %z, i32 %index821 store <8 x i16> %7, ptr %8, align 2822 %index.next = add i32 %index, 8823 %9 = icmp eq i32 %index.next, 1024824 br i1 %9, label %for.cond.cleanup, label %vector.body825 826for.cond.cleanup: ; preds = %vector.body827 ret void828}829 830define void @vrhadd_loop_u32(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {831; CHECK-LABEL: vrhadd_loop_u32:832; CHECK: @ %bb.0: @ %entry833; CHECK-NEXT: .save {r7, lr}834; CHECK-NEXT: push {r7, lr}835; CHECK-NEXT: mov.w lr, #256836; CHECK-NEXT: .LBB35_1: @ %vector.body837; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1838; CHECK-NEXT: vldrw.u32 q0, [r1], #16839; CHECK-NEXT: vldrw.u32 q1, [r0], #16840; CHECK-NEXT: vrhadd.u32 q0, q1, q0841; CHECK-NEXT: vstrb.8 q0, [r2], #16842; CHECK-NEXT: le lr, .LBB35_1843; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup844; CHECK-NEXT: pop {r7, pc}845entry:846 br label %vector.body847 848vector.body: ; preds = %vector.body, %entry849 %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]850 %0 = getelementptr inbounds i32, ptr %x, i32 %index851 %wide.load = load <4 x i32>, ptr %0, align 4852 %1 = zext <4 x i32> %wide.load to <4 x i64>853 %2 = getelementptr inbounds i32, ptr %y, i32 %index854 %wide.load16 = load <4 x i32>, ptr %2, align 4855 %3 = zext <4 x i32> %wide.load16 to <4 x i64>856 %4 = add nuw nsw <4 x i64> %1, <i64 1, i64 1, i64 1, i64 1>857 %5 = add nuw nsw <4 x i64> %4, %3858 %6 = lshr <4 x i64> %5, <i64 1, i64 1, i64 1, i64 1>859 %7 = trunc <4 x i64> %6 to <4 x i32>860 %8 = getelementptr inbounds i32, ptr %z, i32 %index861 store <4 x i32> %7, ptr %8, align 4862 %index.next = add i32 %index, 4863 %9 = icmp eq i32 %index.next, 1024864 br i1 %9, label %for.cond.cleanup, label %vector.body865 866for.cond.cleanup: ; preds = %vector.body867 ret void868}869 870 871define arm_aapcs_vfpcc i16 @vhadds_reduce_v16i8(<16 x i8> %s0, <16 x i8> %s1) {872; CHECK-LABEL: vhadds_reduce_v16i8:873; CHECK: @ %bb.0: @ %entry874; CHECK-NEXT: vhadd.s8 q0, q0, q1875; CHECK-NEXT: vaddv.s8 r0, q0876; CHECK-NEXT: bx lr877entry:878 %s0s = sext <16 x i8> %s0 to <16 x i16>879 %s1s = sext <16 x i8> %s1 to <16 x i16>880 %add = add <16 x i16> %s0s, %s1s881 %s = ashr <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>882 %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)883 ret i16 %result884}885 886define arm_aapcs_vfpcc i16 @vhaddu_reduce_v16i8(<16 x i8> %s0, <16 x i8> %s1) {887; CHECK-LABEL: vhaddu_reduce_v16i8:888; CHECK: @ %bb.0: @ %entry889; CHECK-NEXT: vhadd.u8 q0, q0, q1890; CHECK-NEXT: vaddv.u8 r0, q0891; CHECK-NEXT: bx lr892entry:893 %s0s = zext <16 x i8> %s0 to <16 x i16>894 %s1s = zext <16 x i8> %s1 to <16 x i16>895 %add = add <16 x i16> %s0s, %s1s896 %s = lshr <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>897 %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)898 ret i16 %result899}900 901define arm_aapcs_vfpcc i16 @vrhadds_reduce_v16i8(<16 x i8> %s0, <16 x i8> %s1) {902; CHECK-LABEL: vrhadds_reduce_v16i8:903; CHECK: @ %bb.0: @ %entry904; CHECK-NEXT: vrhadd.s8 q0, q0, q1905; CHECK-NEXT: vaddv.s8 r0, q0906; CHECK-NEXT: bx lr907entry:908 %s0s = sext <16 x i8> %s0 to <16 x i16>909 %s1s = sext <16 x i8> %s1 to <16 x i16>910 %add = add <16 x i16> %s0s, %s1s911 %add2 = add <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>912 %s = ashr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>913 %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)914 ret i16 %result915}916 917define arm_aapcs_vfpcc i16 @vrhaddu_reduce_v16i8(<16 x i8> %s0, <16 x i8> %s1) {918; CHECK-LABEL: vrhaddu_reduce_v16i8:919; CHECK: @ %bb.0: @ %entry920; CHECK-NEXT: vrhadd.u8 q0, q0, q1921; CHECK-NEXT: vaddv.u8 r0, q0922; CHECK-NEXT: bx lr923entry:924 %s0s = zext <16 x i8> %s0 to <16 x i16>925 %s1s = zext <16 x i8> %s1 to <16 x i16>926 %add = add <16 x i16> %s0s, %s1s927 %add2 = add <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>928 %s = lshr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>929 %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)930 ret i16 %result931}932 933declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>)934