383 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=aarch64 | FileCheck %s3 4define <4 x i16> @mlai16_trunc(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2) {5; CHECK-LABEL: mlai16_trunc:6; CHECK: // %bb.0: // %entry7; CHECK-NEXT: smull v0.4s, v1.4h, v0.4h8; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h9; CHECK-NEXT: xtn v0.4h, v0.4s10; CHECK-NEXT: ret11entry:12 %v0 = sext <4 x i16> %vec0 to <4 x i32>13 %v1 = sext <4 x i16> %vec1 to <4 x i32>14 %v2 = sext <4 x i16> %vec2 to <4 x i32>15 %v3 = mul <4 x i32> %v1, %v016 %v4 = add <4 x i32> %v3, %v217 %v5 = trunc <4 x i32> %v4 to <4 x i16>18 ret <4 x i16> %v519}20 21define <4 x i32> @mlai16_and(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2) {22; CHECK-LABEL: mlai16_and:23; CHECK: // %bb.0: // %entry24; CHECK-NEXT: smull v0.4s, v1.4h, v0.4h25; CHECK-NEXT: movi v1.2d, #0x00ffff0000ffff26; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h27; CHECK-NEXT: and v0.16b, v0.16b, v1.16b28; CHECK-NEXT: ret29entry:30 %v0 = sext <4 x i16> %vec0 to <4 x i32>31 %v1 = sext <4 x i16> %vec1 to <4 x i32>32 %v2 = sext <4 x i16> %vec2 to <4 x i32>33 %v3 = mul <4 x i32> %v1, %v034 %v4 = add <4 x i32> %v3, %v235 %v5 = and <4 x i32> %v4, <i32 65535, i32 65535, i32 65535, i32 65535>36 ret <4 x i32> %v537}38 39define void @mlai16_loadstore(ptr %a, ptr %b, ptr %c) {40; CHECK-LABEL: mlai16_loadstore:41; CHECK: // %bb.0: // %entry42; CHECK-NEXT: ldr d0, [x0, #16]43; CHECK-NEXT: ldr d1, [x1, #16]44; CHECK-NEXT: smull v0.4s, v1.4h, v0.4h45; CHECK-NEXT: ldr d1, [x2, #16]46; CHECK-NEXT: uaddw v0.4s, v0.4s, v1.4h47; CHECK-NEXT: xtn v0.4h, v0.4s48; CHECK-NEXT: str d0, [x0, #16]49; CHECK-NEXT: ret50entry:51 %scevgep0 = getelementptr i16, ptr %a, i32 852 %vec0 = load <4 x i16>, ptr %scevgep0, align 853 %v0 = sext <4 x i16> %vec0 to <4 x i32>54 %scevgep1 = getelementptr i16, ptr %b, i32 855 %vec1 = load <4 x i16>, ptr %scevgep1, align 856 %v1 = sext <4 x i16> %vec1 to <4 x i32>57 %scevgep2 = getelementptr i16, ptr %c, i32 858 %vec2 = load <4 x i16>, ptr %scevgep2, align 859 %v2 = sext <4 x i16> %vec2 to <4 x i32>60 %v3 = mul <4 x i32> %v1, %v061 %v4 = add <4 x i32> %v3, %v262 %v5 = trunc <4 x i32> %v4 to <4 x i16>63 %scevgep3 = getelementptr i16, ptr %a, i32 864 store <4 x i16> %v5, ptr %scevgep3, align 865 ret void66}67 68define <4 x i16> @addmuli16_trunc(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2) {69; CHECK-LABEL: addmuli16_trunc:70; CHECK: // %bb.0: // %entry71; CHECK-NEXT: smull v1.4s, v1.4h, v2.4h72; CHECK-NEXT: smlal v1.4s, v0.4h, v2.4h73; CHECK-NEXT: xtn v0.4h, v1.4s74; CHECK-NEXT: ret75entry:76 %v0 = sext <4 x i16> %vec0 to <4 x i32>77 %v1 = sext <4 x i16> %vec1 to <4 x i32>78 %v2 = sext <4 x i16> %vec2 to <4 x i32>79 %v3 = add <4 x i32> %v1, %v080 %v4 = mul <4 x i32> %v3, %v281 %v5 = trunc <4 x i32> %v4 to <4 x i16>82 ret <4 x i16> %v583}84 85define <4 x i32> @addmuli16_and(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2) {86; CHECK-LABEL: addmuli16_and:87; CHECK: // %bb.0: // %entry88; CHECK-NEXT: smull v1.4s, v1.4h, v2.4h89; CHECK-NEXT: smlal v1.4s, v0.4h, v2.4h90; CHECK-NEXT: movi v0.2d, #0x00ffff0000ffff91; CHECK-NEXT: and v0.16b, v1.16b, v0.16b92; CHECK-NEXT: ret93entry:94 %v0 = sext <4 x i16> %vec0 to <4 x i32>95 %v1 = sext <4 x i16> %vec1 to <4 x i32>96 %v2 = sext <4 x i16> %vec2 to <4 x i32>97 %v3 = add <4 x i32> %v1, %v098 %v4 = mul <4 x i32> %v3, %v299 %v5 = and <4 x i32> %v4, <i32 65535, i32 65535, i32 65535, i32 65535>100 ret <4 x i32> %v5101}102 103define void @addmuli16_loadstore(ptr %a, ptr %b, ptr %c) {104; CHECK-LABEL: addmuli16_loadstore:105; CHECK: // %bb.0: // %entry106; CHECK-NEXT: ldr d0, [x1, #16]107; CHECK-NEXT: ldr d1, [x2, #16]108; CHECK-NEXT: ldr d2, [x0, #16]109; CHECK-NEXT: smull v0.4s, v0.4h, v1.4h110; CHECK-NEXT: smlal v0.4s, v2.4h, v1.4h111; CHECK-NEXT: xtn v0.4h, v0.4s112; CHECK-NEXT: str d0, [x0, #16]113; CHECK-NEXT: ret114entry:115 %scevgep0 = getelementptr i16, ptr %a, i32 8116 %vec0 = load <4 x i16>, ptr %scevgep0, align 8117 %v0 = sext <4 x i16> %vec0 to <4 x i32>118 %scevgep1 = getelementptr i16, ptr %b, i32 8119 %vec1 = load <4 x i16>, ptr %scevgep1, align 8120 %v1 = sext <4 x i16> %vec1 to <4 x i32>121 %scevgep2 = getelementptr i16, ptr %c, i32 8122 %vec2 = load <4 x i16>, ptr %scevgep2, align 8123 %v2 = sext <4 x i16> %vec2 to <4 x i32>124 %v3 = add <4 x i32> %v1, %v0125 %v4 = mul <4 x i32> %v3, %v2126 %v5 = trunc <4 x i32> %v4 to <4 x i16>127 %scevgep3 = getelementptr i16, ptr %a, i32 8128 store <4 x i16> %v5, ptr %scevgep3, align 8129 ret void130}131 132define <2 x i32> @mlai32_trunc(<2 x i32> %vec0, <2 x i32> %vec1, <2 x i32> %vec2) {133; CHECK-LABEL: mlai32_trunc:134; CHECK: // %bb.0: // %entry135; CHECK-NEXT: smull v0.2d, v1.2s, v0.2s136; CHECK-NEXT: uaddw v0.2d, v0.2d, v2.2s137; CHECK-NEXT: xtn v0.2s, v0.2d138; CHECK-NEXT: ret139entry:140 %v0 = sext <2 x i32> %vec0 to <2 x i64>141 %v1 = sext <2 x i32> %vec1 to <2 x i64>142 %v2 = sext <2 x i32> %vec2 to <2 x i64>143 %v3 = mul <2 x i64> %v1, %v0144 %v4 = add <2 x i64> %v3, %v2145 %v5 = trunc <2 x i64> %v4 to <2 x i32>146 ret <2 x i32> %v5147}148 149define <2 x i64> @mlai32_and(<2 x i32> %vec0, <2 x i32> %vec1, <2 x i32> %vec2) {150; CHECK-LABEL: mlai32_and:151; CHECK: // %bb.0: // %entry152; CHECK-NEXT: smull v0.2d, v1.2s, v0.2s153; CHECK-NEXT: movi v1.2d, #0x000000ffffffff154; CHECK-NEXT: uaddw v0.2d, v0.2d, v2.2s155; CHECK-NEXT: and v0.16b, v0.16b, v1.16b156; CHECK-NEXT: ret157entry:158 %v0 = sext <2 x i32> %vec0 to <2 x i64>159 %v1 = sext <2 x i32> %vec1 to <2 x i64>160 %v2 = sext <2 x i32> %vec2 to <2 x i64>161 %v3 = mul <2 x i64> %v1, %v0162 %v4 = add <2 x i64> %v3, %v2163 %v5 = and <2 x i64> %v4, <i64 4294967295, i64 4294967295>164 ret <2 x i64> %v5165}166 167define void @mlai32_loadstore(ptr %a, ptr %b, ptr %c) {168; CHECK-LABEL: mlai32_loadstore:169; CHECK: // %bb.0: // %entry170; CHECK-NEXT: ldr d0, [x0, #32]171; CHECK-NEXT: ldr d1, [x1, #32]172; CHECK-NEXT: smull v0.2d, v1.2s, v0.2s173; CHECK-NEXT: ldr d1, [x2, #32]174; CHECK-NEXT: uaddw v0.2d, v0.2d, v1.2s175; CHECK-NEXT: xtn v0.2s, v0.2d176; CHECK-NEXT: str d0, [x0, #32]177; CHECK-NEXT: ret178entry:179 %scevgep0 = getelementptr i32, ptr %a, i32 8180 %vec0 = load <2 x i32>, ptr %scevgep0, align 8181 %v0 = sext <2 x i32> %vec0 to <2 x i64>182 %scevgep1 = getelementptr i32, ptr %b, i32 8183 %vec1 = load <2 x i32>, ptr %scevgep1, align 8184 %v1 = sext <2 x i32> %vec1 to <2 x i64>185 %scevgep2 = getelementptr i32, ptr %c, i32 8186 %vec2 = load <2 x i32>, ptr %scevgep2, align 8187 %v2 = sext <2 x i32> %vec2 to <2 x i64>188 %v3 = mul <2 x i64> %v1, %v0189 %v4 = add <2 x i64> %v3, %v2190 %v5 = trunc <2 x i64> %v4 to <2 x i32>191 %scevgep3 = getelementptr i32, ptr %a, i32 8192 store <2 x i32> %v5, ptr %scevgep3, align 8193 ret void194}195 196define <2 x i32> @addmuli32_trunc(<2 x i32> %vec0, <2 x i32> %vec1, <2 x i32> %vec2) {197; CHECK-LABEL: addmuli32_trunc:198; CHECK: // %bb.0: // %entry199; CHECK-NEXT: smull v1.2d, v1.2s, v2.2s200; CHECK-NEXT: smlal v1.2d, v0.2s, v2.2s201; CHECK-NEXT: xtn v0.2s, v1.2d202; CHECK-NEXT: ret203entry:204 %v0 = sext <2 x i32> %vec0 to <2 x i64>205 %v1 = sext <2 x i32> %vec1 to <2 x i64>206 %v2 = sext <2 x i32> %vec2 to <2 x i64>207 %v3 = add <2 x i64> %v1, %v0208 %v4 = mul <2 x i64> %v3, %v2209 %v5 = trunc <2 x i64> %v4 to <2 x i32>210 ret <2 x i32> %v5211}212 213define <2 x i64> @addmuli32_and(<2 x i32> %vec0, <2 x i32> %vec1, <2 x i32> %vec2) {214; CHECK-LABEL: addmuli32_and:215; CHECK: // %bb.0: // %entry216; CHECK-NEXT: smull v1.2d, v1.2s, v2.2s217; CHECK-NEXT: smlal v1.2d, v0.2s, v2.2s218; CHECK-NEXT: movi v0.2d, #0x000000ffffffff219; CHECK-NEXT: and v0.16b, v1.16b, v0.16b220; CHECK-NEXT: ret221entry:222 %v0 = sext <2 x i32> %vec0 to <2 x i64>223 %v1 = sext <2 x i32> %vec1 to <2 x i64>224 %v2 = sext <2 x i32> %vec2 to <2 x i64>225 %v3 = add <2 x i64> %v1, %v0226 %v4 = mul <2 x i64> %v3, %v2227 %v5 = and <2 x i64> %v4, <i64 4294967295, i64 4294967295>228 ret <2 x i64> %v5229}230 231define void @addmuli32_loadstore(ptr %a, ptr %b, ptr %c) {232; CHECK-LABEL: addmuli32_loadstore:233; CHECK: // %bb.0: // %entry234; CHECK-NEXT: ldr d0, [x1, #32]235; CHECK-NEXT: ldr d1, [x2, #32]236; CHECK-NEXT: ldr d2, [x0, #32]237; CHECK-NEXT: smull v0.2d, v0.2s, v1.2s238; CHECK-NEXT: smlal v0.2d, v2.2s, v1.2s239; CHECK-NEXT: xtn v0.2s, v0.2d240; CHECK-NEXT: str d0, [x0, #32]241; CHECK-NEXT: ret242entry:243 %scevgep0 = getelementptr i32, ptr %a, i32 8244 %vec0 = load <2 x i32>, ptr %scevgep0, align 8245 %v0 = sext <2 x i32> %vec0 to <2 x i64>246 %scevgep1 = getelementptr i32, ptr %b, i32 8247 %vec1 = load <2 x i32>, ptr %scevgep1, align 8248 %v1 = sext <2 x i32> %vec1 to <2 x i64>249 %scevgep2 = getelementptr i32, ptr %c, i32 8250 %vec2 = load <2 x i32>, ptr %scevgep2, align 8251 %v2 = sext <2 x i32> %vec2 to <2 x i64>252 %v3 = add <2 x i64> %v1, %v0253 %v4 = mul <2 x i64> %v3, %v2254 %v5 = trunc <2 x i64> %v4 to <2 x i32>255 %scevgep3 = getelementptr i32, ptr %a, i32 8256 store <2 x i32> %v5, ptr %scevgep3, align 8257 ret void258}259 260define void @func1(ptr %a, ptr %b, ptr %c) {261; CHECK-LABEL: func1:262; CHECK: // %bb.0: // %entry263; CHECK-NEXT: ldr d0, [x2, #16]264; CHECK-NEXT: ldr d1, [x1, #16]265; CHECK-NEXT: ldr d2, [x0, #16]266; CHECK-NEXT: sshll v0.4s, v0.4h, #0267; CHECK-NEXT: uaddw v0.4s, v0.4s, v1.4h268; CHECK-NEXT: xtn v1.4h, v0.4s269; CHECK-NEXT: str d1, [x0, #16]270; CHECK-NEXT: ldr d1, [x2, #16]271; CHECK-NEXT: sshll v1.4s, v1.4h, #0272; CHECK-NEXT: mul v0.4s, v1.4s, v0.4s273; CHECK-NEXT: xtn v1.4h, v0.4s274; CHECK-NEXT: str d1, [x1, #16]275; CHECK-NEXT: ldr d1, [x2, #16]276; CHECK-NEXT: smlal v0.4s, v1.4h, v2.4h277; CHECK-NEXT: xtn v0.4h, v0.4s278; CHECK-NEXT: str d0, [x0, #16]279; CHECK-NEXT: ret280entry:281; The test case trying to vectorize the pseudo code below.282; a[i] = b[i] + c[i];283; b[i] = aptr c[i];284; a[i] = b[i] + aptr c[i];285; Checking that vector load a[i] for "a[i] = b[i] + aptr c[i]" is286; scheduled before the first vector store to "a[i] = b[i] + c[i]".287; Checking that there is no vector load a[i] scheduled between the vector288; stores to a[i], otherwise the load of a[i] will be polluted by the first289; vector store to a[i].290; This test case check that the chain information is updated during291; lowerMUL for the new created Load SDNode.292 293 294 %scevgep0 = getelementptr i16, ptr %a, i32 8295 %vec0 = load <4 x i16>, ptr %scevgep0, align 8296 %scevgep1 = getelementptr i16, ptr %b, i32 8297 %vec1 = load <4 x i16>, ptr %scevgep1, align 8298 %0 = zext <4 x i16> %vec1 to <4 x i32>299 %scevgep2 = getelementptr i16, ptr %c, i32 8300 %vec2 = load <4 x i16>, ptr %scevgep2, align 8301 %1 = sext <4 x i16> %vec2 to <4 x i32>302 %vec3 = add <4 x i32> %1, %0303 %2 = trunc <4 x i32> %vec3 to <4 x i16>304 %scevgep3 = getelementptr i16, ptr %a, i32 8305 store <4 x i16> %2, ptr %scevgep3, align 8306 %vec4 = load <4 x i16>, ptr %scevgep2, align 8307 %3 = sext <4 x i16> %vec4 to <4 x i32>308 %vec5 = mul <4 x i32> %3, %vec3309 %4 = trunc <4 x i32> %vec5 to <4 x i16>310 store <4 x i16> %4, ptr %scevgep1, align 8311 %5 = sext <4 x i16> %vec0 to <4 x i32>312 %vec6 = load <4 x i16>, ptr %scevgep2, align 8313 %6 = sext <4 x i16> %vec6 to <4 x i32>314 %vec7 = mul <4 x i32> %6, %5315 %vec8 = add <4 x i32> %vec7, %vec5316 %7 = trunc <4 x i32> %vec8 to <4 x i16>317 store <4 x i16> %7, ptr %scevgep3, align 8318 ret void319}320 321define void @func2(ptr %a, ptr %b, ptr %c) {322; CHECK-LABEL: func2:323; CHECK: // %bb.0: // %entry324; CHECK-NEXT: ldr d0, [x2, #16]325; CHECK-NEXT: ldr d1, [x1, #16]326; CHECK-NEXT: ldr d2, [x0, #16]327; CHECK-NEXT: sshll v0.4s, v0.4h, #0328; CHECK-NEXT: uaddw v0.4s, v0.4s, v1.4h329; CHECK-NEXT: xtn v1.4h, v0.4s330; CHECK-NEXT: str d1, [x0, #16]331; CHECK-NEXT: ldr d1, [x2, #16]332; CHECK-NEXT: sshll v1.4s, v1.4h, #0333; CHECK-NEXT: mul v0.4s, v1.4s, v0.4s334; CHECK-NEXT: xtn v1.4h, v0.4s335; CHECK-NEXT: str d1, [x1, #16]336; CHECK-NEXT: ldr d1, [x2, #16]337; CHECK-NEXT: smlal v0.4s, v1.4h, v2.4h338; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h339; CHECK-NEXT: xtn v0.4h, v0.4s340; CHECK-NEXT: str d0, [x0, #16]341; CHECK-NEXT: ret342entry:343; The test case trying to vectorize the pseudo code below.344; a[i] = b[i] + c[i];345; b[i] = aptr c[i];346; a[i] = b[i] + aptr c[i] + a[i];347; Checking that vector load a[i] for "a[i] = b[i] + aptr c[i] + a[i]"348; is scheduled before the first vector store to "a[i] = b[i] + c[i]".349; Checking that there is no vector load a[i] scheduled between the first350; vector store to a[i] and the vector add of a[i], otherwise the load of351; a[i] will be polluted by the first vector store to a[i].352; This test case check that both the chain and value of the new created353; Load SDNode are updated during lowerMUL.354 355 356 %scevgep0 = getelementptr i16, ptr %a, i32 8357 %vec0 = load <4 x i16>, ptr %scevgep0, align 8358 %scevgep1 = getelementptr i16, ptr %b, i32 8359 %vec1 = load <4 x i16>, ptr %scevgep1, align 8360 %0 = zext <4 x i16> %vec1 to <4 x i32>361 %scevgep2 = getelementptr i16, ptr %c, i32 8362 %vec2 = load <4 x i16>, ptr %scevgep2, align 8363 %1 = sext <4 x i16> %vec2 to <4 x i32>364 %vec3 = add <4 x i32> %1, %0365 %2 = trunc <4 x i32> %vec3 to <4 x i16>366 %scevgep3 = getelementptr i16, ptr %a, i32 8367 store <4 x i16> %2, ptr %scevgep3, align 8368 %vec4 = load <4 x i16>, ptr %scevgep2, align 8369 %3 = sext <4 x i16> %vec4 to <4 x i32>370 %vec5 = mul <4 x i32> %3, %vec3371 %4 = trunc <4 x i32> %vec5 to <4 x i16>372 store <4 x i16> %4, ptr %scevgep1, align 8373 %5 = sext <4 x i16> %vec0 to <4 x i32>374 %vec6 = load <4 x i16>, ptr %scevgep2, align 8375 %6 = sext <4 x i16> %vec6 to <4 x i32>376 %vec7 = mul <4 x i32> %6, %5377 %vec8 = add <4 x i32> %vec7, %vec5378 %vec9 = add <4 x i32> %vec8, %5379 %7 = trunc <4 x i32> %vec9 to <4 x i16>380 store <4 x i16> %7, ptr %scevgep3, align 8381 ret void382}383