1199 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE23; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE,SSSE34; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE415; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX16; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX27; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F8; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F9; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BW10; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BW11 12declare <1 x i8> @llvm.uadd.sat.v1i8(<1 x i8>, <1 x i8>)13declare <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8>, <2 x i8>)14declare <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8>, <4 x i8>)15declare <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8>, <8 x i8>)16declare <12 x i8> @llvm.uadd.sat.v12i8(<12 x i8>, <12 x i8>)17declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>)18declare <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8>, <32 x i8>)19declare <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8>, <64 x i8>)20 21declare <1 x i16> @llvm.uadd.sat.v1i16(<1 x i16>, <1 x i16>)22declare <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16>, <2 x i16>)23declare <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16>, <4 x i16>)24declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>)25declare <12 x i16> @llvm.uadd.sat.v12i16(<12 x i16>, <12 x i16>)26declare <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16>, <16 x i16>)27declare <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16>, <32 x i16>)28 29declare <16 x i1> @llvm.uadd.sat.v16i1(<16 x i1>, <16 x i1>)30declare <16 x i4> @llvm.uadd.sat.v16i4(<16 x i4>, <16 x i4>)31 32declare <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32>, <2 x i32>)33declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>)34declare <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32>, <8 x i32>)35declare <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32>, <16 x i32>)36declare <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64>, <2 x i64>)37declare <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64>, <4 x i64>)38declare <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64>, <8 x i64>)39 40declare <4 x i24> @llvm.uadd.sat.v4i24(<4 x i24>, <4 x i24>)41declare <2 x i128> @llvm.uadd.sat.v2i128(<2 x i128>, <2 x i128>)42 43; Legal types, depending on architecture.44 45define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {46; SSE-LABEL: v16i8:47; SSE: # %bb.0:48; SSE-NEXT: paddusb %xmm1, %xmm049; SSE-NEXT: retq50;51; AVX-LABEL: v16i8:52; AVX: # %bb.0:53; AVX-NEXT: vpaddusb %xmm1, %xmm0, %xmm054; AVX-NEXT: retq55 %z = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)56 ret <16 x i8> %z57}58 59define <32 x i8> @v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {60; SSE-LABEL: v32i8:61; SSE: # %bb.0:62; SSE-NEXT: paddusb %xmm2, %xmm063; SSE-NEXT: paddusb %xmm3, %xmm164; SSE-NEXT: retq65;66; AVX1-LABEL: v32i8:67; AVX1: # %bb.0:68; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm269; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm370; AVX1-NEXT: vpaddusb %xmm2, %xmm3, %xmm271; AVX1-NEXT: vpaddusb %xmm1, %xmm0, %xmm072; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm073; AVX1-NEXT: retq74;75; AVX2-LABEL: v32i8:76; AVX2: # %bb.0:77; AVX2-NEXT: vpaddusb %ymm1, %ymm0, %ymm078; AVX2-NEXT: retq79;80; AVX512-LABEL: v32i8:81; AVX512: # %bb.0:82; AVX512-NEXT: vpaddusb %ymm1, %ymm0, %ymm083; AVX512-NEXT: retq84 %z = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %x, <32 x i8> %y)85 ret <32 x i8> %z86}87 88define <64 x i8> @v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {89; SSE-LABEL: v64i8:90; SSE: # %bb.0:91; SSE-NEXT: paddusb %xmm4, %xmm092; SSE-NEXT: paddusb %xmm5, %xmm193; SSE-NEXT: paddusb %xmm6, %xmm294; SSE-NEXT: paddusb %xmm7, %xmm395; SSE-NEXT: retq96;97; AVX1-LABEL: v64i8:98; AVX1: # %bb.0:99; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4100; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5101; AVX1-NEXT: vpaddusb %xmm4, %xmm5, %xmm4102; AVX1-NEXT: vpaddusb %xmm2, %xmm0, %xmm0103; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0104; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2105; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4106; AVX1-NEXT: vpaddusb %xmm2, %xmm4, %xmm2107; AVX1-NEXT: vpaddusb %xmm3, %xmm1, %xmm1108; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1109; AVX1-NEXT: retq110;111; AVX2-LABEL: v64i8:112; AVX2: # %bb.0:113; AVX2-NEXT: vpaddusb %ymm2, %ymm0, %ymm0114; AVX2-NEXT: vpaddusb %ymm3, %ymm1, %ymm1115; AVX2-NEXT: retq116;117; AVX512F-LABEL: v64i8:118; AVX512F: # %bb.0:119; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2120; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3121; AVX512F-NEXT: vpaddusb %ymm2, %ymm3, %ymm2122; AVX512F-NEXT: vpaddusb %ymm1, %ymm0, %ymm0123; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0124; AVX512F-NEXT: retq125;126; AVX512BW-LABEL: v64i8:127; AVX512BW: # %bb.0:128; AVX512BW-NEXT: vpaddusb %zmm1, %zmm0, %zmm0129; AVX512BW-NEXT: retq130 %z = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %x, <64 x i8> %y)131 ret <64 x i8> %z132}133 134define <8 x i16> @v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {135; SSE-LABEL: v8i16:136; SSE: # %bb.0:137; SSE-NEXT: paddusw %xmm1, %xmm0138; SSE-NEXT: retq139;140; AVX-LABEL: v8i16:141; AVX: # %bb.0:142; AVX-NEXT: vpaddusw %xmm1, %xmm0, %xmm0143; AVX-NEXT: retq144 %z = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)145 ret <8 x i16> %z146}147 148define <16 x i16> @v16i16(<16 x i16> %x, <16 x i16> %y) nounwind {149; SSE-LABEL: v16i16:150; SSE: # %bb.0:151; SSE-NEXT: paddusw %xmm2, %xmm0152; SSE-NEXT: paddusw %xmm3, %xmm1153; SSE-NEXT: retq154;155; AVX1-LABEL: v16i16:156; AVX1: # %bb.0:157; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2158; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3159; AVX1-NEXT: vpaddusw %xmm2, %xmm3, %xmm2160; AVX1-NEXT: vpaddusw %xmm1, %xmm0, %xmm0161; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0162; AVX1-NEXT: retq163;164; AVX2-LABEL: v16i16:165; AVX2: # %bb.0:166; AVX2-NEXT: vpaddusw %ymm1, %ymm0, %ymm0167; AVX2-NEXT: retq168;169; AVX512-LABEL: v16i16:170; AVX512: # %bb.0:171; AVX512-NEXT: vpaddusw %ymm1, %ymm0, %ymm0172; AVX512-NEXT: retq173 %z = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %x, <16 x i16> %y)174 ret <16 x i16> %z175}176 177define <32 x i16> @v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {178; SSE-LABEL: v32i16:179; SSE: # %bb.0:180; SSE-NEXT: paddusw %xmm4, %xmm0181; SSE-NEXT: paddusw %xmm5, %xmm1182; SSE-NEXT: paddusw %xmm6, %xmm2183; SSE-NEXT: paddusw %xmm7, %xmm3184; SSE-NEXT: retq185;186; AVX1-LABEL: v32i16:187; AVX1: # %bb.0:188; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4189; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5190; AVX1-NEXT: vpaddusw %xmm4, %xmm5, %xmm4191; AVX1-NEXT: vpaddusw %xmm2, %xmm0, %xmm0192; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0193; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2194; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4195; AVX1-NEXT: vpaddusw %xmm2, %xmm4, %xmm2196; AVX1-NEXT: vpaddusw %xmm3, %xmm1, %xmm1197; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1198; AVX1-NEXT: retq199;200; AVX2-LABEL: v32i16:201; AVX2: # %bb.0:202; AVX2-NEXT: vpaddusw %ymm2, %ymm0, %ymm0203; AVX2-NEXT: vpaddusw %ymm3, %ymm1, %ymm1204; AVX2-NEXT: retq205;206; AVX512F-LABEL: v32i16:207; AVX512F: # %bb.0:208; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2209; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3210; AVX512F-NEXT: vpaddusw %ymm2, %ymm3, %ymm2211; AVX512F-NEXT: vpaddusw %ymm1, %ymm0, %ymm0212; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0213; AVX512F-NEXT: retq214;215; AVX512BW-LABEL: v32i16:216; AVX512BW: # %bb.0:217; AVX512BW-NEXT: vpaddusw %zmm1, %zmm0, %zmm0218; AVX512BW-NEXT: retq219 %z = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %x, <32 x i16> %y)220 ret <32 x i16> %z221}222 223; Too narrow vectors, legalized by widening.224 225define void @v8i8(ptr %px, ptr %py, ptr %pz) nounwind {226; SSE-LABEL: v8i8:227; SSE: # %bb.0:228; SSE-NEXT: movq {{.*#+}} xmm0 = mem[0],zero229; SSE-NEXT: movq {{.*#+}} xmm1 = mem[0],zero230; SSE-NEXT: paddusb %xmm0, %xmm1231; SSE-NEXT: movq %xmm1, (%rdx)232; SSE-NEXT: retq233;234; AVX-LABEL: v8i8:235; AVX: # %bb.0:236; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero237; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero238; AVX-NEXT: vpaddusb %xmm1, %xmm0, %xmm0239; AVX-NEXT: vmovq %xmm0, (%rdx)240; AVX-NEXT: retq241 %x = load <8 x i8>, ptr %px242 %y = load <8 x i8>, ptr %py243 %z = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> %x, <8 x i8> %y)244 store <8 x i8> %z, ptr %pz245 ret void246}247 248define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind {249; SSE-LABEL: v4i8:250; SSE: # %bb.0:251; SSE-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero252; SSE-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero253; SSE-NEXT: paddusb %xmm0, %xmm1254; SSE-NEXT: movd %xmm1, (%rdx)255; SSE-NEXT: retq256;257; AVX-LABEL: v4i8:258; AVX: # %bb.0:259; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero260; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero261; AVX-NEXT: vpaddusb %xmm1, %xmm0, %xmm0262; AVX-NEXT: vmovd %xmm0, (%rdx)263; AVX-NEXT: retq264 %x = load <4 x i8>, ptr %px265 %y = load <4 x i8>, ptr %py266 %z = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> %x, <4 x i8> %y)267 store <4 x i8> %z, ptr %pz268 ret void269}270 271define void @v2i8(ptr %px, ptr %py, ptr %pz) nounwind {272; SSE2-LABEL: v2i8:273; SSE2: # %bb.0:274; SSE2-NEXT: movzwl (%rdi), %eax275; SSE2-NEXT: movd %eax, %xmm0276; SSE2-NEXT: movzwl (%rsi), %eax277; SSE2-NEXT: movd %eax, %xmm1278; SSE2-NEXT: paddusb %xmm0, %xmm1279; SSE2-NEXT: movd %xmm1, %eax280; SSE2-NEXT: movw %ax, (%rdx)281; SSE2-NEXT: retq282;283; SSSE3-LABEL: v2i8:284; SSSE3: # %bb.0:285; SSSE3-NEXT: movzwl (%rdi), %eax286; SSSE3-NEXT: movd %eax, %xmm0287; SSSE3-NEXT: movzwl (%rsi), %eax288; SSSE3-NEXT: movd %eax, %xmm1289; SSSE3-NEXT: paddusb %xmm0, %xmm1290; SSSE3-NEXT: movd %xmm1, %eax291; SSSE3-NEXT: movw %ax, (%rdx)292; SSSE3-NEXT: retq293;294; SSE41-LABEL: v2i8:295; SSE41: # %bb.0:296; SSE41-NEXT: movzwl (%rdi), %eax297; SSE41-NEXT: movd %eax, %xmm0298; SSE41-NEXT: movzwl (%rsi), %eax299; SSE41-NEXT: movd %eax, %xmm1300; SSE41-NEXT: paddusb %xmm0, %xmm1301; SSE41-NEXT: pextrw $0, %xmm1, (%rdx)302; SSE41-NEXT: retq303;304; AVX-LABEL: v2i8:305; AVX: # %bb.0:306; AVX-NEXT: movzwl (%rdi), %eax307; AVX-NEXT: vmovd %eax, %xmm0308; AVX-NEXT: movzwl (%rsi), %eax309; AVX-NEXT: vmovd %eax, %xmm1310; AVX-NEXT: vpaddusb %xmm1, %xmm0, %xmm0311; AVX-NEXT: vpextrw $0, %xmm0, (%rdx)312; AVX-NEXT: retq313 %x = load <2 x i8>, ptr %px314 %y = load <2 x i8>, ptr %py315 %z = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %x, <2 x i8> %y)316 store <2 x i8> %z, ptr %pz317 ret void318}319 320define void @v4i16(ptr %px, ptr %py, ptr %pz) nounwind {321; SSE-LABEL: v4i16:322; SSE: # %bb.0:323; SSE-NEXT: movq {{.*#+}} xmm0 = mem[0],zero324; SSE-NEXT: movq {{.*#+}} xmm1 = mem[0],zero325; SSE-NEXT: paddusw %xmm0, %xmm1326; SSE-NEXT: movq %xmm1, (%rdx)327; SSE-NEXT: retq328;329; AVX-LABEL: v4i16:330; AVX: # %bb.0:331; AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero332; AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero333; AVX-NEXT: vpaddusw %xmm1, %xmm0, %xmm0334; AVX-NEXT: vmovq %xmm0, (%rdx)335; AVX-NEXT: retq336 %x = load <4 x i16>, ptr %px337 %y = load <4 x i16>, ptr %py338 %z = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> %x, <4 x i16> %y)339 store <4 x i16> %z, ptr %pz340 ret void341}342 343define void @v2i16(ptr %px, ptr %py, ptr %pz) nounwind {344; SSE-LABEL: v2i16:345; SSE: # %bb.0:346; SSE-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero347; SSE-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero348; SSE-NEXT: paddusw %xmm0, %xmm1349; SSE-NEXT: movd %xmm1, (%rdx)350; SSE-NEXT: retq351;352; AVX-LABEL: v2i16:353; AVX: # %bb.0:354; AVX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero355; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero356; AVX-NEXT: vpaddusw %xmm1, %xmm0, %xmm0357; AVX-NEXT: vmovd %xmm0, (%rdx)358; AVX-NEXT: retq359 %x = load <2 x i16>, ptr %px360 %y = load <2 x i16>, ptr %py361 %z = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %x, <2 x i16> %y)362 store <2 x i16> %z, ptr %pz363 ret void364}365 366define <12 x i8> @v12i8(<12 x i8> %x, <12 x i8> %y) nounwind {367; SSE-LABEL: v12i8:368; SSE: # %bb.0:369; SSE-NEXT: paddusb %xmm1, %xmm0370; SSE-NEXT: retq371;372; AVX-LABEL: v12i8:373; AVX: # %bb.0:374; AVX-NEXT: vpaddusb %xmm1, %xmm0, %xmm0375; AVX-NEXT: retq376 %z = call <12 x i8> @llvm.uadd.sat.v12i8(<12 x i8> %x, <12 x i8> %y)377 ret <12 x i8> %z378}379 380define void @v12i16(ptr %px, ptr %py, ptr %pz) nounwind {381; SSE-LABEL: v12i16:382; SSE: # %bb.0:383; SSE-NEXT: movdqa (%rdi), %xmm0384; SSE-NEXT: movdqa 16(%rdi), %xmm1385; SSE-NEXT: paddusw 16(%rsi), %xmm1386; SSE-NEXT: paddusw (%rsi), %xmm0387; SSE-NEXT: movdqa %xmm0, (%rdx)388; SSE-NEXT: movq %xmm1, 16(%rdx)389; SSE-NEXT: retq390;391; AVX1-LABEL: v12i16:392; AVX1: # %bb.0:393; AVX1-NEXT: vmovdqa (%rdi), %xmm0394; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1395; AVX1-NEXT: vpaddusw 16(%rsi), %xmm1, %xmm1396; AVX1-NEXT: vpaddusw (%rsi), %xmm0, %xmm0397; AVX1-NEXT: vmovdqa %xmm0, (%rdx)398; AVX1-NEXT: vmovq %xmm1, 16(%rdx)399; AVX1-NEXT: retq400;401; AVX2-LABEL: v12i16:402; AVX2: # %bb.0:403; AVX2-NEXT: vmovdqa (%rdi), %ymm0404; AVX2-NEXT: vpaddusw (%rsi), %ymm0, %ymm0405; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1406; AVX2-NEXT: vmovq %xmm1, 16(%rdx)407; AVX2-NEXT: vmovdqa %xmm0, (%rdx)408; AVX2-NEXT: vzeroupper409; AVX2-NEXT: retq410;411; AVX512-LABEL: v12i16:412; AVX512: # %bb.0:413; AVX512-NEXT: vmovdqa (%rdi), %ymm0414; AVX512-NEXT: vpaddusw (%rsi), %ymm0, %ymm0415; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1416; AVX512-NEXT: vmovq %xmm1, 16(%rdx)417; AVX512-NEXT: vmovdqa %xmm0, (%rdx)418; AVX512-NEXT: vzeroupper419; AVX512-NEXT: retq420 %x = load <12 x i16>, ptr %px421 %y = load <12 x i16>, ptr %py422 %z = call <12 x i16> @llvm.uadd.sat.v12i16(<12 x i16> %x, <12 x i16> %y)423 store <12 x i16> %z, ptr %pz424 ret void425}426 427; Scalarization428 429define void @v1i8(ptr %px, ptr %py, ptr %pz) nounwind {430; SSE-LABEL: v1i8:431; SSE: # %bb.0:432; SSE-NEXT: movzbl (%rdi), %eax433; SSE-NEXT: addb (%rsi), %al434; SSE-NEXT: movzbl %al, %eax435; SSE-NEXT: movl $255, %ecx436; SSE-NEXT: cmovael %eax, %ecx437; SSE-NEXT: movb %cl, (%rdx)438; SSE-NEXT: retq439;440; AVX-LABEL: v1i8:441; AVX: # %bb.0:442; AVX-NEXT: movzbl (%rdi), %eax443; AVX-NEXT: addb (%rsi), %al444; AVX-NEXT: movzbl %al, %eax445; AVX-NEXT: movl $255, %ecx446; AVX-NEXT: cmovael %eax, %ecx447; AVX-NEXT: movb %cl, (%rdx)448; AVX-NEXT: retq449 %x = load <1 x i8>, ptr %px450 %y = load <1 x i8>, ptr %py451 %z = call <1 x i8> @llvm.uadd.sat.v1i8(<1 x i8> %x, <1 x i8> %y)452 store <1 x i8> %z, ptr %pz453 ret void454}455 456define void @v1i16(ptr %px, ptr %py, ptr %pz) nounwind {457; SSE-LABEL: v1i16:458; SSE: # %bb.0:459; SSE-NEXT: movzwl (%rdi), %eax460; SSE-NEXT: addw (%rsi), %ax461; SSE-NEXT: movl $65535, %ecx # imm = 0xFFFF462; SSE-NEXT: cmovael %eax, %ecx463; SSE-NEXT: movw %cx, (%rdx)464; SSE-NEXT: retq465;466; AVX-LABEL: v1i16:467; AVX: # %bb.0:468; AVX-NEXT: movzwl (%rdi), %eax469; AVX-NEXT: addw (%rsi), %ax470; AVX-NEXT: movl $65535, %ecx # imm = 0xFFFF471; AVX-NEXT: cmovael %eax, %ecx472; AVX-NEXT: movw %cx, (%rdx)473; AVX-NEXT: retq474 %x = load <1 x i16>, ptr %px475 %y = load <1 x i16>, ptr %py476 %z = call <1 x i16> @llvm.uadd.sat.v1i16(<1 x i16> %x, <1 x i16> %y)477 store <1 x i16> %z, ptr %pz478 ret void479}480 481; Promotion482 483define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {484; SSE-LABEL: v16i4:485; SSE: # %bb.0:486; SSE-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]487; SSE-NEXT: pand %xmm2, %xmm1488; SSE-NEXT: pand %xmm2, %xmm0489; SSE-NEXT: paddb %xmm1, %xmm0490; SSE-NEXT: pminub %xmm2, %xmm0491; SSE-NEXT: retq492;493; AVX1-LABEL: v16i4:494; AVX1: # %bb.0:495; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]496; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1497; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0498; AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0499; AVX1-NEXT: vpminub %xmm2, %xmm0, %xmm0500; AVX1-NEXT: retq501;502; AVX2-LABEL: v16i4:503; AVX2: # %bb.0:504; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]505; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm1506; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0507; AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0508; AVX2-NEXT: vpminub %xmm2, %xmm0, %xmm0509; AVX2-NEXT: retq510;511; AVX512-LABEL: v16i4:512; AVX512: # %bb.0:513; AVX512-NEXT: vpbroadcastb {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]514; AVX512-NEXT: vpand %xmm2, %xmm1, %xmm1515; AVX512-NEXT: vpand %xmm2, %xmm0, %xmm0516; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm0517; AVX512-NEXT: vpminub %xmm2, %xmm0, %xmm0518; AVX512-NEXT: retq519 %z = call <16 x i4> @llvm.uadd.sat.v16i4(<16 x i4> %x, <16 x i4> %y)520 ret <16 x i4> %z521}522 523define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {524; SSE-LABEL: v16i1:525; SSE: # %bb.0:526; SSE-NEXT: orps %xmm1, %xmm0527; SSE-NEXT: retq528;529; AVX-LABEL: v16i1:530; AVX: # %bb.0:531; AVX-NEXT: vorps %xmm1, %xmm0, %xmm0532; AVX-NEXT: retq533 %z = call <16 x i1> @llvm.uadd.sat.v16i1(<16 x i1> %x, <16 x i1> %y)534 ret <16 x i1> %z535}536 537; Expanded538 539define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {540; SSE2-LABEL: v2i32:541; SSE2: # %bb.0:542; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]543; SSE2-NEXT: movdqa %xmm0, %xmm3544; SSE2-NEXT: pxor %xmm2, %xmm3545; SSE2-NEXT: paddd %xmm1, %xmm0546; SSE2-NEXT: pxor %xmm0, %xmm2547; SSE2-NEXT: pcmpgtd %xmm2, %xmm3548; SSE2-NEXT: por %xmm3, %xmm0549; SSE2-NEXT: retq550;551; SSSE3-LABEL: v2i32:552; SSSE3: # %bb.0:553; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]554; SSSE3-NEXT: movdqa %xmm0, %xmm3555; SSSE3-NEXT: pxor %xmm2, %xmm3556; SSSE3-NEXT: paddd %xmm1, %xmm0557; SSSE3-NEXT: pxor %xmm0, %xmm2558; SSSE3-NEXT: pcmpgtd %xmm2, %xmm3559; SSSE3-NEXT: por %xmm3, %xmm0560; SSSE3-NEXT: retq561;562; SSE41-LABEL: v2i32:563; SSE41: # %bb.0:564; SSE41-NEXT: pcmpeqd %xmm2, %xmm2565; SSE41-NEXT: pxor %xmm1, %xmm2566; SSE41-NEXT: pminud %xmm2, %xmm0567; SSE41-NEXT: paddd %xmm1, %xmm0568; SSE41-NEXT: retq569;570; AVX1-LABEL: v2i32:571; AVX1: # %bb.0:572; AVX1-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2573; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm2574; AVX1-NEXT: vpminud %xmm2, %xmm0, %xmm0575; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0576; AVX1-NEXT: retq577;578; AVX2-LABEL: v2i32:579; AVX2: # %bb.0:580; AVX2-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2581; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm2582; AVX2-NEXT: vpminud %xmm2, %xmm0, %xmm0583; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0584; AVX2-NEXT: retq585;586; AVX512F-LABEL: v2i32:587; AVX512F: # %bb.0:588; AVX512F-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1589; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm2590; AVX512F-NEXT: vpternlogq {{.*#+}} zmm2 = ~zmm2591; AVX512F-NEXT: vpminud %xmm2, %xmm0, %xmm0592; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0593; AVX512F-NEXT: vzeroupper594; AVX512F-NEXT: retq595;596; AVX512BW-LABEL: v2i32:597; AVX512BW: # %bb.0:598; AVX512BW-NEXT: vmovdqa %xmm1, %xmm2599; AVX512BW-NEXT: vpternlogq {{.*#+}} xmm2 = ~xmm2600; AVX512BW-NEXT: vpminud %xmm2, %xmm0, %xmm0601; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0602; AVX512BW-NEXT: retq603 %z = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %x, <2 x i32> %y)604 ret <2 x i32> %z605}606 607define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {608; SSE2-LABEL: v4i32:609; SSE2: # %bb.0:610; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]611; SSE2-NEXT: movdqa %xmm0, %xmm3612; SSE2-NEXT: pxor %xmm2, %xmm3613; SSE2-NEXT: paddd %xmm1, %xmm0614; SSE2-NEXT: pxor %xmm0, %xmm2615; SSE2-NEXT: pcmpgtd %xmm2, %xmm3616; SSE2-NEXT: por %xmm3, %xmm0617; SSE2-NEXT: retq618;619; SSSE3-LABEL: v4i32:620; SSSE3: # %bb.0:621; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]622; SSSE3-NEXT: movdqa %xmm0, %xmm3623; SSSE3-NEXT: pxor %xmm2, %xmm3624; SSSE3-NEXT: paddd %xmm1, %xmm0625; SSSE3-NEXT: pxor %xmm0, %xmm2626; SSSE3-NEXT: pcmpgtd %xmm2, %xmm3627; SSSE3-NEXT: por %xmm3, %xmm0628; SSSE3-NEXT: retq629;630; SSE41-LABEL: v4i32:631; SSE41: # %bb.0:632; SSE41-NEXT: pcmpeqd %xmm2, %xmm2633; SSE41-NEXT: pxor %xmm1, %xmm2634; SSE41-NEXT: pminud %xmm2, %xmm0635; SSE41-NEXT: paddd %xmm1, %xmm0636; SSE41-NEXT: retq637;638; AVX1-LABEL: v4i32:639; AVX1: # %bb.0:640; AVX1-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2641; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm2642; AVX1-NEXT: vpminud %xmm2, %xmm0, %xmm0643; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0644; AVX1-NEXT: retq645;646; AVX2-LABEL: v4i32:647; AVX2: # %bb.0:648; AVX2-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2649; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm2650; AVX2-NEXT: vpminud %xmm2, %xmm0, %xmm0651; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0652; AVX2-NEXT: retq653;654; AVX512F-LABEL: v4i32:655; AVX512F: # %bb.0:656; AVX512F-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1657; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm2658; AVX512F-NEXT: vpternlogq {{.*#+}} zmm2 = ~zmm2659; AVX512F-NEXT: vpminud %xmm2, %xmm0, %xmm0660; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0661; AVX512F-NEXT: vzeroupper662; AVX512F-NEXT: retq663;664; AVX512BW-LABEL: v4i32:665; AVX512BW: # %bb.0:666; AVX512BW-NEXT: vmovdqa %xmm1, %xmm2667; AVX512BW-NEXT: vpternlogq {{.*#+}} xmm2 = ~xmm2668; AVX512BW-NEXT: vpminud %xmm2, %xmm0, %xmm0669; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0670; AVX512BW-NEXT: retq671 %z = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)672 ret <4 x i32> %z673}674 675define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {676; SSE2-LABEL: v8i32:677; SSE2: # %bb.0:678; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]679; SSE2-NEXT: movdqa %xmm0, %xmm5680; SSE2-NEXT: pxor %xmm4, %xmm5681; SSE2-NEXT: paddd %xmm2, %xmm0682; SSE2-NEXT: movdqa %xmm0, %xmm2683; SSE2-NEXT: pxor %xmm4, %xmm2684; SSE2-NEXT: pcmpgtd %xmm2, %xmm5685; SSE2-NEXT: por %xmm5, %xmm0686; SSE2-NEXT: movdqa %xmm1, %xmm2687; SSE2-NEXT: pxor %xmm4, %xmm2688; SSE2-NEXT: paddd %xmm3, %xmm1689; SSE2-NEXT: pxor %xmm1, %xmm4690; SSE2-NEXT: pcmpgtd %xmm4, %xmm2691; SSE2-NEXT: por %xmm2, %xmm1692; SSE2-NEXT: retq693;694; SSSE3-LABEL: v8i32:695; SSSE3: # %bb.0:696; SSSE3-NEXT: movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]697; SSSE3-NEXT: movdqa %xmm0, %xmm5698; SSSE3-NEXT: pxor %xmm4, %xmm5699; SSSE3-NEXT: paddd %xmm2, %xmm0700; SSSE3-NEXT: movdqa %xmm0, %xmm2701; SSSE3-NEXT: pxor %xmm4, %xmm2702; SSSE3-NEXT: pcmpgtd %xmm2, %xmm5703; SSSE3-NEXT: por %xmm5, %xmm0704; SSSE3-NEXT: movdqa %xmm1, %xmm2705; SSSE3-NEXT: pxor %xmm4, %xmm2706; SSSE3-NEXT: paddd %xmm3, %xmm1707; SSSE3-NEXT: pxor %xmm1, %xmm4708; SSSE3-NEXT: pcmpgtd %xmm4, %xmm2709; SSSE3-NEXT: por %xmm2, %xmm1710; SSSE3-NEXT: retq711;712; SSE41-LABEL: v8i32:713; SSE41: # %bb.0:714; SSE41-NEXT: pcmpeqd %xmm4, %xmm4715; SSE41-NEXT: movdqa %xmm2, %xmm5716; SSE41-NEXT: pxor %xmm4, %xmm5717; SSE41-NEXT: pminud %xmm5, %xmm0718; SSE41-NEXT: paddd %xmm2, %xmm0719; SSE41-NEXT: pxor %xmm3, %xmm4720; SSE41-NEXT: pminud %xmm4, %xmm1721; SSE41-NEXT: paddd %xmm3, %xmm1722; SSE41-NEXT: retq723;724; AVX1-LABEL: v8i32:725; AVX1: # %bb.0:726; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2727; AVX1-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3728; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm4729; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5730; AVX1-NEXT: vpminud %xmm4, %xmm5, %xmm4731; AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm2732; AVX1-NEXT: vpxor %xmm3, %xmm1, %xmm3733; AVX1-NEXT: vpminud %xmm3, %xmm0, %xmm0734; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0735; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0736; AVX1-NEXT: retq737;738; AVX2-LABEL: v8i32:739; AVX2: # %bb.0:740; AVX2-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2741; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm2742; AVX2-NEXT: vpminud %ymm2, %ymm0, %ymm0743; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0744; AVX2-NEXT: retq745;746; AVX512F-LABEL: v8i32:747; AVX512F: # %bb.0:748; AVX512F-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm1749; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm2750; AVX512F-NEXT: vpternlogq {{.*#+}} zmm2 = ~zmm2751; AVX512F-NEXT: vpminud %ymm2, %ymm0, %ymm0752; AVX512F-NEXT: vpaddd %ymm1, %ymm0, %ymm0753; AVX512F-NEXT: retq754;755; AVX512BW-LABEL: v8i32:756; AVX512BW: # %bb.0:757; AVX512BW-NEXT: vmovdqa %ymm1, %ymm2758; AVX512BW-NEXT: vpternlogq {{.*#+}} ymm2 = ~ymm2759; AVX512BW-NEXT: vpminud %ymm2, %ymm0, %ymm0760; AVX512BW-NEXT: vpaddd %ymm1, %ymm0, %ymm0761; AVX512BW-NEXT: retq762 %z = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> %x, <8 x i32> %y)763 ret <8 x i32> %z764}765 766define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {767; SSE2-LABEL: v16i32:768; SSE2: # %bb.0:769; SSE2-NEXT: movdqa {{.*#+}} xmm8 = [2147483648,2147483648,2147483648,2147483648]770; SSE2-NEXT: movdqa %xmm0, %xmm9771; SSE2-NEXT: pxor %xmm8, %xmm9772; SSE2-NEXT: paddd %xmm4, %xmm0773; SSE2-NEXT: movdqa %xmm0, %xmm4774; SSE2-NEXT: pxor %xmm8, %xmm4775; SSE2-NEXT: pcmpgtd %xmm4, %xmm9776; SSE2-NEXT: por %xmm9, %xmm0777; SSE2-NEXT: movdqa %xmm1, %xmm4778; SSE2-NEXT: pxor %xmm8, %xmm4779; SSE2-NEXT: paddd %xmm5, %xmm1780; SSE2-NEXT: movdqa %xmm1, %xmm5781; SSE2-NEXT: pxor %xmm8, %xmm5782; SSE2-NEXT: pcmpgtd %xmm5, %xmm4783; SSE2-NEXT: por %xmm4, %xmm1784; SSE2-NEXT: movdqa %xmm2, %xmm4785; SSE2-NEXT: pxor %xmm8, %xmm4786; SSE2-NEXT: paddd %xmm6, %xmm2787; SSE2-NEXT: movdqa %xmm2, %xmm5788; SSE2-NEXT: pxor %xmm8, %xmm5789; SSE2-NEXT: pcmpgtd %xmm5, %xmm4790; SSE2-NEXT: por %xmm4, %xmm2791; SSE2-NEXT: movdqa %xmm3, %xmm4792; SSE2-NEXT: pxor %xmm8, %xmm4793; SSE2-NEXT: paddd %xmm7, %xmm3794; SSE2-NEXT: pxor %xmm3, %xmm8795; SSE2-NEXT: pcmpgtd %xmm8, %xmm4796; SSE2-NEXT: por %xmm4, %xmm3797; SSE2-NEXT: retq798;799; SSSE3-LABEL: v16i32:800; SSSE3: # %bb.0:801; SSSE3-NEXT: movdqa {{.*#+}} xmm8 = [2147483648,2147483648,2147483648,2147483648]802; SSSE3-NEXT: movdqa %xmm0, %xmm9803; SSSE3-NEXT: pxor %xmm8, %xmm9804; SSSE3-NEXT: paddd %xmm4, %xmm0805; SSSE3-NEXT: movdqa %xmm0, %xmm4806; SSSE3-NEXT: pxor %xmm8, %xmm4807; SSSE3-NEXT: pcmpgtd %xmm4, %xmm9808; SSSE3-NEXT: por %xmm9, %xmm0809; SSSE3-NEXT: movdqa %xmm1, %xmm4810; SSSE3-NEXT: pxor %xmm8, %xmm4811; SSSE3-NEXT: paddd %xmm5, %xmm1812; SSSE3-NEXT: movdqa %xmm1, %xmm5813; SSSE3-NEXT: pxor %xmm8, %xmm5814; SSSE3-NEXT: pcmpgtd %xmm5, %xmm4815; SSSE3-NEXT: por %xmm4, %xmm1816; SSSE3-NEXT: movdqa %xmm2, %xmm4817; SSSE3-NEXT: pxor %xmm8, %xmm4818; SSSE3-NEXT: paddd %xmm6, %xmm2819; SSSE3-NEXT: movdqa %xmm2, %xmm5820; SSSE3-NEXT: pxor %xmm8, %xmm5821; SSSE3-NEXT: pcmpgtd %xmm5, %xmm4822; SSSE3-NEXT: por %xmm4, %xmm2823; SSSE3-NEXT: movdqa %xmm3, %xmm4824; SSSE3-NEXT: pxor %xmm8, %xmm4825; SSSE3-NEXT: paddd %xmm7, %xmm3826; SSSE3-NEXT: pxor %xmm3, %xmm8827; SSSE3-NEXT: pcmpgtd %xmm8, %xmm4828; SSSE3-NEXT: por %xmm4, %xmm3829; SSSE3-NEXT: retq830;831; SSE41-LABEL: v16i32:832; SSE41: # %bb.0:833; SSE41-NEXT: pcmpeqd %xmm8, %xmm8834; SSE41-NEXT: movdqa %xmm4, %xmm9835; SSE41-NEXT: pxor %xmm8, %xmm9836; SSE41-NEXT: pminud %xmm9, %xmm0837; SSE41-NEXT: paddd %xmm4, %xmm0838; SSE41-NEXT: movdqa %xmm5, %xmm4839; SSE41-NEXT: pxor %xmm8, %xmm4840; SSE41-NEXT: pminud %xmm4, %xmm1841; SSE41-NEXT: paddd %xmm5, %xmm1842; SSE41-NEXT: movdqa %xmm6, %xmm4843; SSE41-NEXT: pxor %xmm8, %xmm4844; SSE41-NEXT: pminud %xmm4, %xmm2845; SSE41-NEXT: paddd %xmm6, %xmm2846; SSE41-NEXT: pxor %xmm7, %xmm8847; SSE41-NEXT: pminud %xmm8, %xmm3848; SSE41-NEXT: paddd %xmm7, %xmm3849; SSE41-NEXT: retq850;851; AVX1-LABEL: v16i32:852; AVX1: # %bb.0:853; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4854; AVX1-NEXT: vpcmpeqd %xmm5, %xmm5, %xmm5855; AVX1-NEXT: vpxor %xmm5, %xmm4, %xmm6856; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm7857; AVX1-NEXT: vpminud %xmm6, %xmm7, %xmm6858; AVX1-NEXT: vpaddd %xmm4, %xmm6, %xmm4859; AVX1-NEXT: vpxor %xmm5, %xmm2, %xmm6860; AVX1-NEXT: vpminud %xmm6, %xmm0, %xmm0861; AVX1-NEXT: vpaddd %xmm2, %xmm0, %xmm0862; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0863; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2864; AVX1-NEXT: vpxor %xmm5, %xmm2, %xmm4865; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm6866; AVX1-NEXT: vpminud %xmm4, %xmm6, %xmm4867; AVX1-NEXT: vpaddd %xmm2, %xmm4, %xmm2868; AVX1-NEXT: vpxor %xmm5, %xmm3, %xmm4869; AVX1-NEXT: vpminud %xmm4, %xmm1, %xmm1870; AVX1-NEXT: vpaddd %xmm3, %xmm1, %xmm1871; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1872; AVX1-NEXT: retq873;874; AVX2-LABEL: v16i32:875; AVX2: # %bb.0:876; AVX2-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4877; AVX2-NEXT: vpxor %ymm4, %ymm2, %ymm5878; AVX2-NEXT: vpminud %ymm5, %ymm0, %ymm0879; AVX2-NEXT: vpaddd %ymm2, %ymm0, %ymm0880; AVX2-NEXT: vpxor %ymm4, %ymm3, %ymm2881; AVX2-NEXT: vpminud %ymm2, %ymm1, %ymm1882; AVX2-NEXT: vpaddd %ymm3, %ymm1, %ymm1883; AVX2-NEXT: retq884;885; AVX512-LABEL: v16i32:886; AVX512: # %bb.0:887; AVX512-NEXT: vmovdqa64 %zmm1, %zmm2888; AVX512-NEXT: vpternlogq {{.*#+}} zmm2 = ~zmm2889; AVX512-NEXT: vpminud %zmm2, %zmm0, %zmm0890; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0891; AVX512-NEXT: retq892 %z = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> %x, <16 x i32> %y)893 ret <16 x i32> %z894}895 896define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {897; SSE-LABEL: v2i64:898; SSE: # %bb.0:899; SSE-NEXT: movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]900; SSE-NEXT: movdqa %xmm0, %xmm3901; SSE-NEXT: pxor %xmm2, %xmm3902; SSE-NEXT: paddq %xmm1, %xmm0903; SSE-NEXT: pxor %xmm0, %xmm2904; SSE-NEXT: movdqa %xmm3, %xmm1905; SSE-NEXT: pcmpgtd %xmm2, %xmm1906; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]907; SSE-NEXT: pcmpeqd %xmm3, %xmm2908; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]909; SSE-NEXT: pand %xmm4, %xmm2910; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]911; SSE-NEXT: por %xmm1, %xmm0912; SSE-NEXT: por %xmm2, %xmm0913; SSE-NEXT: retq914;915; AVX1-LABEL: v2i64:916; AVX1: # %bb.0:917; AVX1-NEXT: vmovddup {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]918; AVX1-NEXT: # xmm2 = mem[0,0]919; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm3920; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0921; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm1922; AVX1-NEXT: vpcmpgtq %xmm1, %xmm3, %xmm1923; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0924; AVX1-NEXT: retq925;926; AVX2-LABEL: v2i64:927; AVX2: # %bb.0:928; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]929; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm3930; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0931; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm1932; AVX2-NEXT: vpcmpgtq %xmm1, %xmm3, %xmm1933; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0934; AVX2-NEXT: retq935;936; AVX512F-LABEL: v2i64:937; AVX512F: # %bb.0:938; AVX512F-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1939; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0940; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm2941; AVX512F-NEXT: vpternlogq {{.*#+}} zmm2 = ~zmm2942; AVX512F-NEXT: vpminuq %zmm2, %zmm0, %zmm0943; AVX512F-NEXT: vpaddq %xmm1, %xmm0, %xmm0944; AVX512F-NEXT: vzeroupper945; AVX512F-NEXT: retq946;947; AVX512BW-LABEL: v2i64:948; AVX512BW: # %bb.0:949; AVX512BW-NEXT: vmovdqa %xmm1, %xmm2950; AVX512BW-NEXT: vpternlogq {{.*#+}} xmm2 = ~xmm2951; AVX512BW-NEXT: vpminuq %xmm2, %xmm0, %xmm0952; AVX512BW-NEXT: vpaddq %xmm1, %xmm0, %xmm0953; AVX512BW-NEXT: retq954 %z = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> %x, <2 x i64> %y)955 ret <2 x i64> %z956}957 958define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {959; SSE-LABEL: v4i64:960; SSE: # %bb.0:961; SSE-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]962; SSE-NEXT: movdqa %xmm0, %xmm5963; SSE-NEXT: pxor %xmm4, %xmm5964; SSE-NEXT: paddq %xmm2, %xmm0965; SSE-NEXT: movdqa %xmm0, %xmm2966; SSE-NEXT: pxor %xmm4, %xmm2967; SSE-NEXT: movdqa %xmm5, %xmm6968; SSE-NEXT: pcmpgtd %xmm2, %xmm6969; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]970; SSE-NEXT: pcmpeqd %xmm5, %xmm2971; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]972; SSE-NEXT: pand %xmm7, %xmm2973; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]974; SSE-NEXT: por %xmm5, %xmm0975; SSE-NEXT: por %xmm2, %xmm0976; SSE-NEXT: movdqa %xmm1, %xmm2977; SSE-NEXT: pxor %xmm4, %xmm2978; SSE-NEXT: paddq %xmm3, %xmm1979; SSE-NEXT: pxor %xmm1, %xmm4980; SSE-NEXT: movdqa %xmm2, %xmm3981; SSE-NEXT: pcmpgtd %xmm4, %xmm3982; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]983; SSE-NEXT: pcmpeqd %xmm2, %xmm4984; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]985; SSE-NEXT: pand %xmm5, %xmm2986; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]987; SSE-NEXT: por %xmm3, %xmm1988; SSE-NEXT: por %xmm2, %xmm1989; SSE-NEXT: retq990;991; AVX1-LABEL: v4i64:992; AVX1: # %bb.0:993; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2994; AVX1-NEXT: vmovddup {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]995; AVX1-NEXT: # xmm3 = mem[0,0]996; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm4997; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5998; AVX1-NEXT: vpaddq %xmm5, %xmm2, %xmm2999; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm51000; AVX1-NEXT: vpcmpgtq %xmm5, %xmm4, %xmm41001; AVX1-NEXT: vpor %xmm4, %xmm2, %xmm21002; AVX1-NEXT: vpxor %xmm3, %xmm0, %xmm41003; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm01004; AVX1-NEXT: vpxor %xmm3, %xmm0, %xmm11005; AVX1-NEXT: vpcmpgtq %xmm1, %xmm4, %xmm11006; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm01007; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm01008; AVX1-NEXT: retq1009;1010; AVX2-LABEL: v4i64:1011; AVX2: # %bb.0:1012; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]1013; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm31014; AVX2-NEXT: vpaddq %ymm1, %ymm0, %ymm01015; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm11016; AVX2-NEXT: vpcmpgtq %ymm1, %ymm3, %ymm11017; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm01018; AVX2-NEXT: retq1019;1020; AVX512F-LABEL: v4i64:1021; AVX512F: # %bb.0:1022; AVX512F-NEXT: # kill: def $ymm1 killed $ymm1 def $zmm11023; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm01024; AVX512F-NEXT: vmovdqa64 %zmm1, %zmm21025; AVX512F-NEXT: vpternlogq {{.*#+}} zmm2 = ~zmm21026; AVX512F-NEXT: vpminuq %zmm2, %zmm0, %zmm01027; AVX512F-NEXT: vpaddq %ymm1, %ymm0, %ymm01028; AVX512F-NEXT: retq1029;1030; AVX512BW-LABEL: v4i64:1031; AVX512BW: # %bb.0:1032; AVX512BW-NEXT: vmovdqa %ymm1, %ymm21033; AVX512BW-NEXT: vpternlogq {{.*#+}} ymm2 = ~ymm21034; AVX512BW-NEXT: vpminuq %ymm2, %ymm0, %ymm01035; AVX512BW-NEXT: vpaddq %ymm1, %ymm0, %ymm01036; AVX512BW-NEXT: retq1037 %z = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> %x, <4 x i64> %y)1038 ret <4 x i64> %z1039}1040 1041define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {1042; SSE-LABEL: v8i64:1043; SSE: # %bb.0:1044; SSE-NEXT: movdqa {{.*#+}} xmm8 = [9223372039002259456,9223372039002259456]1045; SSE-NEXT: movdqa %xmm0, %xmm91046; SSE-NEXT: pxor %xmm8, %xmm91047; SSE-NEXT: paddq %xmm4, %xmm01048; SSE-NEXT: movdqa %xmm0, %xmm41049; SSE-NEXT: pxor %xmm8, %xmm41050; SSE-NEXT: movdqa %xmm9, %xmm101051; SSE-NEXT: pcmpgtd %xmm4, %xmm101052; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]1053; SSE-NEXT: pcmpeqd %xmm9, %xmm41054; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]1055; SSE-NEXT: pand %xmm11, %xmm41056; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]1057; SSE-NEXT: por %xmm9, %xmm01058; SSE-NEXT: por %xmm4, %xmm01059; SSE-NEXT: movdqa %xmm1, %xmm41060; SSE-NEXT: pxor %xmm8, %xmm41061; SSE-NEXT: paddq %xmm5, %xmm11062; SSE-NEXT: movdqa %xmm1, %xmm51063; SSE-NEXT: pxor %xmm8, %xmm51064; SSE-NEXT: movdqa %xmm4, %xmm91065; SSE-NEXT: pcmpgtd %xmm5, %xmm91066; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]1067; SSE-NEXT: pcmpeqd %xmm4, %xmm51068; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]1069; SSE-NEXT: pand %xmm10, %xmm41070; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm9[1,1,3,3]1071; SSE-NEXT: por %xmm5, %xmm11072; SSE-NEXT: por %xmm4, %xmm11073; SSE-NEXT: movdqa %xmm2, %xmm41074; SSE-NEXT: pxor %xmm8, %xmm41075; SSE-NEXT: paddq %xmm6, %xmm21076; SSE-NEXT: movdqa %xmm2, %xmm51077; SSE-NEXT: pxor %xmm8, %xmm51078; SSE-NEXT: movdqa %xmm4, %xmm61079; SSE-NEXT: pcmpgtd %xmm5, %xmm61080; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm6[0,0,2,2]1081; SSE-NEXT: pcmpeqd %xmm4, %xmm51082; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]1083; SSE-NEXT: pand %xmm9, %xmm41084; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]1085; SSE-NEXT: por %xmm5, %xmm21086; SSE-NEXT: por %xmm4, %xmm21087; SSE-NEXT: movdqa %xmm3, %xmm41088; SSE-NEXT: pxor %xmm8, %xmm41089; SSE-NEXT: paddq %xmm7, %xmm31090; SSE-NEXT: pxor %xmm3, %xmm81091; SSE-NEXT: movdqa %xmm4, %xmm51092; SSE-NEXT: pcmpgtd %xmm8, %xmm51093; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]1094; SSE-NEXT: pcmpeqd %xmm4, %xmm81095; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm8[1,1,3,3]1096; SSE-NEXT: pand %xmm6, %xmm41097; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]1098; SSE-NEXT: por %xmm5, %xmm31099; SSE-NEXT: por %xmm4, %xmm31100; SSE-NEXT: retq1101;1102; AVX1-LABEL: v8i64:1103; AVX1: # %bb.0:1104; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm41105; AVX1-NEXT: vmovddup {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808]1106; AVX1-NEXT: # xmm5 = mem[0,0]1107; AVX1-NEXT: vpxor %xmm5, %xmm4, %xmm61108; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm71109; AVX1-NEXT: vpaddq %xmm7, %xmm4, %xmm41110; AVX1-NEXT: vpxor %xmm5, %xmm4, %xmm71111; AVX1-NEXT: vpcmpgtq %xmm7, %xmm6, %xmm61112; AVX1-NEXT: vpor %xmm6, %xmm4, %xmm41113; AVX1-NEXT: vpxor %xmm5, %xmm0, %xmm61114; AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm01115; AVX1-NEXT: vpxor %xmm5, %xmm0, %xmm21116; AVX1-NEXT: vpcmpgtq %xmm2, %xmm6, %xmm21117; AVX1-NEXT: vpor %xmm2, %xmm0, %xmm01118; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm01119; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm21120; AVX1-NEXT: vpxor %xmm5, %xmm2, %xmm41121; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm61122; AVX1-NEXT: vpaddq %xmm6, %xmm2, %xmm21123; AVX1-NEXT: vpxor %xmm5, %xmm2, %xmm61124; AVX1-NEXT: vpcmpgtq %xmm6, %xmm4, %xmm41125; AVX1-NEXT: vpor %xmm4, %xmm2, %xmm21126; AVX1-NEXT: vpxor %xmm5, %xmm1, %xmm41127; AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm11128; AVX1-NEXT: vpxor %xmm5, %xmm1, %xmm31129; AVX1-NEXT: vpcmpgtq %xmm3, %xmm4, %xmm31130; AVX1-NEXT: vpor %xmm3, %xmm1, %xmm11131; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm11132; AVX1-NEXT: retq1133;1134; AVX2-LABEL: v8i64:1135; AVX2: # %bb.0:1136; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm4 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]1137; AVX2-NEXT: vpxor %ymm4, %ymm0, %ymm51138; AVX2-NEXT: vpaddq %ymm2, %ymm0, %ymm01139; AVX2-NEXT: vpxor %ymm4, %ymm0, %ymm21140; AVX2-NEXT: vpcmpgtq %ymm2, %ymm5, %ymm21141; AVX2-NEXT: vpor %ymm2, %ymm0, %ymm01142; AVX2-NEXT: vpxor %ymm4, %ymm1, %ymm21143; AVX2-NEXT: vpaddq %ymm3, %ymm1, %ymm11144; AVX2-NEXT: vpxor %ymm4, %ymm1, %ymm31145; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm21146; AVX2-NEXT: vpor %ymm2, %ymm1, %ymm11147; AVX2-NEXT: retq1148;1149; AVX512-LABEL: v8i64:1150; AVX512: # %bb.0:1151; AVX512-NEXT: vmovdqa64 %zmm1, %zmm21152; AVX512-NEXT: vpternlogq {{.*#+}} zmm2 = ~zmm21153; AVX512-NEXT: vpminuq %zmm2, %zmm0, %zmm01154; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm01155; AVX512-NEXT: retq1156 %z = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> %x, <8 x i64> %y)1157 ret <8 x i64> %z1158}1159 1160define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {1161; SSE-LABEL: v2i128:1162; SSE: # %bb.0:1163; SSE-NEXT: movq %rdi, %rax1164; SSE-NEXT: addq %r9, %rsi1165; SSE-NEXT: adcq {{[0-9]+}}(%rsp), %rdx1166; SSE-NEXT: movq $-1, %rdi1167; SSE-NEXT: cmovbq %rdi, %rsi1168; SSE-NEXT: cmovbq %rdi, %rdx1169; SSE-NEXT: addq {{[0-9]+}}(%rsp), %rcx1170; SSE-NEXT: adcq {{[0-9]+}}(%rsp), %r81171; SSE-NEXT: cmovbq %rdi, %r81172; SSE-NEXT: cmovbq %rdi, %rcx1173; SSE-NEXT: movq %r8, 24(%rax)1174; SSE-NEXT: movq %rcx, 16(%rax)1175; SSE-NEXT: movq %rdx, 8(%rax)1176; SSE-NEXT: movq %rsi, (%rax)1177; SSE-NEXT: retq1178;1179; AVX-LABEL: v2i128:1180; AVX: # %bb.0:1181; AVX-NEXT: movq %rdi, %rax1182; AVX-NEXT: addq %r9, %rsi1183; AVX-NEXT: adcq {{[0-9]+}}(%rsp), %rdx1184; AVX-NEXT: movq $-1, %rdi1185; AVX-NEXT: cmovbq %rdi, %rsi1186; AVX-NEXT: cmovbq %rdi, %rdx1187; AVX-NEXT: addq {{[0-9]+}}(%rsp), %rcx1188; AVX-NEXT: adcq {{[0-9]+}}(%rsp), %r81189; AVX-NEXT: cmovbq %rdi, %r81190; AVX-NEXT: cmovbq %rdi, %rcx1191; AVX-NEXT: movq %r8, 24(%rax)1192; AVX-NEXT: movq %rcx, 16(%rax)1193; AVX-NEXT: movq %rdx, 8(%rax)1194; AVX-NEXT: movq %rsi, (%rax)1195; AVX-NEXT: retq1196 %z = call <2 x i128> @llvm.uadd.sat.v2i128(<2 x i128> %x, <2 x i128> %y)1197 ret <2 x i128> %z1198}1199