736 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc < %s -mtriple=x86_64-- -mattr=+avxifma | FileCheck %s --check-prefixes=X64,AVX,AVXIFMA3; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512ifma | FileCheck %s --check-prefixes=X64,AVX512,AVX512-NOVL4; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512ifma,+avx512vl | FileCheck %s --check-prefixes=X64,AVX512,AVX512VL5; RUN: llc < %s -mtriple=x86_64-- -mattr=+avxifma,+avx512vl | FileCheck %s --check-prefixes=X64,AVX,AVX512-NOIFMA6 7; 67108863 == (1 << 26) - 18; 4503599627370496 == (1 << 52)9; 4503599627370495 == (1 << 52) - 110 11define <8 x i64> @test_512_combine(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {12; AVXIFMA-LABEL: test_512_combine:13; AVXIFMA: # %bb.0:14; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm6 = [67108863,67108863,67108863,67108863]15; AVXIFMA-NEXT: vpand %ymm6, %ymm2, %ymm216; AVXIFMA-NEXT: vpand %ymm6, %ymm0, %ymm017; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm2, %ymm0, %ymm418; AVXIFMA-NEXT: vpand %ymm6, %ymm3, %ymm019; AVXIFMA-NEXT: vpand %ymm6, %ymm1, %ymm120; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm0, %ymm1, %ymm521; AVXIFMA-NEXT: vmovdqa %ymm4, %ymm022; AVXIFMA-NEXT: vmovdqa %ymm5, %ymm123; AVXIFMA-NEXT: retq24;25; AVX512-LABEL: test_512_combine:26; AVX512: # %bb.0:27; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm3 = [67108863,67108863,67108863,67108863,67108863,67108863,67108863,67108863]28; AVX512-NEXT: vpandq %zmm3, %zmm0, %zmm029; AVX512-NEXT: vpandq %zmm3, %zmm1, %zmm130; AVX512-NEXT: vpmadd52luq %zmm1, %zmm0, %zmm231; AVX512-NEXT: vmovdqa64 %zmm2, %zmm032; AVX512-NEXT: retq33;34; AVX512-NOIFMA-LABEL: test_512_combine:35; AVX512-NOIFMA: # %bb.0:36; AVX512-NOIFMA-NEXT: vpbroadcastq {{.*#+}} zmm3 = [67108863,67108863,67108863,67108863,67108863,67108863,67108863,67108863]37; AVX512-NOIFMA-NEXT: vpandq %zmm3, %zmm0, %zmm038; AVX512-NOIFMA-NEXT: vpandq %zmm3, %zmm1, %zmm139; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm1, %ymm340; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm0, %ymm441; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm2, %ymm542; AVX512-NOIFMA-NEXT: {vex} vpmadd52luq %ymm3, %ymm4, %ymm543; AVX512-NOIFMA-NEXT: {vex} vpmadd52luq %ymm1, %ymm0, %ymm244; AVX512-NOIFMA-NEXT: vinserti64x4 $1, %ymm5, %zmm2, %zmm045; AVX512-NOIFMA-NEXT: retq46 %x_masked = and <8 x i64> %x, splat (i64 67108863)47 %y_masked = and <8 x i64> %y, splat (i64 67108863)48 %mul = mul nuw nsw <8 x i64> %x_masked, %y_masked49 %res = add nuw nsw <8 x i64> %mul, %z50 ret <8 x i64> %res51}52 53define <8 x i64> @test_512_combine_v2(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {54; AVXIFMA-LABEL: test_512_combine_v2:55; AVXIFMA: # %bb.0:56; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm6 = [3,3,3,3]57; AVXIFMA-NEXT: vpand %ymm6, %ymm2, %ymm258; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm7 = [1125899906842623,1125899906842623,1125899906842623,1125899906842623]59; AVXIFMA-NEXT: vpand %ymm7, %ymm0, %ymm060; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm2, %ymm0, %ymm461; AVXIFMA-NEXT: vpand %ymm6, %ymm3, %ymm062; AVXIFMA-NEXT: vpand %ymm7, %ymm1, %ymm163; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm0, %ymm1, %ymm564; AVXIFMA-NEXT: vmovdqa %ymm4, %ymm065; AVXIFMA-NEXT: vmovdqa %ymm5, %ymm166; AVXIFMA-NEXT: retq67;68; AVX512-LABEL: test_512_combine_v2:69; AVX512: # %bb.0:70; AVX512-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm071; AVX512-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm172; AVX512-NEXT: vpmadd52luq %zmm1, %zmm0, %zmm273; AVX512-NEXT: vmovdqa64 %zmm2, %zmm074; AVX512-NEXT: retq75;76; AVX512-NOIFMA-LABEL: test_512_combine_v2:77; AVX512-NOIFMA: # %bb.0:78; AVX512-NOIFMA-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm079; AVX512-NOIFMA-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm180; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm2, %ymm381; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm1, %ymm482; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm0, %ymm583; AVX512-NOIFMA-NEXT: {vex} vpmadd52luq %ymm4, %ymm5, %ymm384; AVX512-NOIFMA-NEXT: {vex} vpmadd52luq %ymm1, %ymm0, %ymm285; AVX512-NOIFMA-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm086; AVX512-NOIFMA-NEXT: retq87 %x_masked = and <8 x i64> %x, splat (i64 1125899906842623) ; (1 << 50) - 188 %y_masked = and <8 x i64> %y, splat (i64 3)89 %mul = mul nuw nsw <8 x i64> %x_masked, %y_masked90 %res = add nuw nsw <8 x i64> %mul, %z91 ret <8 x i64> %res92}93 94define <8 x i64> @test_512_no_combine(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {95; AVXIFMA-LABEL: test_512_no_combine:96; AVXIFMA: # %bb.0:97; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm6 = [4503599627370495,4503599627370495,4503599627370495,4503599627370495]98; AVXIFMA-NEXT: vpand %ymm6, %ymm0, %ymm799; AVXIFMA-NEXT: vpand %ymm6, %ymm1, %ymm8100; AVXIFMA-NEXT: vpand %ymm6, %ymm2, %ymm9101; AVXIFMA-NEXT: vpand %ymm6, %ymm3, %ymm6102; AVXIFMA-NEXT: vpsrlq $32, %ymm8, %ymm8103; AVXIFMA-NEXT: vpmuludq %ymm3, %ymm8, %ymm8104; AVXIFMA-NEXT: vpsrlq $32, %ymm6, %ymm6105; AVXIFMA-NEXT: vpmuludq %ymm6, %ymm1, %ymm6106; AVXIFMA-NEXT: vpaddq %ymm6, %ymm8, %ymm6107; AVXIFMA-NEXT: vpsllq $32, %ymm6, %ymm6108; AVXIFMA-NEXT: vpmuludq %ymm3, %ymm1, %ymm1109; AVXIFMA-NEXT: vpsrlq $32, %ymm7, %ymm3110; AVXIFMA-NEXT: vpmuludq %ymm2, %ymm3, %ymm3111; AVXIFMA-NEXT: vpsrlq $32, %ymm9, %ymm7112; AVXIFMA-NEXT: vpmuludq %ymm7, %ymm0, %ymm7113; AVXIFMA-NEXT: vpaddq %ymm3, %ymm7, %ymm3114; AVXIFMA-NEXT: vpsllq $32, %ymm3, %ymm3115; AVXIFMA-NEXT: vpmuludq %ymm2, %ymm0, %ymm0116; AVXIFMA-NEXT: vpaddq %ymm4, %ymm0, %ymm0117; AVXIFMA-NEXT: vpaddq %ymm3, %ymm0, %ymm0118; AVXIFMA-NEXT: vpaddq %ymm5, %ymm1, %ymm1119; AVXIFMA-NEXT: vpaddq %ymm6, %ymm1, %ymm1120; AVXIFMA-NEXT: retq121;122; AVX512-LABEL: test_512_no_combine:123; AVX512: # %bb.0:124; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm3 = [4503599627370495,4503599627370495,4503599627370495,4503599627370495,4503599627370495,4503599627370495,4503599627370495,4503599627370495]125; AVX512-NEXT: vpandq %zmm3, %zmm0, %zmm4126; AVX512-NEXT: vpandq %zmm3, %zmm1, %zmm3127; AVX512-NEXT: vpsrlq $32, %zmm4, %zmm4128; AVX512-NEXT: vpmuludq %zmm1, %zmm4, %zmm4129; AVX512-NEXT: vpsrlq $32, %zmm3, %zmm3130; AVX512-NEXT: vpmuludq %zmm3, %zmm0, %zmm3131; AVX512-NEXT: vpaddq %zmm4, %zmm3, %zmm3132; AVX512-NEXT: vpsllq $32, %zmm3, %zmm3133; AVX512-NEXT: vpmuludq %zmm1, %zmm0, %zmm0134; AVX512-NEXT: vpaddq %zmm2, %zmm0, %zmm0135; AVX512-NEXT: vpaddq %zmm3, %zmm0, %zmm0136; AVX512-NEXT: retq137;138; AVX512-NOIFMA-LABEL: test_512_no_combine:139; AVX512-NOIFMA: # %bb.0:140; AVX512-NOIFMA-NEXT: vpbroadcastq {{.*#+}} zmm3 = [4503599627370495,4503599627370495,4503599627370495,4503599627370495,4503599627370495,4503599627370495,4503599627370495,4503599627370495]141; AVX512-NOIFMA-NEXT: vpandq %zmm3, %zmm0, %zmm4142; AVX512-NOIFMA-NEXT: vpandq %zmm3, %zmm1, %zmm3143; AVX512-NOIFMA-NEXT: vpsrlq $32, %zmm4, %zmm4144; AVX512-NOIFMA-NEXT: vpmuludq %zmm1, %zmm4, %zmm4145; AVX512-NOIFMA-NEXT: vpsrlq $32, %zmm3, %zmm3146; AVX512-NOIFMA-NEXT: vpmuludq %zmm3, %zmm0, %zmm3147; AVX512-NOIFMA-NEXT: vpaddq %zmm4, %zmm3, %zmm3148; AVX512-NOIFMA-NEXT: vpsllq $32, %zmm3, %zmm3149; AVX512-NOIFMA-NEXT: vpmuludq %zmm1, %zmm0, %zmm0150; AVX512-NOIFMA-NEXT: vpaddq %zmm2, %zmm0, %zmm0151; AVX512-NOIFMA-NEXT: vpaddq %zmm3, %zmm0, %zmm0152; AVX512-NOIFMA-NEXT: retq153 %x_masked = and <8 x i64> %x, splat (i64 4503599627370495)154 %y_masked = and <8 x i64> %y, splat (i64 4503599627370495)155 %mul = mul nuw nsw <8 x i64> %x_masked, %y_masked156 %res = add nuw nsw <8 x i64> %mul, %z157 ret <8 x i64> %res158}159 160define <8 x i64> @test_512_no_combine_v2(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {161; AVXIFMA-LABEL: test_512_no_combine_v2:162; AVXIFMA: # %bb.0:163; AVXIFMA-NEXT: vpsrlq $32, %ymm1, %ymm6164; AVXIFMA-NEXT: vpmuludq %ymm3, %ymm6, %ymm6165; AVXIFMA-NEXT: vpsrlq $32, %ymm3, %ymm7166; AVXIFMA-NEXT: vpmuludq %ymm7, %ymm1, %ymm7167; AVXIFMA-NEXT: vpaddq %ymm6, %ymm7, %ymm6168; AVXIFMA-NEXT: vpsllq $32, %ymm6, %ymm6169; AVXIFMA-NEXT: vpmuludq %ymm3, %ymm1, %ymm1170; AVXIFMA-NEXT: vpsrlq $32, %ymm0, %ymm3171; AVXIFMA-NEXT: vpmuludq %ymm2, %ymm3, %ymm3172; AVXIFMA-NEXT: vpsrlq $32, %ymm2, %ymm7173; AVXIFMA-NEXT: vpmuludq %ymm7, %ymm0, %ymm7174; AVXIFMA-NEXT: vpaddq %ymm3, %ymm7, %ymm3175; AVXIFMA-NEXT: vpsllq $32, %ymm3, %ymm3176; AVXIFMA-NEXT: vpmuludq %ymm2, %ymm0, %ymm0177; AVXIFMA-NEXT: vpaddq %ymm4, %ymm0, %ymm0178; AVXIFMA-NEXT: vpaddq %ymm3, %ymm0, %ymm0179; AVXIFMA-NEXT: vpaddq %ymm5, %ymm1, %ymm1180; AVXIFMA-NEXT: vpaddq %ymm6, %ymm1, %ymm1181; AVXIFMA-NEXT: retq182;183; AVX512-LABEL: test_512_no_combine_v2:184; AVX512: # %bb.0:185; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm3186; AVX512-NEXT: vpmuludq %zmm1, %zmm3, %zmm3187; AVX512-NEXT: vpsrlq $32, %zmm1, %zmm4188; AVX512-NEXT: vpmuludq %zmm4, %zmm0, %zmm4189; AVX512-NEXT: vpaddq %zmm3, %zmm4, %zmm3190; AVX512-NEXT: vpsllq $32, %zmm3, %zmm3191; AVX512-NEXT: vpmuludq %zmm1, %zmm0, %zmm0192; AVX512-NEXT: vpaddq %zmm2, %zmm0, %zmm0193; AVX512-NEXT: vpaddq %zmm3, %zmm0, %zmm0194; AVX512-NEXT: retq195;196; AVX512-NOIFMA-LABEL: test_512_no_combine_v2:197; AVX512-NOIFMA: # %bb.0:198; AVX512-NOIFMA-NEXT: vpsrlq $32, %zmm0, %zmm3199; AVX512-NOIFMA-NEXT: vpmuludq %zmm1, %zmm3, %zmm3200; AVX512-NOIFMA-NEXT: vpsrlq $32, %zmm1, %zmm4201; AVX512-NOIFMA-NEXT: vpmuludq %zmm4, %zmm0, %zmm4202; AVX512-NOIFMA-NEXT: vpaddq %zmm3, %zmm4, %zmm3203; AVX512-NOIFMA-NEXT: vpsllq $32, %zmm3, %zmm3204; AVX512-NOIFMA-NEXT: vpmuludq %zmm1, %zmm0, %zmm0205; AVX512-NOIFMA-NEXT: vpaddq %zmm2, %zmm0, %zmm0206; AVX512-NOIFMA-NEXT: vpaddq %zmm3, %zmm0, %zmm0207; AVX512-NOIFMA-NEXT: retq208 %mul = mul <8 x i64> %x, %y209 %res = add <8 x i64> %mul, %z210 ret <8 x i64> %res211}212 213define <4 x i64> @test_256_combine(<4 x i64> %x, <4 x i64> %y, <4 x i64> %z) {214; AVX-LABEL: test_256_combine:215; AVX: # %bb.0:216; AVX-NEXT: vpbroadcastq {{.*#+}} ymm3 = [67108863,67108863,67108863,67108863]217; AVX-NEXT: vpand %ymm3, %ymm0, %ymm0218; AVX-NEXT: vpand %ymm3, %ymm1, %ymm1219; AVX-NEXT: {vex} vpmadd52luq %ymm1, %ymm0, %ymm2220; AVX-NEXT: vmovdqa %ymm2, %ymm0221; AVX-NEXT: retq222;223; AVX512-NOVL-LABEL: test_256_combine:224; AVX512-NOVL: # %bb.0:225; AVX512-NOVL-NEXT: vpbroadcastq {{.*#+}} ymm3 = [67108863,67108863,67108863,67108863]226; AVX512-NOVL-NEXT: vpand %ymm3, %ymm0, %ymm0227; AVX512-NOVL-NEXT: vpand %ymm3, %ymm1, %ymm1228; AVX512-NOVL-NEXT: vpmuldq %ymm1, %ymm0, %ymm0229; AVX512-NOVL-NEXT: vpaddq %ymm0, %ymm2, %ymm0230; AVX512-NOVL-NEXT: retq231;232; AVX512VL-LABEL: test_256_combine:233; AVX512VL: # %bb.0:234; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm3 = [67108863,67108863,67108863,67108863]235; AVX512VL-NEXT: vpand %ymm3, %ymm0, %ymm0236; AVX512VL-NEXT: vpand %ymm3, %ymm1, %ymm1237; AVX512VL-NEXT: vpmadd52luq %ymm1, %ymm0, %ymm2238; AVX512VL-NEXT: vmovdqa %ymm2, %ymm0239; AVX512VL-NEXT: retq240 %x_masked = and <4 x i64> %x, splat(i64 67108863)241 %y_masked = and <4 x i64> %y, splat(i64 67108863)242 %mul = mul nuw nsw <4 x i64> %x_masked, %y_masked243 %res = add nuw nsw <4 x i64> %z, %mul244 ret <4 x i64> %res245}246 247define <4 x i64> @test_256_no_combine(<4 x i64> %x, <4 x i64> %y, <4 x i64> %z) {248; X64-LABEL: test_256_no_combine:249; X64: # %bb.0:250; X64-NEXT: vpsrlq $32, %ymm0, %ymm3251; X64-NEXT: vpmuludq %ymm1, %ymm3, %ymm3252; X64-NEXT: vpsrlq $32, %ymm1, %ymm4253; X64-NEXT: vpmuludq %ymm4, %ymm0, %ymm4254; X64-NEXT: vpaddq %ymm3, %ymm4, %ymm3255; X64-NEXT: vpsllq $32, %ymm3, %ymm3256; X64-NEXT: vpmuludq %ymm1, %ymm0, %ymm0257; X64-NEXT: vpaddq %ymm2, %ymm0, %ymm0258; X64-NEXT: vpaddq %ymm3, %ymm0, %ymm0259; X64-NEXT: retq260 %mul = mul <4 x i64> %x, %y261 %res = add <4 x i64> %mul, %z262 ret <4 x i64> %res263}264 265define <2 x i64> @test_128_combine(<2 x i64> %x, <2 x i64> %y, <2 x i64> %z) {266; AVX-LABEL: test_128_combine:267; AVX: # %bb.0:268; AVX-NEXT: vpbroadcastq {{.*#+}} xmm3 = [67108863,67108863]269; AVX-NEXT: vpand %xmm3, %xmm0, %xmm0270; AVX-NEXT: vpand %xmm3, %xmm1, %xmm1271; AVX-NEXT: {vex} vpmadd52luq %xmm1, %xmm0, %xmm2272; AVX-NEXT: vmovdqa %xmm2, %xmm0273; AVX-NEXT: retq274;275; AVX512-NOVL-LABEL: test_128_combine:276; AVX512-NOVL: # %bb.0:277; AVX512-NOVL-NEXT: vpbroadcastq {{.*#+}} xmm3 = [67108863,67108863]278; AVX512-NOVL-NEXT: vpand %xmm3, %xmm0, %xmm0279; AVX512-NOVL-NEXT: vpand %xmm3, %xmm1, %xmm1280; AVX512-NOVL-NEXT: vpmuldq %xmm1, %xmm0, %xmm0281; AVX512-NOVL-NEXT: vpaddq %xmm0, %xmm2, %xmm0282; AVX512-NOVL-NEXT: retq283;284; AVX512VL-LABEL: test_128_combine:285; AVX512VL: # %bb.0:286; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm3 = [67108863,67108863]287; AVX512VL-NEXT: vpand %xmm3, %xmm0, %xmm0288; AVX512VL-NEXT: vpand %xmm3, %xmm1, %xmm1289; AVX512VL-NEXT: vpmadd52luq %xmm1, %xmm0, %xmm2290; AVX512VL-NEXT: vmovdqa %xmm2, %xmm0291; AVX512VL-NEXT: retq292 %x_masked = and <2 x i64> %x, splat (i64 67108863)293 %y_masked = and <2 x i64> %y, splat (i64 67108863)294 %mul = mul <2 x i64> %x_masked, %y_masked295 %res = add <2 x i64> %z, %mul296 ret <2 x i64> %res297}298 299; Sanity check we're not applying this here300define <1 x i64> @test_scalar_no_ifma(<1 x i64> %x, <1 x i64> %y, <1 x i64> %z) {301; X64-LABEL: test_scalar_no_ifma:302; X64: # %bb.0:303; X64-NEXT: imulq %rsi, %rdi304; X64-NEXT: leaq (%rdi,%rdx), %rax305; X64-NEXT: retq306 %mul = mul <1 x i64> %x, %y307 %res = add <1 x i64> %mul, %z308 ret <1 x i64> %res309}310 311; 40-bit and 13-bit, too wide312define <8 x i64> @test_mixed_width_too_wide(<8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {313; AVXIFMA-LABEL: test_mixed_width_too_wide:314; AVXIFMA: # %bb.0:315; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm6 = [8191,8191,8191,8191]316; AVXIFMA-NEXT: vpand %ymm6, %ymm2, %ymm2317; AVXIFMA-NEXT: vpand %ymm6, %ymm3, %ymm3318; AVXIFMA-NEXT: vpmovzxdq {{.*#+}} ymm6 = [2155905028,2155905036,2155905044,2155905052]319; AVXIFMA-NEXT: vpshufb %ymm6, %ymm1, %ymm7320; AVXIFMA-NEXT: vpmuludq %ymm3, %ymm7, %ymm7321; AVXIFMA-NEXT: vpsllq $32, %ymm7, %ymm7322; AVXIFMA-NEXT: vpmuludq %ymm3, %ymm1, %ymm1323; AVXIFMA-NEXT: vpshufb %ymm6, %ymm0, %ymm3324; AVXIFMA-NEXT: vpmuludq %ymm2, %ymm3, %ymm3325; AVXIFMA-NEXT: vpsllq $32, %ymm3, %ymm3326; AVXIFMA-NEXT: vpmuludq %ymm2, %ymm0, %ymm0327; AVXIFMA-NEXT: vpaddq %ymm0, %ymm4, %ymm0328; AVXIFMA-NEXT: vpaddq %ymm3, %ymm0, %ymm0329; AVXIFMA-NEXT: vpaddq %ymm1, %ymm5, %ymm1330; AVXIFMA-NEXT: vpaddq %ymm7, %ymm1, %ymm1331; AVXIFMA-NEXT: retq332;333; AVX512-LABEL: test_mixed_width_too_wide:334; AVX512: # %bb.0:335; AVX512-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm1336; AVX512-NEXT: vpmuludq %zmm1, %zmm0, %zmm3337; AVX512-NEXT: vpsrlq $32, %zmm0, %zmm0338; AVX512-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0339; AVX512-NEXT: vpmuludq %zmm1, %zmm0, %zmm0340; AVX512-NEXT: vpsllq $32, %zmm0, %zmm0341; AVX512-NEXT: vpaddq %zmm3, %zmm2, %zmm1342; AVX512-NEXT: vpaddq %zmm0, %zmm1, %zmm0343; AVX512-NEXT: retq344;345; AVX512-NOIFMA-LABEL: test_mixed_width_too_wide:346; AVX512-NOIFMA: # %bb.0:347; AVX512-NOIFMA-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm1348; AVX512-NOIFMA-NEXT: vpmuludq %zmm1, %zmm0, %zmm3349; AVX512-NOIFMA-NEXT: vpsrlq $32, %zmm0, %zmm0350; AVX512-NOIFMA-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0351; AVX512-NOIFMA-NEXT: vpmuludq %zmm1, %zmm0, %zmm0352; AVX512-NOIFMA-NEXT: vpsllq $32, %zmm0, %zmm0353; AVX512-NOIFMA-NEXT: vpaddq %zmm3, %zmm2, %zmm1354; AVX512-NOIFMA-NEXT: vpaddq %zmm0, %zmm1, %zmm0355; AVX512-NOIFMA-NEXT: retq356 %x40 = and <8 x i64> %x, splat (i64 1099511627775)357 %y13 = and <8 x i64> %y, splat (i64 8191)358 %mul = mul <8 x i64> %x40, %y13359 %res = add <8 x i64> %z, %mul360 ret <8 x i64> %res361}362 363define <8 x i64> @test_zext32_inputs_not_safe(<8 x i32> %xi32, <8 x i32> %yi32, <8 x i64> %z) {364; AVXIFMA-LABEL: test_zext32_inputs_not_safe:365; AVXIFMA: # %bb.0:366; AVXIFMA-NEXT: vpmovzxdq {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero367; AVXIFMA-NEXT: vextracti128 $1, %ymm0, %xmm0368; AVXIFMA-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero369; AVXIFMA-NEXT: vpmovzxdq {{.*#+}} ymm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero370; AVXIFMA-NEXT: vpmuludq %ymm5, %ymm4, %ymm4371; AVXIFMA-NEXT: vextracti128 $1, %ymm1, %xmm1372; AVXIFMA-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero373; AVXIFMA-NEXT: vpmuludq %ymm1, %ymm0, %ymm1374; AVXIFMA-NEXT: vpaddq %ymm4, %ymm2, %ymm0375; AVXIFMA-NEXT: vpaddq %ymm1, %ymm3, %ymm1376; AVXIFMA-NEXT: retq377;378; AVX512-LABEL: test_zext32_inputs_not_safe:379; AVX512: # %bb.0:380; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero381; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero382; AVX512-NEXT: vpmuludq %zmm1, %zmm0, %zmm0383; AVX512-NEXT: vpaddq %zmm0, %zmm2, %zmm0384; AVX512-NEXT: retq385;386; AVX512-NOIFMA-LABEL: test_zext32_inputs_not_safe:387; AVX512-NOIFMA: # %bb.0:388; AVX512-NOIFMA-NEXT: vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero389; AVX512-NOIFMA-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero390; AVX512-NOIFMA-NEXT: vpmuludq %zmm1, %zmm0, %zmm0391; AVX512-NOIFMA-NEXT: vpaddq %zmm0, %zmm2, %zmm0392; AVX512-NOIFMA-NEXT: retq393 %x = zext <8 x i32> %xi32 to <8 x i64>394 %y = zext <8 x i32> %yi32 to <8 x i64>395 %mul = mul <8 x i64> %x, %y396 %res = add <8 x i64> %z, %mul397 ret <8 x i64> %res398}399 400define <16 x i64> @test_1024_combine_split(<16 x i64> %x, <16 x i64> %y, <16 x i64> %z) nounwind {401; AVXIFMA-LABEL: test_1024_combine_split:402; AVXIFMA: # %bb.0:403; AVXIFMA-NEXT: pushq %rbp404; AVXIFMA-NEXT: movq %rsp, %rbp405; AVXIFMA-NEXT: andq $-32, %rsp406; AVXIFMA-NEXT: subq $32, %rsp407; AVXIFMA-NEXT: vmovdqa 112(%rbp), %ymm8408; AVXIFMA-NEXT: vmovdqa 80(%rbp), %ymm9409; AVXIFMA-NEXT: vmovdqa 48(%rbp), %ymm10410; AVXIFMA-NEXT: vmovdqa 16(%rbp), %ymm11411; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm12 = [67108863,67108863,67108863,67108863]412; AVXIFMA-NEXT: vpand %ymm3, %ymm12, %ymm3413; AVXIFMA-NEXT: vpand %ymm2, %ymm12, %ymm2414; AVXIFMA-NEXT: vpand %ymm1, %ymm12, %ymm1415; AVXIFMA-NEXT: vpand %ymm0, %ymm12, %ymm0416; AVXIFMA-NEXT: vpand %ymm7, %ymm12, %ymm7417; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm7, %ymm3, %ymm8418; AVXIFMA-NEXT: vpand %ymm6, %ymm12, %ymm3419; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm3, %ymm2, %ymm9420; AVXIFMA-NEXT: vpand %ymm5, %ymm12, %ymm2421; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm2, %ymm1, %ymm10422; AVXIFMA-NEXT: vpand %ymm4, %ymm12, %ymm1423; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm1, %ymm0, %ymm11424; AVXIFMA-NEXT: vmovdqa %ymm11, %ymm0425; AVXIFMA-NEXT: vmovdqa %ymm10, %ymm1426; AVXIFMA-NEXT: vmovdqa %ymm9, %ymm2427; AVXIFMA-NEXT: vmovdqa %ymm8, %ymm3428; AVXIFMA-NEXT: movq %rbp, %rsp429; AVXIFMA-NEXT: popq %rbp430; AVXIFMA-NEXT: retq431;432; AVX512-LABEL: test_1024_combine_split:433; AVX512: # %bb.0:434; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm6 = [67108863,67108863,67108863,67108863,67108863,67108863,67108863,67108863]435; AVX512-NEXT: vpandq %zmm6, %zmm2, %zmm2436; AVX512-NEXT: vpandq %zmm6, %zmm0, %zmm0437; AVX512-NEXT: vpmadd52luq %zmm2, %zmm0, %zmm4438; AVX512-NEXT: vpandq %zmm6, %zmm3, %zmm0439; AVX512-NEXT: vpandq %zmm6, %zmm1, %zmm1440; AVX512-NEXT: vpmadd52luq %zmm0, %zmm1, %zmm5441; AVX512-NEXT: vmovdqa64 %zmm4, %zmm0442; AVX512-NEXT: vmovdqa64 %zmm5, %zmm1443; AVX512-NEXT: retq444;445; AVX512-NOIFMA-LABEL: test_1024_combine_split:446; AVX512-NOIFMA: # %bb.0:447; AVX512-NOIFMA-NEXT: vpbroadcastq {{.*#+}} zmm6 = [67108863,67108863,67108863,67108863,67108863,67108863,67108863,67108863]448; AVX512-NOIFMA-NEXT: vpandq %zmm6, %zmm1, %zmm1449; AVX512-NOIFMA-NEXT: vpandq %zmm6, %zmm0, %zmm0450; AVX512-NOIFMA-NEXT: vpandq %zmm6, %zmm3, %zmm3451; AVX512-NOIFMA-NEXT: vpandq %zmm6, %zmm2, %zmm2452; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm2, %ymm6453; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm0, %ymm7454; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm4, %ymm8455; AVX512-NOIFMA-NEXT: {vex} vpmadd52luq %ymm6, %ymm7, %ymm8456; AVX512-NOIFMA-NEXT: {vex} vpmadd52luq %ymm2, %ymm0, %ymm4457; AVX512-NOIFMA-NEXT: vinserti64x4 $1, %ymm8, %zmm4, %zmm0458; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm3, %ymm2459; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm1, %ymm4460; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm5, %ymm6461; AVX512-NOIFMA-NEXT: {vex} vpmadd52luq %ymm2, %ymm4, %ymm6462; AVX512-NOIFMA-NEXT: {vex} vpmadd52luq %ymm3, %ymm1, %ymm5463; AVX512-NOIFMA-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm1464; AVX512-NOIFMA-NEXT: retq465 %x_masked = and <16 x i64> %x, splat (i64 67108863)466 %y_masked = and <16 x i64> %y, splat (i64 67108863)467 %mul = mul <16 x i64> %x_masked, %y_masked468 %res = add <16 x i64> %z, %mul469 ret <16 x i64> %res470}471 472define <1 x i64> @test_not_v1i64(<1 x i64> %x, <1 x i64> %y, <1 x i64> %z) {473; X64-LABEL: test_not_v1i64:474; X64: # %bb.0:475; X64-NEXT: andl $67108863, %edi # imm = 0x3FFFFFF476; X64-NEXT: imulq %rdi, %rdi477; X64-NEXT: leaq (%rdi,%rdx), %rax478; X64-NEXT: retq479 %x_masked = and <1 x i64> %x, splat (i64 67108863)480 %y_masked = and <1 x i64> %x, splat (i64 67108863)481 %mul = mul <1 x i64> %x_masked, %y_masked482 %res = add <1 x i64> %mul, %z483 ret <1 x i64> %res484}485 486define <3 x i64> @test_v3i64(<3 x i64> %x, <3 x i64> %y, <3 x i64> %z) {487; AVXIFMA-LABEL: test_v3i64:488; AVXIFMA: # %bb.0:489; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm1 = [67108863,67108863,67108863,67108863]490; AVXIFMA-NEXT: vpand %ymm1, %ymm0, %ymm0491; AVXIFMA-NEXT: vpmuludq %ymm0, %ymm0, %ymm0492; AVXIFMA-NEXT: vpaddq %ymm2, %ymm0, %ymm0493; AVXIFMA-NEXT: retq494;495; AVX512-NOVL-LABEL: test_v3i64:496; AVX512-NOVL: # %bb.0:497; AVX512-NOVL-NEXT: vpbroadcastq {{.*#+}} ymm1 = [67108863,67108863,67108863,67108863]498; AVX512-NOVL-NEXT: vpand %ymm1, %ymm0, %ymm0499; AVX512-NOVL-NEXT: vpmuludq %ymm0, %ymm0, %ymm0500; AVX512-NOVL-NEXT: vpaddq %ymm2, %ymm0, %ymm0501; AVX512-NOVL-NEXT: retq502;503; AVX512VL-LABEL: test_v3i64:504; AVX512VL: # %bb.0:505; AVX512VL-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0506; AVX512VL-NEXT: vpmuludq %ymm0, %ymm0, %ymm0507; AVX512VL-NEXT: vpaddq %ymm2, %ymm0, %ymm0508; AVX512VL-NEXT: retq509;510; AVX512-NOIFMA-LABEL: test_v3i64:511; AVX512-NOIFMA: # %bb.0:512; AVX512-NOIFMA-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0513; AVX512-NOIFMA-NEXT: vpmuludq %ymm0, %ymm0, %ymm0514; AVX512-NOIFMA-NEXT: vpaddq %ymm2, %ymm0, %ymm0515; AVX512-NOIFMA-NEXT: retq516 %x_masked = and <3 x i64> %x, splat (i64 67108863)517 %y_masked = and <3 x i64> %x, splat (i64 67108863)518 %mul = mul <3 x i64> %x_masked, %y_masked519 %res = add <3 x i64> %mul, %z520 ret <3 x i64> %res521}522 523define <5 x i64> @test_v5i64(<5 x i64> %x, <5 x i64> %y, <5 x i64> %z) {524; AVXIFMA-LABEL: test_v5i64:525; AVXIFMA: # %bb.0:526; AVXIFMA-NEXT: movq %rdi, %rax527; AVXIFMA-NEXT: vmovq %r8, %xmm0528; AVXIFMA-NEXT: vmovq %rcx, %xmm1529; AVXIFMA-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]530; AVXIFMA-NEXT: vmovq %rdx, %xmm1531; AVXIFMA-NEXT: vmovq %rsi, %xmm2532; AVXIFMA-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]533; AVXIFMA-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0534; AVXIFMA-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero535; AVXIFMA-NEXT: vmovdqu {{[0-9]+}}(%rsp), %ymm2536; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm3 = [67108863,67108863,67108863,67108863]537; AVXIFMA-NEXT: vpand %ymm3, %ymm0, %ymm0538; AVXIFMA-NEXT: movl $67108863, %ecx # imm = 0x3FFFFFF539; AVXIFMA-NEXT: vmovq %rcx, %xmm3540; AVXIFMA-NEXT: vmovq %r9, %xmm4541; AVXIFMA-NEXT: vpand %xmm3, %xmm4, %xmm3542; AVXIFMA-NEXT: vpsrlq $32, %xmm3, %xmm4543; AVXIFMA-NEXT: vpmuludq %xmm4, %xmm3, %xmm4544; AVXIFMA-NEXT: vpsllq $33, %xmm4, %xmm4545; AVXIFMA-NEXT: vpmuludq %xmm3, %xmm3, %xmm3546; AVXIFMA-NEXT: vpaddq %xmm1, %xmm3, %xmm1547; AVXIFMA-NEXT: vpaddq %xmm4, %xmm1, %xmm1548; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm0, %ymm0, %ymm2549; AVXIFMA-NEXT: vmovdqa %ymm2, (%rdi)550; AVXIFMA-NEXT: vmovq %xmm1, 32(%rdi)551; AVXIFMA-NEXT: vzeroupper552; AVXIFMA-NEXT: retq553;554; AVX512-LABEL: test_v5i64:555; AVX512: # %bb.0:556; AVX512-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0557; AVX512-NEXT: vpmuludq %zmm0, %zmm0, %zmm0558; AVX512-NEXT: vpaddq %zmm2, %zmm0, %zmm0559; AVX512-NEXT: retq560;561; AVX512-NOIFMA-LABEL: test_v5i64:562; AVX512-NOIFMA: # %bb.0:563; AVX512-NOIFMA-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0564; AVX512-NOIFMA-NEXT: vpmuludq %zmm0, %zmm0, %zmm0565; AVX512-NOIFMA-NEXT: vpaddq %zmm2, %zmm0, %zmm0566; AVX512-NOIFMA-NEXT: retq567 %x_masked = and <5 x i64> %x, splat (i64 67108863)568 %y_masked = and <5 x i64> %x, splat (i64 67108863)569 %mul = mul <5 x i64> %x_masked, %y_masked570 %res = add <5 x i64> %mul, %z571 ret <5 x i64> %res572}573 574define <6 x i64> @test_v6i64(<6 x i64> %x, <6 x i64> %y, <6 x i64> %z) {575; AVXIFMA-LABEL: test_v6i64:576; AVXIFMA: # %bb.0:577; AVXIFMA-NEXT: movq %rdi, %rax578; AVXIFMA-NEXT: vmovq %r8, %xmm0579; AVXIFMA-NEXT: vmovq %rcx, %xmm1580; AVXIFMA-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]581; AVXIFMA-NEXT: vmovq %rdx, %xmm1582; AVXIFMA-NEXT: vmovq %rsi, %xmm2583; AVXIFMA-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]584; AVXIFMA-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0585; AVXIFMA-NEXT: vmovdqu {{[0-9]+}}(%rsp), %ymm1586; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm2 = [67108863,67108863,67108863,67108863]587; AVXIFMA-NEXT: vpand %ymm2, %ymm0, %ymm0588; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm0, %ymm0, %ymm1589; AVXIFMA-NEXT: vmovq %r9, %xmm0590; AVXIFMA-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero591; AVXIFMA-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]592; AVXIFMA-NEXT: vpand %xmm2, %xmm0, %xmm0593; AVXIFMA-NEXT: vpmuldq %xmm0, %xmm0, %xmm0594; AVXIFMA-NEXT: vpaddq {{[0-9]+}}(%rsp), %xmm0, %xmm0595; AVXIFMA-NEXT: vmovdqa %xmm0, 32(%rdi)596; AVXIFMA-NEXT: vmovdqa %ymm1, (%rdi)597; AVXIFMA-NEXT: vzeroupper598; AVXIFMA-NEXT: retq599;600; AVX512-LABEL: test_v6i64:601; AVX512: # %bb.0:602; AVX512-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0603; AVX512-NEXT: vpmuludq %zmm0, %zmm0, %zmm0604; AVX512-NEXT: vpaddq %zmm2, %zmm0, %zmm0605; AVX512-NEXT: retq606;607; AVX512-NOIFMA-LABEL: test_v6i64:608; AVX512-NOIFMA: # %bb.0:609; AVX512-NOIFMA-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0610; AVX512-NOIFMA-NEXT: vpmuludq %zmm0, %zmm0, %zmm0611; AVX512-NOIFMA-NEXT: vpaddq %zmm2, %zmm0, %zmm0612; AVX512-NOIFMA-NEXT: retq613 %x_masked = and <6 x i64> %x, splat (i64 67108863)614 %y_masked = and <6 x i64> %x, splat (i64 67108863)615 %mul = mul <6 x i64> %x_masked, %y_masked616 %res = add <6 x i64> %mul, %z617 ret <6 x i64> %res618}619 620define <9 x i64> @test_v9i64(<9 x i64> %x, <9 x i64> %y, <9 x i64> %z) {621; AVXIFMA-LABEL: test_v9i64:622; AVXIFMA: # %bb.0:623; AVXIFMA-NEXT: movq %rdi, %rax624; AVXIFMA-NEXT: vmovq %r8, %xmm0625; AVXIFMA-NEXT: vmovq %rcx, %xmm1626; AVXIFMA-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]627; AVXIFMA-NEXT: vmovq %rdx, %xmm1628; AVXIFMA-NEXT: vmovq %rsi, %xmm2629; AVXIFMA-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]630; AVXIFMA-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0631; AVXIFMA-NEXT: vmovq %r9, %xmm1632; AVXIFMA-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero633; AVXIFMA-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]634; AVXIFMA-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1635; AVXIFMA-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero636; AVXIFMA-NEXT: vmovdqu {{[0-9]+}}(%rsp), %ymm3637; AVXIFMA-NEXT: vmovdqu {{[0-9]+}}(%rsp), %ymm4638; AVXIFMA-NEXT: vpbroadcastq {{.*#+}} ymm5 = [67108863,67108863,67108863,67108863]639; AVXIFMA-NEXT: vpand %ymm5, %ymm0, %ymm0640; AVXIFMA-NEXT: vpand %ymm5, %ymm1, %ymm1641; AVXIFMA-NEXT: movl $67108863, %ecx # imm = 0x3FFFFFF642; AVXIFMA-NEXT: vmovq %rcx, %xmm5643; AVXIFMA-NEXT: vmovq {{.*#+}} xmm6 = mem[0],zero644; AVXIFMA-NEXT: vpand %xmm5, %xmm6, %xmm5645; AVXIFMA-NEXT: vpsrlq $32, %xmm5, %xmm6646; AVXIFMA-NEXT: vpmuludq %xmm6, %xmm5, %xmm6647; AVXIFMA-NEXT: vpsllq $33, %xmm6, %xmm6648; AVXIFMA-NEXT: vpmuludq %xmm5, %xmm5, %xmm5649; AVXIFMA-NEXT: vpaddq %xmm2, %xmm5, %xmm2650; AVXIFMA-NEXT: vpaddq %xmm6, %xmm2, %xmm2651; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm0, %ymm0, %ymm4652; AVXIFMA-NEXT: {vex} vpmadd52luq %ymm1, %ymm1, %ymm3653; AVXIFMA-NEXT: vmovdqa %ymm3, 32(%rdi)654; AVXIFMA-NEXT: vmovdqa %ymm4, (%rdi)655; AVXIFMA-NEXT: vmovq %xmm2, 64(%rdi)656; AVXIFMA-NEXT: vzeroupper657; AVXIFMA-NEXT: retq658;659; AVX512-LABEL: test_v9i64:660; AVX512: # %bb.0:661; AVX512-NEXT: movq %rdi, %rax662; AVX512-NEXT: vmovq %r8, %xmm0663; AVX512-NEXT: vmovq %rcx, %xmm1664; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]665; AVX512-NEXT: vmovq %rdx, %xmm1666; AVX512-NEXT: vmovq %rsi, %xmm2667; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]668; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0669; AVX512-NEXT: vmovq %r9, %xmm1670; AVX512-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero671; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]672; AVX512-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1673; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0674; AVX512-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero675; AVX512-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm2676; AVX512-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0677; AVX512-NEXT: movl $67108863, %ecx # imm = 0x3FFFFFF678; AVX512-NEXT: vmovq %rcx, %xmm3679; AVX512-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero680; AVX512-NEXT: vpand %xmm3, %xmm4, %xmm3681; AVX512-NEXT: vpsrlq $32, %xmm3, %xmm4682; AVX512-NEXT: vpmuludq %xmm4, %xmm3, %xmm4683; AVX512-NEXT: vpsllq $33, %xmm4, %xmm4684; AVX512-NEXT: vpmuludq %xmm3, %xmm3, %xmm3685; AVX512-NEXT: vpaddq %xmm1, %xmm3, %xmm1686; AVX512-NEXT: vpaddq %xmm4, %xmm1, %xmm1687; AVX512-NEXT: vpmadd52luq %zmm0, %zmm0, %zmm2688; AVX512-NEXT: vmovq %xmm1, 64(%rdi)689; AVX512-NEXT: vmovdqa64 %zmm2, (%rdi)690; AVX512-NEXT: vzeroupper691; AVX512-NEXT: retq692;693; AVX512-NOIFMA-LABEL: test_v9i64:694; AVX512-NOIFMA: # %bb.0:695; AVX512-NOIFMA-NEXT: movq %rdi, %rax696; AVX512-NOIFMA-NEXT: vmovq %r8, %xmm0697; AVX512-NOIFMA-NEXT: vmovq %rcx, %xmm1698; AVX512-NOIFMA-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]699; AVX512-NOIFMA-NEXT: vmovq %rdx, %xmm1700; AVX512-NOIFMA-NEXT: vmovq %rsi, %xmm2701; AVX512-NOIFMA-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]702; AVX512-NOIFMA-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0703; AVX512-NOIFMA-NEXT: vmovq %r9, %xmm1704; AVX512-NOIFMA-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero705; AVX512-NOIFMA-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]706; AVX512-NOIFMA-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1707; AVX512-NOIFMA-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0708; AVX512-NOIFMA-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero709; AVX512-NOIFMA-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm0710; AVX512-NOIFMA-NEXT: movl $67108863, %ecx # imm = 0x3FFFFFF711; AVX512-NOIFMA-NEXT: vmovq %rcx, %xmm2712; AVX512-NOIFMA-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero713; AVX512-NOIFMA-NEXT: vpand %xmm2, %xmm3, %xmm2714; AVX512-NOIFMA-NEXT: vpsrlq $32, %xmm2, %xmm3715; AVX512-NOIFMA-NEXT: vpmuludq %xmm3, %xmm2, %xmm3716; AVX512-NOIFMA-NEXT: vpsllq $33, %xmm3, %xmm3717; AVX512-NOIFMA-NEXT: vpmuludq %xmm2, %xmm2, %xmm2718; AVX512-NOIFMA-NEXT: vpaddq %xmm1, %xmm2, %xmm1719; AVX512-NOIFMA-NEXT: vpaddq %xmm3, %xmm1, %xmm1720; AVX512-NOIFMA-NEXT: vextracti64x4 $1, %zmm0, %ymm2721; AVX512-NOIFMA-NEXT: vmovdqu {{[0-9]+}}(%rsp), %ymm3722; AVX512-NOIFMA-NEXT: {vex} vpmadd52luq %ymm2, %ymm2, %ymm3723; AVX512-NOIFMA-NEXT: vmovdqu {{[0-9]+}}(%rsp), %ymm2724; AVX512-NOIFMA-NEXT: {vex} vpmadd52luq %ymm0, %ymm0, %ymm2725; AVX512-NOIFMA-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm0726; AVX512-NOIFMA-NEXT: vmovq %xmm1, 64(%rdi)727; AVX512-NOIFMA-NEXT: vmovdqa64 %zmm0, (%rdi)728; AVX512-NOIFMA-NEXT: vzeroupper729; AVX512-NOIFMA-NEXT: retq730 %x_masked = and <9 x i64> %x, splat (i64 67108863)731 %y_masked = and <9 x i64> %x, splat (i64 67108863)732 %mul = mul <9 x i64> %x_masked, %y_masked733 %res = add <9 x i64> %mul, %z734 ret <9 x i64> %res735}736