204 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-- -mcpu=x86-64 %s | FileCheck %s --check-prefixes=SSE,SSE23; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-- -mcpu=x86-64-v2 %s | FileCheck %s --check-prefixes=SSE,SSE44; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-- -mcpu=x86-64-v3 %s | FileCheck %s --check-prefixes=AVX25; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-- -mcpu=x86-64-v4 %s | FileCheck %s --check-prefixes=AVX5126 7; PR124993 - ensure scalarized CTLZ calls remain scalarized unless there is a definite cost improvement, the cost of scalarization was being over estimated.8 9define <2 x i64> @scalarize_ctlz_v2i64(<2 x i64> %v) {10; SSE-LABEL: define <2 x i64> @scalarize_ctlz_v2i64(11; SSE-SAME: <2 x i64> [[V:%.*]]) #[[ATTR0:[0-9]+]] {12; SSE-NEXT: [[V0:%.*]] = extractelement <2 x i64> [[V]], i64 013; SSE-NEXT: [[V1:%.*]] = extractelement <2 x i64> [[V]], i64 114; SSE-NEXT: [[C0:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V0]], i1 false)15; SSE-NEXT: [[C1:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V1]], i1 false)16; SSE-NEXT: [[R0:%.*]] = insertelement <2 x i64> poison, i64 [[C0]], i64 017; SSE-NEXT: [[R1:%.*]] = insertelement <2 x i64> [[R0]], i64 [[C1]], i64 118; SSE-NEXT: ret <2 x i64> [[R1]]19;20; AVX2-LABEL: define <2 x i64> @scalarize_ctlz_v2i64(21; AVX2-SAME: <2 x i64> [[V:%.*]]) #[[ATTR0:[0-9]+]] {22; AVX2-NEXT: [[V0:%.*]] = extractelement <2 x i64> [[V]], i64 023; AVX2-NEXT: [[V1:%.*]] = extractelement <2 x i64> [[V]], i64 124; AVX2-NEXT: [[C0:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V0]], i1 false)25; AVX2-NEXT: [[C1:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V1]], i1 false)26; AVX2-NEXT: [[R0:%.*]] = insertelement <2 x i64> poison, i64 [[C0]], i64 027; AVX2-NEXT: [[R1:%.*]] = insertelement <2 x i64> [[R0]], i64 [[C1]], i64 128; AVX2-NEXT: ret <2 x i64> [[R1]]29;30; AVX512-LABEL: define <2 x i64> @scalarize_ctlz_v2i64(31; AVX512-SAME: <2 x i64> [[V:%.*]]) #[[ATTR0:[0-9]+]] {32; AVX512-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[V]], i1 false)33; AVX512-NEXT: ret <2 x i64> [[TMP1]]34;35 %v0 = extractelement <2 x i64> %v, i64 036 %v1 = extractelement <2 x i64> %v, i64 137 %c0 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v0, i1 false)38 %c1 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v1, i1 false)39 %r0 = insertelement <2 x i64> poison, i64 %c0, i64 040 %r1 = insertelement <2 x i64> %r0, i64 %c1, i64 141 ret <2 x i64> %r142}43 44define <4 x i64> @scalarize_ctlz_v4i64(<4 x i64> %v) {45; SSE-LABEL: define <4 x i64> @scalarize_ctlz_v4i64(46; SSE-SAME: <4 x i64> [[V:%.*]]) #[[ATTR0]] {47; SSE-NEXT: [[V0:%.*]] = extractelement <4 x i64> [[V]], i64 048; SSE-NEXT: [[V1:%.*]] = extractelement <4 x i64> [[V]], i64 149; SSE-NEXT: [[V2:%.*]] = extractelement <4 x i64> [[V]], i64 250; SSE-NEXT: [[V3:%.*]] = extractelement <4 x i64> [[V]], i64 351; SSE-NEXT: [[C0:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V0]], i1 false)52; SSE-NEXT: [[C1:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V1]], i1 false)53; SSE-NEXT: [[C2:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V2]], i1 false)54; SSE-NEXT: [[C3:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V3]], i1 false)55; SSE-NEXT: [[R0:%.*]] = insertelement <4 x i64> poison, i64 [[C0]], i64 056; SSE-NEXT: [[R1:%.*]] = insertelement <4 x i64> [[R0]], i64 [[C1]], i64 157; SSE-NEXT: [[R2:%.*]] = insertelement <4 x i64> [[R1]], i64 [[C2]], i64 258; SSE-NEXT: [[R3:%.*]] = insertelement <4 x i64> [[R2]], i64 [[C3]], i64 359; SSE-NEXT: ret <4 x i64> [[R3]]60;61; AVX2-LABEL: define <4 x i64> @scalarize_ctlz_v4i64(62; AVX2-SAME: <4 x i64> [[V:%.*]]) #[[ATTR0]] {63; AVX2-NEXT: [[V0:%.*]] = extractelement <4 x i64> [[V]], i64 064; AVX2-NEXT: [[V1:%.*]] = extractelement <4 x i64> [[V]], i64 165; AVX2-NEXT: [[V2:%.*]] = extractelement <4 x i64> [[V]], i64 266; AVX2-NEXT: [[V3:%.*]] = extractelement <4 x i64> [[V]], i64 367; AVX2-NEXT: [[C0:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V0]], i1 false)68; AVX2-NEXT: [[C1:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V1]], i1 false)69; AVX2-NEXT: [[C2:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V2]], i1 false)70; AVX2-NEXT: [[C3:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V3]], i1 false)71; AVX2-NEXT: [[R0:%.*]] = insertelement <4 x i64> poison, i64 [[C0]], i64 072; AVX2-NEXT: [[R1:%.*]] = insertelement <4 x i64> [[R0]], i64 [[C1]], i64 173; AVX2-NEXT: [[R2:%.*]] = insertelement <4 x i64> [[R1]], i64 [[C2]], i64 274; AVX2-NEXT: [[R3:%.*]] = insertelement <4 x i64> [[R2]], i64 [[C3]], i64 375; AVX2-NEXT: ret <4 x i64> [[R3]]76;77; AVX512-LABEL: define <4 x i64> @scalarize_ctlz_v4i64(78; AVX512-SAME: <4 x i64> [[V:%.*]]) #[[ATTR0]] {79; AVX512-NEXT: [[TMP1:%.*]] = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> [[V]], i1 false)80; AVX512-NEXT: ret <4 x i64> [[TMP1]]81;82 %v0 = extractelement <4 x i64> %v, i64 083 %v1 = extractelement <4 x i64> %v, i64 184 %v2 = extractelement <4 x i64> %v, i64 285 %v3 = extractelement <4 x i64> %v, i64 386 %c0 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v0, i1 false)87 %c1 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v1, i1 false)88 %c2 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v2, i1 false)89 %c3 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v3, i1 false)90 %r0 = insertelement <4 x i64> poison, i64 %c0, i64 091 %r1 = insertelement <4 x i64> %r0, i64 %c1, i64 192 %r2 = insertelement <4 x i64> %r1, i64 %c2, i64 293 %r3 = insertelement <4 x i64> %r2, i64 %c3, i64 394 ret <4 x i64> %r395}96 97define <8 x i64> @scalarize_ctlz_v8i64(<8 x i64> %v) {98; SSE2-LABEL: define <8 x i64> @scalarize_ctlz_v8i64(99; SSE2-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] {100; SSE2-NEXT: [[TMP1:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> <i32 0, i32 1>101; SSE2-NEXT: [[TMP2:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP1]], i1 false)102; SSE2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> <i32 2, i32 3>103; SSE2-NEXT: [[TMP4:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP3]], i1 false)104; SSE2-NEXT: [[TMP5:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> <i32 4, i32 5>105; SSE2-NEXT: [[TMP6:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP5]], i1 false)106; SSE2-NEXT: [[TMP7:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> <i32 6, i32 7>107; SSE2-NEXT: [[TMP8:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP7]], i1 false)108; SSE2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>109; SSE2-NEXT: [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>110; SSE2-NEXT: [[R31:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>111; SSE2-NEXT: [[TMP11:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>112; SSE2-NEXT: [[R52:%.*]] = shufflevector <8 x i64> [[R31]], <8 x i64> [[TMP11]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>113; SSE2-NEXT: [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>114; SSE2-NEXT: [[R73:%.*]] = shufflevector <8 x i64> [[R52]], <8 x i64> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>115; SSE2-NEXT: ret <8 x i64> [[R73]]116;117; SSE4-LABEL: define <8 x i64> @scalarize_ctlz_v8i64(118; SSE4-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] {119; SSE4-NEXT: [[V0:%.*]] = extractelement <8 x i64> [[V]], i64 0120; SSE4-NEXT: [[V1:%.*]] = extractelement <8 x i64> [[V]], i64 1121; SSE4-NEXT: [[V2:%.*]] = extractelement <8 x i64> [[V]], i64 2122; SSE4-NEXT: [[V3:%.*]] = extractelement <8 x i64> [[V]], i64 3123; SSE4-NEXT: [[V4:%.*]] = extractelement <8 x i64> [[V]], i64 4124; SSE4-NEXT: [[V5:%.*]] = extractelement <8 x i64> [[V]], i64 5125; SSE4-NEXT: [[V6:%.*]] = extractelement <8 x i64> [[V]], i64 6126; SSE4-NEXT: [[V7:%.*]] = extractelement <8 x i64> [[V]], i64 7127; SSE4-NEXT: [[C0:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V0]], i1 false)128; SSE4-NEXT: [[C1:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V1]], i1 false)129; SSE4-NEXT: [[C2:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V2]], i1 false)130; SSE4-NEXT: [[C3:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V3]], i1 false)131; SSE4-NEXT: [[C4:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V4]], i1 false)132; SSE4-NEXT: [[C5:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V5]], i1 false)133; SSE4-NEXT: [[C6:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V6]], i1 false)134; SSE4-NEXT: [[C7:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V7]], i1 false)135; SSE4-NEXT: [[R0:%.*]] = insertelement <8 x i64> poison, i64 [[C0]], i64 0136; SSE4-NEXT: [[R1:%.*]] = insertelement <8 x i64> [[R0]], i64 [[C1]], i64 1137; SSE4-NEXT: [[R2:%.*]] = insertelement <8 x i64> [[R1]], i64 [[C2]], i64 2138; SSE4-NEXT: [[R3:%.*]] = insertelement <8 x i64> [[R2]], i64 [[C3]], i64 3139; SSE4-NEXT: [[R4:%.*]] = insertelement <8 x i64> [[R3]], i64 [[C4]], i64 4140; SSE4-NEXT: [[R5:%.*]] = insertelement <8 x i64> [[R4]], i64 [[C5]], i64 5141; SSE4-NEXT: [[R6:%.*]] = insertelement <8 x i64> [[R5]], i64 [[C6]], i64 6142; SSE4-NEXT: [[R7:%.*]] = insertelement <8 x i64> [[R6]], i64 [[C7]], i64 7143; SSE4-NEXT: ret <8 x i64> [[R7]]144;145; AVX2-LABEL: define <8 x i64> @scalarize_ctlz_v8i64(146; AVX2-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] {147; AVX2-NEXT: [[V0:%.*]] = extractelement <8 x i64> [[V]], i64 0148; AVX2-NEXT: [[V1:%.*]] = extractelement <8 x i64> [[V]], i64 1149; AVX2-NEXT: [[V2:%.*]] = extractelement <8 x i64> [[V]], i64 2150; AVX2-NEXT: [[V3:%.*]] = extractelement <8 x i64> [[V]], i64 3151; AVX2-NEXT: [[V4:%.*]] = extractelement <8 x i64> [[V]], i64 4152; AVX2-NEXT: [[V5:%.*]] = extractelement <8 x i64> [[V]], i64 5153; AVX2-NEXT: [[V6:%.*]] = extractelement <8 x i64> [[V]], i64 6154; AVX2-NEXT: [[V7:%.*]] = extractelement <8 x i64> [[V]], i64 7155; AVX2-NEXT: [[C0:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V0]], i1 false)156; AVX2-NEXT: [[C1:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V1]], i1 false)157; AVX2-NEXT: [[C2:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V2]], i1 false)158; AVX2-NEXT: [[C3:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V3]], i1 false)159; AVX2-NEXT: [[C4:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V4]], i1 false)160; AVX2-NEXT: [[C5:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V5]], i1 false)161; AVX2-NEXT: [[C6:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V6]], i1 false)162; AVX2-NEXT: [[C7:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V7]], i1 false)163; AVX2-NEXT: [[R0:%.*]] = insertelement <8 x i64> poison, i64 [[C0]], i64 0164; AVX2-NEXT: [[R1:%.*]] = insertelement <8 x i64> [[R0]], i64 [[C1]], i64 1165; AVX2-NEXT: [[R2:%.*]] = insertelement <8 x i64> [[R1]], i64 [[C2]], i64 2166; AVX2-NEXT: [[R3:%.*]] = insertelement <8 x i64> [[R2]], i64 [[C3]], i64 3167; AVX2-NEXT: [[R4:%.*]] = insertelement <8 x i64> [[R3]], i64 [[C4]], i64 4168; AVX2-NEXT: [[R5:%.*]] = insertelement <8 x i64> [[R4]], i64 [[C5]], i64 5169; AVX2-NEXT: [[R6:%.*]] = insertelement <8 x i64> [[R5]], i64 [[C6]], i64 6170; AVX2-NEXT: [[R7:%.*]] = insertelement <8 x i64> [[R6]], i64 [[C7]], i64 7171; AVX2-NEXT: ret <8 x i64> [[R7]]172;173; AVX512-LABEL: define <8 x i64> @scalarize_ctlz_v8i64(174; AVX512-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] {175; AVX512-NEXT: [[TMP1:%.*]] = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> [[V]], i1 false)176; AVX512-NEXT: ret <8 x i64> [[TMP1]]177;178 %v0 = extractelement <8 x i64> %v, i64 0179 %v1 = extractelement <8 x i64> %v, i64 1180 %v2 = extractelement <8 x i64> %v, i64 2181 %v3 = extractelement <8 x i64> %v, i64 3182 %v4 = extractelement <8 x i64> %v, i64 4183 %v5 = extractelement <8 x i64> %v, i64 5184 %v6 = extractelement <8 x i64> %v, i64 6185 %v7 = extractelement <8 x i64> %v, i64 7186 %c0 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v0, i1 false)187 %c1 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v1, i1 false)188 %c2 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v2, i1 false)189 %c3 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v3, i1 false)190 %c4 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v4, i1 false)191 %c5 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v5, i1 false)192 %c6 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v6, i1 false)193 %c7 = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 %v7, i1 false)194 %r0 = insertelement <8 x i64> poison, i64 %c0, i64 0195 %r1 = insertelement <8 x i64> %r0, i64 %c1, i64 1196 %r2 = insertelement <8 x i64> %r1, i64 %c2, i64 2197 %r3 = insertelement <8 x i64> %r2, i64 %c3, i64 3198 %r4 = insertelement <8 x i64> %r3, i64 %c4, i64 4199 %r5 = insertelement <8 x i64> %r4, i64 %c5, i64 5200 %r6 = insertelement <8 x i64> %r5, i64 %c6, i64 6201 %r7 = insertelement <8 x i64> %r6, i64 %c7, i64 7202 ret <8 x i64> %r7203}204