624 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown -mcpu=x86-64 -S | FileCheck %s --check-prefixes=SSE,SSE23; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown -mcpu=x86-64-v2 -S | FileCheck %s --check-prefixes=SSE,SSE44; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown -mcpu=x86-64-v3 -S | FileCheck %s --check-prefixes=AVX5; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown -mcpu=x86-64-v4 -S | FileCheck %s --check-prefixes=AVX5126 7; // PR426528; unsigned long bitmask_16xi8(const char *src) {9; unsigned long mask = 0;10; for (unsigned i = 0; i != 16; ++i) {11; if (src[i])12; mask |= (1ull << i);13; }14; return mask;15; }16 17define i64 @bitmask_16xi8(ptr nocapture noundef readonly %src) {18; SSE-LABEL: @bitmask_16xi8(19; SSE-NEXT: entry:20; SSE-NEXT: [[TMP0:%.*]] = load i8, ptr [[SRC:%.*]], align 121; SSE-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i8 [[TMP0]], 022; SSE-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i6423; SSE-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 124; SSE-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[ARRAYIDX_1]], align 125; SSE-NEXT: [[TMP2:%.*]] = icmp eq <8 x i8> [[TMP1]], zeroinitializer26; SSE-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i64> zeroinitializer, <8 x i64> <i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256>27; SSE-NEXT: [[ARRAYIDX_9:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 928; SSE-NEXT: [[TMP4:%.*]] = load <4 x i8>, ptr [[ARRAYIDX_9]], align 129; SSE-NEXT: [[TMP5:%.*]] = icmp eq <4 x i8> [[TMP4]], zeroinitializer30; SSE-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP5]], <4 x i64> zeroinitializer, <4 x i64> <i64 512, i64 1024, i64 2048, i64 4096>31; SSE-NEXT: [[ARRAYIDX_13:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1332; SSE-NEXT: [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX_13]], align 133; SSE-NEXT: [[TOBOOL_NOT_13:%.*]] = icmp eq i8 [[TMP7]], 034; SSE-NEXT: [[OR_13:%.*]] = select i1 [[TOBOOL_NOT_13]], i64 0, i64 819235; SSE-NEXT: [[ARRAYIDX_14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1436; SSE-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX_14]], align 137; SSE-NEXT: [[TOBOOL_NOT_14:%.*]] = icmp eq i8 [[TMP8]], 038; SSE-NEXT: [[OR_14:%.*]] = select i1 [[TOBOOL_NOT_14]], i64 0, i64 1638439; SSE-NEXT: [[ARRAYIDX_15:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1540; SSE-NEXT: [[TMP9:%.*]] = load i8, ptr [[ARRAYIDX_15]], align 141; SSE-NEXT: [[TOBOOL_NOT_15:%.*]] = icmp eq i8 [[TMP9]], 042; SSE-NEXT: [[OR_15:%.*]] = select i1 [[TOBOOL_NOT_15]], i64 0, i64 3276843; SSE-NEXT: [[TMP10:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>44; SSE-NEXT: [[RDX_OP:%.*]] = or <4 x i64> [[TMP10]], [[TMP6]]45; SSE-NEXT: [[TMP12:%.*]] = shufflevector <4 x i64> [[RDX_OP]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>46; SSE-NEXT: [[TMP11:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> [[TMP12]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>47; SSE-NEXT: [[TMP16:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP11]])48; SSE-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP16]], [[OR_13]]49; SSE-NEXT: [[OP_RDX5:%.*]] = or i64 [[OR_14]], [[OR_15]]50; SSE-NEXT: [[OP_RDX6:%.*]] = or i64 [[OP_RDX]], [[OP_RDX5]]51; SSE-NEXT: [[OP_RDX7:%.*]] = or i64 [[OP_RDX6]], [[OR]]52; SSE-NEXT: ret i64 [[OP_RDX7]]53;54; AVX-LABEL: @bitmask_16xi8(55; AVX-NEXT: entry:56; AVX-NEXT: [[TMP0:%.*]] = load i8, ptr [[SRC:%.*]], align 157; AVX-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i8 [[TMP0]], 058; AVX-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i6459; AVX-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 160; AVX-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[ARRAYIDX_1]], align 161; AVX-NEXT: [[TMP2:%.*]] = icmp eq <8 x i8> [[TMP1]], zeroinitializer62; AVX-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i64> zeroinitializer, <8 x i64> <i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256>63; AVX-NEXT: [[ARRAYIDX_9:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 964; AVX-NEXT: [[TMP4:%.*]] = load <4 x i8>, ptr [[ARRAYIDX_9]], align 165; AVX-NEXT: [[TMP5:%.*]] = icmp eq <4 x i8> [[TMP4]], zeroinitializer66; AVX-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP5]], <4 x i64> zeroinitializer, <4 x i64> <i64 512, i64 1024, i64 2048, i64 4096>67; AVX-NEXT: [[ARRAYIDX_13:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1368; AVX-NEXT: [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX_13]], align 169; AVX-NEXT: [[TOBOOL_NOT_13:%.*]] = icmp eq i8 [[TMP7]], 070; AVX-NEXT: [[OR_13:%.*]] = select i1 [[TOBOOL_NOT_13]], i64 0, i64 819271; AVX-NEXT: [[ARRAYIDX_14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1472; AVX-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX_14]], align 173; AVX-NEXT: [[TOBOOL_NOT_14:%.*]] = icmp eq i8 [[TMP8]], 074; AVX-NEXT: [[OR_14:%.*]] = select i1 [[TOBOOL_NOT_14]], i64 0, i64 1638475; AVX-NEXT: [[ARRAYIDX_15:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1576; AVX-NEXT: [[TMP9:%.*]] = load i8, ptr [[ARRAYIDX_15]], align 177; AVX-NEXT: [[TOBOOL_NOT_15:%.*]] = icmp eq i8 [[TMP9]], 078; AVX-NEXT: [[OR_15:%.*]] = select i1 [[TOBOOL_NOT_15]], i64 0, i64 3276879; AVX-NEXT: [[TMP10:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>80; AVX-NEXT: [[RDX_OP:%.*]] = or <4 x i64> [[TMP10]], [[TMP6]]81; AVX-NEXT: [[TMP13:%.*]] = shufflevector <4 x i64> [[RDX_OP]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>82; AVX-NEXT: [[TMP11:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> [[TMP13]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>83; AVX-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP11]])84; AVX-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP12]], [[OR_13]]85; AVX-NEXT: [[OP_RDX2:%.*]] = or i64 [[OR_14]], [[OR_15]]86; AVX-NEXT: [[OP_RDX3:%.*]] = or i64 [[OP_RDX]], [[OP_RDX2]]87; AVX-NEXT: [[OP_RDX4:%.*]] = or i64 [[OP_RDX3]], [[OR]]88; AVX-NEXT: ret i64 [[OP_RDX4]]89;90; AVX512-LABEL: @bitmask_16xi8(91; AVX512-NEXT: entry:92; AVX512-NEXT: [[TMP0:%.*]] = load i8, ptr [[SRC:%.*]], align 193; AVX512-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i8 [[TMP0]], 094; AVX512-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i6495; AVX512-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 196; AVX512-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[ARRAYIDX_1]], align 197; AVX512-NEXT: [[TMP2:%.*]] = icmp eq <8 x i8> [[TMP1]], zeroinitializer98; AVX512-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i64> zeroinitializer, <8 x i64> <i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256>99; AVX512-NEXT: [[ARRAYIDX_9:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 9100; AVX512-NEXT: [[TMP4:%.*]] = load <4 x i8>, ptr [[ARRAYIDX_9]], align 1101; AVX512-NEXT: [[TMP5:%.*]] = icmp eq <4 x i8> [[TMP4]], zeroinitializer102; AVX512-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP5]], <4 x i64> zeroinitializer, <4 x i64> <i64 512, i64 1024, i64 2048, i64 4096>103; AVX512-NEXT: [[ARRAYIDX_13:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 13104; AVX512-NEXT: [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX_13]], align 1105; AVX512-NEXT: [[TOBOOL_NOT_13:%.*]] = icmp eq i8 [[TMP7]], 0106; AVX512-NEXT: [[OR_13:%.*]] = select i1 [[TOBOOL_NOT_13]], i64 0, i64 8192107; AVX512-NEXT: [[ARRAYIDX_14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 14108; AVX512-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX_14]], align 1109; AVX512-NEXT: [[TOBOOL_NOT_14:%.*]] = icmp eq i8 [[TMP8]], 0110; AVX512-NEXT: [[OR_14:%.*]] = select i1 [[TOBOOL_NOT_14]], i64 0, i64 16384111; AVX512-NEXT: [[ARRAYIDX_15:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 15112; AVX512-NEXT: [[TMP9:%.*]] = load i8, ptr [[ARRAYIDX_15]], align 1113; AVX512-NEXT: [[TOBOOL_NOT_15:%.*]] = icmp eq i8 [[TMP9]], 0114; AVX512-NEXT: [[OR_15:%.*]] = select i1 [[TOBOOL_NOT_15]], i64 0, i64 32768115; AVX512-NEXT: [[TMP10:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>116; AVX512-NEXT: [[RDX_OP:%.*]] = or <4 x i64> [[TMP10]], [[TMP6]]117; AVX512-NEXT: [[TMP13:%.*]] = shufflevector <4 x i64> [[RDX_OP]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>118; AVX512-NEXT: [[TMP11:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> [[TMP13]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>119; AVX512-NEXT: [[TMP12:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP11]])120; AVX512-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP12]], [[OR_13]]121; AVX512-NEXT: [[OP_RDX2:%.*]] = or i64 [[OR_14]], [[OR_15]]122; AVX512-NEXT: [[OP_RDX3:%.*]] = or i64 [[OP_RDX]], [[OP_RDX2]]123; AVX512-NEXT: [[OP_RDX4:%.*]] = or i64 [[OP_RDX3]], [[OR]]124; AVX512-NEXT: ret i64 [[OP_RDX4]]125;126entry:127 %0 = load i8, ptr %src, align 1128 %tobool.not = icmp ne i8 %0, 0129 %or = zext i1 %tobool.not to i64130 %arrayidx.1 = getelementptr inbounds i8, ptr %src, i64 1131 %1 = load i8, ptr %arrayidx.1, align 1132 %tobool.not.1 = icmp eq i8 %1, 0133 %or.1 = select i1 %tobool.not.1, i64 0, i64 2134 %mask.1.1 = or i64 %or.1, %or135 %arrayidx.2 = getelementptr inbounds i8, ptr %src, i64 2136 %2 = load i8, ptr %arrayidx.2, align 1137 %tobool.not.2 = icmp eq i8 %2, 0138 %or.2 = select i1 %tobool.not.2, i64 0, i64 4139 %mask.1.2 = or i64 %or.2, %mask.1.1140 %arrayidx.3 = getelementptr inbounds i8, ptr %src, i64 3141 %3 = load i8, ptr %arrayidx.3, align 1142 %tobool.not.3 = icmp eq i8 %3, 0143 %or.3 = select i1 %tobool.not.3, i64 0, i64 8144 %mask.1.3 = or i64 %or.3, %mask.1.2145 %arrayidx.4 = getelementptr inbounds i8, ptr %src, i64 4146 %4 = load i8, ptr %arrayidx.4, align 1147 %tobool.not.4 = icmp eq i8 %4, 0148 %or.4 = select i1 %tobool.not.4, i64 0, i64 16149 %mask.1.4 = or i64 %or.4, %mask.1.3150 %arrayidx.5 = getelementptr inbounds i8, ptr %src, i64 5151 %5 = load i8, ptr %arrayidx.5, align 1152 %tobool.not.5 = icmp eq i8 %5, 0153 %or.5 = select i1 %tobool.not.5, i64 0, i64 32154 %mask.1.5 = or i64 %or.5, %mask.1.4155 %arrayidx.6 = getelementptr inbounds i8, ptr %src, i64 6156 %6 = load i8, ptr %arrayidx.6, align 1157 %tobool.not.6 = icmp eq i8 %6, 0158 %or.6 = select i1 %tobool.not.6, i64 0, i64 64159 %mask.1.6 = or i64 %or.6, %mask.1.5160 %arrayidx.7 = getelementptr inbounds i8, ptr %src, i64 7161 %7 = load i8, ptr %arrayidx.7, align 1162 %tobool.not.7 = icmp eq i8 %7, 0163 %or.7 = select i1 %tobool.not.7, i64 0, i64 128164 %mask.1.7 = or i64 %or.7, %mask.1.6165 %arrayidx.8 = getelementptr inbounds i8, ptr %src, i64 8166 %8 = load i8, ptr %arrayidx.8, align 1167 %tobool.not.8 = icmp eq i8 %8, 0168 %or.8 = select i1 %tobool.not.8, i64 0, i64 256169 %mask.1.8 = or i64 %or.8, %mask.1.7170 %arrayidx.9 = getelementptr inbounds i8, ptr %src, i64 9171 %9 = load i8, ptr %arrayidx.9, align 1172 %tobool.not.9 = icmp eq i8 %9, 0173 %or.9 = select i1 %tobool.not.9, i64 0, i64 512174 %mask.1.9 = or i64 %or.9, %mask.1.8175 %arrayidx.10 = getelementptr inbounds i8, ptr %src, i64 10176 %10 = load i8, ptr %arrayidx.10, align 1177 %tobool.not.10 = icmp eq i8 %10, 0178 %or.10 = select i1 %tobool.not.10, i64 0, i64 1024179 %mask.1.10 = or i64 %or.10, %mask.1.9180 %arrayidx.11 = getelementptr inbounds i8, ptr %src, i64 11181 %11 = load i8, ptr %arrayidx.11, align 1182 %tobool.not.11 = icmp eq i8 %11, 0183 %or.11 = select i1 %tobool.not.11, i64 0, i64 2048184 %mask.1.11 = or i64 %or.11, %mask.1.10185 %arrayidx.12 = getelementptr inbounds i8, ptr %src, i64 12186 %12 = load i8, ptr %arrayidx.12, align 1187 %tobool.not.12 = icmp eq i8 %12, 0188 %or.12 = select i1 %tobool.not.12, i64 0, i64 4096189 %mask.1.12 = or i64 %or.12, %mask.1.11190 %arrayidx.13 = getelementptr inbounds i8, ptr %src, i64 13191 %13 = load i8, ptr %arrayidx.13, align 1192 %tobool.not.13 = icmp eq i8 %13, 0193 %or.13 = select i1 %tobool.not.13, i64 0, i64 8192194 %mask.1.13 = or i64 %or.13, %mask.1.12195 %arrayidx.14 = getelementptr inbounds i8, ptr %src, i64 14196 %14 = load i8, ptr %arrayidx.14, align 1197 %tobool.not.14 = icmp eq i8 %14, 0198 %or.14 = select i1 %tobool.not.14, i64 0, i64 16384199 %mask.1.14 = or i64 %or.14, %mask.1.13200 %arrayidx.15 = getelementptr inbounds i8, ptr %src, i64 15201 %15 = load i8, ptr %arrayidx.15, align 1202 %tobool.not.15 = icmp eq i8 %15, 0203 %or.15 = select i1 %tobool.not.15, i64 0, i64 32768204 %mask.1.15 = or i64 %or.15, %mask.1.14205 ret i64 %mask.1.15206}207 208define i64 @bitmask_4xi16(ptr nocapture noundef readonly %src) {209; SSE-LABEL: @bitmask_4xi16(210; SSE-NEXT: entry:211; SSE-NEXT: [[TMP0:%.*]] = load i16, ptr [[SRC:%.*]], align 2212; SSE-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i16 [[TMP0]], 0213; SSE-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i64214; SSE-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 1215; SSE-NEXT: [[TMP1:%.*]] = load <4 x i16>, ptr [[ARRAYIDX_1]], align 2216; SSE-NEXT: [[TMP2:%.*]] = icmp eq <4 x i16> [[TMP1]], zeroinitializer217; SSE-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> zeroinitializer, <4 x i64> <i64 2, i64 4, i64 8, i64 16>218; SSE-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 5219; SSE-NEXT: [[TMP4:%.*]] = load i16, ptr [[ARRAYIDX_5]], align 2220; SSE-NEXT: [[TOBOOL_NOT_5:%.*]] = icmp eq i16 [[TMP4]], 0221; SSE-NEXT: [[OR_5:%.*]] = select i1 [[TOBOOL_NOT_5]], i64 0, i64 32222; SSE-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 6223; SSE-NEXT: [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX_6]], align 2224; SSE-NEXT: [[TOBOOL_NOT_6:%.*]] = icmp eq i16 [[TMP5]], 0225; SSE-NEXT: [[OR_6:%.*]] = select i1 [[TOBOOL_NOT_6]], i64 0, i64 64226; SSE-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 7227; SSE-NEXT: [[TMP6:%.*]] = load i16, ptr [[ARRAYIDX_7]], align 2228; SSE-NEXT: [[TOBOOL_NOT_7:%.*]] = icmp eq i16 [[TMP6]], 0229; SSE-NEXT: [[OR_7:%.*]] = select i1 [[TOBOOL_NOT_7]], i64 0, i64 128230; SSE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP3]])231; SSE-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP7]], [[OR_5]]232; SSE-NEXT: [[OP_RDX1:%.*]] = or i64 [[OR_6]], [[OR_7]]233; SSE-NEXT: [[OP_RDX2:%.*]] = or i64 [[OP_RDX]], [[OP_RDX1]]234; SSE-NEXT: [[OP_RDX3:%.*]] = or i64 [[OP_RDX2]], [[OR]]235; SSE-NEXT: ret i64 [[OP_RDX3]]236;237; AVX-LABEL: @bitmask_4xi16(238; AVX-NEXT: entry:239; AVX-NEXT: [[TMP0:%.*]] = load i16, ptr [[SRC:%.*]], align 2240; AVX-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i16 [[TMP0]], 0241; AVX-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i64242; AVX-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 1243; AVX-NEXT: [[TMP1:%.*]] = load <4 x i16>, ptr [[ARRAYIDX_1]], align 2244; AVX-NEXT: [[TMP2:%.*]] = icmp eq <4 x i16> [[TMP1]], zeroinitializer245; AVX-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> zeroinitializer, <4 x i64> <i64 2, i64 4, i64 8, i64 16>246; AVX-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 5247; AVX-NEXT: [[TMP4:%.*]] = load i16, ptr [[ARRAYIDX_5]], align 2248; AVX-NEXT: [[TOBOOL_NOT_5:%.*]] = icmp eq i16 [[TMP4]], 0249; AVX-NEXT: [[OR_5:%.*]] = select i1 [[TOBOOL_NOT_5]], i64 0, i64 32250; AVX-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 6251; AVX-NEXT: [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX_6]], align 2252; AVX-NEXT: [[TOBOOL_NOT_6:%.*]] = icmp eq i16 [[TMP5]], 0253; AVX-NEXT: [[OR_6:%.*]] = select i1 [[TOBOOL_NOT_6]], i64 0, i64 64254; AVX-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 7255; AVX-NEXT: [[TMP6:%.*]] = load i16, ptr [[ARRAYIDX_7]], align 2256; AVX-NEXT: [[TOBOOL_NOT_7:%.*]] = icmp eq i16 [[TMP6]], 0257; AVX-NEXT: [[OR_7:%.*]] = select i1 [[TOBOOL_NOT_7]], i64 0, i64 128258; AVX-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP3]])259; AVX-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP7]], [[OR_5]]260; AVX-NEXT: [[OP_RDX1:%.*]] = or i64 [[OR_6]], [[OR_7]]261; AVX-NEXT: [[OP_RDX2:%.*]] = or i64 [[OP_RDX]], [[OP_RDX1]]262; AVX-NEXT: [[OP_RDX3:%.*]] = or i64 [[OP_RDX2]], [[OR]]263; AVX-NEXT: ret i64 [[OP_RDX3]]264;265; AVX512-LABEL: @bitmask_4xi16(266; AVX512-NEXT: entry:267; AVX512-NEXT: [[TMP0:%.*]] = load i16, ptr [[SRC:%.*]], align 2268; AVX512-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i16 [[TMP0]], 0269; AVX512-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i64270; AVX512-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 1271; AVX512-NEXT: [[TMP1:%.*]] = load <4 x i16>, ptr [[ARRAYIDX_1]], align 2272; AVX512-NEXT: [[TMP2:%.*]] = icmp eq <4 x i16> [[TMP1]], zeroinitializer273; AVX512-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> zeroinitializer, <4 x i64> <i64 2, i64 4, i64 8, i64 16>274; AVX512-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 5275; AVX512-NEXT: [[TMP4:%.*]] = load i16, ptr [[ARRAYIDX_5]], align 2276; AVX512-NEXT: [[TOBOOL_NOT_5:%.*]] = icmp eq i16 [[TMP4]], 0277; AVX512-NEXT: [[OR_5:%.*]] = select i1 [[TOBOOL_NOT_5]], i64 0, i64 32278; AVX512-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 6279; AVX512-NEXT: [[TMP5:%.*]] = load <2 x i16>, ptr [[ARRAYIDX_6]], align 2280; AVX512-NEXT: [[TMP6:%.*]] = icmp eq <2 x i16> [[TMP5]], zeroinitializer281; AVX512-NEXT: [[TMP7:%.*]] = select <2 x i1> [[TMP6]], <2 x i64> zeroinitializer, <2 x i64> <i64 64, i64 128>282; AVX512-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP3]])283; AVX512-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP8]], [[OR_5]]284; AVX512-NEXT: [[TMP9:%.*]] = extractelement <2 x i64> [[TMP7]], i32 0285; AVX512-NEXT: [[TMP10:%.*]] = extractelement <2 x i64> [[TMP7]], i32 1286; AVX512-NEXT: [[OP_RDX1:%.*]] = or i64 [[TMP9]], [[TMP10]]287; AVX512-NEXT: [[OP_RDX2:%.*]] = or i64 [[OP_RDX]], [[OP_RDX1]]288; AVX512-NEXT: [[OP_RDX3:%.*]] = or i64 [[OP_RDX2]], [[OR]]289; AVX512-NEXT: ret i64 [[OP_RDX3]]290;291entry:292 %0 = load i16, ptr %src, align 2293 %tobool.not = icmp ne i16 %0, 0294 %or = zext i1 %tobool.not to i64295 %arrayidx.1 = getelementptr inbounds i16, ptr %src, i64 1296 %1 = load i16, ptr %arrayidx.1, align 2297 %tobool.not.1 = icmp eq i16 %1, 0298 %or.1 = select i1 %tobool.not.1, i64 0, i64 2299 %mask.1.1 = or i64 %or.1, %or300 %arrayidx.2 = getelementptr inbounds i16, ptr %src, i64 2301 %2 = load i16, ptr %arrayidx.2, align 2302 %tobool.not.2 = icmp eq i16 %2, 0303 %or.2 = select i1 %tobool.not.2, i64 0, i64 4304 %mask.1.2 = or i64 %or.2, %mask.1.1305 %arrayidx.3 = getelementptr inbounds i16, ptr %src, i64 3306 %3 = load i16, ptr %arrayidx.3, align 2307 %tobool.not.3 = icmp eq i16 %3, 0308 %or.3 = select i1 %tobool.not.3, i64 0, i64 8309 %mask.1.3 = or i64 %or.3, %mask.1.2310 %arrayidx.4 = getelementptr inbounds i16, ptr %src, i64 4311 %4 = load i16, ptr %arrayidx.4, align 2312 %tobool.not.4 = icmp eq i16 %4, 0313 %or.4 = select i1 %tobool.not.4, i64 0, i64 16314 %mask.1.4 = or i64 %or.4, %mask.1.3315 %arrayidx.5 = getelementptr inbounds i16, ptr %src, i64 5316 %5 = load i16, ptr %arrayidx.5, align 2317 %tobool.not.5 = icmp eq i16 %5, 0318 %or.5 = select i1 %tobool.not.5, i64 0, i64 32319 %mask.1.5 = or i64 %or.5, %mask.1.4320 %arrayidx.6 = getelementptr inbounds i16, ptr %src, i64 6321 %6 = load i16, ptr %arrayidx.6, align 2322 %tobool.not.6 = icmp eq i16 %6, 0323 %or.6 = select i1 %tobool.not.6, i64 0, i64 64324 %mask.1.6 = or i64 %or.6, %mask.1.5325 %arrayidx.7 = getelementptr inbounds i16, ptr %src, i64 7326 %7 = load i16, ptr %arrayidx.7, align 2327 %tobool.not.7 = icmp eq i16 %7, 0328 %or.7 = select i1 %tobool.not.7, i64 0, i64 128329 %mask.1.7 = or i64 %or.7, %mask.1.6330 ret i64 %mask.1.7331}332 333define i64 @bitmask_8xi32(ptr nocapture noundef readonly %src) {334; SSE-LABEL: @bitmask_8xi32(335; SSE-NEXT: entry:336; SSE-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC:%.*]], align 4337; SSE-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i32 [[TMP0]], 0338; SSE-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i64339; SSE-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 1340; SSE-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[ARRAYIDX_1]], align 4341; SSE-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[TMP1]], zeroinitializer342; SSE-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> zeroinitializer, <4 x i64> <i64 2, i64 4, i64 8, i64 16>343; SSE-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 5344; SSE-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX_5]], align 4345; SSE-NEXT: [[TOBOOL_NOT_5:%.*]] = icmp eq i32 [[TMP4]], 0346; SSE-NEXT: [[OR_5:%.*]] = select i1 [[TOBOOL_NOT_5]], i64 0, i64 32347; SSE-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 6348; SSE-NEXT: [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX_6]], align 4349; SSE-NEXT: [[TOBOOL_NOT_6:%.*]] = icmp eq i32 [[TMP5]], 0350; SSE-NEXT: [[OR_6:%.*]] = select i1 [[TOBOOL_NOT_6]], i64 0, i64 64351; SSE-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 7352; SSE-NEXT: [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX_7]], align 4353; SSE-NEXT: [[TOBOOL_NOT_7:%.*]] = icmp eq i32 [[TMP6]], 0354; SSE-NEXT: [[OR_7:%.*]] = select i1 [[TOBOOL_NOT_7]], i64 0, i64 128355; SSE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP3]])356; SSE-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP7]], [[OR_5]]357; SSE-NEXT: [[OP_RDX1:%.*]] = or i64 [[OR_6]], [[OR_7]]358; SSE-NEXT: [[OP_RDX2:%.*]] = or i64 [[OP_RDX]], [[OP_RDX1]]359; SSE-NEXT: [[OP_RDX3:%.*]] = or i64 [[OP_RDX2]], [[OR]]360; SSE-NEXT: ret i64 [[OP_RDX3]]361;362; AVX-LABEL: @bitmask_8xi32(363; AVX-NEXT: entry:364; AVX-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC:%.*]], align 4365; AVX-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i32 [[TMP0]], 0366; AVX-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i64367; AVX-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 1368; AVX-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[ARRAYIDX_1]], align 4369; AVX-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[TMP1]], zeroinitializer370; AVX-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> zeroinitializer, <4 x i64> <i64 2, i64 4, i64 8, i64 16>371; AVX-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 5372; AVX-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX_5]], align 4373; AVX-NEXT: [[TOBOOL_NOT_5:%.*]] = icmp eq i32 [[TMP4]], 0374; AVX-NEXT: [[OR_5:%.*]] = select i1 [[TOBOOL_NOT_5]], i64 0, i64 32375; AVX-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 6376; AVX-NEXT: [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX_6]], align 4377; AVX-NEXT: [[TOBOOL_NOT_6:%.*]] = icmp eq i32 [[TMP5]], 0378; AVX-NEXT: [[OR_6:%.*]] = select i1 [[TOBOOL_NOT_6]], i64 0, i64 64379; AVX-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 7380; AVX-NEXT: [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX_7]], align 4381; AVX-NEXT: [[TOBOOL_NOT_7:%.*]] = icmp eq i32 [[TMP6]], 0382; AVX-NEXT: [[OR_7:%.*]] = select i1 [[TOBOOL_NOT_7]], i64 0, i64 128383; AVX-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP3]])384; AVX-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP7]], [[OR_5]]385; AVX-NEXT: [[OP_RDX1:%.*]] = or i64 [[OR_6]], [[OR_7]]386; AVX-NEXT: [[OP_RDX2:%.*]] = or i64 [[OP_RDX]], [[OP_RDX1]]387; AVX-NEXT: [[OP_RDX3:%.*]] = or i64 [[OP_RDX2]], [[OR]]388; AVX-NEXT: ret i64 [[OP_RDX3]]389;390; AVX512-LABEL: @bitmask_8xi32(391; AVX512-NEXT: entry:392; AVX512-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC:%.*]], align 4393; AVX512-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i32 [[TMP0]], 0394; AVX512-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i64395; AVX512-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 1396; AVX512-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[ARRAYIDX_1]], align 4397; AVX512-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[TMP1]], zeroinitializer398; AVX512-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> zeroinitializer, <4 x i64> <i64 2, i64 4, i64 8, i64 16>399; AVX512-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 5400; AVX512-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX_5]], align 4401; AVX512-NEXT: [[TOBOOL_NOT_5:%.*]] = icmp eq i32 [[TMP4]], 0402; AVX512-NEXT: [[OR_5:%.*]] = select i1 [[TOBOOL_NOT_5]], i64 0, i64 32403; AVX512-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 6404; AVX512-NEXT: [[TMP5:%.*]] = load <2 x i32>, ptr [[ARRAYIDX_6]], align 4405; AVX512-NEXT: [[TMP6:%.*]] = icmp eq <2 x i32> [[TMP5]], zeroinitializer406; AVX512-NEXT: [[TMP7:%.*]] = select <2 x i1> [[TMP6]], <2 x i64> zeroinitializer, <2 x i64> <i64 64, i64 128>407; AVX512-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP3]])408; AVX512-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP8]], [[OR_5]]409; AVX512-NEXT: [[TMP9:%.*]] = extractelement <2 x i64> [[TMP7]], i32 0410; AVX512-NEXT: [[TMP10:%.*]] = extractelement <2 x i64> [[TMP7]], i32 1411; AVX512-NEXT: [[OP_RDX1:%.*]] = or i64 [[TMP9]], [[TMP10]]412; AVX512-NEXT: [[OP_RDX2:%.*]] = or i64 [[OP_RDX]], [[OP_RDX1]]413; AVX512-NEXT: [[OP_RDX3:%.*]] = or i64 [[OP_RDX2]], [[OR]]414; AVX512-NEXT: ret i64 [[OP_RDX3]]415;416entry:417 %0 = load i32, ptr %src, align 4418 %tobool.not = icmp ne i32 %0, 0419 %or = zext i1 %tobool.not to i64420 %arrayidx.1 = getelementptr inbounds i32, ptr %src, i64 1421 %1 = load i32, ptr %arrayidx.1, align 4422 %tobool.not.1 = icmp eq i32 %1, 0423 %or.1 = select i1 %tobool.not.1, i64 0, i64 2424 %mask.1.1 = or i64 %or.1, %or425 %arrayidx.2 = getelementptr inbounds i32, ptr %src, i64 2426 %2 = load i32, ptr %arrayidx.2, align 4427 %tobool.not.2 = icmp eq i32 %2, 0428 %or.2 = select i1 %tobool.not.2, i64 0, i64 4429 %mask.1.2 = or i64 %or.2, %mask.1.1430 %arrayidx.3 = getelementptr inbounds i32, ptr %src, i64 3431 %3 = load i32, ptr %arrayidx.3, align 4432 %tobool.not.3 = icmp eq i32 %3, 0433 %or.3 = select i1 %tobool.not.3, i64 0, i64 8434 %mask.1.3 = or i64 %or.3, %mask.1.2435 %arrayidx.4 = getelementptr inbounds i32, ptr %src, i64 4436 %4 = load i32, ptr %arrayidx.4, align 4437 %tobool.not.4 = icmp eq i32 %4, 0438 %or.4 = select i1 %tobool.not.4, i64 0, i64 16439 %mask.1.4 = or i64 %or.4, %mask.1.3440 %arrayidx.5 = getelementptr inbounds i32, ptr %src, i64 5441 %5 = load i32, ptr %arrayidx.5, align 4442 %tobool.not.5 = icmp eq i32 %5, 0443 %or.5 = select i1 %tobool.not.5, i64 0, i64 32444 %mask.1.5 = or i64 %or.5, %mask.1.4445 %arrayidx.6 = getelementptr inbounds i32, ptr %src, i64 6446 %6 = load i32, ptr %arrayidx.6, align 4447 %tobool.not.6 = icmp eq i32 %6, 0448 %or.6 = select i1 %tobool.not.6, i64 0, i64 64449 %mask.1.6 = or i64 %or.6, %mask.1.5450 %arrayidx.7 = getelementptr inbounds i32, ptr %src, i64 7451 %7 = load i32, ptr %arrayidx.7, align 4452 %tobool.not.7 = icmp eq i32 %7, 0453 %or.7 = select i1 %tobool.not.7, i64 0, i64 128454 %mask.1.7 = or i64 %or.7, %mask.1.6455 ret i64 %mask.1.7456}457 458define i64 @bitmask_8xi64(ptr nocapture noundef readonly %src) {459; SSE2-LABEL: @bitmask_8xi64(460; SSE2-NEXT: entry:461; SSE2-NEXT: [[TMP0:%.*]] = load i64, ptr [[SRC:%.*]], align 8462; SSE2-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i64 [[TMP0]], 0463; SSE2-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i64464; SSE2-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 1465; SSE2-NEXT: [[TMP1:%.*]] = load i64, ptr [[ARRAYIDX_1]], align 8466; SSE2-NEXT: [[TOBOOL_NOT_1:%.*]] = icmp eq i64 [[TMP1]], 0467; SSE2-NEXT: [[OR_1:%.*]] = select i1 [[TOBOOL_NOT_1]], i64 0, i64 2468; SSE2-NEXT: [[MASK_1_1:%.*]] = or i64 [[OR_1]], [[OR]]469; SSE2-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 2470; SSE2-NEXT: [[TMP2:%.*]] = load i64, ptr [[ARRAYIDX_2]], align 8471; SSE2-NEXT: [[TOBOOL_NOT_2:%.*]] = icmp eq i64 [[TMP2]], 0472; SSE2-NEXT: [[OR_2:%.*]] = select i1 [[TOBOOL_NOT_2]], i64 0, i64 4473; SSE2-NEXT: [[MASK_1_2:%.*]] = or i64 [[OR_2]], [[MASK_1_1]]474; SSE2-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 3475; SSE2-NEXT: [[TMP3:%.*]] = load i64, ptr [[ARRAYIDX_3]], align 8476; SSE2-NEXT: [[TOBOOL_NOT_3:%.*]] = icmp eq i64 [[TMP3]], 0477; SSE2-NEXT: [[OR_3:%.*]] = select i1 [[TOBOOL_NOT_3]], i64 0, i64 8478; SSE2-NEXT: [[MASK_1_3:%.*]] = or i64 [[OR_3]], [[MASK_1_2]]479; SSE2-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 4480; SSE2-NEXT: [[TMP4:%.*]] = load i64, ptr [[ARRAYIDX_4]], align 8481; SSE2-NEXT: [[TOBOOL_NOT_4:%.*]] = icmp eq i64 [[TMP4]], 0482; SSE2-NEXT: [[OR_4:%.*]] = select i1 [[TOBOOL_NOT_4]], i64 0, i64 16483; SSE2-NEXT: [[MASK_1_4:%.*]] = or i64 [[OR_4]], [[MASK_1_3]]484; SSE2-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 5485; SSE2-NEXT: [[TMP5:%.*]] = load i64, ptr [[ARRAYIDX_5]], align 8486; SSE2-NEXT: [[TOBOOL_NOT_5:%.*]] = icmp eq i64 [[TMP5]], 0487; SSE2-NEXT: [[OR_5:%.*]] = select i1 [[TOBOOL_NOT_5]], i64 0, i64 32488; SSE2-NEXT: [[MASK_1_5:%.*]] = or i64 [[OR_5]], [[MASK_1_4]]489; SSE2-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 6490; SSE2-NEXT: [[TMP6:%.*]] = load i64, ptr [[ARRAYIDX_6]], align 8491; SSE2-NEXT: [[TOBOOL_NOT_6:%.*]] = icmp eq i64 [[TMP6]], 0492; SSE2-NEXT: [[OR_6:%.*]] = select i1 [[TOBOOL_NOT_6]], i64 0, i64 64493; SSE2-NEXT: [[MASK_1_6:%.*]] = or i64 [[OR_6]], [[MASK_1_5]]494; SSE2-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 7495; SSE2-NEXT: [[TMP7:%.*]] = load i64, ptr [[ARRAYIDX_7]], align 8496; SSE2-NEXT: [[TOBOOL_NOT_7:%.*]] = icmp eq i64 [[TMP7]], 0497; SSE2-NEXT: [[OR_7:%.*]] = select i1 [[TOBOOL_NOT_7]], i64 0, i64 128498; SSE2-NEXT: [[MASK_1_7:%.*]] = or i64 [[OR_7]], [[MASK_1_6]]499; SSE2-NEXT: ret i64 [[MASK_1_7]]500;501; SSE4-LABEL: @bitmask_8xi64(502; SSE4-NEXT: entry:503; SSE4-NEXT: [[TMP0:%.*]] = load i64, ptr [[SRC:%.*]], align 8504; SSE4-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i64 [[TMP0]], 0505; SSE4-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i64506; SSE4-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 1507; SSE4-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[ARRAYIDX_1]], align 8508; SSE4-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[TMP1]], zeroinitializer509; SSE4-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> zeroinitializer, <4 x i64> <i64 2, i64 4, i64 8, i64 16>510; SSE4-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 5511; SSE4-NEXT: [[TMP4:%.*]] = load i64, ptr [[ARRAYIDX_5]], align 8512; SSE4-NEXT: [[TOBOOL_NOT_5:%.*]] = icmp eq i64 [[TMP4]], 0513; SSE4-NEXT: [[OR_5:%.*]] = select i1 [[TOBOOL_NOT_5]], i64 0, i64 32514; SSE4-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 6515; SSE4-NEXT: [[TMP5:%.*]] = load i64, ptr [[ARRAYIDX_6]], align 8516; SSE4-NEXT: [[TOBOOL_NOT_6:%.*]] = icmp eq i64 [[TMP5]], 0517; SSE4-NEXT: [[OR_6:%.*]] = select i1 [[TOBOOL_NOT_6]], i64 0, i64 64518; SSE4-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 7519; SSE4-NEXT: [[TMP6:%.*]] = load i64, ptr [[ARRAYIDX_7]], align 8520; SSE4-NEXT: [[TOBOOL_NOT_7:%.*]] = icmp eq i64 [[TMP6]], 0521; SSE4-NEXT: [[OR_7:%.*]] = select i1 [[TOBOOL_NOT_7]], i64 0, i64 128522; SSE4-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP3]])523; SSE4-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP7]], [[OR_5]]524; SSE4-NEXT: [[OP_RDX1:%.*]] = or i64 [[OR_6]], [[OR_7]]525; SSE4-NEXT: [[OP_RDX2:%.*]] = or i64 [[OP_RDX]], [[OP_RDX1]]526; SSE4-NEXT: [[OP_RDX3:%.*]] = or i64 [[OP_RDX2]], [[OR]]527; SSE4-NEXT: ret i64 [[OP_RDX3]]528;529; AVX-LABEL: @bitmask_8xi64(530; AVX-NEXT: entry:531; AVX-NEXT: [[TMP0:%.*]] = load i64, ptr [[SRC:%.*]], align 8532; AVX-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i64 [[TMP0]], 0533; AVX-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i64534; AVX-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 1535; AVX-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[ARRAYIDX_1]], align 8536; AVX-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[TMP1]], zeroinitializer537; AVX-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> zeroinitializer, <4 x i64> <i64 2, i64 4, i64 8, i64 16>538; AVX-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 5539; AVX-NEXT: [[TMP4:%.*]] = load i64, ptr [[ARRAYIDX_5]], align 8540; AVX-NEXT: [[TOBOOL_NOT_5:%.*]] = icmp eq i64 [[TMP4]], 0541; AVX-NEXT: [[OR_5:%.*]] = select i1 [[TOBOOL_NOT_5]], i64 0, i64 32542; AVX-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 6543; AVX-NEXT: [[TMP5:%.*]] = load i64, ptr [[ARRAYIDX_6]], align 8544; AVX-NEXT: [[TOBOOL_NOT_6:%.*]] = icmp eq i64 [[TMP5]], 0545; AVX-NEXT: [[OR_6:%.*]] = select i1 [[TOBOOL_NOT_6]], i64 0, i64 64546; AVX-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 7547; AVX-NEXT: [[TMP6:%.*]] = load i64, ptr [[ARRAYIDX_7]], align 8548; AVX-NEXT: [[TOBOOL_NOT_7:%.*]] = icmp eq i64 [[TMP6]], 0549; AVX-NEXT: [[OR_7:%.*]] = select i1 [[TOBOOL_NOT_7]], i64 0, i64 128550; AVX-NEXT: [[TMP7:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP3]])551; AVX-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP7]], [[OR_5]]552; AVX-NEXT: [[OP_RDX1:%.*]] = or i64 [[OR_6]], [[OR_7]]553; AVX-NEXT: [[OP_RDX2:%.*]] = or i64 [[OP_RDX]], [[OP_RDX1]]554; AVX-NEXT: [[OP_RDX3:%.*]] = or i64 [[OP_RDX2]], [[OR]]555; AVX-NEXT: ret i64 [[OP_RDX3]]556;557; AVX512-LABEL: @bitmask_8xi64(558; AVX512-NEXT: entry:559; AVX512-NEXT: [[TMP0:%.*]] = load i64, ptr [[SRC:%.*]], align 8560; AVX512-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i64 [[TMP0]], 0561; AVX512-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i64562; AVX512-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 1563; AVX512-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[ARRAYIDX_1]], align 8564; AVX512-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[TMP1]], zeroinitializer565; AVX512-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i64> zeroinitializer, <4 x i64> <i64 2, i64 4, i64 8, i64 16>566; AVX512-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 5567; AVX512-NEXT: [[TMP4:%.*]] = load i64, ptr [[ARRAYIDX_5]], align 8568; AVX512-NEXT: [[TOBOOL_NOT_5:%.*]] = icmp eq i64 [[TMP4]], 0569; AVX512-NEXT: [[OR_5:%.*]] = select i1 [[TOBOOL_NOT_5]], i64 0, i64 32570; AVX512-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 6571; AVX512-NEXT: [[TMP5:%.*]] = load <2 x i64>, ptr [[ARRAYIDX_6]], align 8572; AVX512-NEXT: [[TMP6:%.*]] = icmp eq <2 x i64> [[TMP5]], zeroinitializer573; AVX512-NEXT: [[TMP7:%.*]] = select <2 x i1> [[TMP6]], <2 x i64> zeroinitializer, <2 x i64> <i64 64, i64 128>574; AVX512-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP3]])575; AVX512-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP8]], [[OR_5]]576; AVX512-NEXT: [[TMP9:%.*]] = extractelement <2 x i64> [[TMP7]], i32 0577; AVX512-NEXT: [[TMP10:%.*]] = extractelement <2 x i64> [[TMP7]], i32 1578; AVX512-NEXT: [[OP_RDX1:%.*]] = or i64 [[TMP9]], [[TMP10]]579; AVX512-NEXT: [[OP_RDX2:%.*]] = or i64 [[OP_RDX]], [[OP_RDX1]]580; AVX512-NEXT: [[OP_RDX3:%.*]] = or i64 [[OP_RDX2]], [[OR]]581; AVX512-NEXT: ret i64 [[OP_RDX3]]582;583entry:584 %0 = load i64, ptr %src, align 8585 %tobool.not = icmp ne i64 %0, 0586 %or = zext i1 %tobool.not to i64587 %arrayidx.1 = getelementptr inbounds i64, ptr %src, i64 1588 %1 = load i64, ptr %arrayidx.1, align 8589 %tobool.not.1 = icmp eq i64 %1, 0590 %or.1 = select i1 %tobool.not.1, i64 0, i64 2591 %mask.1.1 = or i64 %or.1, %or592 %arrayidx.2 = getelementptr inbounds i64, ptr %src, i64 2593 %2 = load i64, ptr %arrayidx.2, align 8594 %tobool.not.2 = icmp eq i64 %2, 0595 %or.2 = select i1 %tobool.not.2, i64 0, i64 4596 %mask.1.2 = or i64 %or.2, %mask.1.1597 %arrayidx.3 = getelementptr inbounds i64, ptr %src, i64 3598 %3 = load i64, ptr %arrayidx.3, align 8599 %tobool.not.3 = icmp eq i64 %3, 0600 %or.3 = select i1 %tobool.not.3, i64 0, i64 8601 %mask.1.3 = or i64 %or.3, %mask.1.2602 %arrayidx.4 = getelementptr inbounds i64, ptr %src, i64 4603 %4 = load i64, ptr %arrayidx.4, align 8604 %tobool.not.4 = icmp eq i64 %4, 0605 %or.4 = select i1 %tobool.not.4, i64 0, i64 16606 %mask.1.4 = or i64 %or.4, %mask.1.3607 %arrayidx.5 = getelementptr inbounds i64, ptr %src, i64 5608 %5 = load i64, ptr %arrayidx.5, align 8609 %tobool.not.5 = icmp eq i64 %5, 0610 %or.5 = select i1 %tobool.not.5, i64 0, i64 32611 %mask.1.5 = or i64 %or.5, %mask.1.4612 %arrayidx.6 = getelementptr inbounds i64, ptr %src, i64 6613 %6 = load i64, ptr %arrayidx.6, align 8614 %tobool.not.6 = icmp eq i64 %6, 0615 %or.6 = select i1 %tobool.not.6, i64 0, i64 64616 %mask.1.6 = or i64 %or.6, %mask.1.5617 %arrayidx.7 = getelementptr inbounds i64, ptr %src, i64 7618 %7 = load i64, ptr %arrayidx.7, align 8619 %tobool.not.7 = icmp eq i64 %7, 0620 %or.7 = select i1 %tobool.not.7, i64 0, i64 128621 %mask.1.7 = or i64 %or.7, %mask.1.6622 ret i64 %mask.1.7623}624