520 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -S -passes=slp-vectorizer < %s | FileCheck %s --check-prefix=GENERIC3; RUN: opt -S -mcpu=kryo -passes=slp-vectorizer < %s | FileCheck %s --check-prefix=KRYO4 5target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"6target triple = "aarch64--linux-gnu"7 8; These tests check that we vectorize the index calculations in the9; gather-reduce pattern shown below. We check cases having i32 and i6410; subtraction.11;12; int gather_reduce_8x16(short *a, short *b, short *g, int n) {13; int sum = 0;14; for (int i = 0; i < n ; ++i) {15; sum += g[*a++ - b[0]]; sum += g[*a++ - b[1]];16; sum += g[*a++ - b[2]]; sum += g[*a++ - b[3]];17; sum += g[*a++ - b[4]]; sum += g[*a++ - b[5]];18; sum += g[*a++ - b[6]]; sum += g[*a++ - b[7]];19; }20; return sum;21; }22 23define i32 @gather_reduce_8x16_i32(ptr nocapture readonly %a, ptr nocapture readonly %b, ptr nocapture readonly %g, i32 %n) {24; GENERIC-LABEL: @gather_reduce_8x16_i32(25; GENERIC-NEXT: entry:26; GENERIC-NEXT: [[CMP_99:%.*]] = icmp sgt i32 [[N:%.*]], 027; GENERIC-NEXT: br i1 [[CMP_99]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]28; GENERIC: for.body.preheader:29; GENERIC-NEXT: br label [[FOR_BODY:%.*]]30; GENERIC: for.cond.cleanup.loopexit:31; GENERIC-NEXT: br label [[FOR_COND_CLEANUP]]32; GENERIC: for.cond.cleanup:33; GENERIC-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD66:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]34; GENERIC-NEXT: ret i32 [[SUM_0_LCSSA]]35; GENERIC: for.body:36; GENERIC-NEXT: [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]37; GENERIC-NEXT: [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]38; GENERIC-NEXT: [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]39; GENERIC-NEXT: [[INCDEC_PTR58]] = getelementptr inbounds i16, ptr [[A_ADDR_0101]], i64 840; GENERIC-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 241; GENERIC-NEXT: [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>42; GENERIC-NEXT: [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 243; GENERIC-NEXT: [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>44; GENERIC-NEXT: [[TMP4:%.*]] = sub nsw <8 x i32> [[TMP1]], [[TMP3]]45; GENERIC-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i32 046; GENERIC-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i32 [[TMP5]]47; GENERIC-NEXT: [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 248; GENERIC-NEXT: [[CONV3:%.*]] = zext i16 [[TMP7]] to i3249; GENERIC-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV3]], [[SUM_0102]]50; GENERIC-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i32 151; GENERIC-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP8]]52; GENERIC-NEXT: [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX10]], align 253; GENERIC-NEXT: [[CONV11:%.*]] = zext i16 [[TMP10]] to i3254; GENERIC-NEXT: [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]55; GENERIC-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP4]], i32 256; GENERIC-NEXT: [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP9]]57; GENERIC-NEXT: [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX19]], align 258; GENERIC-NEXT: [[CONV20:%.*]] = zext i16 [[TMP13]] to i3259; GENERIC-NEXT: [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]60; GENERIC-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i32 361; GENERIC-NEXT: [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP11]]62; GENERIC-NEXT: [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX28]], align 263; GENERIC-NEXT: [[CONV29:%.*]] = zext i16 [[TMP16]] to i3264; GENERIC-NEXT: [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]65; GENERIC-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i32 466; GENERIC-NEXT: [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP14]]67; GENERIC-NEXT: [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX37]], align 268; GENERIC-NEXT: [[CONV38:%.*]] = zext i16 [[TMP19]] to i3269; GENERIC-NEXT: [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]70; GENERIC-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[TMP4]], i32 571; GENERIC-NEXT: [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP15]]72; GENERIC-NEXT: [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX46]], align 273; GENERIC-NEXT: [[CONV47:%.*]] = zext i16 [[TMP22]] to i3274; GENERIC-NEXT: [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]75; GENERIC-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i32 676; GENERIC-NEXT: [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP17]]77; GENERIC-NEXT: [[TMP25:%.*]] = load i16, ptr [[ARRAYIDX55]], align 278; GENERIC-NEXT: [[CONV56:%.*]] = zext i16 [[TMP25]] to i3279; GENERIC-NEXT: [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]80; GENERIC-NEXT: [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i32 781; GENERIC-NEXT: [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP20]]82; GENERIC-NEXT: [[TMP28:%.*]] = load i16, ptr [[ARRAYIDX64]], align 283; GENERIC-NEXT: [[CONV65:%.*]] = zext i16 [[TMP28]] to i3284; GENERIC-NEXT: [[ADD66]] = add nsw i32 [[ADD57]], [[CONV65]]85; GENERIC-NEXT: [[INC]] = add nuw nsw i32 [[I_0103]], 186; GENERIC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]87; GENERIC-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]88;89; KRYO-LABEL: @gather_reduce_8x16_i32(90; KRYO-NEXT: entry:91; KRYO-NEXT: [[CMP_99:%.*]] = icmp sgt i32 [[N:%.*]], 092; KRYO-NEXT: br i1 [[CMP_99]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]93; KRYO: for.body.preheader:94; KRYO-NEXT: br label [[FOR_BODY:%.*]]95; KRYO: for.cond.cleanup.loopexit:96; KRYO-NEXT: br label [[FOR_COND_CLEANUP]]97; KRYO: for.cond.cleanup:98; KRYO-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD66:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]99; KRYO-NEXT: ret i32 [[SUM_0_LCSSA]]100; KRYO: for.body:101; KRYO-NEXT: [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]102; KRYO-NEXT: [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]103; KRYO-NEXT: [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]104; KRYO-NEXT: [[INCDEC_PTR58]] = getelementptr inbounds i16, ptr [[A_ADDR_0101]], i64 8105; KRYO-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2106; KRYO-NEXT: [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>107; KRYO-NEXT: [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2108; KRYO-NEXT: [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>109; KRYO-NEXT: [[TMP4:%.*]] = sub nsw <8 x i32> [[TMP1]], [[TMP3]]110; KRYO-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i32 0111; KRYO-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i32 [[TMP5]]112; KRYO-NEXT: [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 2113; KRYO-NEXT: [[CONV3:%.*]] = zext i16 [[TMP7]] to i32114; KRYO-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV3]], [[SUM_0102]]115; KRYO-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i32 1116; KRYO-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP8]]117; KRYO-NEXT: [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2118; KRYO-NEXT: [[CONV11:%.*]] = zext i16 [[TMP10]] to i32119; KRYO-NEXT: [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]120; KRYO-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP4]], i32 2121; KRYO-NEXT: [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP9]]122; KRYO-NEXT: [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX19]], align 2123; KRYO-NEXT: [[CONV20:%.*]] = zext i16 [[TMP13]] to i32124; KRYO-NEXT: [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]125; KRYO-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i32 3126; KRYO-NEXT: [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP11]]127; KRYO-NEXT: [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX28]], align 2128; KRYO-NEXT: [[CONV29:%.*]] = zext i16 [[TMP16]] to i32129; KRYO-NEXT: [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]130; KRYO-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i32 4131; KRYO-NEXT: [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP14]]132; KRYO-NEXT: [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX37]], align 2133; KRYO-NEXT: [[CONV38:%.*]] = zext i16 [[TMP19]] to i32134; KRYO-NEXT: [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]135; KRYO-NEXT: [[TMP15:%.*]] = extractelement <8 x i32> [[TMP4]], i32 5136; KRYO-NEXT: [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP15]]137; KRYO-NEXT: [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX46]], align 2138; KRYO-NEXT: [[CONV47:%.*]] = zext i16 [[TMP22]] to i32139; KRYO-NEXT: [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]140; KRYO-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i32 6141; KRYO-NEXT: [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP17]]142; KRYO-NEXT: [[TMP25:%.*]] = load i16, ptr [[ARRAYIDX55]], align 2143; KRYO-NEXT: [[CONV56:%.*]] = zext i16 [[TMP25]] to i32144; KRYO-NEXT: [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]145; KRYO-NEXT: [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i32 7146; KRYO-NEXT: [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP20]]147; KRYO-NEXT: [[TMP28:%.*]] = load i16, ptr [[ARRAYIDX64]], align 2148; KRYO-NEXT: [[CONV65:%.*]] = zext i16 [[TMP28]] to i32149; KRYO-NEXT: [[ADD66]] = add nsw i32 [[ADD57]], [[CONV65]]150; KRYO-NEXT: [[INC]] = add nuw nsw i32 [[I_0103]], 1151; KRYO-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]152; KRYO-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]153;154entry:155 %cmp.99 = icmp sgt i32 %n, 0156 br i1 %cmp.99, label %for.body.preheader, label %for.cond.cleanup157 158for.body.preheader:159 br label %for.body160 161for.cond.cleanup.loopexit:162 br label %for.cond.cleanup163 164for.cond.cleanup:165 %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add66, %for.cond.cleanup.loopexit ]166 ret i32 %sum.0.lcssa167 168for.body:169 %i.0103 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]170 %sum.0102 = phi i32 [ %add66, %for.body ], [ 0, %for.body.preheader ]171 %a.addr.0101 = phi ptr [ %incdec.ptr58, %for.body ], [ %a, %for.body.preheader ]172 %incdec.ptr = getelementptr inbounds i16, ptr %a.addr.0101, i64 1173 %0 = load i16, ptr %a.addr.0101, align 2174 %conv = zext i16 %0 to i32175 %incdec.ptr1 = getelementptr inbounds i16, ptr %b, i64 1176 %1 = load i16, ptr %b, align 2177 %conv2 = zext i16 %1 to i32178 %sub = sub nsw i32 %conv, %conv2179 %arrayidx = getelementptr inbounds i16, ptr %g, i32 %sub180 %2 = load i16, ptr %arrayidx, align 2181 %conv3 = zext i16 %2 to i32182 %add = add nsw i32 %conv3, %sum.0102183 %incdec.ptr4 = getelementptr inbounds i16, ptr %a.addr.0101, i64 2184 %3 = load i16, ptr %incdec.ptr, align 2185 %conv5 = zext i16 %3 to i32186 %incdec.ptr6 = getelementptr inbounds i16, ptr %b, i64 2187 %4 = load i16, ptr %incdec.ptr1, align 2188 %conv7 = zext i16 %4 to i32189 %sub8 = sub nsw i32 %conv5, %conv7190 %arrayidx10 = getelementptr inbounds i16, ptr %g, i32 %sub8191 %5 = load i16, ptr %arrayidx10, align 2192 %conv11 = zext i16 %5 to i32193 %add12 = add nsw i32 %add, %conv11194 %incdec.ptr13 = getelementptr inbounds i16, ptr %a.addr.0101, i64 3195 %6 = load i16, ptr %incdec.ptr4, align 2196 %conv14 = zext i16 %6 to i32197 %incdec.ptr15 = getelementptr inbounds i16, ptr %b, i64 3198 %7 = load i16, ptr %incdec.ptr6, align 2199 %conv16 = zext i16 %7 to i32200 %sub17 = sub nsw i32 %conv14, %conv16201 %arrayidx19 = getelementptr inbounds i16, ptr %g, i32 %sub17202 %8 = load i16, ptr %arrayidx19, align 2203 %conv20 = zext i16 %8 to i32204 %add21 = add nsw i32 %add12, %conv20205 %incdec.ptr22 = getelementptr inbounds i16, ptr %a.addr.0101, i64 4206 %9 = load i16, ptr %incdec.ptr13, align 2207 %conv23 = zext i16 %9 to i32208 %incdec.ptr24 = getelementptr inbounds i16, ptr %b, i64 4209 %10 = load i16, ptr %incdec.ptr15, align 2210 %conv25 = zext i16 %10 to i32211 %sub26 = sub nsw i32 %conv23, %conv25212 %arrayidx28 = getelementptr inbounds i16, ptr %g, i32 %sub26213 %11 = load i16, ptr %arrayidx28, align 2214 %conv29 = zext i16 %11 to i32215 %add30 = add nsw i32 %add21, %conv29216 %incdec.ptr31 = getelementptr inbounds i16, ptr %a.addr.0101, i64 5217 %12 = load i16, ptr %incdec.ptr22, align 2218 %conv32 = zext i16 %12 to i32219 %incdec.ptr33 = getelementptr inbounds i16, ptr %b, i64 5220 %13 = load i16, ptr %incdec.ptr24, align 2221 %conv34 = zext i16 %13 to i32222 %sub35 = sub nsw i32 %conv32, %conv34223 %arrayidx37 = getelementptr inbounds i16, ptr %g, i32 %sub35224 %14 = load i16, ptr %arrayidx37, align 2225 %conv38 = zext i16 %14 to i32226 %add39 = add nsw i32 %add30, %conv38227 %incdec.ptr40 = getelementptr inbounds i16, ptr %a.addr.0101, i64 6228 %15 = load i16, ptr %incdec.ptr31, align 2229 %conv41 = zext i16 %15 to i32230 %incdec.ptr42 = getelementptr inbounds i16, ptr %b, i64 6231 %16 = load i16, ptr %incdec.ptr33, align 2232 %conv43 = zext i16 %16 to i32233 %sub44 = sub nsw i32 %conv41, %conv43234 %arrayidx46 = getelementptr inbounds i16, ptr %g, i32 %sub44235 %17 = load i16, ptr %arrayidx46, align 2236 %conv47 = zext i16 %17 to i32237 %add48 = add nsw i32 %add39, %conv47238 %incdec.ptr49 = getelementptr inbounds i16, ptr %a.addr.0101, i64 7239 %18 = load i16, ptr %incdec.ptr40, align 2240 %conv50 = zext i16 %18 to i32241 %incdec.ptr51 = getelementptr inbounds i16, ptr %b, i64 7242 %19 = load i16, ptr %incdec.ptr42, align 2243 %conv52 = zext i16 %19 to i32244 %sub53 = sub nsw i32 %conv50, %conv52245 %arrayidx55 = getelementptr inbounds i16, ptr %g, i32 %sub53246 %20 = load i16, ptr %arrayidx55, align 2247 %conv56 = zext i16 %20 to i32248 %add57 = add nsw i32 %add48, %conv56249 %incdec.ptr58 = getelementptr inbounds i16, ptr %a.addr.0101, i64 8250 %21 = load i16, ptr %incdec.ptr49, align 2251 %conv59 = zext i16 %21 to i32252 %22 = load i16, ptr %incdec.ptr51, align 2253 %conv61 = zext i16 %22 to i32254 %sub62 = sub nsw i32 %conv59, %conv61255 %arrayidx64 = getelementptr inbounds i16, ptr %g, i32 %sub62256 %23 = load i16, ptr %arrayidx64, align 2257 %conv65 = zext i16 %23 to i32258 %add66 = add nsw i32 %add57, %conv65259 %inc = add nuw nsw i32 %i.0103, 1260 %exitcond = icmp eq i32 %inc, %n261 br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body262}263 264define i32 @gather_reduce_8x16_i64(ptr nocapture readonly %a, ptr nocapture readonly %b, ptr nocapture readonly %g, i32 %n) {265; GENERIC-LABEL: @gather_reduce_8x16_i64(266; GENERIC-NEXT: entry:267; GENERIC-NEXT: [[CMP_99:%.*]] = icmp sgt i32 [[N:%.*]], 0268; GENERIC-NEXT: br i1 [[CMP_99]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]269; GENERIC: for.body.preheader:270; GENERIC-NEXT: br label [[FOR_BODY:%.*]]271; GENERIC: for.cond.cleanup.loopexit:272; GENERIC-NEXT: br label [[FOR_COND_CLEANUP]]273; GENERIC: for.cond.cleanup:274; GENERIC-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD66:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]275; GENERIC-NEXT: ret i32 [[SUM_0_LCSSA]]276; GENERIC: for.body:277; GENERIC-NEXT: [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]278; GENERIC-NEXT: [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]279; GENERIC-NEXT: [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]280; GENERIC-NEXT: [[INCDEC_PTR58]] = getelementptr inbounds i16, ptr [[A_ADDR_0101]], i64 8281; GENERIC-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2282; GENERIC-NEXT: [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>283; GENERIC-NEXT: [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2284; GENERIC-NEXT: [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>285; GENERIC-NEXT: [[TMP4:%.*]] = sub <8 x i32> [[TMP1]], [[TMP3]]286; GENERIC-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i32 0287; GENERIC-NEXT: [[TMP6:%.*]] = sext i32 [[TMP5]] to i64288; GENERIC-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i64 [[TMP6]]289; GENERIC-NEXT: [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 2290; GENERIC-NEXT: [[CONV3:%.*]] = zext i16 [[TMP7]] to i32291; GENERIC-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV3]], [[SUM_0102]]292; GENERIC-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i32 1293; GENERIC-NEXT: [[TMP9:%.*]] = sext i32 [[TMP8]] to i64294; GENERIC-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP9]]295; GENERIC-NEXT: [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2296; GENERIC-NEXT: [[CONV11:%.*]] = zext i16 [[TMP10]] to i32297; GENERIC-NEXT: [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]298; GENERIC-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i32 2299; GENERIC-NEXT: [[TMP12:%.*]] = sext i32 [[TMP11]] to i64300; GENERIC-NEXT: [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP12]]301; GENERIC-NEXT: [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX19]], align 2302; GENERIC-NEXT: [[CONV20:%.*]] = zext i16 [[TMP13]] to i32303; GENERIC-NEXT: [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]304; GENERIC-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i32 3305; GENERIC-NEXT: [[TMP15:%.*]] = sext i32 [[TMP14]] to i64306; GENERIC-NEXT: [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP15]]307; GENERIC-NEXT: [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX28]], align 2308; GENERIC-NEXT: [[CONV29:%.*]] = zext i16 [[TMP16]] to i32309; GENERIC-NEXT: [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]310; GENERIC-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i32 4311; GENERIC-NEXT: [[TMP18:%.*]] = sext i32 [[TMP17]] to i64312; GENERIC-NEXT: [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP18]]313; GENERIC-NEXT: [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX37]], align 2314; GENERIC-NEXT: [[CONV38:%.*]] = zext i16 [[TMP19]] to i32315; GENERIC-NEXT: [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]316; GENERIC-NEXT: [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i32 5317; GENERIC-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64318; GENERIC-NEXT: [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP21]]319; GENERIC-NEXT: [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX46]], align 2320; GENERIC-NEXT: [[CONV47:%.*]] = zext i16 [[TMP22]] to i32321; GENERIC-NEXT: [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]322; GENERIC-NEXT: [[TMP23:%.*]] = extractelement <8 x i32> [[TMP4]], i32 6323; GENERIC-NEXT: [[TMP24:%.*]] = sext i32 [[TMP23]] to i64324; GENERIC-NEXT: [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP24]]325; GENERIC-NEXT: [[TMP25:%.*]] = load i16, ptr [[ARRAYIDX55]], align 2326; GENERIC-NEXT: [[CONV56:%.*]] = zext i16 [[TMP25]] to i32327; GENERIC-NEXT: [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]328; GENERIC-NEXT: [[TMP26:%.*]] = extractelement <8 x i32> [[TMP4]], i32 7329; GENERIC-NEXT: [[TMP27:%.*]] = sext i32 [[TMP26]] to i64330; GENERIC-NEXT: [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP27]]331; GENERIC-NEXT: [[TMP28:%.*]] = load i16, ptr [[ARRAYIDX64]], align 2332; GENERIC-NEXT: [[CONV65:%.*]] = zext i16 [[TMP28]] to i32333; GENERIC-NEXT: [[ADD66]] = add nsw i32 [[ADD57]], [[CONV65]]334; GENERIC-NEXT: [[INC]] = add nuw nsw i32 [[I_0103]], 1335; GENERIC-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]336; GENERIC-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]337;338; KRYO-LABEL: @gather_reduce_8x16_i64(339; KRYO-NEXT: entry:340; KRYO-NEXT: [[CMP_99:%.*]] = icmp sgt i32 [[N:%.*]], 0341; KRYO-NEXT: br i1 [[CMP_99]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]342; KRYO: for.body.preheader:343; KRYO-NEXT: br label [[FOR_BODY:%.*]]344; KRYO: for.cond.cleanup.loopexit:345; KRYO-NEXT: br label [[FOR_COND_CLEANUP]]346; KRYO: for.cond.cleanup:347; KRYO-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD66:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]348; KRYO-NEXT: ret i32 [[SUM_0_LCSSA]]349; KRYO: for.body:350; KRYO-NEXT: [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]351; KRYO-NEXT: [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]352; KRYO-NEXT: [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]353; KRYO-NEXT: [[INCDEC_PTR58]] = getelementptr inbounds i16, ptr [[A_ADDR_0101]], i64 8354; KRYO-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2355; KRYO-NEXT: [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>356; KRYO-NEXT: [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2357; KRYO-NEXT: [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>358; KRYO-NEXT: [[TMP4:%.*]] = sub <8 x i32> [[TMP1]], [[TMP3]]359; KRYO-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i32 0360; KRYO-NEXT: [[TMP6:%.*]] = sext i32 [[TMP5]] to i64361; KRYO-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i64 [[TMP6]]362; KRYO-NEXT: [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 2363; KRYO-NEXT: [[CONV3:%.*]] = zext i16 [[TMP7]] to i32364; KRYO-NEXT: [[ADD:%.*]] = add nsw i32 [[CONV3]], [[SUM_0102]]365; KRYO-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i32 1366; KRYO-NEXT: [[TMP9:%.*]] = sext i32 [[TMP8]] to i64367; KRYO-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP9]]368; KRYO-NEXT: [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2369; KRYO-NEXT: [[CONV11:%.*]] = zext i16 [[TMP10]] to i32370; KRYO-NEXT: [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]371; KRYO-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i32 2372; KRYO-NEXT: [[TMP12:%.*]] = sext i32 [[TMP11]] to i64373; KRYO-NEXT: [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP12]]374; KRYO-NEXT: [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX19]], align 2375; KRYO-NEXT: [[CONV20:%.*]] = zext i16 [[TMP13]] to i32376; KRYO-NEXT: [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]377; KRYO-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i32 3378; KRYO-NEXT: [[TMP15:%.*]] = sext i32 [[TMP14]] to i64379; KRYO-NEXT: [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP15]]380; KRYO-NEXT: [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX28]], align 2381; KRYO-NEXT: [[CONV29:%.*]] = zext i16 [[TMP16]] to i32382; KRYO-NEXT: [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]383; KRYO-NEXT: [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i32 4384; KRYO-NEXT: [[TMP18:%.*]] = sext i32 [[TMP17]] to i64385; KRYO-NEXT: [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP18]]386; KRYO-NEXT: [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX37]], align 2387; KRYO-NEXT: [[CONV38:%.*]] = zext i16 [[TMP19]] to i32388; KRYO-NEXT: [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]389; KRYO-NEXT: [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i32 5390; KRYO-NEXT: [[TMP21:%.*]] = sext i32 [[TMP20]] to i64391; KRYO-NEXT: [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP21]]392; KRYO-NEXT: [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX46]], align 2393; KRYO-NEXT: [[CONV47:%.*]] = zext i16 [[TMP22]] to i32394; KRYO-NEXT: [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]395; KRYO-NEXT: [[TMP23:%.*]] = extractelement <8 x i32> [[TMP4]], i32 6396; KRYO-NEXT: [[TMP24:%.*]] = sext i32 [[TMP23]] to i64397; KRYO-NEXT: [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP24]]398; KRYO-NEXT: [[TMP25:%.*]] = load i16, ptr [[ARRAYIDX55]], align 2399; KRYO-NEXT: [[CONV56:%.*]] = zext i16 [[TMP25]] to i32400; KRYO-NEXT: [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]401; KRYO-NEXT: [[TMP26:%.*]] = extractelement <8 x i32> [[TMP4]], i32 7402; KRYO-NEXT: [[TMP27:%.*]] = sext i32 [[TMP26]] to i64403; KRYO-NEXT: [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP27]]404; KRYO-NEXT: [[TMP28:%.*]] = load i16, ptr [[ARRAYIDX64]], align 2405; KRYO-NEXT: [[CONV65:%.*]] = zext i16 [[TMP28]] to i32406; KRYO-NEXT: [[ADD66]] = add nsw i32 [[ADD57]], [[CONV65]]407; KRYO-NEXT: [[INC]] = add nuw nsw i32 [[I_0103]], 1408; KRYO-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]409; KRYO-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]410;411entry:412 %cmp.99 = icmp sgt i32 %n, 0413 br i1 %cmp.99, label %for.body.preheader, label %for.cond.cleanup414 415for.body.preheader:416 br label %for.body417 418for.cond.cleanup.loopexit:419 br label %for.cond.cleanup420 421for.cond.cleanup:422 %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add66, %for.cond.cleanup.loopexit ]423 ret i32 %sum.0.lcssa424 425for.body:426 %i.0103 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ]427 %sum.0102 = phi i32 [ %add66, %for.body ], [ 0, %for.body.preheader ]428 %a.addr.0101 = phi ptr [ %incdec.ptr58, %for.body ], [ %a, %for.body.preheader ]429 %incdec.ptr = getelementptr inbounds i16, ptr %a.addr.0101, i64 1430 %0 = load i16, ptr %a.addr.0101, align 2431 %conv = zext i16 %0 to i64432 %incdec.ptr1 = getelementptr inbounds i16, ptr %b, i64 1433 %1 = load i16, ptr %b, align 2434 %conv2 = zext i16 %1 to i64435 %sub = sub nsw i64 %conv, %conv2436 %arrayidx = getelementptr inbounds i16, ptr %g, i64 %sub437 %2 = load i16, ptr %arrayidx, align 2438 %conv3 = zext i16 %2 to i32439 %add = add nsw i32 %conv3, %sum.0102440 %incdec.ptr4 = getelementptr inbounds i16, ptr %a.addr.0101, i64 2441 %3 = load i16, ptr %incdec.ptr, align 2442 %conv5 = zext i16 %3 to i64443 %incdec.ptr6 = getelementptr inbounds i16, ptr %b, i64 2444 %4 = load i16, ptr %incdec.ptr1, align 2445 %conv7 = zext i16 %4 to i64446 %sub8 = sub nsw i64 %conv5, %conv7447 %arrayidx10 = getelementptr inbounds i16, ptr %g, i64 %sub8448 %5 = load i16, ptr %arrayidx10, align 2449 %conv11 = zext i16 %5 to i32450 %add12 = add nsw i32 %add, %conv11451 %incdec.ptr13 = getelementptr inbounds i16, ptr %a.addr.0101, i64 3452 %6 = load i16, ptr %incdec.ptr4, align 2453 %conv14 = zext i16 %6 to i64454 %incdec.ptr15 = getelementptr inbounds i16, ptr %b, i64 3455 %7 = load i16, ptr %incdec.ptr6, align 2456 %conv16 = zext i16 %7 to i64457 %sub17 = sub nsw i64 %conv14, %conv16458 %arrayidx19 = getelementptr inbounds i16, ptr %g, i64 %sub17459 %8 = load i16, ptr %arrayidx19, align 2460 %conv20 = zext i16 %8 to i32461 %add21 = add nsw i32 %add12, %conv20462 %incdec.ptr22 = getelementptr inbounds i16, ptr %a.addr.0101, i64 4463 %9 = load i16, ptr %incdec.ptr13, align 2464 %conv23 = zext i16 %9 to i64465 %incdec.ptr24 = getelementptr inbounds i16, ptr %b, i64 4466 %10 = load i16, ptr %incdec.ptr15, align 2467 %conv25 = zext i16 %10 to i64468 %sub26 = sub nsw i64 %conv23, %conv25469 %arrayidx28 = getelementptr inbounds i16, ptr %g, i64 %sub26470 %11 = load i16, ptr %arrayidx28, align 2471 %conv29 = zext i16 %11 to i32472 %add30 = add nsw i32 %add21, %conv29473 %incdec.ptr31 = getelementptr inbounds i16, ptr %a.addr.0101, i64 5474 %12 = load i16, ptr %incdec.ptr22, align 2475 %conv32 = zext i16 %12 to i64476 %incdec.ptr33 = getelementptr inbounds i16, ptr %b, i64 5477 %13 = load i16, ptr %incdec.ptr24, align 2478 %conv34 = zext i16 %13 to i64479 %sub35 = sub nsw i64 %conv32, %conv34480 %arrayidx37 = getelementptr inbounds i16, ptr %g, i64 %sub35481 %14 = load i16, ptr %arrayidx37, align 2482 %conv38 = zext i16 %14 to i32483 %add39 = add nsw i32 %add30, %conv38484 %incdec.ptr40 = getelementptr inbounds i16, ptr %a.addr.0101, i64 6485 %15 = load i16, ptr %incdec.ptr31, align 2486 %conv41 = zext i16 %15 to i64487 %incdec.ptr42 = getelementptr inbounds i16, ptr %b, i64 6488 %16 = load i16, ptr %incdec.ptr33, align 2489 %conv43 = zext i16 %16 to i64490 %sub44 = sub nsw i64 %conv41, %conv43491 %arrayidx46 = getelementptr inbounds i16, ptr %g, i64 %sub44492 %17 = load i16, ptr %arrayidx46, align 2493 %conv47 = zext i16 %17 to i32494 %add48 = add nsw i32 %add39, %conv47495 %incdec.ptr49 = getelementptr inbounds i16, ptr %a.addr.0101, i64 7496 %18 = load i16, ptr %incdec.ptr40, align 2497 %conv50 = zext i16 %18 to i64498 %incdec.ptr51 = getelementptr inbounds i16, ptr %b, i64 7499 %19 = load i16, ptr %incdec.ptr42, align 2500 %conv52 = zext i16 %19 to i64501 %sub53 = sub nsw i64 %conv50, %conv52502 %arrayidx55 = getelementptr inbounds i16, ptr %g, i64 %sub53503 %20 = load i16, ptr %arrayidx55, align 2504 %conv56 = zext i16 %20 to i32505 %add57 = add nsw i32 %add48, %conv56506 %incdec.ptr58 = getelementptr inbounds i16, ptr %a.addr.0101, i64 8507 %21 = load i16, ptr %incdec.ptr49, align 2508 %conv59 = zext i16 %21 to i64509 %22 = load i16, ptr %incdec.ptr51, align 2510 %conv61 = zext i16 %22 to i64511 %sub62 = sub nsw i64 %conv59, %conv61512 %arrayidx64 = getelementptr inbounds i16, ptr %g, i64 %sub62513 %23 = load i16, ptr %arrayidx64, align 2514 %conv65 = zext i16 %23 to i32515 %add66 = add nsw i32 %add57, %conv65516 %inc = add nuw nsw i32 %i.0103, 1517 %exitcond = icmp eq i32 %inc, %n518 br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body519}520