218 lines · plain
1; REQUIRES: asserts2; RUN: opt -S -passes=loop-vectorize -force-vector-width=8 -force-vector-interleave=1 -enable-interleaved-mem-accesses -debug-only=loop-vectorize,vectorutils -disable-output < %s 2>&1 | FileCheck %s -check-prefix=STRIDED_UNMASKED3; RUN: opt -S -passes=loop-vectorize -force-vector-width=8 -force-vector-interleave=1 -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses -debug-only=loop-vectorize,vectorutils -disable-output < %s 2>&1 | FileCheck %s -check-prefix=STRIDED_MASKED4 5target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"6 7; We test here that the loop-vectorizer forms an interleave-groups from8; predicated memory accesses only if they are both in the same (predicated)9; block (first scenario below).10; If the accesses are not in the same predicated block, an interleave-group11; is not formed (scenarios 2,3 below).12 13; Scenario 1: Check the case where it is legal to create masked interleave-14; groups. Altogether two groups are created (one for loads and one for stores)15; when masked-interleaved-acceses are enabled. When masked-interleaved-acceses16; are disabled we do not create any interleave-group.17;18; void masked_strided1(const unsigned char* restrict p,19; unsigned char* restrict q,20; unsigned char guard) {21; for(ix=0; ix < 1024; ++ix) {22; if (ix > guard) {23; char left = p[2*ix];24; char right = p[2*ix + 1];25; char max = max(left, right);26; q[2*ix] = max;27; q[2*ix+1] = 0 - max;28; }29; }30;}31 32 33; STRIDED_UNMASKED: LV: Checking a loop in 'masked_strided1'34; STRIDED_UNMASKED: LV: Analyzing interleaved accesses...35; STRIDED_UNMASKED-NOT: LV: Creating an interleave group36 37; STRIDED_MASKED: LV: Checking a loop in 'masked_strided1'38; STRIDED_MASKED: LV: Analyzing interleaved accesses...39; STRIDED_MASKED-NEXT: LV: Creating an interleave group with: store i8 %{{.*}}, ptr %{{.*}}, align 140; STRIDED_MASKED-NEXT: LV: Inserted: store i8 %{{.*}}, ptr %{{.*}}, align 141; STRIDED_MASKED-NEXT: into the interleave group with store i8 %{{.*}}, ptr %{{.*}}, align 142; STRIDED_MASKED-NEXT: LV: Creating an interleave group with: %{{.*}} = load i8, ptr %{{.*}}, align 143; STRIDED_MASKED-NEXT: LV: Inserted: %{{.*}} = load i8, ptr %{{.*}}, align 144; STRIDED_MASKED-NEXT: into the interleave group with %{{.*}} = load i8, ptr %{{.*}}, align 145 46; Scenario 2: Check the case where it is illegal to create a masked interleave-47; group because the first access is predicated, and the second isn't.48; We therefore create a separate interleave-group with gaps for each of the49; stores (if masked-interleaved-accesses are enabled).50; If masked-interleaved-accesses is not enabled we create only one interleave51; group of stores (for the non-predicated store) and it is later invalidated52; due to gaps.53;54; void masked_strided2(const unsigned char* restrict p,55; unsigned char* restrict q,56; unsigned char guard1,57; unsigned char guard2) {58; for(ix=0; ix < 1024; ++ix) {59; if (ix > guard1) {60; q[2*ix] = 1;61; }62; q[2*ix+1] = 2;63; }64;}65 66; STRIDED_UNMASKED: LV: Checking a loop in 'masked_strided2'67; STRIDED_UNMASKED: LV: Analyzing interleaved accesses...68; STRIDED_UNMASKED-NEXT: LV: Creating an interleave group with: store i8 1, ptr %{{.*}}, align 169; STRIDED_UNMASKED-NEXT: LV: Invalidate candidate interleaved store group due to gaps.70; STRIDED_UNMASKED-NOT: LV: Creating an interleave group71 72; STRIDED_MASKED: LV: Checking a loop in 'masked_strided2'73; STRIDED_MASKED: LV: Analyzing interleaved accesses...74; STRIDED_MASKED-NEXT: LV: Creating an interleave group with: store i8 2, ptr %{{.*}}, align 175; STRIDED_MASKED-NEXT: LV: Creating an interleave group with: store i8 1, ptr %{{.*}}, align 176; STRIDED_MASKED-NOT: LV: Invalidate candidate interleaved store group due to gaps.77 78 79; Scenario 3: Check the case where it is illegal to create a masked interleave-80; group because the two accesses are in separate predicated blocks.81; We therefore create a separate interleave-group with gaps for each of the accesses,82; If masked-interleaved-accesses is not enabled we don't create any interleave83; group because all accesses are predicated.84;85; void masked_strided3(const unsigned char* restrict p,86; unsigned char* restrict q,87; unsigned char guard1,88; unsigned char guard2) {89; for(ix=0; ix < 1024; ++ix) {90; if (ix > guard1) {91; q[2*ix] = 1;92; }93; if (ix > guard2) {94; q[2*ix+1] = 2;95; }96; }97;}98 99 100; STRIDED_UNMASKED: LV: Checking a loop in 'masked_strided3'101; STRIDED_UNMASKED: LV: Analyzing interleaved accesses...102; STRIDED_UNMASKED-NOT: LV: Creating an interleave group103 104; STRIDED_MASKED: LV: Checking a loop in 'masked_strided3'105; STRIDED_MASKED: LV: Analyzing interleaved accesses...106; STRIDED_MASKED-NEXT: LV: Creating an interleave group with: store i8 2, ptr %{{.*}}, align 1107; STRIDED_MASKED-NEXT: LV: Creating an interleave group with: store i8 1, ptr %{{.*}}, align 1108; STRIDED_MASKED-NOT: LV: Invalidate candidate interleaved store group due to gaps.109 110 111; ModuleID = 'test.c'112source_filename = "test.c"113target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"114target triple = "i386-unknown-linux-gnu"115 116define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) local_unnamed_addr #0 {117entry:118 %conv = zext i8 %guard to i32119 br label %for.body120 121for.body:122 %ix.024 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]123 %cmp1 = icmp ugt i32 %ix.024, %conv124 br i1 %cmp1, label %if.then, label %for.inc125 126if.then:127 %mul = shl nuw nsw i32 %ix.024, 1128 %arrayidx = getelementptr inbounds i8, ptr %p, i32 %mul129 %0 = load i8, ptr %arrayidx, align 1130 %add = or disjoint i32 %mul, 1131 %arrayidx4 = getelementptr inbounds i8, ptr %p, i32 %add132 %1 = load i8, ptr %arrayidx4, align 1133 %cmp.i = icmp slt i8 %0, %1134 %spec.select.i = select i1 %cmp.i, i8 %1, i8 %0135 %arrayidx6 = getelementptr inbounds i8, ptr %q, i32 %mul136 store i8 %spec.select.i, ptr %arrayidx6, align 1137 %sub = sub i8 0, %spec.select.i138 %arrayidx11 = getelementptr inbounds i8, ptr %q, i32 %add139 store i8 %sub, ptr %arrayidx11, align 1140 br label %for.inc141 142for.inc:143 %inc = add nuw nsw i32 %ix.024, 1144 %exitcond = icmp eq i32 %inc, 1024145 br i1 %exitcond, label %for.end, label %for.body146 147for.end:148 ret void149}150 151 152define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr noalias nocapture %q, i8 zeroext %guard) local_unnamed_addr #0 {153entry:154 %conv = zext i8 %guard to i32155 br label %for.body156 157for.body:158 %ix.012 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]159 %mul = shl nuw nsw i32 %ix.012, 1160 %arrayidx = getelementptr inbounds i8, ptr %q, i32 %mul161 store i8 1, ptr %arrayidx, align 1162 %cmp1 = icmp ugt i32 %ix.012, %conv163 br i1 %cmp1, label %if.then, label %for.inc164 165if.then:166 %add = or disjoint i32 %mul, 1167 %arrayidx3 = getelementptr inbounds i8, ptr %q, i32 %add168 store i8 2, ptr %arrayidx3, align 1169 br label %for.inc170 171for.inc:172 %inc = add nuw nsw i32 %ix.012, 1173 %exitcond = icmp eq i32 %inc, 1024174 br i1 %exitcond, label %for.end, label %for.body175 176for.end:177 ret void178}179 180 181define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr noalias nocapture %q, i8 zeroext %guard1, i8 zeroext %guard2) local_unnamed_addr #0 {182entry:183 %conv = zext i8 %guard1 to i32184 %conv3 = zext i8 %guard2 to i32185 br label %for.body186 187for.body:188 %ix.018 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]189 %mul = shl nuw nsw i32 %ix.018, 1190 %cmp1 = icmp ugt i32 %ix.018, %conv191 br i1 %cmp1, label %if.then, label %if.end192 193if.then:194 %arrayidx = getelementptr inbounds i8, ptr %q, i32 %mul195 store i8 1, ptr %arrayidx, align 1196 br label %if.end197 198if.end:199 %cmp4 = icmp ugt i32 %ix.018, %conv3200 br i1 %cmp4, label %if.then6, label %for.inc201 202if.then6:203 %add = or disjoint i32 %mul, 1204 %arrayidx7 = getelementptr inbounds i8, ptr %q, i32 %add205 store i8 2, ptr %arrayidx7, align 1206 br label %for.inc207 208for.inc:209 %inc = add nuw nsw i32 %ix.018, 1210 %exitcond = icmp eq i32 %inc, 1024211 br i1 %exitcond, label %for.end, label %for.body212 213for.end:214 ret void215}216 217attributes #0 = { "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" }218