195 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "LV: Found an estimated cost of [0-9]+ for VF [0-9]+ For instruction:\s*%i[2,4] = load i16, ptr %[a-zA-Z0-7]+, align 2"2; RUN: opt -passes=loop-vectorize -enable-interleaved-mem-accesses -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -mcpu=skx --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=DISABLED_MASKED_STRIDED3; RUN: opt -passes=loop-vectorize -enable-interleaved-mem-accesses -enable-masked-interleaved-mem-accesses -prefer-predicate-over-epilogue=predicate-dont-vectorize -S -mcpu=skx --debug-only=loop-vectorize --disable-output < %s 2>&1 | FileCheck %s --check-prefix=ENABLED_MASKED_STRIDED4; REQUIRES: asserts5 6target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"7target triple = "x86_64-unknown-linux-gnu"8 9; (1) Interleave-group with factor 4, storing only 2 members out of the 4.10; Check that when we allow masked-memops to support interleave-group with gaps,11; the store is vectorized using a wide masked store, with a 1,1,0,0,1,1,0,0,... mask.12; Check that when we don't allow masked-memops to support interleave-group with gaps,13; the store is scalarized.14; The input IR was generated from this source:15; for(i=0;i<1024;i++){16; x[i] = points[i*4];17; y[i] = points[i*4 + 1];18; }19; (relates to the testcase in PR50566)20 21define void @test1(ptr noalias nocapture %points, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y) {22; DISABLED_MASKED_STRIDED-LABEL: 'test1'23; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i2 = load i16, ptr %arrayidx2, align 224; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i4 = load i16, ptr %arrayidx7, align 225; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 6 for VF 2 For instruction: %i2 = load i16, ptr %arrayidx2, align 226; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 6 for VF 2 For instruction: %i4 = load i16, ptr %arrayidx7, align 227; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 13 for VF 4 For instruction: %i2 = load i16, ptr %arrayidx2, align 228; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 13 for VF 4 For instruction: %i4 = load i16, ptr %arrayidx7, align 229; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 27 for VF 8 For instruction: %i2 = load i16, ptr %arrayidx2, align 230; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 27 for VF 8 For instruction: %i4 = load i16, ptr %arrayidx7, align 231; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 55 for VF 16 For instruction: %i2 = load i16, ptr %arrayidx2, align 232; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 55 for VF 16 For instruction: %i4 = load i16, ptr %arrayidx7, align 233;34; ENABLED_MASKED_STRIDED-LABEL: 'test1'35; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i2 = load i16, ptr %arrayidx2, align 236; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i4 = load i16, ptr %arrayidx7, align 237; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 8 for VF 2 For instruction: %i2 = load i16, ptr %arrayidx2, align 238; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 2 For instruction: %i4 = load i16, ptr %arrayidx7, align 239; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 11 for VF 4 For instruction: %i2 = load i16, ptr %arrayidx2, align 240; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 4 For instruction: %i4 = load i16, ptr %arrayidx7, align 241; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 11 for VF 8 For instruction: %i2 = load i16, ptr %arrayidx2, align 242; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 8 For instruction: %i4 = load i16, ptr %arrayidx7, align 243; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 17 for VF 16 For instruction: %i2 = load i16, ptr %arrayidx2, align 244; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 16 For instruction: %i4 = load i16, ptr %arrayidx7, align 245;46entry:47 br label %for.body48 49for.body:50 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]51 %i1 = shl nuw nsw i64 %indvars.iv, 252 %arrayidx2 = getelementptr inbounds i16, ptr %points, i64 %i153 %i2 = load i16, ptr %arrayidx2, align 254 %i3 = or disjoint i64 %i1, 155 %arrayidx7 = getelementptr inbounds i16, ptr %points, i64 %i356 %i4 = load i16, ptr %arrayidx7, align 257 %arrayidx = getelementptr inbounds i16, ptr %x, i64 %indvars.iv58 store i16 %i2, ptr %arrayidx, align 259 %arrayidx4 = getelementptr inbounds i16, ptr %y, i64 %indvars.iv60 store i16 %i4, ptr %arrayidx4, align 261 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 162 %exitcond.not = icmp eq i64 %indvars.iv.next, 102463 br i1 %exitcond.not, label %for.end, label %for.body64 65for.end:66 ret void67}68 69; (2) Same as above, but this time the gaps mask of the store is also And-ed with the70; fold-tail mask. If using masked memops to vectorize interleaved-group with gaps is71; not allowed, the store is scalarized and predicated.72; The input IR was generated from this source:73; for(i=0;i<numPoints;i++){74; x[i] = points[i*4];75; y[i] = points[i*4 + 1];76; }77 78define void @test2(ptr noalias nocapture %points, i32 %numPoints, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y) {79; DISABLED_MASKED_STRIDED-LABEL: 'test2'80; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i2 = load i16, ptr %arrayidx2, align 281; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i4 = load i16, ptr %arrayidx7, align 282; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %i2 = load i16, ptr %arrayidx2, align 283; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %i4 = load i16, ptr %arrayidx7, align 284; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %i2 = load i16, ptr %arrayidx2, align 285; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %i4 = load i16, ptr %arrayidx7, align 286; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %i2 = load i16, ptr %arrayidx2, align 287; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %i4 = load i16, ptr %arrayidx7, align 288; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %i2 = load i16, ptr %arrayidx2, align 289; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %i4 = load i16, ptr %arrayidx7, align 290;91; ENABLED_MASKED_STRIDED-LABEL: 'test2'92; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i2 = load i16, ptr %arrayidx2, align 293; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i4 = load i16, ptr %arrayidx7, align 294; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 8 for VF 2 For instruction: %i2 = load i16, ptr %arrayidx2, align 295; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 2 For instruction: %i4 = load i16, ptr %arrayidx7, align 296; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 11 for VF 4 For instruction: %i2 = load i16, ptr %arrayidx2, align 297; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 4 For instruction: %i4 = load i16, ptr %arrayidx7, align 298; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 11 for VF 8 For instruction: %i2 = load i16, ptr %arrayidx2, align 299; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 8 For instruction: %i4 = load i16, ptr %arrayidx7, align 2100; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 17 for VF 16 For instruction: %i2 = load i16, ptr %arrayidx2, align 2101; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 0 for VF 16 For instruction: %i4 = load i16, ptr %arrayidx7, align 2102;103entry:104 %cmp15 = icmp sgt i32 %numPoints, 0105 br i1 %cmp15, label %for.body.preheader, label %for.end106 107for.body.preheader:108 %wide.trip.count = zext i32 %numPoints to i64109 br label %for.body110 111for.body:112 %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]113 %i1 = shl nuw nsw i64 %indvars.iv, 2114 %arrayidx2 = getelementptr inbounds i16, ptr %points, i64 %i1115 %i2 = load i16, ptr %arrayidx2, align 2116 %i3 = or disjoint i64 %i1, 1117 %arrayidx7 = getelementptr inbounds i16, ptr %points, i64 %i3118 %i4 = load i16, ptr %arrayidx7, align 2119 %arrayidx = getelementptr inbounds i16, ptr %x, i64 %indvars.iv120 store i16 %i2, ptr %arrayidx, align 2121 %arrayidx4 = getelementptr inbounds i16, ptr %y, i64 %indvars.iv122 store i16 %i4, ptr %arrayidx4, align 2123 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1124 %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count125 br i1 %exitcond.not, label %for.end.loopexit, label %for.body126 127for.end.loopexit:128 br label %for.end129 130for.end:131 ret void132}133 134; (3) Testing a scenario of a conditional store. The gaps mask of the store is also135; And-ed with the condition mask (x[i] > 0).136; If using masked memops to vectorize interleaved-group with gaps is137; not allowed, the store is scalarized and predicated.138; Here the Interleave-group is with factor 3, storing only 1 member out of the 3.139; The input IR was generated from this source:140; for(i=0;i<1024;i++){141; if (x[i] > 0)142; x[i] = points[i*3];143; }144 145define void @test(ptr noalias nocapture %points, ptr noalias nocapture readonly %x, ptr noalias nocapture readnone %y) {146; DISABLED_MASKED_STRIDED-LABEL: 'test'147; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i2 = load i16, ptr %arrayidx, align 2148; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i4 = load i16, ptr %arrayidx6, align 2149; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 2 For instruction: %i2 = load i16, ptr %arrayidx, align 2150; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %i4 = load i16, ptr %arrayidx6, align 2151; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 4 For instruction: %i2 = load i16, ptr %arrayidx, align 2152; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %i4 = load i16, ptr %arrayidx6, align 2153; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 8 For instruction: %i2 = load i16, ptr %arrayidx, align 2154; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %i4 = load i16, ptr %arrayidx6, align 2155; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 16 For instruction: %i2 = load i16, ptr %arrayidx, align 2156; DISABLED_MASKED_STRIDED: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %i4 = load i16, ptr %arrayidx6, align 2157;158; ENABLED_MASKED_STRIDED-LABEL: 'test'159; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i2 = load i16, ptr %arrayidx, align 2160; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 1 For instruction: %i4 = load i16, ptr %arrayidx6, align 2161; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 2 For instruction: %i2 = load i16, ptr %arrayidx, align 2162; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 7 for VF 2 For instruction: %i4 = load i16, ptr %arrayidx6, align 2163; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 4 For instruction: %i2 = load i16, ptr %arrayidx, align 2164; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 9 for VF 4 For instruction: %i4 = load i16, ptr %arrayidx6, align 2165; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 8 For instruction: %i2 = load i16, ptr %arrayidx, align 2166; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 9 for VF 8 For instruction: %i4 = load i16, ptr %arrayidx6, align 2167; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 1 for VF 16 For instruction: %i2 = load i16, ptr %arrayidx, align 2168; ENABLED_MASKED_STRIDED: LV: Found an estimated cost of 14 for VF 16 For instruction: %i4 = load i16, ptr %arrayidx6, align 2169;170entry:171 br label %for.body172 173for.body:174 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]175 %arrayidx = getelementptr inbounds i16, ptr %x, i64 %indvars.iv176 %i2 = load i16, ptr %arrayidx, align 2177 %cmp1 = icmp sgt i16 %i2, 0178 br i1 %cmp1, label %if.then, label %for.inc179 180if.then:181 %i1 = mul nuw nsw i64 %indvars.iv, 3182 %arrayidx6 = getelementptr inbounds i16, ptr %points, i64 %i1183 %i4 = load i16, ptr %arrayidx6, align 2184 store i16 %i4, ptr %arrayidx, align 2185 br label %for.inc186 187for.inc:188 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1189 %exitcond.not = icmp eq i64 %indvars.iv.next, 1024190 br i1 %exitcond.not, label %for.end, label %for.body191 192for.end:193 ret void194}195