268 lines · plain
1; REQUIRES: asserts2; RUN: opt < %s -force-vector-width=2 -passes=loop-vectorize -debug-only=loop-vectorize -disable-output 2>&1 | FileCheck %s3 4target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"5target triple = "aarch64--linux-gnu"6 7; Check predication-related cost calculations, including scalarization overhead8; and block probability scaling. Note that the functionality being tested is9; not specific to AArch64. We specify a target to get actual values for the10; instruction costs.11 12; CHECK-LABEL: predicated_udiv13;14; This test checks that we correctly compute the cost of the predicated udiv15; instruction. If we assume the block probability is 50%, we compute the cost16; as:17;18; Cost of udiv:19; (udiv(2) + extractelement(8) + insertelement(4)) / 2 = 720;21; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp2, %tmp322; CHECK: Cost of 7 for VF 2: profitable to scalarize %tmp4 = udiv i32 %tmp2, %tmp323;24define i32 @predicated_udiv(ptr %a, ptr %b, i1 %c, i64 %n) {25entry:26 br label %for.body27 28for.body:29 %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]30 %r = phi i32 [ 0, %entry ], [ %tmp6, %for.inc ]31 %tmp0 = getelementptr inbounds i32, ptr %a, i64 %i32 %tmp1 = getelementptr inbounds i32, ptr %b, i64 %i33 %tmp2 = load i32, ptr %tmp0, align 434 %tmp3 = load i32, ptr %tmp1, align 435 br i1 %c, label %if.then, label %for.inc36 37if.then:38 %tmp4 = udiv i32 %tmp2, %tmp339 br label %for.inc40 41for.inc:42 %tmp5 = phi i32 [ %tmp3, %for.body ], [ %tmp4, %if.then]43 %tmp6 = add i32 %r, %tmp544 %i.next = add nuw nsw i64 %i, 145 %cond = icmp slt i64 %i.next, %n46 br i1 %cond, label %for.body, label %for.end47 48for.end:49 %tmp7 = phi i32 [ %tmp6, %for.inc ]50 ret i32 %tmp751}52 53; CHECK-LABEL: predicated_store54;55; This test checks that we correctly compute the cost of the predicated store56; instruction. If we assume the block probability is 50%, we compute the cost57; as:58;59; Cost of store:60; (store(4) + extractelement(4)) / 2 = 461;62; CHECK: Scalarizing and predicating: store i32 %tmp2, ptr %tmp0, align 463; CHECK: Cost of 4 for VF 2: profitable to scalarize store i32 %tmp2, ptr %tmp0, align 464;65define void @predicated_store(ptr %a, i1 %c, i32 %x, i64 %n) {66entry:67 br label %for.body68 69for.body:70 %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]71 %tmp0 = getelementptr inbounds i32, ptr %a, i64 %i72 %tmp1 = load i32, ptr %tmp0, align 473 %tmp2 = add nsw i32 %tmp1, %x74 br i1 %c, label %if.then, label %for.inc75 76if.then:77 store i32 %tmp2, ptr %tmp0, align 478 br label %for.inc79 80for.inc:81 %i.next = add nuw nsw i64 %i, 182 %cond = icmp slt i64 %i.next, %n83 br i1 %cond, label %for.body, label %for.end84 85for.end:86 ret void87}88 89; CHECK-LABEL: predicated_store_phi90;91; Same as predicate_store except we use a pointer PHI to maintain the address92;93; CHECK: Found scalar instruction: %addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ]94; CHECK: Found scalar instruction: %addr.next = getelementptr inbounds i32, ptr %addr, i64 195; CHECK: Scalarizing and predicating: store i32 %tmp2, ptr %addr, align 496; CHECK: Cost of 0 for VF 2: induction instruction %addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ]97; CHECK: Cost of 4 for VF 2: profitable to scalarize store i32 %tmp2, ptr %addr, align 498;99define void @predicated_store_phi(ptr %a, i1 %c, i32 %x, i64 %n) {100entry:101 br label %for.body102 103for.body:104 %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]105 %addr = phi ptr [ %a, %entry ], [ %addr.next, %for.inc ]106 %tmp1 = load i32, ptr %addr, align 4107 %tmp2 = add nsw i32 %tmp1, %x108 br i1 %c, label %if.then, label %for.inc109 110if.then:111 store i32 %tmp2, ptr %addr, align 4112 br label %for.inc113 114for.inc:115 %i.next = add nuw nsw i64 %i, 1116 %cond = icmp slt i64 %i.next, %n117 %addr.next = getelementptr inbounds i32, ptr %addr, i64 1118 br i1 %cond, label %for.body, label %for.end119 120for.end:121 ret void122}123 124; CHECK-LABEL: predicated_udiv_scalarized_operand125;126; This test checks that we correctly compute the cost of the predicated udiv127; instruction and the add instruction it uses. The add is scalarized and sunk128; inside the predicated block. If we assume the block probability is 50%, we129; compute the cost as:130;131; Cost of add:132; (add(2) + extractelement(4)) / 2 = 3133; Cost of udiv:134; (udiv(2) + extractelement(4) + insertelement(4)) / 2 = 5135;136; CHECK: Scalarizing: %tmp3 = add nsw i32 %tmp2, %x137; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp2, %tmp3138; CHECK: Cost of 5 for VF 2: profitable to scalarize %tmp4 = udiv i32 %tmp2, %tmp3139; CHECK: Cost of 3 for VF 2: profitable to scalarize %tmp3 = add nsw i32 %tmp2, %x140;141 142define i32 @predicated_udiv_scalarized_operand(ptr %a, i1 %c, i32 %x, i64 %n) {143entry:144 br label %for.body145 146for.body:147 %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]148 %r = phi i32 [ 0, %entry ], [ %tmp6, %for.inc ]149 %tmp0 = getelementptr inbounds i32, ptr %a, i64 %i150 %tmp2 = load i32, ptr %tmp0, align 4151 br i1 %c, label %if.then, label %for.inc152 153if.then:154 %tmp3 = add nsw i32 %tmp2, %x155 %tmp4 = udiv i32 %tmp2, %tmp3156 br label %for.inc157 158for.inc:159 %tmp5 = phi i32 [ %tmp2, %for.body ], [ %tmp4, %if.then]160 %tmp6 = add i32 %r, %tmp5161 %i.next = add nuw nsw i64 %i, 1162 %cond = icmp slt i64 %i.next, %n163 br i1 %cond, label %for.body, label %for.end164 165for.end:166 %tmp7 = phi i32 [ %tmp6, %for.inc ]167 ret i32 %tmp7168}169 170; CHECK-LABEL: predicated_store_scalarized_operand171;172; This test checks that we correctly compute the cost of the predicated store173; instruction and the add instruction it uses. The add is scalarized and sunk174; inside the predicated block. If we assume the block probability is 50%, we175; compute the cost as:176;177; Cost of add:178; (add(2) + extractelement(4)) / 2 = 3179; Cost of store:180; store(4) / 2 = 2181;182; CHECK: Scalarizing: %tmp2 = add nsw i32 %tmp1, %x183; CHECK: Scalarizing and predicating: store i32 %tmp2, ptr %tmp0, align 4184; CHECK: Cost of 2 for VF 2: profitable to scalarize store i32 %tmp2, ptr %tmp0, align 4185; CHECK: Cost of 3 for VF 2: profitable to scalarize %tmp2 = add nsw i32 %tmp1, %x186;187define void @predicated_store_scalarized_operand(ptr %a, i1 %c, i32 %x, i64 %n) {188entry:189 br label %for.body190 191for.body:192 %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]193 %tmp0 = getelementptr inbounds i32, ptr %a, i64 %i194 %tmp1 = load i32, ptr %tmp0, align 4195 br i1 %c, label %if.then, label %for.inc196 197if.then:198 %tmp2 = add nsw i32 %tmp1, %x199 store i32 %tmp2, ptr %tmp0, align 4200 br label %for.inc201 202for.inc:203 %i.next = add nuw nsw i64 %i, 1204 %cond = icmp slt i64 %i.next, %n205 br i1 %cond, label %for.body, label %for.end206 207for.end:208 ret void209}210 211; CHECK-LABEL: predication_multi_context212;213; This test checks that we correctly compute the cost of multiple predicated214; instructions in the same block. The sdiv, udiv, and store must be scalarized215; and predicated. The sub feeding the store is scalarized and sunk inside the216; store's predicated block. However, the add feeding the sdiv and udiv cannot217; be sunk and is not scalarized. If we assume the block probability is 50%, we218; compute the cost as:219;220; Cost of add:221; add(1) = 1222; Cost of sdiv:223; (sdiv(2) + extractelement(8) + insertelement(4)) / 2 = 7224; Cost of udiv:225; (udiv(2) + extractelement(8) + insertelement(4)) / 2 = 7226; Cost of sub:227; (sub(2) + extractelement(4)) / 2 = 3228; Cost of store:229; store(4) / 2 = 2230;231; CHECK-NOT: Scalarizing: %tmp2 = add i32 %tmp1, %x232; CHECK: Scalarizing and predicating: %tmp3 = sdiv i32 %tmp1, %tmp2233; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp3, %tmp2234; CHECK: Scalarizing: %tmp5 = sub i32 %tmp4, %x235; CHECK: Scalarizing and predicating: store i32 %tmp5, ptr %tmp0, align 4236; CHECK: Cost of 7 for VF 2: profitable to scalarize %tmp3 = sdiv i32 %tmp1, %tmp2237; CHECK: Cost of 7 for VF 2: profitable to scalarize %tmp4 = udiv i32 %tmp3, %tmp2238; CHECK: Cost of 2 for VF 2: profitable to scalarize store i32 %tmp5, ptr %tmp0, align 4239; CHECK: Cost of 3 for VF 2: profitable to scalarize %tmp5 = sub i32 %tmp4, %x240; CHECK: Cost of 1 for VF 2: WIDEN ir<%tmp2> = add ir<%tmp1>, ir<%x>241;242define void @predication_multi_context(ptr %a, i1 %c, i32 %x, i64 %n) {243entry:244 br label %for.body245 246for.body:247 %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]248 %tmp0 = getelementptr inbounds i32, ptr %a, i64 %i249 %tmp1 = load i32, ptr %tmp0, align 4250 br i1 %c, label %if.then, label %for.inc251 252if.then:253 %tmp2 = add i32 %tmp1, %x254 %tmp3 = sdiv i32 %tmp1, %tmp2255 %tmp4 = udiv i32 %tmp3, %tmp2256 %tmp5 = sub i32 %tmp4, %x257 store i32 %tmp5, ptr %tmp0, align 4258 br label %for.inc259 260for.inc:261 %i.next = add nuw nsw i64 %i, 1262 %cond = icmp slt i64 %i.next, %n263 br i1 %cond, label %for.body, label %for.end264 265for.end:266 ret void267}268