237 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 62; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 < %s | FileCheck -check-prefix=OPT %s3 4; testing insert case5define amdgpu_kernel void @runningSum(ptr addrspace(1) %out0, ptr addrspace(1) %out1, i32 %inputElement1, i32 %inputIter) {6; OPT-LABEL: define amdgpu_kernel void @runningSum(7; OPT-SAME: ptr addrspace(1) [[OUT0:%.*]], ptr addrspace(1) [[OUT1:%.*]], i32 [[INPUTELEMENT1:%.*]], i32 [[INPUTITER:%.*]]) #[[ATTR0:[0-9]+]] {8; OPT-NEXT: [[PREHEADER:.*]]:9; OPT-NEXT: [[VECELEMENT1:%.*]] = insertelement <2 x i32> poison, i32 [[INPUTELEMENT1]], i64 010; OPT-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[VECELEMENT1]], <2 x i32> poison, <2 x i32> zeroinitializer11; OPT-NEXT: br label %[[LOOPBODY:.*]]12; OPT: [[LOOPBODY]]:13; OPT-NEXT: [[PREVIOUSSUM:%.*]] = phi <2 x i32> [ [[TMP1]], %[[PREHEADER]] ], [ [[RUNNINGSUM:%.*]], %[[LOOPBODY]] ]14; OPT-NEXT: [[ITERCOUNT:%.*]] = phi i32 [ [[INPUTITER]], %[[PREHEADER]] ], [ [[ITERSLEFT:%.*]], %[[LOOPBODY]] ]15; OPT-NEXT: [[RUNNINGSUM]] = add <2 x i32> [[TMP1]], [[PREVIOUSSUM]]16; OPT-NEXT: [[ITERSLEFT]] = sub i32 [[ITERCOUNT]], 117; OPT-NEXT: [[COND:%.*]] = icmp eq i32 [[ITERSLEFT]], 018; OPT-NEXT: br i1 [[COND]], label %[[LOOPEXIT:.*]], label %[[LOOPBODY]]19; OPT: [[LOOPEXIT]]:20; OPT-NEXT: [[SUMELEMENT0:%.*]] = extractelement <2 x i32> [[RUNNINGSUM]], i64 021; OPT-NEXT: [[SUMELEMENT1:%.*]] = extractelement <2 x i32> [[RUNNINGSUM]], i64 122; OPT-NEXT: store i32 [[SUMELEMENT0]], ptr addrspace(1) [[OUT0]], align 423; OPT-NEXT: store i32 [[SUMELEMENT1]], ptr addrspace(1) [[OUT1]], align 424; OPT-NEXT: ret void25;26preheader:27 %vecElement1 = insertelement <2 x i32> poison, i32 %inputElement1, i64 028 %broadcast1 = shufflevector <2 x i32> %vecElement1, <2 x i32> poison, <2 x i32> zeroinitializer29 br label %loopBody30 31loopBody:32 %previousSum = phi <2 x i32> [ %broadcast1, %preheader ], [ %runningSum, %loopBody ]33 %iterCount = phi i32 [ %inputIter, %preheader ], [ %itersLeft, %loopBody ]34 %runningSum = add <2 x i32> %broadcast1, %previousSum35 %itersLeft = sub i32 %iterCount, 136 %cond = icmp eq i32 %itersLeft, 037 br i1 %cond, label %loopExit, label %loopBody38 39loopExit:40 %sumElement0 = extractelement <2 x i32> %runningSum, i64 041 %sumElement1 = extractelement <2 x i32> %runningSum, i64 142 store i32 %sumElement0, ptr addrspace(1) %out043 store i32 %sumElement1, ptr addrspace(1) %out144 ret void45}46 47; testing extract case with single use - with divergent control flow48; The vector has SINGLE use (extractelement), both sink into if.then49define amdgpu_kernel void @test_sink_extract_single_use_operands(ptr addrspace(1) %out0, <2 x i32> %inputVec, i32 %tid, i32 %cond) {50; OPT-LABEL: define amdgpu_kernel void @test_sink_extract_single_use_operands(51; OPT-SAME: ptr addrspace(1) [[OUT0:%.*]], <2 x i32> [[INPUTVEC:%.*]], i32 [[TID:%.*]], i32 [[COND:%.*]]) #[[ATTR0]] {52; OPT-NEXT: [[ENTRY:.*:]]53; OPT-NEXT: [[RUNNINGSUM:%.*]] = add <2 x i32> [[INPUTVEC]], splat (i32 1)54; OPT-NEXT: [[TMP0:%.*]] = extractelement <2 x i32> [[RUNNINGSUM]], i64 055; OPT-NEXT: [[CMP:%.*]] = icmp slt i32 [[TID]], [[COND]]56; OPT-NEXT: br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]57; OPT: [[IF_THEN]]:58; OPT-NEXT: [[RESULT:%.*]] = add i32 [[TMP0]], 10059; OPT-NEXT: store i32 [[RESULT]], ptr addrspace(1) [[OUT0]], align 460; OPT-NEXT: br label %[[IF_END]]61; OPT: [[IF_END]]:62; OPT-NEXT: ret void63;64entry:65 %runningSum = add <2 x i32> %inputVec, <i32 1, i32 1>66 %sumElement0 = extractelement <2 x i32> %runningSum, i64 067 %cmp = icmp slt i32 %tid, %cond68 br i1 %cmp, label %if.then, label %if.end69 70if.then:71 %result = add i32 %sumElement0, 10072 store i32 %result, ptr addrspace(1) %out073 br label %if.end74 75if.end:76 ret void77}78 79; testing extract case - extracting two elements with divergent control flow80; The vector has TWO uses (two extractelements), all sink into if.then81define amdgpu_kernel void @test_sink_extract_operands(ptr addrspace(1) %out0, ptr addrspace(1) %out1, <4 x i32> %input_vec, i32 %tid, i32 %cond) {82; OPT-LABEL: define amdgpu_kernel void @test_sink_extract_operands(83; OPT-SAME: ptr addrspace(1) [[OUT0:%.*]], ptr addrspace(1) [[OUT1:%.*]], <4 x i32> [[INPUT_VEC:%.*]], i32 [[TID:%.*]], i32 [[COND:%.*]]) #[[ATTR0]] {84; OPT-NEXT: [[ENTRY:.*:]]85; OPT-NEXT: [[VEC_FULL:%.*]] = add <4 x i32> [[INPUT_VEC]], <i32 42, i32 43, i32 44, i32 45>86; OPT-NEXT: [[TMP0:%.*]] = extractelement <4 x i32> [[VEC_FULL]], i64 087; OPT-NEXT: [[TMP1:%.*]] = extractelement <4 x i32> [[VEC_FULL]], i64 188; OPT-NEXT: [[CMP:%.*]] = icmp slt i32 [[TID]], [[COND]]89; OPT-NEXT: br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]90; OPT: [[IF_THEN]]:91; OPT-NEXT: [[RESULT0:%.*]] = add i32 [[TMP0]], 10092; OPT-NEXT: [[RESULT1:%.*]] = add i32 [[TMP1]], 20093; OPT-NEXT: store i32 [[RESULT0]], ptr addrspace(1) [[OUT0]], align 494; OPT-NEXT: store i32 [[RESULT1]], ptr addrspace(1) [[OUT1]], align 495; OPT-NEXT: br label %[[IF_END]]96; OPT: [[IF_END]]:97; OPT-NEXT: ret void98;99entry:100 %vec_full = add <4 x i32> %input_vec, <i32 42, i32 43, i32 44, i32 45>101 %extract0 = extractelement <4 x i32> %vec_full, i64 0102 %extract1 = extractelement <4 x i32> %vec_full, i64 1103 %cmp = icmp slt i32 %tid, %cond104 br i1 %cmp, label %if.then, label %if.end105 106if.then:107 %result0 = add i32 %extract0, 100108 %result1 = add i32 %extract1, 200109 store i32 %result0, ptr addrspace(1) %out0110 store i32 %result1, ptr addrspace(1) %out1111 br label %if.end112 113if.end:114 ret void115}116 117; testing shuffle case with divergent control flow - shuffles sink into if.then118define amdgpu_kernel void @test_shuffle_insert_subvector(ptr addrspace(1) %ptr, <4 x i16> %vec1, <4 x i16> %vec2, i32 %tid, i32 %cond) {119; OPT-LABEL: define amdgpu_kernel void @test_shuffle_insert_subvector(120; OPT-SAME: ptr addrspace(1) [[PTR:%.*]], <4 x i16> [[VEC1:%.*]], <4 x i16> [[VEC2:%.*]], i32 [[TID:%.*]], i32 [[COND:%.*]]) #[[ATTR0]] {121; OPT-NEXT: [[ENTRY:.*:]]122; OPT-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[VEC1]], <4 x i16> [[VEC2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>123; OPT-NEXT: [[SHUFFLE2:%.*]] = shufflevector <4 x i16> [[VEC1]], <4 x i16> [[VEC2]], <4 x i32> <i32 2, i32 3, i32 6, i32 7>124; OPT-NEXT: [[SHUFFLE3:%.*]] = shufflevector <4 x i16> [[VEC1]], <4 x i16> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>125; OPT-NEXT: [[SHUFFLE4:%.*]] = shufflevector <4 x i16> [[VEC2]], <4 x i16> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>126; OPT-NEXT: [[SHUFFLE5:%.*]] = shufflevector <4 x i16> [[SHUFFLE]], <4 x i16> [[SHUFFLE2]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>127; OPT-NEXT: [[CMP:%.*]] = icmp slt i32 [[TID]], [[COND]]128; OPT-NEXT: br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]129; OPT: [[IF_THEN]]:130; OPT-NEXT: [[RESULT_VEC:%.*]] = add <4 x i16> [[SHUFFLE5]], <i16 100, i16 200, i16 300, i16 400>131; OPT-NEXT: [[OTHER_RESULT:%.*]] = mul <4 x i16> [[SHUFFLE3]], splat (i16 2)132; OPT-NEXT: [[MORE_RESULT:%.*]] = sub <4 x i16> [[SHUFFLE4]], splat (i16 5)133; OPT-NEXT: store <4 x i16> [[RESULT_VEC]], ptr addrspace(1) [[PTR]], align 8134; OPT-NEXT: store <4 x i16> [[OTHER_RESULT]], ptr addrspace(1) [[PTR]], align 8135; OPT-NEXT: store <4 x i16> [[MORE_RESULT]], ptr addrspace(1) [[PTR]], align 8136; OPT-NEXT: br label %[[IF_END]]137; OPT: [[IF_END]]:138; OPT-NEXT: ret void139;140entry:141 %shuffle = shufflevector <4 x i16> %vec1, <4 x i16> %vec2, <4 x i32> <i32 0, i32 1, i32 4, i32 5>142 %shuffle2 = shufflevector <4 x i16> %vec1, <4 x i16> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>143 %shuffle3 = shufflevector <4 x i16> %vec1, <4 x i16> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>144 %shuffle4 = shufflevector <4 x i16> %vec2, <4 x i16> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>145 %shuffle5 = shufflevector <4 x i16> %shuffle, <4 x i16> %shuffle2, <4 x i32> <i32 0, i32 2, i32 4, i32 6>146 %cmp = icmp slt i32 %tid, %cond147 br i1 %cmp, label %if.then, label %if.end148 149if.then:150 %result_vec = add <4 x i16> %shuffle5, <i16 100, i16 200, i16 300, i16 400>151 %other_result = mul <4 x i16> %shuffle3, <i16 2, i16 2, i16 2, i16 2>152 %more_result = sub <4 x i16> %shuffle4, <i16 5, i16 5, i16 5, i16 5>153 store <4 x i16> %result_vec, ptr addrspace(1) %ptr154 store <4 x i16> %other_result, ptr addrspace(1) %ptr155 store <4 x i16> %more_result, ptr addrspace(1) %ptr156 br label %if.end157 158if.end:159 ret void160}161 162; testing shuffle extract subvector with divergent control flow - shuffles sink into if.then163define amdgpu_kernel void @test_shuffle_extract_subvector(ptr addrspace(1) %ptr, <4 x i16> %input_vec, i32 %tid, i32 %cond) {164; OPT-LABEL: define amdgpu_kernel void @test_shuffle_extract_subvector(165; OPT-SAME: ptr addrspace(1) [[PTR:%.*]], <4 x i16> [[INPUT_VEC:%.*]], i32 [[TID:%.*]], i32 [[COND:%.*]]) #[[ATTR0]] {166; OPT-NEXT: [[ENTRY:.*:]]167; OPT-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[INPUT_VEC]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>168; OPT-NEXT: [[SHUFFLE2:%.*]] = shufflevector <4 x i16> [[INPUT_VEC]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>169; OPT-NEXT: [[SHUFFLE3:%.*]] = shufflevector <4 x i16> [[INPUT_VEC]], <4 x i16> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>170; OPT-NEXT: [[CMP:%.*]] = icmp slt i32 [[TID]], [[COND]]171; OPT-NEXT: br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]172; OPT: [[IF_THEN]]:173; OPT-NEXT: [[RESULT_VEC:%.*]] = add <2 x i16> [[SHUFFLE]], <i16 100, i16 200>174; OPT-NEXT: [[RESULT_VEC2:%.*]] = mul <2 x i16> [[SHUFFLE2]], splat (i16 3)175; OPT-NEXT: [[RESULT_VEC3:%.*]] = sub <4 x i16> [[SHUFFLE3]], splat (i16 10)176; OPT-NEXT: store <2 x i16> [[RESULT_VEC]], ptr addrspace(1) [[PTR]], align 4177; OPT-NEXT: store <2 x i16> [[RESULT_VEC2]], ptr addrspace(1) [[PTR]], align 4178; OPT-NEXT: store <4 x i16> [[RESULT_VEC3]], ptr addrspace(1) [[PTR]], align 8179; OPT-NEXT: br label %[[IF_END]]180; OPT: [[IF_END]]:181; OPT-NEXT: ret void182;183entry:184 %shuffle = shufflevector <4 x i16> %input_vec, <4 x i16> poison, <2 x i32> <i32 2, i32 3>185 %shuffle2 = shufflevector <4 x i16> %input_vec, <4 x i16> poison, <2 x i32> <i32 0, i32 1>186 %shuffle3 = shufflevector <4 x i16> %input_vec, <4 x i16> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>187 %cmp = icmp slt i32 %tid, %cond188 br i1 %cmp, label %if.then, label %if.end189 190if.then:191 %result_vec = add <2 x i16> %shuffle, <i16 100, i16 200>192 %result_vec2 = mul <2 x i16> %shuffle2, <i16 3, i16 3>193 %result_vec3 = sub <4 x i16> %shuffle3, <i16 10, i16 10, i16 10, i16 10>194 store <2 x i16> %result_vec, ptr addrspace(1) %ptr195 store <2 x i16> %result_vec2, ptr addrspace(1) %ptr196 store <4 x i16> %result_vec3, ptr addrspace(1) %ptr197 br label %if.end198 199if.end:200 ret void201}202 203; testing shuffle sink with widening operations and divergent control flow204define amdgpu_kernel void @test_shuffle_sink_operands(ptr addrspace(1) %ptr, <2 x i16> %input_vec, <2 x i16> %input_vec2, i32 %tid, i32 %cond) {205; OPT-LABEL: define amdgpu_kernel void @test_shuffle_sink_operands(206; OPT-SAME: ptr addrspace(1) [[PTR:%.*]], <2 x i16> [[INPUT_VEC:%.*]], <2 x i16> [[INPUT_VEC2:%.*]], i32 [[TID:%.*]], i32 [[COND:%.*]]) #[[ATTR0]] {207; OPT-NEXT: [[ENTRY:.*:]]208; OPT-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i16> [[INPUT_VEC]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>209; OPT-NEXT: [[SHUFFLE2:%.*]] = shufflevector <2 x i16> [[INPUT_VEC2]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>210; OPT-NEXT: [[CMP:%.*]] = icmp slt i32 [[TID]], [[COND]]211; OPT-NEXT: br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]212; OPT: [[IF_THEN]]:213; OPT-NEXT: [[RESULT_VEC:%.*]] = add <4 x i16> [[SHUFFLE]], <i16 100, i16 200, i16 300, i16 400>214; OPT-NEXT: [[RESULT_VEC2:%.*]] = mul <4 x i16> [[SHUFFLE2]], splat (i16 5)215; OPT-NEXT: store <4 x i16> [[RESULT_VEC]], ptr addrspace(1) [[PTR]], align 8216; OPT-NEXT: store <4 x i16> [[RESULT_VEC2]], ptr addrspace(1) [[PTR]], align 8217; OPT-NEXT: br label %[[IF_END]]218; OPT: [[IF_END]]:219; OPT-NEXT: ret void220;221entry:222 %shuffle = shufflevector <2 x i16> %input_vec, <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>223 %shuffle2 = shufflevector <2 x i16> %input_vec2, <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>224 %cmp = icmp slt i32 %tid, %cond225 br i1 %cmp, label %if.then, label %if.end226 227if.then:228 %result_vec = add <4 x i16> %shuffle, <i16 100, i16 200, i16 300, i16 400>229 %result_vec2 = mul <4 x i16> %shuffle2, <i16 5, i16 5, i16 5, i16 5>230 store <4 x i16> %result_vec, ptr addrspace(1) %ptr231 store <4 x i16> %result_vec2, ptr addrspace(1) %ptr232 br label %if.end233 234if.end:235 ret void236}237