419 lines · plain
1; RUN: opt -S -mtriple=amdgcn- -passes=sroa %s -o %t.sroa.ll2; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -mattr=-promote-alloca < %t.sroa.ll | FileCheck -enable-var-scope --check-prefixes=GCN,GCN-ALLOCA %s3; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -mattr=+promote-alloca < %t.sroa.ll | FileCheck -enable-var-scope --check-prefixes=GCN,GCN-PROMOTE %s4; RUN: opt -S -mtriple=amdgcn-- -passes='sroa,amdgpu-promote-alloca,instcombine' < %s | FileCheck -check-prefix=OPT %s5 6; OPT-LABEL: @vector_read_alloca_bitcast(7; OPT-NOT: alloca8; OPT: %0 = extractelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 %index9; OPT-NEXT: store i32 %0, ptr addrspace(1) %out, align 410 11; GCN-LABEL: {{^}}vector_read_alloca_bitcast:12; GCN-ALLOCA-COUNT-4: buffer_store_dword13; GCN-ALLOCA: buffer_load_dword14 15; GCN-PROMOTE: s_cmp_eq_u32 s{{[0-9]+}}, 116; GCN-PROMOTE: s_cselect_b32 [[IND1:s[0-9]+]], 1, 017; GCN-PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 218; GCN-PROMOTE: s_cselect_b32 [[IND2:s[0-9]+]], [[IND1]], 219; GCN-PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 320; GCN-PROMOTE: s_cselect_b32 [[IND3:s[0-9]+]], [[IND2]], 321; GCN-PROMOTE: ScratchSize: 022 23define amdgpu_kernel void @vector_read_alloca_bitcast(ptr addrspace(1) %out, i32 %index) {24entry:25 %tmp = alloca [4 x i32], addrspace(5)26 %y = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 127 %z = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 228 %w = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 329 store i32 0, ptr addrspace(5) %tmp30 store i32 1, ptr addrspace(5) %y31 store i32 2, ptr addrspace(5) %z32 store i32 3, ptr addrspace(5) %w33 %tmp1 = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 %index34 %tmp2 = load i32, ptr addrspace(5) %tmp135 store i32 %tmp2, ptr addrspace(1) %out36 ret void37}38 39; OPT-LABEL: @vector_write_alloca_bitcast(40; OPT-NOT: alloca41; OPT: %0 = insertelement <4 x i32> zeroinitializer, i32 1, i32 %w_index42; OPT-NEXT: %1 = extractelement <4 x i32> %0, i32 %r_index43; OPT-NEXT: store i32 %1, ptr addrspace(1) %out, align44 45; GCN-LABEL: {{^}}vector_write_alloca_bitcast:46; GCN-ALLOCA-COUNT-5: buffer_store_dword47; GCN-ALLOCA: buffer_load_dword48 49; GCN-PROMOTE-COUNT-7: s_cselect_b3250 51; GCN-PROMOTE: ScratchSize: 052 53define amdgpu_kernel void @vector_write_alloca_bitcast(ptr addrspace(1) %out, i32 %w_index, i32 %r_index) {54entry:55 %tmp = alloca [4 x i32], addrspace(5)56 %y = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 157 %z = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 258 %w = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 359 store i32 0, ptr addrspace(5) %tmp60 store i32 0, ptr addrspace(5) %y61 store i32 0, ptr addrspace(5) %z62 store i32 0, ptr addrspace(5) %w63 %tmp1 = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 %w_index64 store i32 1, ptr addrspace(5) %tmp165 %tmp2 = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 %r_index66 %tmp3 = load i32, ptr addrspace(5) %tmp267 store i32 %tmp3, ptr addrspace(1) %out68 ret void69}70 71; OPT-LABEL: @vector_write_read_bitcast_to_float(72; OPT-NOT: alloca73; OPT: bb2:74; OPT: %promotealloca = phi <6 x float> [ zeroinitializer, %bb ], [ %0, %bb2 ]75; OPT: %0 = insertelement <6 x float> %promotealloca, float %tmp71, i32 %tmp1076; OPT: .preheader:77; OPT: %bc = bitcast <6 x float> %0 to <6 x i32>78; OPT: %1 = extractelement <6 x i32> %bc, i32 %tmp2079 80; GCN-LABEL: {{^}}vector_write_read_bitcast_to_float:81; GCN-ALLOCA: buffer_store_dword82 83; GCN-PROMOTE: s_cmp_eq_u3284 85; GCN: s_cbranch86 87; GCN-ALLOCA: buffer_load_dword88 89; GCN-PROMOTE: ScratchSize: 090 91define amdgpu_kernel void @vector_write_read_bitcast_to_float(ptr addrspace(1) %arg) {92bb:93 %tmp = alloca [6 x float], align 4, addrspace(5)94 call void @llvm.lifetime.start.p5(i64 24, ptr addrspace(5) %tmp) #295 br label %bb296 97bb2: ; preds = %bb2, %bb98 %tmp3 = phi i32 [ 0, %bb ], [ %tmp13, %bb2 ]99 %tmp4 = zext i32 %tmp3 to i64100 %tmp5 = getelementptr inbounds float, ptr addrspace(1) %arg, i64 %tmp4101 %tmp7 = load i32, ptr addrspace(1) %tmp5, align 4102 %tmp8 = trunc i32 %tmp3 to i16103 %tmp9 = urem i16 %tmp8, 6104 %tmp10 = zext i16 %tmp9 to i32105 %tmp11 = getelementptr inbounds [6 x float], ptr addrspace(5) %tmp, i32 0, i32 %tmp10106 store i32 %tmp7, ptr addrspace(5) %tmp11, align 4107 %tmp13 = add nuw nsw i32 %tmp3, 1108 %tmp14 = icmp eq i32 %tmp13, 1000109 br i1 %tmp14, label %.preheader, label %bb2110 111bb15: ; preds = %.preheader112 call void @llvm.lifetime.end.p5(i64 24, ptr addrspace(5) %tmp) #2113 ret void114 115.preheader: ; preds = %.preheader, %bb2116 %tmp16 = phi i32 [ %tmp27, %.preheader ], [ 0, %bb2 ]117 %tmp17 = trunc i32 %tmp16 to i16118 %tmp18 = urem i16 %tmp17, 6119 %tmp19 = sub nuw nsw i16 5, %tmp18120 %tmp20 = zext i16 %tmp19 to i32121 %tmp21 = getelementptr inbounds [6 x float], ptr addrspace(5) %tmp, i32 0, i32 %tmp20122 %tmp23 = load i32, ptr addrspace(5) %tmp21, align 4123 %tmp24 = zext i32 %tmp16 to i64124 %tmp25 = getelementptr inbounds float, ptr addrspace(1) %arg, i64 %tmp24125 store i32 %tmp23, ptr addrspace(1) %tmp25, align 4126 %tmp27 = add nuw nsw i32 %tmp16, 1127 %tmp28 = icmp eq i32 %tmp27, 1000128 br i1 %tmp28, label %bb15, label %.preheader129}130 131; OPT-LABEL: @vector_write_read_bitcast_to_double(132; OPT-NOT: alloca133; OPT: bb2:134; OPT: %promotealloca = phi <6 x double> [ zeroinitializer, %bb ], [ %0, %bb2 ]135; OPT: %0 = insertelement <6 x double> %promotealloca, double %tmp71, i32 %tmp10136; OPT: .preheader:137; OPT: %bc = bitcast <6 x double> %0 to <6 x i64>138; OPT: %1 = extractelement <6 x i64> %bc, i32 %tmp20139 140; GCN-LABEL: {{^}}vector_write_read_bitcast_to_double:141 142; GCN-ALLOCA-COUNT-2: buffer_store_dword143; GCN-PROMOTE-COUNT-2: v_movreld_b32_e32144 145; GCN: s_cbranch146 147; GCN-ALLOCA-COUNT-2: buffer_load_dword148; GCN-PROMOTE-COUNT-2: v_movrels_b32_e32149 150; GCN-PROMOTE: ScratchSize: 0151 152define amdgpu_kernel void @vector_write_read_bitcast_to_double(ptr addrspace(1) %arg) {153bb:154 %tmp = alloca [6 x double], align 8, addrspace(5)155 call void @llvm.lifetime.start.p5(i64 48, ptr addrspace(5) %tmp) #2156 br label %bb2157 158bb2: ; preds = %bb2, %bb159 %tmp3 = phi i32 [ 0, %bb ], [ %tmp13, %bb2 ]160 %tmp4 = zext i32 %tmp3 to i64161 %tmp5 = getelementptr inbounds double, ptr addrspace(1) %arg, i64 %tmp4162 %tmp7 = load i64, ptr addrspace(1) %tmp5, align 8163 %tmp8 = trunc i32 %tmp3 to i16164 %tmp9 = urem i16 %tmp8, 6165 %tmp10 = zext i16 %tmp9 to i32166 %tmp11 = getelementptr inbounds [6 x double], ptr addrspace(5) %tmp, i32 0, i32 %tmp10167 store i64 %tmp7, ptr addrspace(5) %tmp11, align 8168 %tmp13 = add nuw nsw i32 %tmp3, 1169 %tmp14 = icmp eq i32 %tmp13, 1000170 br i1 %tmp14, label %.preheader, label %bb2171 172bb15: ; preds = %.preheader173 call void @llvm.lifetime.end.p5(i64 48, ptr addrspace(5) %tmp) #2174 ret void175 176.preheader: ; preds = %.preheader, %bb2177 %tmp16 = phi i32 [ %tmp27, %.preheader ], [ 0, %bb2 ]178 %tmp17 = trunc i32 %tmp16 to i16179 %tmp18 = urem i16 %tmp17, 6180 %tmp19 = sub nuw nsw i16 5, %tmp18181 %tmp20 = zext i16 %tmp19 to i32182 %tmp21 = getelementptr inbounds [6 x double], ptr addrspace(5) %tmp, i32 0, i32 %tmp20183 %tmp23 = load i64, ptr addrspace(5) %tmp21, align 8184 %tmp24 = zext i32 %tmp16 to i64185 %tmp25 = getelementptr inbounds double, ptr addrspace(1) %arg, i64 %tmp24186 store i64 %tmp23, ptr addrspace(1) %tmp25, align 8187 %tmp27 = add nuw nsw i32 %tmp16, 1188 %tmp28 = icmp eq i32 %tmp27, 1000189 br i1 %tmp28, label %bb15, label %.preheader190}191 192; OPT-LABEL: @vector_write_read_bitcast_to_i64(193; OPT-NOT: alloca194; OPT: bb2:195; OPT: %promotealloca = phi <6 x i64> [ zeroinitializer, %bb ], [ %0, %bb2 ]196; OPT: %0 = insertelement <6 x i64> %promotealloca, i64 %tmp6, i32 %tmp9197; OPT: .preheader:198; OPT: %1 = extractelement <6 x i64> %0, i32 %tmp18199 200; GCN-LABEL: {{^}}vector_write_read_bitcast_to_i64:201 202; GCN-ALLOCA-COUNT-2: buffer_store_dword203; GCN-PROMOTE-COUNT-2: v_movreld_b32_e32204 205; GCN: s_cbranch206 207; GCN-ALLOCA-COUNT-2: buffer_load_dword208; GCN-PROMOTE-COUNT-2: v_movrels_b32_e32209 210; GCN-PROMOTE: ScratchSize: 0211 212define amdgpu_kernel void @vector_write_read_bitcast_to_i64(ptr addrspace(1) %arg) {213bb:214 %tmp = alloca [6 x i64], align 8, addrspace(5)215 call void @llvm.lifetime.start.p5(i64 48, ptr addrspace(5) %tmp) #2216 br label %bb2217 218bb2: ; preds = %bb2, %bb219 %tmp3 = phi i32 [ 0, %bb ], [ %tmp11, %bb2 ]220 %tmp4 = zext i32 %tmp3 to i64221 %tmp5 = getelementptr inbounds i64, ptr addrspace(1) %arg, i64 %tmp4222 %tmp6 = load i64, ptr addrspace(1) %tmp5, align 8223 %tmp7 = trunc i32 %tmp3 to i16224 %tmp8 = urem i16 %tmp7, 6225 %tmp9 = zext i16 %tmp8 to i32226 %tmp10 = getelementptr inbounds [6 x i64], ptr addrspace(5) %tmp, i32 0, i32 %tmp9227 store i64 %tmp6, ptr addrspace(5) %tmp10, align 8228 %tmp11 = add nuw nsw i32 %tmp3, 1229 %tmp12 = icmp eq i32 %tmp11, 1000230 br i1 %tmp12, label %.preheader, label %bb2231 232bb13: ; preds = %.preheader233 call void @llvm.lifetime.end.p5(i64 48, ptr addrspace(5) %tmp) #2234 ret void235 236.preheader: ; preds = %.preheader, %bb2237 %tmp14 = phi i32 [ %tmp23, %.preheader ], [ 0, %bb2 ]238 %tmp15 = trunc i32 %tmp14 to i16239 %tmp16 = urem i16 %tmp15, 6240 %tmp17 = sub nuw nsw i16 5, %tmp16241 %tmp18 = zext i16 %tmp17 to i32242 %tmp19 = getelementptr inbounds [6 x i64], ptr addrspace(5) %tmp, i32 0, i32 %tmp18243 %tmp20 = load i64, ptr addrspace(5) %tmp19, align 8244 %tmp21 = zext i32 %tmp14 to i64245 %tmp22 = getelementptr inbounds i64, ptr addrspace(1) %arg, i64 %tmp21246 store i64 %tmp20, ptr addrspace(1) %tmp22, align 8247 %tmp23 = add nuw nsw i32 %tmp14, 1248 %tmp24 = icmp eq i32 %tmp23, 1000249 br i1 %tmp24, label %bb13, label %.preheader250}251 252; TODO: llvm.assume can be ingored253 254; OPT-LABEL: @vector_read_alloca_bitcast_assume(255; OPT: %0 = extractelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 %index256; OPT: store i32 %0, ptr addrspace(1) %out, align 4257 258; GCN-LABEL: {{^}}vector_read_alloca_bitcast_assume:259; GCN-COUNT: buffer_store_dword260 261define amdgpu_kernel void @vector_read_alloca_bitcast_assume(ptr addrspace(1) %out, i32 %index) {262entry:263 %tmp = alloca [4 x i32], addrspace(5)264 %cmp = icmp ne ptr addrspace(5) %tmp, null265 call void @llvm.assume(i1 %cmp)266 %y = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 1267 %z = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 2268 %w = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 3269 store i32 0, ptr addrspace(5) %tmp270 store i32 1, ptr addrspace(5) %y271 store i32 2, ptr addrspace(5) %z272 store i32 3, ptr addrspace(5) %w273 %tmp1 = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 %index274 %tmp2 = load i32, ptr addrspace(5) %tmp1275 store i32 %tmp2, ptr addrspace(1) %out276 ret void277}278 279; OPT-LABEL: @vector_read_alloca_multiuse(280; OPT-NOT: alloca281; OPT: %0 = extractelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 %index282; OPT-NEXT: %add2 = add nuw nsw i32 %0, 1283; OPT-NEXT: store i32 %add2, ptr addrspace(1) %out, align 4284 285; GCN-LABEL: {{^}}vector_read_alloca_multiuse:286; GCN-ALLOCA-COUNT-4: buffer_store_dword287; GCN-ALLOCA: buffer_load_dword288 289; GCN-PROMOTE: s_cmp_eq_u32 s{{[0-9]+}}, 1290; GCN-PROMOTE: s_cselect_b32 [[IND1:s[0-9]+]], 1, 0291; GCN-PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 2292; GCN-PROMOTE: s_cselect_b32 [[IND2:s[0-9]+]], [[IND1]], 2293; GCN-PROMOTE: s_cmp_lg_u32 s{{[0-9]+}}, 3294; GCN-PROMOTE: s_cselect_b32 [[IND3:s[0-9]+]], [[IND2]], 3295 296; GCN-PROMOTE: ScratchSize: 0297 298define amdgpu_kernel void @vector_read_alloca_multiuse(ptr addrspace(1) %out, i32 %index) {299entry:300 %tmp = alloca [4 x i32], addrspace(5)301 %y = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 1302 %z = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 2303 %w = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 3304 store i32 0, ptr addrspace(5) %tmp305 store i32 1, ptr addrspace(5) %y306 store i32 2, ptr addrspace(5) %z307 store i32 3, ptr addrspace(5) %w308 %tmp1 = getelementptr [4 x i32], ptr addrspace(5) %tmp, i32 0, i32 %index309 %tmp2 = load i32, ptr addrspace(5) %tmp1310 %tmp3 = load i32, ptr addrspace(5) %tmp311 %tmp4 = load i32, ptr addrspace(5) %y312 %add1 = add i32 %tmp2, %tmp3313 %add2 = add i32 %add1, %tmp4314 store i32 %add2, ptr addrspace(1) %out315 ret void316}317 318; OPT-LABEL: @bitcast_vector_to_vector(319; OPT-NOT: alloca320; OPT: store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr addrspace(1) %out, align 16321 322; GCN-LABEL: {{^}}bitcast_vector_to_vector:323; GCN: v_mov_b32_e32 v0, 1324; GCN: v_mov_b32_e32 v1, 2325; GCN: v_mov_b32_e32 v2, 3326; GCN: v_mov_b32_e32 v3, 4327 328; GCN: ScratchSize: 0329 330define amdgpu_kernel void @bitcast_vector_to_vector(ptr addrspace(1) %out) {331.entry:332 %alloca = alloca <4 x float>, align 16, addrspace(5)333 store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr addrspace(5) %alloca334 %load = load <4 x i32>, ptr addrspace(5) %alloca, align 16335 store <4 x i32> %load, ptr addrspace(1) %out336 ret void337}338 339; OPT-LABEL: @vector_bitcast_from_alloca_array(340; OPT-NOT: alloca341; OPT: store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr addrspace(1) %out, align 16342 343; GCN-LABEL: {{^}}vector_bitcast_from_alloca_array:344; GCN: v_mov_b32_e32 v0, 1345; GCN: v_mov_b32_e32 v1, 2346; GCN: v_mov_b32_e32 v2, 3347; GCN: v_mov_b32_e32 v3, 4348 349; GCN: ScratchSize: 0350 351define amdgpu_kernel void @vector_bitcast_from_alloca_array(ptr addrspace(1) %out) {352.entry:353 %alloca = alloca [4 x float], align 16, addrspace(5)354 store <4 x i32> <i32 1, i32 2, i32 3, i32 4>, ptr addrspace(5) %alloca355 %load = load <4 x i32>, ptr addrspace(5) %alloca, align 16356 store <4 x i32> %load, ptr addrspace(1) %out357 ret void358}359 360; OPT-LABEL: @vector_bitcast_to_array_from_alloca_array(361; OPT-NOT: alloca362; OPT-NEXT: store i32 1, ptr addrspace(1) %out, align 4363; OPT-NEXT: %out.repack1 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i64 4364; OPT-NEXT: store i32 2, ptr addrspace(1) %out.repack1, align 4365; OPT-NEXT: %out.repack2 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i64 8366; OPT-NEXT: store i32 3, ptr addrspace(1) %out.repack2, align 4367; OPT-NEXT: %out.repack3 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i64 12368; OPT-NEXT: store i32 4, ptr addrspace(1) %out.repack3, align 4369 370; GCN-LABEL: {{^}}vector_bitcast_to_array_from_alloca_array:371; GCN: v_mov_b32_e32 v0, 1372; GCN: v_mov_b32_e32 v1, 2373; GCN: v_mov_b32_e32 v2, 3374; GCN: v_mov_b32_e32 v3, 4375 376; GCN: ScratchSize: 0377 378define amdgpu_kernel void @vector_bitcast_to_array_from_alloca_array(ptr addrspace(1) %out) {379 %alloca = alloca [4 x float], align 16, addrspace(5)380 store [4 x i32] [i32 1, i32 2, i32 3, i32 4], ptr addrspace(5) %alloca381 %load = load [4 x i32], ptr addrspace(5) %alloca, align 16382 store [4 x i32] %load, ptr addrspace(1) %out383 ret void384}385 386; OPT-LABEL: @vector_bitcast_to_struct_from_alloca_array(387; OPT-NOT: alloca388; OPT-NEXT: store i32 1, ptr addrspace(1) %out, align 4389; OPT-NEXT: %out.repack1 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i64 4390; OPT-NEXT: store i32 2, ptr addrspace(1) %out.repack1, align 4391; OPT-NEXT: %out.repack2 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i64 8392; OPT-NEXT: store i32 3, ptr addrspace(1) %out.repack2, align 4393; OPT-NEXT: %out.repack3 = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i64 12394; OPT-NEXT: store i32 4, ptr addrspace(1) %out.repack3, align 4395 396; GCN-LABEL: {{^}}vector_bitcast_to_struct_from_alloca_array:397; GCN: v_mov_b32_e32 v0, 1398; GCN: v_mov_b32_e32 v1, 2399; GCN: v_mov_b32_e32 v2, 3400; GCN: v_mov_b32_e32 v3, 4401 402; GCN: ScratchSize: 0403 404%struct.v4 = type { i32, i32, i32, i32 }405 406define amdgpu_kernel void @vector_bitcast_to_struct_from_alloca_array(ptr addrspace(1) %out) {407 %alloca = alloca [4 x float], align 16, addrspace(5)408 store %struct.v4 { i32 1, i32 2, i32 3, i32 4 }, ptr addrspace(5) %alloca409 %load = load %struct.v4, ptr addrspace(5) %alloca, align 16410 store %struct.v4 %load, ptr addrspace(1) %out411 ret void412}413 414declare void @llvm.lifetime.start.p5(i64 immarg, ptr addrspace(5) nocapture)415 416declare void @llvm.lifetime.end.p5(i64 immarg, ptr addrspace(5) nocapture)417 418declare void @llvm.assume(i1)419