717 lines · plain
1; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn-- < %s | FileCheck -enable-var-scope --check-prefixes=GCN,SI,FUNC %s2; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX89,FUNC %s3; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX9,GFX89,FUNC %s4; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=r600-- -mcpu=cypress < %s | FileCheck -enable-var-scope --check-prefixes=EG,FUNC %s5 6; FIXME: i16 promotion pass ruins the scalar cases when legal.7; FIXME: r600 fails verifier8 9; FUNC-LABEL: {{^}}sext_in_reg_i1_i32:10; GCN: s_load_dword [[ARG:s[0-9]+]],11; GCN: s_bfe_i32 [[SEXTRACT:s[0-9]+]], [[ARG]], 0x1000012; GCN: v_mov_b32_e32 [[EXTRACT:v[0-9]+]], [[SEXTRACT]]13; GCN: buffer_store_dword [[EXTRACT]],14 15; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]]16; EG: LSHR * [[ADDR]]17; EG: BFE_INT * [[RES]], {{.*}}, 0.0, 118define amdgpu_kernel void @sext_in_reg_i1_i32(ptr addrspace(1) %out, i32 %in) #0 {19 %shl = shl i32 %in, 3120 %sext = ashr i32 %shl, 3121 store i32 %sext, ptr addrspace(1) %out22 ret void23}24 25; FUNC-LABEL: {{^}}sext_in_reg_i8_to_i32:26; GCN: s_add_i32 [[VAL:s[0-9]+]],27; GCN: s_sext_i32_i8 [[EXTRACT:s[0-9]+]], [[VAL]]28; GCN: v_mov_b32_e32 [[VEXTRACT:v[0-9]+]], [[EXTRACT]]29; GCN: buffer_store_dword [[VEXTRACT]],30 31; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]]32; EG: ADD_INT33; EG-NEXT: BFE_INT [[RES]], {{.*}}, 0.0, literal34; EG-NEXT: LSHR * [[ADDR]]35define amdgpu_kernel void @sext_in_reg_i8_to_i32(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {36 %c = add i32 %a, %b ; add to prevent folding into extload37 %shl = shl i32 %c, 2438 %ashr = ashr i32 %shl, 2439 store i32 %ashr, ptr addrspace(1) %out, align 440 ret void41}42 43; FUNC-LABEL: {{^}}sext_in_reg_i16_to_i32:44; GCN: s_add_i32 [[VAL:s[0-9]+]],45; GCN: s_sext_i32_i16 [[EXTRACT:s[0-9]+]], [[VAL]]46; GCN: v_mov_b32_e32 [[VEXTRACT:v[0-9]+]], [[EXTRACT]]47; GCN: buffer_store_dword [[VEXTRACT]],48 49; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]]50; EG: ADD_INT51; EG-NEXT: BFE_INT [[RES]], {{.*}}, 0.0, literal52; EG-NEXT: LSHR * [[ADDR]]53define amdgpu_kernel void @sext_in_reg_i16_to_i32(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {54 %c = add i32 %a, %b ; add to prevent folding into extload55 %shl = shl i32 %c, 1656 %ashr = ashr i32 %shl, 1657 store i32 %ashr, ptr addrspace(1) %out, align 458 ret void59}60 61; FUNC-LABEL: {{^}}sext_in_reg_i8_to_v1i32:62; GCN: s_add_i32 [[VAL:s[0-9]+]],63; GCN: s_sext_i32_i8 [[EXTRACT:s[0-9]+]], [[VAL]]64; GCN: v_mov_b32_e32 [[VEXTRACT:v[0-9]+]], [[EXTRACT]]65; GCN: buffer_store_dword [[VEXTRACT]],66 67; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]]68; EG: ADD_INT69; EG-NEXT: BFE_INT [[RES]], {{.*}}, 0.0, literal70; EG-NEXT: LSHR * [[ADDR]]71define amdgpu_kernel void @sext_in_reg_i8_to_v1i32(ptr addrspace(1) %out, <1 x i32> %a, <1 x i32> %b) #0 {72 %c = add <1 x i32> %a, %b ; add to prevent folding into extload73 %shl = shl <1 x i32> %c, <i32 24>74 %ashr = ashr <1 x i32> %shl, <i32 24>75 store <1 x i32> %ashr, ptr addrspace(1) %out, align 476 ret void77}78 79; FUNC-LABEL: {{^}}sext_in_reg_i1_to_i64:80; GCN: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]]81; GCN-DAG: s_bfe_i64 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], [[VAL]], 0x1000082; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]]83; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]]84; GCN: buffer_store_dwordx2 v[[[VLO]]:[[VHI]]]85define amdgpu_kernel void @sext_in_reg_i1_to_i64(ptr addrspace(1) %out, i64 %a, i64 %b) #0 {86 %c = shl i64 %a, %b87 %shl = shl i64 %c, 6388 %ashr = ashr i64 %shl, 6389 store i64 %ashr, ptr addrspace(1) %out, align 890 ret void91}92 93; FUNC-LABEL: {{^}}sext_in_reg_i8_to_i64:94; GCN: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]]95; GCN-DAG: s_bfe_i64 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], [[VAL]], 0x8000096; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]]97; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]]98; GCN: buffer_store_dwordx2 v[[[VLO]]:[[VHI]]]99define amdgpu_kernel void @sext_in_reg_i8_to_i64(ptr addrspace(1) %out, i64 %a, i64 %b) #0 {100 %c = shl i64 %a, %b101 %shl = shl i64 %c, 56102 %ashr = ashr i64 %shl, 56103 store i64 %ashr, ptr addrspace(1) %out, align 8104 ret void105}106 107; FUNC-LABEL: {{^}}sext_in_reg_i16_to_i64:108; GCN: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]]109; GCN-DAG: s_bfe_i64 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], [[VAL]], 0x100000110; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]]111; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]]112; GCN: buffer_store_dwordx2 v[[[VLO]]:[[VHI]]]113 114define amdgpu_kernel void @sext_in_reg_i16_to_i64(ptr addrspace(1) %out, i64 %a, i64 %b) #0 {115 %c = shl i64 %a, %b116 %shl = shl i64 %c, 48117 %ashr = ashr i64 %shl, 48118 store i64 %ashr, ptr addrspace(1) %out, align 8119 ret void120}121 122; FUNC-LABEL: {{^}}sext_in_reg_i32_to_i64:123; GCN: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]]124; GCN-DAG: s_bfe_i64 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], [[VAL]], 0x200000125; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]]126; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]]127; GCN: buffer_store_dwordx2 v[[[VLO]]:[[VHI]]]128define amdgpu_kernel void @sext_in_reg_i32_to_i64(ptr addrspace(1) %out, i64 %a, i64 %b) #0 {129 %c = shl i64 %a, %b130 %shl = shl i64 %c, 32131 %ashr = ashr i64 %shl, 32132 store i64 %ashr, ptr addrspace(1) %out, align 8133 ret void134}135 136; This is broken on Evergreen for some reason related to the <1 x i64> kernel arguments.137; XFUNC-LABEL: {{^}}sext_in_reg_i8_to_v1i64:138; XGCN: s_bfe_i32 [[EXTRACT:s[0-9]+]], {{s[0-9]+}}, 524288139; XGCN: s_ashr_i32 {{v[0-9]+}}, [[EXTRACT]], 31140; XGCN: buffer_store_dword141; XEG: BFE_INT142; XEG: ASHR143; define amdgpu_kernel void @sext_in_reg_i8_to_v1i64(ptr addrspace(1) %out, <1 x i64> %a, <1 x i64> %b) #0 {144; %c = add <1 x i64> %a, %b145; %shl = shl <1 x i64> %c, <i64 56>146; %ashr = ashr <1 x i64> %shl, <i64 56>147; store <1 x i64> %ashr, ptr addrspace(1) %out, align 8148; ret void149; }150 151; FUNC-LABEL: {{^}}v_sext_in_reg_i1_to_i64:152; SI: buffer_load_dwordx2153; SI: v_lshl_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]154 155; GFX89: {{flat|global}}_load_dwordx2156; GFX89: v_lshlrev_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]157 158; GCN: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 1159; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]160 161; SI: buffer_store_dwordx2 v[[[LO]]:[[HI]]]162; GFX89: {{flat|global}}_store_dwordx2 v{{.+}}, v[[[LO]]:[[HI]]]163define amdgpu_kernel void @v_sext_in_reg_i1_to_i64(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) #0 {164 %tid = call i32 @llvm.amdgcn.workitem.id.x()165 %a.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid166 %b.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid167 %out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %tid168 %a = load i64, ptr addrspace(1) %a.gep, align 8169 %b = load i64, ptr addrspace(1) %b.gep, align 8170 171 %c = shl i64 %a, %b172 %shl = shl i64 %c, 63173 %ashr = ashr i64 %shl, 63174 store i64 %ashr, ptr addrspace(1) %out.gep, align 8175 ret void176}177 178; FUNC-LABEL: {{^}}v_sext_in_reg_i8_to_i64:179; SI: buffer_load_dwordx2180; SI: v_lshl_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]181 182; GFX89: {{flat|global}}_load_dwordx2183; GFX89: v_lshlrev_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]184 185; GCN: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 8186; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]187 188; SI: buffer_store_dwordx2 v[[[LO]]:[[HI]]]189; GFX89: {{flat|global}}_store_dwordx2 v{{.+}}, v[[[LO]]:[[HI]]]190define amdgpu_kernel void @v_sext_in_reg_i8_to_i64(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) #0 {191 %tid = call i32 @llvm.amdgcn.workitem.id.x()192 %a.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid193 %b.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid194 %out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %tid195 %a = load i64, ptr addrspace(1) %a.gep, align 8196 %b = load i64, ptr addrspace(1) %b.gep, align 8197 198 %c = shl i64 %a, %b199 %shl = shl i64 %c, 56200 %ashr = ashr i64 %shl, 56201 store i64 %ashr, ptr addrspace(1) %out.gep, align 8202 ret void203}204 205; FUNC-LABEL: {{^}}v_sext_in_reg_i16_to_i64:206; SI: buffer_load_dwordx2207; SI: v_lshl_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]208 209; GFX89: {{flat|global}}_load_dwordx2210; GFX89: v_lshlrev_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]211 212; GCN: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 16213; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]214 215; SI: buffer_store_dwordx2 v[[[LO]]:[[HI]]]216; GFX89: {{flat|global}}_store_dwordx2 v{{.+}}, v[[[LO]]:[[HI]]]217define amdgpu_kernel void @v_sext_in_reg_i16_to_i64(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) #0 {218 %tid = call i32 @llvm.amdgcn.workitem.id.x()219 %a.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid220 %b.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid221 %out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %tid222 %a = load i64, ptr addrspace(1) %a.gep, align 8223 %b = load i64, ptr addrspace(1) %b.gep, align 8224 225 %c = shl i64 %a, %b226 %shl = shl i64 %c, 48227 %ashr = ashr i64 %shl, 48228 store i64 %ashr, ptr addrspace(1) %out.gep, align 8229 ret void230}231 232; FUNC-LABEL: {{^}}v_sext_in_reg_i32_to_i64:233; SI: buffer_load_dwordx2234; SI: v_lshl_b64 v[[[LO:[0-9]+]]:[[HI:[0-9]+]]],235 236; GFX89: {{flat|global}}_load_dwordx2237; GFX89: v_lshlrev_b64 v[[[LO:[0-9]+]]:[[HI:[0-9]+]]],238 239; GCN: v_ashrrev_i32_e32 v[[SHR:[0-9]+]], 31, v[[LO]]240; GFX89: {{flat|global}}_store_dwordx2 v{{.+}}, v[[[LO]]:[[SHR]]]241define amdgpu_kernel void @v_sext_in_reg_i32_to_i64(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) #0 {242 %tid = call i32 @llvm.amdgcn.workitem.id.x()243 %a.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid244 %b.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid245 %out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %tid246 %a = load i64, ptr addrspace(1) %a.gep, align 8247 %b = load i64, ptr addrspace(1) %b.gep, align 8248 249 %c = shl i64 %a, %b250 %shl = shl i64 %c, 32251 %ashr = ashr i64 %shl, 32252 store i64 %ashr, ptr addrspace(1) %out.gep, align 8253 ret void254}255 256; FUNC-LABEL: {{^}}sext_in_reg_i1_in_i32_other_amount:257; GCN-NOT: s_lshl258; GCN-NOT: s_ashr259; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x190001260 261; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]]262; EG-NOT: BFE263; EG: ADD_INT264; EG: LSHL265; EG: ASHR [[RES]]266; EG: LSHR {{\*?}} [[ADDR]]267define amdgpu_kernel void @sext_in_reg_i1_in_i32_other_amount(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {268 %c = add i32 %a, %b269 %x = shl i32 %c, 6270 %y = ashr i32 %x, 7271 store i32 %y, ptr addrspace(1) %out272 ret void273}274 275; FUNC-LABEL: {{^}}sext_in_reg_v2i1_in_v2i32_other_amount:276; GCN-NOT: s_lshl277; GCN-NOT: s_ashr278; GCN-DAG: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x190001279; GCN-DAG: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x190001280; GCN: s_endpgm281 282; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]]283; EG-NOT: BFE284; EG: ADD_INT285; EG: LSHL286; EG: ASHR [[RES]]287; EG: LSHL288; EG: ASHR [[RES]]289; EG: LSHR {{\*?}} [[ADDR]]290define amdgpu_kernel void @sext_in_reg_v2i1_in_v2i32_other_amount(ptr addrspace(1) %out, <2 x i32> %a, <2 x i32> %b) #0 {291 %c = add <2 x i32> %a, %b292 %x = shl <2 x i32> %c, <i32 6, i32 6>293 %y = ashr <2 x i32> %x, <i32 7, i32 7>294 store <2 x i32> %y, ptr addrspace(1) %out295 ret void296}297 298 299; FUNC-LABEL: {{^}}sext_in_reg_v2i1_to_v2i32:300; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000301; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000302; GCN: buffer_store_dwordx2303 304; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]]305; EG: BFE_INT [[RES]]306; EG: BFE_INT [[RES]]307; EG: LSHR {{\*?}} [[ADDR]]308define amdgpu_kernel void @sext_in_reg_v2i1_to_v2i32(ptr addrspace(1) %out, <2 x i32> %a, <2 x i32> %b) #0 {309 %c = add <2 x i32> %a, %b ; add to prevent folding into extload310 %shl = shl <2 x i32> %c, <i32 31, i32 31>311 %ashr = ashr <2 x i32> %shl, <i32 31, i32 31>312 store <2 x i32> %ashr, ptr addrspace(1) %out, align 8313 ret void314}315 316; FUNC-LABEL: {{^}}sext_in_reg_v4i1_to_v4i32:317; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000318; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000319; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000320; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000321; GCN: buffer_store_dwordx4322 323; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW][XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]]324; EG: BFE_INT [[RES]]325; EG: BFE_INT [[RES]]326; EG: BFE_INT [[RES]]327; EG: BFE_INT [[RES]]328; EG: LSHR {{\*?}} [[ADDR]]329define amdgpu_kernel void @sext_in_reg_v4i1_to_v4i32(ptr addrspace(1) %out, <4 x i32> %a, <4 x i32> %b) #0 {330 %c = add <4 x i32> %a, %b ; add to prevent folding into extload331 %shl = shl <4 x i32> %c, <i32 31, i32 31, i32 31, i32 31>332 %ashr = ashr <4 x i32> %shl, <i32 31, i32 31, i32 31, i32 31>333 store <4 x i32> %ashr, ptr addrspace(1) %out, align 8334 ret void335}336 337; FUNC-LABEL: {{^}}sext_in_reg_v2i8_to_v2i32:338; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}}339; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}}340; GCN: buffer_store_dwordx2341 342; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]]343; EG: BFE_INT [[RES]]344; EG: BFE_INT [[RES]]345; EG: LSHR {{\*?}} [[ADDR]]346define amdgpu_kernel void @sext_in_reg_v2i8_to_v2i32(ptr addrspace(1) %out, <2 x i32> %a, <2 x i32> %b) #0 {347 %c = add <2 x i32> %a, %b ; add to prevent folding into extload348 %shl = shl <2 x i32> %c, <i32 24, i32 24>349 %ashr = ashr <2 x i32> %shl, <i32 24, i32 24>350 store <2 x i32> %ashr, ptr addrspace(1) %out, align 8351 ret void352}353 354; FUNC-LABEL: {{^}}sext_in_reg_v4i8_to_v4i32:355; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}}356; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}}357; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}}358; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}}359; GCN: buffer_store_dwordx4360 361; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW][XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]]362; EG: BFE_INT [[RES]]363; EG: BFE_INT [[RES]]364; EG: BFE_INT [[RES]]365; EG: BFE_INT [[RES]]366; EG: LSHR {{\*?}} [[ADDR]]367define amdgpu_kernel void @sext_in_reg_v4i8_to_v4i32(ptr addrspace(1) %out, <4 x i32> %a, <4 x i32> %b) #0 {368 %c = add <4 x i32> %a, %b ; add to prevent folding into extload369 %shl = shl <4 x i32> %c, <i32 24, i32 24, i32 24, i32 24>370 %ashr = ashr <4 x i32> %shl, <i32 24, i32 24, i32 24, i32 24>371 store <4 x i32> %ashr, ptr addrspace(1) %out, align 8372 ret void373}374 375; FUNC-LABEL: {{^}}sext_in_reg_v2i16_to_v2i32:376; GCN: s_sext_i32_i16 {{s[0-9]+}}, {{s[0-9]+}}377; GCN: s_sext_i32_i16 {{s[0-9]+}}, {{s[0-9]+}}378; GCN: buffer_store_dwordx2379 380; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]]381; EG: BFE_INT [[RES]]382; EG: BFE_INT [[RES]]383; EG: LSHR {{\*?}} [[ADDR]]384define amdgpu_kernel void @sext_in_reg_v2i16_to_v2i32(ptr addrspace(1) %out, <2 x i32> %a, <2 x i32> %b) #0 {385 %c = add <2 x i32> %a, %b ; add to prevent folding into extload386 %shl = shl <2 x i32> %c, <i32 16, i32 16>387 %ashr = ashr <2 x i32> %shl, <i32 16, i32 16>388 store <2 x i32> %ashr, ptr addrspace(1) %out, align 8389 ret void390}391 392; FUNC-LABEL: {{^}}testcase:393define amdgpu_kernel void @testcase(ptr addrspace(1) %out, i8 %a) #0 {394 %and_a_1 = and i8 %a, 1395 %cmp_eq = icmp eq i8 %and_a_1, 0396 %cmp_slt = icmp slt i8 %a, 0397 %sel0 = select i1 %cmp_slt, i8 0, i8 %a398 %sel1 = select i1 %cmp_eq, i8 0, i8 %a399 %xor = xor i8 %sel0, %sel1400 store i8 %xor, ptr addrspace(1) %out401 ret void402}403 404; FUNC-LABEL: {{^}}testcase_3:405define amdgpu_kernel void @testcase_3(ptr addrspace(1) %out, i8 %a) #0 {406 %and_a_1 = and i8 %a, 1407 %cmp_eq = icmp eq i8 %and_a_1, 0408 %cmp_slt = icmp slt i8 %a, 0409 %sel0 = select i1 %cmp_slt, i8 0, i8 %a410 %sel1 = select i1 %cmp_eq, i8 0, i8 %a411 %xor = xor i8 %sel0, %sel1412 store i8 %xor, ptr addrspace(1) %out413 ret void414}415 416; FUNC-LABEL: {{^}}vgpr_sext_in_reg_v4i8_to_v4i32:417; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8418; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8419; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8420; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8421define amdgpu_kernel void @vgpr_sext_in_reg_v4i8_to_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) #0 {422 %loada = load <4 x i32>, ptr addrspace(1) %a, align 16423 %loadb = load <4 x i32>, ptr addrspace(1) %b, align 16424 %c = add <4 x i32> %loada, %loadb ; add to prevent folding into extload425 %shl = shl <4 x i32> %c, <i32 24, i32 24, i32 24, i32 24>426 %ashr = ashr <4 x i32> %shl, <i32 24, i32 24, i32 24, i32 24>427 store <4 x i32> %ashr, ptr addrspace(1) %out, align 8428 ret void429}430 431; FUNC-LABEL: {{^}}vgpr_sext_in_reg_v4i16_to_v4i32:432; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 16433; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 16434define amdgpu_kernel void @vgpr_sext_in_reg_v4i16_to_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) #0 {435 %loada = load <4 x i32>, ptr addrspace(1) %a, align 16436 %loadb = load <4 x i32>, ptr addrspace(1) %b, align 16437 %c = add <4 x i32> %loada, %loadb ; add to prevent folding into extload438 %shl = shl <4 x i32> %c, <i32 16, i32 16, i32 16, i32 16>439 %ashr = ashr <4 x i32> %shl, <i32 16, i32 16, i32 16, i32 16>440 store <4 x i32> %ashr, ptr addrspace(1) %out, align 8441 ret void442}443 444; FUNC-LABEL: {{^}}sext_in_reg_to_illegal_type:445; GCN: buffer_load_sbyte446; GCN: v_max_i32447; GCN-NOT: bfe448; GCN: buffer_store_short449define amdgpu_kernel void @sext_in_reg_to_illegal_type(ptr addrspace(1) nocapture %out, ptr addrspace(1) nocapture %src) #0 {450 %tmp5 = load i8, ptr addrspace(1) %src, align 1451 %tmp2 = sext i8 %tmp5 to i32452 %tmp2.5 = icmp sgt i32 %tmp2, 0453 %tmp3 = select i1 %tmp2.5, i32 %tmp2, i32 0454 %tmp4 = trunc i32 %tmp3 to i8455 %tmp6 = sext i8 %tmp4 to i16456 store i16 %tmp6, ptr addrspace(1) %out, align 2457 ret void458}459 460; Make sure we propagate the VALUness to users of a moved scalar BFE.461 462; FUNC-LABEL: {{^}}v_sext_in_reg_i1_to_i64_move_use:463; SI: buffer_load_dwordx2464; SI: v_lshl_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]465 466; GFX89: {{flat|global}}_load_dwordx2467; GFX89: v_lshlrev_b64 v[[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]]468 469; GCN-DAG: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 1470; GCN-DAG: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]471; GCN-DAG: v_and_b32_e32 v[[RESULT_LO:[0-9]+]], s{{[0-9]+}}, v[[LO]]472; GCN-DAG: v_and_b32_e32 v[[RESULT_HI:[0-9]+]], s{{[0-9]+}}, v[[HI]]473; SI: buffer_store_dwordx2 v[[[RESULT_LO]]:[[RESULT_HI]]]474; GFX89: {{flat|global}}_store_dwordx2 v{{.+}}, v[[[RESULT_LO]]:[[RESULT_HI]]]475define amdgpu_kernel void @v_sext_in_reg_i1_to_i64_move_use(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, i64 %s.val) #0 {476 %tid = call i32 @llvm.amdgcn.workitem.id.x()477 %a.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid478 %b.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid479 %out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %tid480 %a = load i64, ptr addrspace(1) %a.gep, align 8481 %b = load i64, ptr addrspace(1) %b.gep, align 8482 483 %c = shl i64 %a, %b484 %shl = shl i64 %c, 63485 %ashr = ashr i64 %shl, 63486 487 %and = and i64 %ashr, %s.val488 store i64 %and, ptr addrspace(1) %out.gep, align 8489 ret void490}491 492; FUNC-LABEL: {{^}}v_sext_in_reg_i32_to_i64_move_use:493; SI: buffer_load_dwordx2494; SI: v_lshl_b64 v[[[LO:[0-9]+]]:[[HI:[0-9]+]]],495 496; GFX89: {{flat|global}}_load_dwordx2497; GFX89: v_lshlrev_b64 v[[[LO:[0-9]+]]:[[HI:[0-9]+]]],498 499; GCN-DAG: v_ashrrev_i32_e32 v[[SHR:[0-9]+]], 31, v[[LO]]500; GCN-DAG: v_and_b32_e32 v[[RESULT_LO:[0-9]+]], s{{[0-9]+}}, v[[LO]]501; GCN-DAG: v_and_b32_e32 v[[RESULT_HI:[0-9]+]], s{{[0-9]+}}, v[[SHR]]502 503; SI: buffer_store_dwordx2 v[[[RESULT_LO]]:[[RESULT_HI]]]504; GFX89: {{flat|global}}_store_dwordx2 v{{.+}}, v[[[RESULT_LO]]:[[RESULT_HI]]]505define amdgpu_kernel void @v_sext_in_reg_i32_to_i64_move_use(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, i64 %s.val) #0 {506 %tid = call i32 @llvm.amdgcn.workitem.id.x()507 %a.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid508 %b.gep = getelementptr i64, ptr addrspace(1) %aptr, i32 %tid509 %out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %tid510 %a = load i64, ptr addrspace(1) %a.gep, align 8511 %b = load i64, ptr addrspace(1) %b.gep, align 8512 513 %c = shl i64 %a, %b514 %shl = shl i64 %c, 32515 %ashr = ashr i64 %shl, 32516 %and = and i64 %ashr, %s.val517 store i64 %and, ptr addrspace(1) %out.gep, align 8518 ret void519}520 521; FUNC-LABEL: {{^}}s_sext_in_reg_i1_i16:522; GCN: s_load_dword [[VAL:s[0-9]+]]523 524; GCN: s_bfe_i32 [[BFE:s[0-9]+]], [[VAL]], 0x10000525; GCN: v_mov_b32_e32 [[VBFE:v[0-9]+]], [[BFE]]526; GCN: buffer_store_short [[VBFE]]527 528define amdgpu_kernel void @s_sext_in_reg_i1_i16(ptr addrspace(1) %out, ptr addrspace(4) %ptr) #0 {529 %ld = load i32, ptr addrspace(4) %ptr530 %in = trunc i32 %ld to i16531 %shl = shl i16 %in, 15532 %sext = ashr i16 %shl, 15533 store i16 %sext, ptr addrspace(1) %out534 ret void535}536 537; FUNC-LABEL: {{^}}s_sext_in_reg_i2_i16:538; GCN: s_load_dword [[VAL:s[0-9]+]]539 540; SI: s_bfe_i32 [[BFE:s[0-9]+]], [[VAL]], 0x20000541; SI: v_mov_b32_e32 [[VBFE:v[0-9]+]], [[BFE]]542; SI: buffer_store_short [[VBFE]]543 544; GFX89: s_lshl_b32 s{{[0-9]+}}, s{{[0-9]+}}, 14545; GFX89: s_sext_i32_i16 s{{[0-9]+}}, s{{[0-9]+}}546; GFX89: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 14547define amdgpu_kernel void @s_sext_in_reg_i2_i16(ptr addrspace(1) %out, ptr addrspace(4) %ptr) #0 {548 %ld = load i32, ptr addrspace(4) %ptr549 %in = trunc i32 %ld to i16550 %shl = shl i16 %in, 14551 %sext = ashr i16 %shl, 14552 store i16 %sext, ptr addrspace(1) %out553 ret void554}555 556; FUNC-LABEL: {{^}}v_sext_in_reg_i1_i16:557; GCN: {{buffer|flat|global}}_load_ushort [[VAL:v[0-9]+]]558; GCN: v_bfe_i32 [[BFE:v[0-9]+]], [[VAL]], 0, 1{{$}}559 560; GCN: ds_write_b16 v{{[0-9]+}}, [[BFE]]561define amdgpu_kernel void @v_sext_in_reg_i1_i16(ptr addrspace(3) %out, ptr addrspace(1) %ptr) #0 {562 %tid = call i32 @llvm.amdgcn.workitem.id.x()563 %gep = getelementptr i16, ptr addrspace(1) %ptr, i32 %tid564 %out.gep = getelementptr i16, ptr addrspace(3) %out, i32 %tid565 566 %in = load i16, ptr addrspace(1) %gep567 %shl = shl i16 %in, 15568 %sext = ashr i16 %shl, 15569 store i16 %sext, ptr addrspace(3) %out.gep570 ret void571}572 573; FUNC-LABEL: {{^}}v_sext_in_reg_i1_i16_nonload:574; GCN: {{buffer|flat|global}}_load_ushort [[VAL0:v[0-9]+]]575; GCN: {{buffer|flat|global}}_load_ushort [[VAL1:v[0-9]+]]576 577; SI: v_lshlrev_b32_e32 [[REG:v[0-9]+]], [[VAL1]], [[VAL0]]578; GFX89: v_lshlrev_b16_e32 [[REG:v[0-9]+]], [[VAL1]], [[VAL0]]579 580; GCN: v_bfe_i32 [[BFE:v[0-9]+]], [[REG]], 0, 1{{$}}581; GCN: ds_write_b16 v{{[0-9]+}}, [[BFE]]582define amdgpu_kernel void @v_sext_in_reg_i1_i16_nonload(ptr addrspace(3) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, i16 %s.val) nounwind {583 %tid = call i32 @llvm.amdgcn.workitem.id.x()584 %a.gep = getelementptr i16, ptr addrspace(1) %aptr, i32 %tid585 %b.gep = getelementptr i16, ptr addrspace(1) %bptr, i32 %tid586 %out.gep = getelementptr i16, ptr addrspace(3) %out, i32 %tid587 %a = load volatile i16, ptr addrspace(1) %a.gep, align 2588 %b = load volatile i16, ptr addrspace(1) %b.gep, align 2589 590 %c = shl i16 %a, %b591 %shl = shl i16 %c, 15592 %ashr = ashr i16 %shl, 15593 594 store i16 %ashr, ptr addrspace(3) %out.gep, align 2595 ret void596}597 598; FUNC-LABEL: {{^}}s_sext_in_reg_i2_i16_arg:599; GCN: s_load_dword [[VAL:s[0-9]+]]600 601; SI: s_bfe_i32 [[BFE:s[0-9]+]], [[VAL]], 0x20000602; SI: v_mov_b32_e32 [[VBFE:v[0-9]+]], [[BFE]]603; SI: buffer_store_short [[VBFE]]604 605; GFX89: s_lshl_b32 s{{[0-9]+}}, s{{[0-9]+}}, 14{{$}}606; GFX89: s_sext_i32_i16 s{{[0-9]+}}, s{{[0-9]+}}607; GFX89: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 14{{$}}608define amdgpu_kernel void @s_sext_in_reg_i2_i16_arg(ptr addrspace(1) %out, i16 %in) #0 {609 %shl = shl i16 %in, 14610 %sext = ashr i16 %shl, 14611 store i16 %sext, ptr addrspace(1) %out612 ret void613}614 615; FUNC-LABEL: {{^}}s_sext_in_reg_i8_i16_arg:616; GCN: s_load_dword [[VAL:s[0-9]+]]617 618; SI: s_sext_i32_i8 [[SSEXT:s[0-9]+]], [[VAL]]619; SI: v_mov_b32_e32 [[VSEXT:v[0-9]+]], [[SSEXT]]620; SI: buffer_store_short [[VSEXT]]621 622; GFX89: s_bfe_i32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80000623define amdgpu_kernel void @s_sext_in_reg_i8_i16_arg(ptr addrspace(1) %out, i16 %in) #0 {624 %shl = shl i16 %in, 8625 %sext = ashr i16 %shl, 8626 store i16 %sext, ptr addrspace(1) %out627 ret void628}629 630; FUNC-LABEL: {{^}}s_sext_in_reg_i15_i16_arg:631; GCN: s_load_dword [[VAL:s[0-9]+]]632 633; SI: s_bfe_i32 [[BFE:s[0-9]+]], [[VAL]], 0xf0000634; SI: v_mov_b32_e32 [[VBFE:v[0-9]+]], [[BFE]]635; SI: buffer_store_short [[VBFE]]636 637; GFX89: s_lshl_b32 s{{[0-9]+}}, s{{[0-9]+}}, 1{{$}}638; GFX89: s_sext_i32_i16 s{{[0-9]+}}, s{{[0-9]+}}639; GFX89: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 1{{$}}640define amdgpu_kernel void @s_sext_in_reg_i15_i16_arg(ptr addrspace(1) %out, i16 %in) #0 {641 %shl = shl i16 %in, 1642 %sext = ashr i16 %shl, 1643 store i16 %sext, ptr addrspace(1) %out644 ret void645}646 647; FUNC-LABEL: {{^}}sext_in_reg_v2i1_to_v2i16:648; GFX9: v_pk_add_u16 [[ADD:v[0-9]+]]649; GFX9: v_pk_lshlrev_b16 [[SHL:v[0-9]+]], 15, [[ADD]]650; GFX9: v_pk_ashrrev_i16 [[SRA:v[0-9]+]], 15, [[SHL]]651define amdgpu_kernel void @sext_in_reg_v2i1_to_v2i16(ptr addrspace(1) %out, <2 x i16> %a, <2 x i16> %b) #0 {652 %c = add <2 x i16> %a, %b ; add to prevent folding into extload653 %shl = shl <2 x i16> %c, <i16 15, i16 15>654 %ashr = ashr <2 x i16> %shl, <i16 15, i16 15>655 store <2 x i16> %ashr, ptr addrspace(1) %out656 ret void657}658 659; FUNC-LABEL: {{^}}sext_in_reg_v3i1_to_v3i16:660; GFX9: v_pk_add_u16661; GFX9: v_pk_add_u16662; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 15, v{{[0-9]+}}663; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 15, v{{[0-9]+}}664; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, 15, v{{[0-9]+}}665; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, 15, v{{[0-9]+}}666define amdgpu_kernel void @sext_in_reg_v3i1_to_v3i16(ptr addrspace(1) %out, <3 x i16> %a, <3 x i16> %b) #0 {667 %c = add <3 x i16> %a, %b ; add to prevent folding into extload668 %shl = shl <3 x i16> %c, <i16 15, i16 15, i16 15>669 %ashr = ashr <3 x i16> %shl, <i16 15, i16 15, i16 15>670 store <3 x i16> %ashr, ptr addrspace(1) %out671 ret void672}673 674; FUNC-LABEL: {{^}}sext_in_reg_v2i2_to_v2i16:675; GFX9: v_pk_add_u16 [[ADD:v[0-9]+]]676; GFX9: v_pk_lshlrev_b16 [[SHL:v[0-9]+]], 14, [[ADD]]677; GFX9: v_pk_ashrrev_i16 [[SRA:v[0-9]+]], 14, [[SHL]]678define amdgpu_kernel void @sext_in_reg_v2i2_to_v2i16(ptr addrspace(1) %out, <2 x i16> %a, <2 x i16> %b) #0 {679 %c = add <2 x i16> %a, %b ; add to prevent folding into extload680 %shl = shl <2 x i16> %c, <i16 14, i16 14>681 %ashr = ashr <2 x i16> %shl, <i16 14, i16 14>682 store <2 x i16> %ashr, ptr addrspace(1) %out683 ret void684}685 686; FUNC-LABEL: {{^}}sext_in_reg_v2i8_to_v2i16:687; GFX9: v_pk_add_u16 [[ADD:v[0-9]+]]688; GFX9: v_pk_lshlrev_b16 [[SHL:v[0-9]+]], 8, [[ADD]]689; GFX9: v_pk_ashrrev_i16 [[SRA:v[0-9]+]], 8, [[SHL]]690define amdgpu_kernel void @sext_in_reg_v2i8_to_v2i16(ptr addrspace(1) %out, <2 x i16> %a, <2 x i16> %b) #0 {691 %c = add <2 x i16> %a, %b ; add to prevent folding into extload692 %shl = shl <2 x i16> %c, <i16 8, i16 8>693 %ashr = ashr <2 x i16> %shl, <i16 8, i16 8>694 store <2 x i16> %ashr, ptr addrspace(1) %out695 ret void696}697 698; FUNC-LABEL: {{^}}sext_in_reg_v3i8_to_v3i16:699; GFX9: v_pk_add_u16700; GFX9: v_pk_add_u16701; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}702; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}703; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, 8, v{{[0-9]+}}704; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, 8, v{{[0-9]+}}705define amdgpu_kernel void @sext_in_reg_v3i8_to_v3i16(ptr addrspace(1) %out, <3 x i16> %a, <3 x i16> %b) #0 {706 %c = add <3 x i16> %a, %b ; add to prevent folding into extload707 %shl = shl <3 x i16> %c, <i16 8, i16 8, i16 8>708 %ashr = ashr <3 x i16> %shl, <i16 8, i16 8, i16 8>709 store <3 x i16> %ashr, ptr addrspace(1) %out710 ret void711}712 713declare i32 @llvm.amdgcn.workitem.id.x() #1714 715attributes #0 = { nounwind }716attributes #1 = { nounwind readnone }717