749 lines · plain
1; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=tahiti < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-SI -check-prefix=OPT-SICIVI %s2; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-CI -check-prefix=OPT-SICIVI %s3; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-VI -check-prefix=OPT-SICIVI %s4; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -mtriple=amdgcn-unknown-unknown -mcpu=gfx900 < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-GFX9 %s5; RUN: llc -mtriple=amdgcn -mcpu=tahiti -mattr=-promote-alloca -amdgpu-scalarize-global-loads=false < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SICIVI %s6; RUN: llc -mtriple=amdgcn -mcpu=bonaire -mattr=-promote-alloca -amdgpu-scalarize-global-loads=false < %s | FileCheck -check-prefix=GCN -check-prefix=CI -check-prefix=SICIVI %s7; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-scalarize-global-loads=false -mattr=-promote-alloca < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=SICIVI %s8; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=-promote-alloca -amdgpu-scalarize-global-loads=false < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s9 10; OPT-LABEL: @test_sink_global_small_offset_i32(11; OPT-CI-NOT: getelementptr i32, ptr addrspace(1) %in12; OPT-VI: getelementptr i32, ptr addrspace(1) %in13; OPT: br i114; OPT-CI: getelementptr i8,15 16; GCN-LABEL: {{^}}test_sink_global_small_offset_i32:17define amdgpu_kernel void @test_sink_global_small_offset_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) {18entry:19 %out.gep = getelementptr i32, ptr addrspace(1) %out, i64 99999920 %in.gep = getelementptr i32, ptr addrspace(1) %in, i64 721 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #022 %tmp0 = icmp eq i32 %tid, 023 br i1 %tmp0, label %endif, label %if24 25if:26 %tmp1 = load i32, ptr addrspace(1) %in.gep27 br label %endif28 29endif:30 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]31 store i32 %x, ptr addrspace(1) %out.gep32 br label %done33 34done:35 ret void36}37 38; OPT-LABEL: @test_sink_global_small_max_i32_ds_offset(39; OPT: %in.gep = getelementptr i8, ptr addrspace(1) %in, i64 6553540; OPT: br i141 42; GCN-LABEL: {{^}}test_sink_global_small_max_i32_ds_offset:43; GCN: s_and_saveexec_b6444; SICIVI: buffer_load_sbyte {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}}45 46; GFX9: v_mov_b32_e32 [[VOFFSET:v[0-9]+]], 0xf000{{$}}47; GFX9: global_load_sbyte {{v[0-9]+}}, [[VOFFSET]], {{s\[[0-9]+:[0-9]+\]}} offset:4095{{$}}48; GCN: {{^}}.LBB1_2:49; GCN: s_or_b64 exec50define amdgpu_kernel void @test_sink_global_small_max_i32_ds_offset(ptr addrspace(1) %out, ptr addrspace(1) %in) {51entry:52 %out.gep = getelementptr i32, ptr addrspace(1) %out, i64 9999953 %in.gep = getelementptr i8, ptr addrspace(1) %in, i64 6553554 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #055 %tmp0 = icmp eq i32 %tid, 056 br i1 %tmp0, label %endif, label %if57 58if:59 %tmp1 = load i8, ptr addrspace(1) %in.gep60 %tmp2 = sext i8 %tmp1 to i3261 br label %endif62 63endif:64 %x = phi i32 [ %tmp2, %if ], [ 0, %entry ]65 store i32 %x, ptr addrspace(1) %out.gep66 br label %done67 68done:69 ret void70}71 72; GCN-LABEL: {{^}}test_sink_global_small_max_mubuf_offset:73; GCN: s_and_saveexec_b6474; SICIVI: buffer_load_sbyte {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:4095{{$}}75; GFX9: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}76; GFX9: global_load_sbyte {{v[0-9]+}}, [[ZERO]], {{s\[[0-9]+:[0-9]+\]}} offset:4095{{$}}77; GCN: {{^}}.LBB2_2:78; GCN: s_or_b64 exec79define amdgpu_kernel void @test_sink_global_small_max_mubuf_offset(ptr addrspace(1) %out, ptr addrspace(1) %in) {80entry:81 %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 102482 %in.gep = getelementptr i8, ptr addrspace(1) %in, i64 409583 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #084 %tmp0 = icmp eq i32 %tid, 085 br i1 %tmp0, label %endif, label %if86 87if:88 %tmp1 = load i8, ptr addrspace(1) %in.gep89 %tmp2 = sext i8 %tmp1 to i3290 br label %endif91 92endif:93 %x = phi i32 [ %tmp2, %if ], [ 0, %entry ]94 store i32 %x, ptr addrspace(1) %out.gep95 br label %done96 97done:98 ret void99}100 101; GCN-LABEL: {{^}}test_sink_global_small_max_plus_1_mubuf_offset:102; GCN: s_and_saveexec_b64103; SICIVI: buffer_load_sbyte {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}}104; GFX9: v_mov_b32_e32 [[VOFFSET:v[0-9]+]], 0x1000{{$}}105; GFX9: global_load_sbyte {{v[0-9]+}}, [[VOFFSET]], {{s\[[0-9]+:[0-9]+\]$}}106; GCN: {{^}}.LBB3_2:107; GCN: s_or_b64 exec108define amdgpu_kernel void @test_sink_global_small_max_plus_1_mubuf_offset(ptr addrspace(1) %out, ptr addrspace(1) %in) {109entry:110 %out.gep = getelementptr i32, ptr addrspace(1) %out, i64 99999111 %in.gep = getelementptr i8, ptr addrspace(1) %in, i64 4096112 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0113 %tmp0 = icmp eq i32 %tid, 0114 br i1 %tmp0, label %endif, label %if115 116if:117 %tmp1 = load i8, ptr addrspace(1) %in.gep118 %tmp2 = sext i8 %tmp1 to i32119 br label %endif120 121endif:122 %x = phi i32 [ %tmp2, %if ], [ 0, %entry ]123 store i32 %x, ptr addrspace(1) %out.gep124 br label %done125 126done:127 ret void128}129 130; OPT-LABEL: @test_sink_scratch_small_offset_i32(131; OPT-NOT: getelementptr [512 x i32]132; OPT: br i1133; OPT: getelementptr i8,134 135; GCN-LABEL: {{^}}test_sink_scratch_small_offset_i32:136; GCN: s_and_saveexec_b64137; GCN: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:4088{{$}}138; GCN: buffer_load_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:4088 glc{{$}}139; GCN: {{^}}.LBB4_2:140define amdgpu_kernel void @test_sink_scratch_small_offset_i32(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %arg) {141entry:142 %alloca = alloca [512 x i32], align 4, addrspace(5)143 %out.gep.0 = getelementptr i32, ptr addrspace(1) %out, i64 999998144 %out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i64 999999145 %add.arg = add i32 %arg, 8146 %alloca.gep = getelementptr [512 x i32], ptr addrspace(5) %alloca, i32 0, i32 1022147 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0148 %tmp0 = icmp eq i32 %tid, 0149 br i1 %tmp0, label %endif, label %if150 151if:152 store volatile i32 123, ptr addrspace(5) %alloca.gep153 %tmp1 = load volatile i32, ptr addrspace(5) %alloca.gep154 br label %endif155 156endif:157 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]158 store i32 %x, ptr addrspace(1) %out.gep.0159 %load = load volatile i32, ptr addrspace(5) %alloca.gep160 store i32 %load, ptr addrspace(1) %out.gep.1161 br label %done162 163done:164 ret void165}166 167; This used to be a special case when the scavenge slot was168; fixed at offset 0.169; OPT-LABEL: @test_sink_scratch_small_offset_i32_reserved(170; OPT-NOT: getelementptr [512 x i32]171; OPT: br i1172; OPT: getelementptr i8,173 174; GCN-LABEL: {{^}}test_sink_scratch_small_offset_i32_reserved:175; GCN: s_and_saveexec_b64176; GCN: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:4092{{$}}177; GCN: buffer_load_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:4092 glc{{$}}178; GCN: {{^.LBB[0-9]+}}_2:179 180define amdgpu_kernel void @test_sink_scratch_small_offset_i32_reserved(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %arg) {181entry:182 %alloca = alloca [512 x i32], align 4, addrspace(5)183 %out.gep.0 = getelementptr i32, ptr addrspace(1) %out, i64 999998184 %out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i64 999999185 %add.arg = add i32 %arg, 8186 %alloca.gep = getelementptr [512 x i32], ptr addrspace(5) %alloca, i32 0, i32 1023187 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0188 %tmp0 = icmp eq i32 %tid, 0189 br i1 %tmp0, label %endif, label %if190 191if:192 store volatile i32 123, ptr addrspace(5) %alloca.gep193 %tmp1 = load volatile i32, ptr addrspace(5) %alloca.gep194 br label %endif195 196endif:197 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]198 store i32 %x, ptr addrspace(1) %out.gep.0199 %load = load volatile i32, ptr addrspace(5) %alloca.gep200 store i32 %load, ptr addrspace(1) %out.gep.1201 br label %done202 203done:204 ret void205}206 207; OPT-LABEL: @test_no_sink_scratch_large_offset_i32(208; OPT: %alloca.gep = getelementptr [512 x i32], ptr addrspace(5) %alloca, i32 0, i32 1024209; OPT: br i1210; OPT-NOT: ptrtoint211 212; GCN-LABEL: {{^}}test_no_sink_scratch_large_offset_i32:213; GCN: s_and_saveexec_b64214; GCN: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 offen{{$}}215; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 offen glc{{$}}216; GCN: {{^.LBB[0-9]+}}_2:217define amdgpu_kernel void @test_no_sink_scratch_large_offset_i32(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %arg) {218entry:219 %alloca = alloca [512 x i32], align 4, addrspace(5)220 %out.gep.0 = getelementptr i32, ptr addrspace(1) %out, i64 999998221 %out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i64 999999222 %add.arg = add i32 %arg, 8223 %alloca.gep = getelementptr [512 x i32], ptr addrspace(5) %alloca, i32 0, i32 1024224 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0225 %tmp0 = icmp eq i32 %tid, 0226 br i1 %tmp0, label %endif, label %if227 228if:229 store volatile i32 123, ptr addrspace(5) %alloca.gep230 %tmp1 = load volatile i32, ptr addrspace(5) %alloca.gep231 br label %endif232 233endif:234 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]235 store i32 %x, ptr addrspace(1) %out.gep.0236 %load = load volatile i32, ptr addrspace(5) %alloca.gep237 store i32 %load, ptr addrspace(1) %out.gep.1238 br label %done239 240done:241 ret void242}243 244; GCN-LABEL: {{^}}test_sink_global_vreg_sreg_i32:245; GCN: s_and_saveexec_b64246; CI: buffer_load_dword {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}247; VI: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}]248; GCN: {{^.LBB[0-9]+}}_2:249define amdgpu_kernel void @test_sink_global_vreg_sreg_i32(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %offset) {250entry:251 %offset.ext = zext i32 %offset to i64252 %out.gep = getelementptr i32, ptr addrspace(1) %out, i64 999999253 %in.gep = getelementptr i32, ptr addrspace(1) %in, i64 %offset.ext254 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0255 %tmp0 = icmp eq i32 %tid, 0256 br i1 %tmp0, label %endif, label %if257 258if:259 %tmp1 = load i32, ptr addrspace(1) %in.gep260 br label %endif261 262endif:263 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]264 store i32 %x, ptr addrspace(1) %out.gep265 br label %done266 267done:268 ret void269}270 271; OPT-LABEL: @test_sink_constant_small_offset_i32272; OPT-NOT: getelementptr i32, ptr addrspace(4)273; OPT: br i1274 275; GCN-LABEL: {{^}}test_sink_constant_small_offset_i32:276; GCN: s_and_saveexec_b64277; SI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0x7{{$}}278; GCN: s_or_b64 exec, exec279define amdgpu_kernel void @test_sink_constant_small_offset_i32(ptr addrspace(1) %out, ptr addrspace(4) %in) {280entry:281 %out.gep = getelementptr i32, ptr addrspace(1) %out, i64 999999282 %in.gep = getelementptr i32, ptr addrspace(4) %in, i64 7283 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0284 %tmp0 = icmp eq i32 %tid, 0285 br i1 %tmp0, label %endif, label %if286 287if:288 %tmp1 = load i32, ptr addrspace(4) %in.gep289 br label %endif290 291endif:292 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]293 store i32 %x, ptr addrspace(1) %out.gep294 br label %done295 296done:297 ret void298}299 300; OPT-LABEL: @test_sink_constant_max_8_bit_offset_i32301; OPT-NOT: getelementptr i32, ptr addrspace(4)302; OPT: br i1303 304; GCN-LABEL: {{^}}test_sink_constant_max_8_bit_offset_i32:305; GCN: s_and_saveexec_b64306; SI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0xff{{$}}307; GCN: s_or_b64 exec, exec308define amdgpu_kernel void @test_sink_constant_max_8_bit_offset_i32(ptr addrspace(1) %out, ptr addrspace(4) %in) {309entry:310 %out.gep = getelementptr i32, ptr addrspace(1) %out, i64 999999311 %in.gep = getelementptr i32, ptr addrspace(4) %in, i64 255312 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0313 %tmp0 = icmp eq i32 %tid, 0314 br i1 %tmp0, label %endif, label %if315 316if:317 %tmp1 = load i32, ptr addrspace(4) %in.gep318 br label %endif319 320endif:321 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]322 store i32 %x, ptr addrspace(1) %out.gep323 br label %done324 325done:326 ret void327}328 329; OPT-LABEL: @test_sink_constant_max_8_bit_offset_p1_i32330; OPT-SI: getelementptr i32, ptr addrspace(4)331; OPT-CI-NOT: getelementptr i32, ptr addrspace(4)332; OPT-VI-NOT: getelementptr i32, ptr addrspace(4)333; OPT: br i1334 335; GCN-LABEL: {{^}}test_sink_constant_max_8_bit_offset_p1_i32:336; GCN: s_and_saveexec_b64337; SI: s_movk_i32 [[OFFSET:s[0-9]+]], 0x400338 339; SI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, [[OFFSET]]{{$}}340; GCN: s_or_b64 exec, exec341define amdgpu_kernel void @test_sink_constant_max_8_bit_offset_p1_i32(ptr addrspace(1) %out, ptr addrspace(4) %in) {342entry:343 %out.gep = getelementptr i32, ptr addrspace(1) %out, i64 999999344 %in.gep = getelementptr i32, ptr addrspace(4) %in, i64 256345 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0346 %tmp0 = icmp eq i32 %tid, 0347 br i1 %tmp0, label %endif, label %if348 349if:350 %tmp1 = load i32, ptr addrspace(4) %in.gep351 br label %endif352 353endif:354 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]355 store i32 %x, ptr addrspace(1) %out.gep356 br label %done357 358done:359 ret void360}361 362; OPT-LABEL: @test_sink_constant_max_32_bit_offset_i32363; OPT-SI: getelementptr i32, ptr addrspace(4)364; OPT-CI-NOT: getelementptr i32, ptr addrspace(4)365; OPT: br i1366 367; GCN-LABEL: {{^}}test_sink_constant_max_32_bit_offset_i32:368; GCN: s_and_saveexec_b64369; SI: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, -4{{$}}370; SI: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, 3{{$}}371; SI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0x0{{$}}372 373; VI: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, -4{{$}}374; VI: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, 3{{$}}375; VI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0x0{{$}}376 377; CI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0xffffffff{{$}}378 379; GCN: s_or_b64 exec, exec380define amdgpu_kernel void @test_sink_constant_max_32_bit_offset_i32(ptr addrspace(1) %out, ptr addrspace(4) %in) {381entry:382 %out.gep = getelementptr i32, ptr addrspace(1) %out, i64 999999383 %in.gep = getelementptr i32, ptr addrspace(4) %in, i64 4294967295384 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0385 %tmp0 = icmp eq i32 %tid, 0386 br i1 %tmp0, label %endif, label %if387 388if:389 %tmp1 = load i32, ptr addrspace(4) %in.gep390 br label %endif391 392endif:393 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]394 store i32 %x, ptr addrspace(1) %out.gep395 br label %done396 397done:398 ret void399}400 401; OPT-LABEL: @test_sink_constant_max_32_bit_offset_p1_i32402; OPT: getelementptr i32, ptr addrspace(4)403; OPT: br i1404 405; GCN-LABEL: {{^}}test_sink_constant_max_32_bit_offset_p1_i32:406; GCN: s_and_saveexec_b64407; GCN: s_add_u32408; GCN: s_addc_u32409; SI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0x0{{$}}410; GCN: s_or_b64 exec, exec411define amdgpu_kernel void @test_sink_constant_max_32_bit_offset_p1_i32(ptr addrspace(1) %out, ptr addrspace(4) %in) {412entry:413 %out.gep = getelementptr i32, ptr addrspace(1) %out, i64 999999414 %in.gep = getelementptr i32, ptr addrspace(4) %in, i64 17179869181415 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0416 %tmp0 = icmp eq i32 %tid, 0417 br i1 %tmp0, label %endif, label %if418 419if:420 %tmp1 = load i32, ptr addrspace(4) %in.gep421 br label %endif422 423endif:424 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]425 store i32 %x, ptr addrspace(1) %out.gep426 br label %done427 428done:429 ret void430}431 432; GCN-LABEL: {{^}}test_sink_constant_max_20_bit_byte_offset_i32:433; GCN: s_and_saveexec_b64434; SI: s_mov_b32 [[OFFSET:s[0-9]+]], 0xffffc{{$}}435; SI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, [[OFFSET]]{{$}}436 437; CI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0x3ffff{{$}}438; VI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0xffffc{{$}}439 440; GCN: s_or_b64 exec, exec441define amdgpu_kernel void @test_sink_constant_max_20_bit_byte_offset_i32(ptr addrspace(1) %out, ptr addrspace(4) %in) {442entry:443 %out.gep = getelementptr i32, ptr addrspace(1) %out, i64 999999444 %in.gep = getelementptr i32, ptr addrspace(4) %in, i64 262143445 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0446 %tmp0 = icmp eq i32 %tid, 0447 br i1 %tmp0, label %endif, label %if448 449if:450 %tmp1 = load i32, ptr addrspace(4) %in.gep451 br label %endif452 453endif:454 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]455 store i32 %x, ptr addrspace(1) %out.gep456 br label %done457 458done:459 ret void460}461 462; OPT-LABEL: @test_sink_constant_max_20_bit_byte_offset_p1_i32463; OPT-SI: getelementptr i32, ptr addrspace(4)464; OPT-CI-NOT: getelementptr i32, ptr addrspace(4)465; OPT-VI: getelementptr i32, ptr addrspace(4)466; OPT: br i1467 468; GCN-LABEL: {{^}}test_sink_constant_max_20_bit_byte_offset_p1_i32:469; GCN: s_and_saveexec_b64470; SI: s_mov_b32 [[OFFSET:s[0-9]+]], 0x100000{{$}}471; SI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, [[OFFSET]]{{$}}472 473; CI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0x40000{{$}}474 475; VI: s_mov_b32 [[OFFSET:s[0-9]+]], 0x100000{{$}}476; VI: s_load_dword s{{[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, [[OFFSET]]{{$}}477 478; GCN: s_or_b64 exec, exec479define amdgpu_kernel void @test_sink_constant_max_20_bit_byte_offset_p1_i32(ptr addrspace(1) %out, ptr addrspace(4) %in) {480entry:481 %out.gep = getelementptr i32, ptr addrspace(1) %out, i64 999999482 %in.gep = getelementptr i32, ptr addrspace(4) %in, i64 262144483 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0484 %tmp0 = icmp eq i32 %tid, 0485 br i1 %tmp0, label %endif, label %if486 487if:488 %tmp1 = load i32, ptr addrspace(4) %in.gep489 br label %endif490 491endif:492 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]493 store i32 %x, ptr addrspace(1) %out.gep494 br label %done495 496done:497 ret void498}499 500%struct.foo = type { [3 x float], [3 x float] }501 502; OPT-LABEL: @sink_ds_address(503; OPT: getelementptr inbounds i8,504 505; GCN-LABEL: {{^}}sink_ds_address:506; GCN: s_load_dword [[SREG1:s[0-9]+]],507; GCN: v_mov_b32_e32 [[VREG1:v[0-9]+]], [[SREG1]]508; GCN-DAG: ds_read2_b32 v[{{[0-9+:[0-9]+}}], [[VREG1]] offset0:3 offset1:5509define amdgpu_kernel void @sink_ds_address(ptr addrspace(3) nocapture %ptr) nounwind {510entry:511 %x = getelementptr inbounds %struct.foo, ptr addrspace(3) %ptr, i32 0, i32 1, i32 0512 %y = getelementptr inbounds %struct.foo, ptr addrspace(3) %ptr, i32 0, i32 1, i32 2513 br label %bb32514 515bb32:516 %a = load float, ptr addrspace(3) %x, align 4517 %b = load float, ptr addrspace(3) %y, align 4518 %cmp = fcmp one float %a, %b519 br i1 %cmp, label %bb34, label %bb33520 521bb33:522 unreachable523 524bb34:525 unreachable526}527 528; Address offset is not a multiple of 4. This is a valid mubuf offset,529; but not smrd.530 531; OPT-LABEL: @test_sink_constant_small_max_mubuf_offset_load_i32_align_1(532; OPT: br i1 %tmp0,533; OPT: if:534; OPT: getelementptr i8, {{.*}} 4095535define amdgpu_kernel void @test_sink_constant_small_max_mubuf_offset_load_i32_align_1(ptr addrspace(1) %out, ptr addrspace(4) %in) {536entry:537 %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 1024538 %in.gep = getelementptr i8, ptr addrspace(4) %in, i64 4095539 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0540 %tmp0 = icmp eq i32 %tid, 0541 br i1 %tmp0, label %endif, label %if542 543if:544 %tmp1 = load i32, ptr addrspace(4) %in.gep, align 1545 br label %endif546 547endif:548 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]549 store i32 %x, ptr addrspace(1) %out.gep550 br label %done551 552done:553 ret void554}555 556; OPT-LABEL: @test_sink_local_small_offset_atomicrmw_i32(557; OPT: %sunkaddr = getelementptr i8, ptr addrspace(3) %in, i32 28558; OPT: %tmp1 = atomicrmw add ptr addrspace(3) %sunkaddr, i32 2 seq_cst559define amdgpu_kernel void @test_sink_local_small_offset_atomicrmw_i32(ptr addrspace(3) %out, ptr addrspace(3) %in) {560entry:561 %out.gep = getelementptr i32, ptr addrspace(3) %out, i32 999999562 %in.gep = getelementptr i32, ptr addrspace(3) %in, i32 7563 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0564 %tmp0 = icmp eq i32 %tid, 0565 br i1 %tmp0, label %endif, label %if566 567if:568 %tmp1 = atomicrmw add ptr addrspace(3) %in.gep, i32 2 seq_cst569 br label %endif570 571endif:572 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]573 store i32 %x, ptr addrspace(3) %out.gep574 br label %done575 576done:577 ret void578}579 580; OPT-LABEL: @test_sink_local_small_offset_cmpxchg_i32(581; OPT: %sunkaddr = getelementptr i8, ptr addrspace(3) %in, i32 28582; OPT: %tmp1.struct = cmpxchg ptr addrspace(3) %sunkaddr, i32 poison, i32 2 seq_cst monotonic583define amdgpu_kernel void @test_sink_local_small_offset_cmpxchg_i32(ptr addrspace(3) %out, ptr addrspace(3) %in) {584entry:585 %out.gep = getelementptr i32, ptr addrspace(3) %out, i32 999999586 %in.gep = getelementptr i32, ptr addrspace(3) %in, i32 7587 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0588 %tmp0 = icmp eq i32 %tid, 0589 br i1 %tmp0, label %endif, label %if590 591if:592 %tmp1.struct = cmpxchg ptr addrspace(3) %in.gep, i32 poison, i32 2 seq_cst monotonic593 %tmp1 = extractvalue { i32, i1 } %tmp1.struct, 0594 br label %endif595 596endif:597 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]598 store i32 %x, ptr addrspace(3) %out.gep599 br label %done600 601done:602 ret void603}604 605; OPT-LABEL: @test_wrong_operand_local_small_offset_cmpxchg_i32(606; OPT: %in.gep = getelementptr i32, ptr addrspace(3) %in, i32 7607; OPT: br i1608; OPT: cmpxchg ptr addrspace(3) poison, ptr addrspace(3) %in.gep, ptr addrspace(3) poison seq_cst monotonic609define amdgpu_kernel void @test_wrong_operand_local_small_offset_cmpxchg_i32(ptr addrspace(3) %out, ptr addrspace(3) %in) {610entry:611 %out.gep = getelementptr ptr addrspace(3), ptr addrspace(3) %out, i32 999999612 %in.gep = getelementptr i32, ptr addrspace(3) %in, i32 7613 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0614 %tmp0 = icmp eq i32 %tid, 0615 br i1 %tmp0, label %endif, label %if616 617if:618 %tmp1.struct = cmpxchg ptr addrspace(3) poison, ptr addrspace(3) %in.gep, ptr addrspace(3) poison seq_cst monotonic619 %tmp1 = extractvalue { ptr addrspace(3), i1 } %tmp1.struct, 0620 br label %endif621 622endif:623 %x = phi ptr addrspace(3) [ %tmp1, %if ], [ null, %entry ]624 store ptr addrspace(3) %x, ptr addrspace(3) %out.gep625 br label %done626 627done:628 ret void629}630 631; OPT-LABEL: @test_sink_global_small_min_scratch_global_offset(632; OPT-SICIVI: %in.gep = getelementptr i8, ptr addrspace(1) %in, i64 -4096633; OPT-SICIV: br634; OPT-SICIVI: %tmp1 = load i8, ptr addrspace(1) %in.gep635 636; OPT-GFX9: br637; OPT-GFX9: %sunkaddr = getelementptr i8, ptr addrspace(1) %in, i64 -4096638; OPT-GFX9: load i8, ptr addrspace(1) %sunkaddr639 640; GCN-LABEL: {{^}}test_sink_global_small_min_scratch_global_offset:641; GFX9: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}642; GFX9: global_load_sbyte v{{[0-9]+}}, [[ZERO]], s{{\[[0-9]+:[0-9]+\]}} offset:-4096{{$}}643define amdgpu_kernel void @test_sink_global_small_min_scratch_global_offset(ptr addrspace(1) %out, ptr addrspace(1) %in) {644entry:645 %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 1024646 %in.gep = getelementptr i8, ptr addrspace(1) %in, i64 -4096647 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0648 %tmp0 = icmp eq i32 %tid, 0649 br i1 %tmp0, label %endif, label %if650 651if:652 %tmp1 = load i8, ptr addrspace(1) %in.gep653 %tmp2 = sext i8 %tmp1 to i32654 br label %endif655 656endif:657 %x = phi i32 [ %tmp2, %if ], [ 0, %entry ]658 store i32 %x, ptr addrspace(1) %out.gep659 br label %done660 661done:662 ret void663}664 665; OPT-LABEL: @test_sink_global_small_min_scratch_global_neg1_offset(666; OPT: %in.gep = getelementptr i8, ptr addrspace(1) %in, i64 -4097667; OPT: br668; OPT: load i8, ptr addrspace(1) %in.gep669 670; GCN-LABEL: {{^}}test_sink_global_small_min_scratch_global_neg1_offset:671define amdgpu_kernel void @test_sink_global_small_min_scratch_global_neg1_offset(ptr addrspace(1) %out, ptr addrspace(1) %in) {672entry:673 %out.gep = getelementptr i32, ptr addrspace(1) %out, i64 99999674 %in.gep = getelementptr i8, ptr addrspace(1) %in, i64 -4097675 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0676 %tmp0 = icmp eq i32 %tid, 0677 br i1 %tmp0, label %endif, label %if678 679if:680 %tmp1 = load i8, ptr addrspace(1) %in.gep681 %tmp2 = sext i8 %tmp1 to i32682 br label %endif683 684endif:685 %x = phi i32 [ %tmp2, %if ], [ 0, %entry ]686 store i32 %x, ptr addrspace(1) %out.gep687 br label %done688 689done:690 ret void691}692 693; OPT-LABEL: @test_sink_small_offset_ds_append(694; OPT: %sunkaddr = getelementptr i8, ptr addrspace(3) %in, i32 28695; OPT: %tmp1 = call i32 @llvm.amdgcn.ds.append.p3(ptr addrspace(3) %sunkaddr, i1 false)696define amdgpu_kernel void @test_sink_small_offset_ds_append(ptr addrspace(3) %out, ptr addrspace(3) %in) {697entry:698 %out.gep = getelementptr i32, ptr addrspace(3) %out, i32 999999699 %in.gep = getelementptr i32, ptr addrspace(3) %in, i32 7700 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0701 %tmp0 = icmp eq i32 %tid, 0702 br i1 %tmp0, label %endif, label %if703 704if:705 %tmp1 = call i32 @llvm.amdgcn.ds.append.p3(ptr addrspace(3) %in.gep, i1 false)706 br label %endif707 708endif:709 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]710 store i32 %x, ptr addrspace(3) %out.gep711 br label %done712 713done:714 ret void715}716 717; OPT-LABEL: @test_sink_small_offset_ds_consume(718; OPT: %sunkaddr = getelementptr i8, ptr addrspace(3) %in, i32 28719; OPT: %tmp1 = call i32 @llvm.amdgcn.ds.consume.p3(ptr addrspace(3) %sunkaddr, i1 false)720define amdgpu_kernel void @test_sink_small_offset_ds_consume(ptr addrspace(3) %out, ptr addrspace(3) %in) {721entry:722 %out.gep = getelementptr i32, ptr addrspace(3) %out, i32 999999723 %in.gep = getelementptr i32, ptr addrspace(3) %in, i32 7724 %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0725 %tmp0 = icmp eq i32 %tid, 0726 br i1 %tmp0, label %endif, label %if727 728if:729 %tmp1 = call i32 @llvm.amdgcn.ds.consume.p3(ptr addrspace(3) %in.gep, i1 false)730 br label %endif731 732endif:733 %x = phi i32 [ %tmp1, %if ], [ 0, %entry ]734 store i32 %x, ptr addrspace(3) %out.gep735 br label %done736 737done:738 ret void739}740 741declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #0742declare i32 @llvm.amdgcn.ds.append.p3(ptr addrspace(3) nocapture, i1 immarg) #3743declare i32 @llvm.amdgcn.ds.consume.p3(ptr addrspace(3) nocapture, i1 immarg) #3744 745attributes #0 = { nounwind readnone }746attributes #1 = { nounwind }747attributes #2 = { nounwind argmemonly }748attributes #3 = { argmemonly convergent nounwind willreturn }749