122 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -S -mtriple=amdgcn-- -amdgpu-atomic-optimizer-strategy=Iterative -passes='amdgpu-atomic-optimizer,verify<domtree>' %s | FileCheck -check-prefix=IR %s3 4define amdgpu_kernel void @uniform_value(ptr addrspace(1) , ptr addrspace(1) %val) #0 {5; IR-LABEL: @uniform_value(6; IR-NEXT: entry:7; IR-NEXT: [[UNIFORM_VALUE_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(52) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()8; IR-NEXT: [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[UNIFORM_VALUE_KERNARG_SEGMENT]], i64 369; IR-NEXT: [[LOADED_OUT_KERNARG_OFFSET:%.*]] = load <2 x i64>, ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 410; IR-NEXT: [[OUT_LOAD1:%.*]] = extractelement <2 x i64> [[LOADED_OUT_KERNARG_OFFSET]], i32 011; IR-NEXT: [[MEM_LOCATION:%.*]] = inttoptr i64 [[OUT_LOAD1]] to ptr addrspace(1)12; IR-NEXT: [[VAL_LOAD2:%.*]] = extractelement <2 x i64> [[LOADED_OUT_KERNARG_OFFSET]], i32 113; IR-NEXT: [[VALUE_ADDRESS:%.*]] = inttoptr i64 [[VAL_LOAD2]] to ptr addrspace(1)14; IR-NEXT: [[LANE:%.*]] = tail call i32 @llvm.amdgcn.workgroup.id.x()15; IR-NEXT: [[IDXPROM:%.*]] = sext i32 [[LANE]] to i6416; IR-NEXT: [[ELE:%.*]] = getelementptr i32, ptr addrspace(1) [[VALUE_ADDRESS]], i64 [[IDXPROM]]17; IR-NEXT: [[VALUE:%.*]] = load i32, ptr addrspace(1) [[ELE]], align 418; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[MEM_LOCATION]], i32 419; IR-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)20; IR-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i3221; IR-NEXT: [[TMP3:%.*]] = lshr i64 [[TMP1]], 3222; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i3223; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)24; IR-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])25; IR-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])26; IR-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i3227; IR-NEXT: [[TMP9:%.*]] = mul i32 [[VALUE]], [[TMP8]]28; IR-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP6]], 029; IR-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]30; IR: 11:31; IR-NEXT: [[TMP12:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP9]] seq_cst, align 432; IR-NEXT: br label [[TMP13]]33; IR: 13:34; IR-NEXT: ret void35;36entry:37 %uniform_value.kernarg.segment = call nonnull align 16 dereferenceable(52) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()38 %out.kernarg.offset = getelementptr inbounds i8, ptr addrspace(4) %uniform_value.kernarg.segment, i64 3639 %loaded.out.kernarg.offset = load <2 x i64>, ptr addrspace(4) %out.kernarg.offset, align 440 %out.load1 = extractelement <2 x i64> %loaded.out.kernarg.offset, i32 041 %mem.location = inttoptr i64 %out.load1 to ptr addrspace(1)42 %val.load2 = extractelement <2 x i64> %loaded.out.kernarg.offset, i32 143 %value.address = inttoptr i64 %val.load2 to ptr addrspace(1)44 %lane = tail call i32 @llvm.amdgcn.workgroup.id.x()45 %idxprom = sext i32 %lane to i6446 %ele = getelementptr i32, ptr addrspace(1) %value.address, i64 %idxprom47 %value = load i32, ptr addrspace(1) %ele, align 448 %gep = getelementptr i32, ptr addrspace(1) %mem.location, i32 449 %old = atomicrmw volatile add ptr addrspace(1) %gep, i32 %value seq_cst, align 450 ret void51}52 53define amdgpu_kernel void @divergent_value(ptr addrspace(1) %out, ptr addrspace(1) %val) #0 {54; IR-LABEL: @divergent_value(55; IR-NEXT: entry:56; IR-NEXT: [[DIVERGENT_VALUE_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(52) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()57; IR-NEXT: [[OUT_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[DIVERGENT_VALUE_KERNARG_SEGMENT]], i64 3658; IR-NEXT: [[LOADED_OUT_KERNARG_OFFSET:%.*]] = load <2 x i64>, ptr addrspace(4) [[OUT_KERNARG_OFFSET]], align 459; IR-NEXT: [[OUT_LOAD1:%.*]] = extractelement <2 x i64> [[LOADED_OUT_KERNARG_OFFSET]], i32 060; IR-NEXT: [[MEM_LOCATION:%.*]] = inttoptr i64 [[OUT_LOAD1]] to ptr addrspace(1)61; IR-NEXT: [[VAL_LOAD2:%.*]] = extractelement <2 x i64> [[LOADED_OUT_KERNARG_OFFSET]], i32 162; IR-NEXT: [[VALUE_ADDRESS:%.*]] = inttoptr i64 [[VAL_LOAD2]] to ptr addrspace(1)63; IR-NEXT: [[LANE:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()64; IR-NEXT: [[IDXPROM:%.*]] = sext i32 [[LANE]] to i6465; IR-NEXT: [[ELE:%.*]] = getelementptr i32, ptr addrspace(1) [[VALUE_ADDRESS]], i64 [[IDXPROM]]66; IR-NEXT: [[VALUE:%.*]] = load i32, ptr addrspace(1) [[ELE]], align 467; IR-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[MEM_LOCATION]], i32 468; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)69; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i3270; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 3271; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i3272; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)73; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])74; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)75; IR-NEXT: br label [[COMPUTELOOP:%.*]]76; IR: 7:77; IR-NEXT: [[TMP8:%.*]] = atomicrmw volatile add ptr addrspace(1) [[GEP]], i32 [[TMP13:%.*]] seq_cst, align 478; IR-NEXT: br label [[TMP9:%.*]]79; IR: 9:80; IR-NEXT: ret void81; IR: ComputeLoop:82; IR-NEXT: [[ACCUMULATOR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP13]], [[COMPUTELOOP]] ]83; IR-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP6]], [[ENTRY]] ], [ [[TMP16:%.*]], [[COMPUTELOOP]] ]84; IR-NEXT: [[TMP10:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)85; IR-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP10]] to i3286; IR-NEXT: [[TMP12:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[VALUE]], i32 [[TMP11]])87; IR-NEXT: [[TMP13]] = add i32 [[ACCUMULATOR]], [[TMP12]]88; IR-NEXT: [[TMP14:%.*]] = shl i64 1, [[TMP10]]89; IR-NEXT: [[TMP15:%.*]] = xor i64 [[TMP14]], -190; IR-NEXT: [[TMP16]] = and i64 [[ACTIVEBITS]], [[TMP15]]91; IR-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 092; IR-NEXT: br i1 [[TMP17]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]93; IR: ComputeEnd:94; IR-NEXT: [[TMP18:%.*]] = icmp eq i32 [[TMP5]], 095; IR-NEXT: br i1 [[TMP18]], label [[TMP7:%.*]], label [[TMP9]]96;97entry:98 %divergent_value.kernarg.segment = call nonnull align 16 dereferenceable(52) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()99 %out.kernarg.offset = getelementptr inbounds i8, ptr addrspace(4) %divergent_value.kernarg.segment, i64 36100 %loaded.out.kernarg.offset = load <2 x i64>, ptr addrspace(4) %out.kernarg.offset, align 4101 %out.load1 = extractelement <2 x i64> %loaded.out.kernarg.offset, i32 0102 %mem.location = inttoptr i64 %out.load1 to ptr addrspace(1)103 %val.load2 = extractelement <2 x i64> %loaded.out.kernarg.offset, i32 1104 %value.address = inttoptr i64 %val.load2 to ptr addrspace(1)105 %lane = tail call i32 @llvm.amdgcn.workitem.id.x()106 %idxprom = sext i32 %lane to i64107 %ele = getelementptr i32, ptr addrspace(1) %value.address, i64 %idxprom108 %value = load i32, ptr addrspace(1) %ele, align 4109 %gep = getelementptr i32, ptr addrspace(1) %mem.location, i32 4110 %old = atomicrmw volatile add ptr addrspace(1) %gep, i32 %value seq_cst, align 4111 ret void112}113 114declare i32 @llvm.amdgcn.workitem.id.x() #1115declare i32 @llvm.amdgcn.workgroup.id.x() #1116 117declare align 4 ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()118 119attributes #0 = {"target-cpu"="gfx906"}120attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none)}121 122