177 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -passes=amdgpu-atomic-optimizer %s | FileCheck %s3 4define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i8 %val) {5; CHECK-LABEL: define amdgpu_kernel void @uniform_or_i8(6; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], i8 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {7; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)8; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i329; CHECK-NEXT: [[TMP3:%.*]] = lshr i64 [[TMP1]], 3210; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i3211; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)12; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])13; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 014; CHECK-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]15; CHECK: [[BB8]]:16; CHECK-NEXT: [[TMP9:%.*]] = atomicrmw or ptr addrspace(1) [[UNIFORM_PTR]], i8 [[VAL]] monotonic, align 117; CHECK-NEXT: br label %[[BB10]]18; CHECK: [[BB10]]:19; CHECK-NEXT: [[TMP11:%.*]] = phi i8 [ poison, [[TMP0:%.*]] ], [ [[TMP9]], %[[BB8]] ]20; CHECK-NEXT: [[TMP16:%.*]] = zext i8 [[TMP11]] to i3221; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP16]])22; CHECK-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP17]] to i823; CHECK-NEXT: [[TMP13:%.*]] = trunc i32 [[TMP6]] to i824; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP7]], i8 0, i8 [[VAL]]25; CHECK-NEXT: [[TMP15:%.*]] = or i8 [[TMP12]], [[TMP14]]26; CHECK-NEXT: store i8 [[TMP15]], ptr addrspace(1) [[RESULT]], align 127; CHECK-NEXT: ret void28;29 %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 130 store i8 %rmw, ptr addrspace(1) %result31 ret void32}33 34define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i8 %val) {35; CHECK-LABEL: define amdgpu_kernel void @uniform_add_i8(36; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], i8 [[VAL:%.*]]) #[[ATTR0]] {37; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)38; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i3239; CHECK-NEXT: [[TMP3:%.*]] = lshr i64 [[TMP1]], 3240; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i3241; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)42; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])43; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])44; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i845; CHECK-NEXT: [[TMP9:%.*]] = mul i8 [[VAL]], [[TMP8]]46; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP6]], 047; CHECK-NEXT: br i1 [[TMP10]], label %[[BB11:.*]], label %[[BB13:.*]]48; CHECK: [[BB11]]:49; CHECK-NEXT: [[TMP12:%.*]] = atomicrmw add ptr addrspace(1) [[UNIFORM_PTR]], i8 [[TMP9]] monotonic, align 150; CHECK-NEXT: br label %[[BB13]]51; CHECK: [[BB13]]:52; CHECK-NEXT: [[TMP14:%.*]] = phi i8 [ poison, [[TMP0:%.*]] ], [ [[TMP12]], %[[BB11]] ]53; CHECK-NEXT: [[TMP19:%.*]] = zext i8 [[TMP14]] to i3254; CHECK-NEXT: [[TMP20:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP19]])55; CHECK-NEXT: [[TMP15:%.*]] = trunc i32 [[TMP20]] to i856; CHECK-NEXT: [[TMP16:%.*]] = trunc i32 [[TMP6]] to i857; CHECK-NEXT: [[TMP17:%.*]] = mul i8 [[VAL]], [[TMP16]]58; CHECK-NEXT: [[TMP18:%.*]] = add i8 [[TMP15]], [[TMP17]]59; CHECK-NEXT: store i8 [[TMP18]], ptr addrspace(1) [[RESULT]], align 160; CHECK-NEXT: ret void61;62 %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 163 store i8 %rmw, ptr addrspace(1) %result64 ret void65}66 67define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i8 %val) {68; CHECK-LABEL: define amdgpu_kernel void @uniform_xchg_i8(69; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], i8 [[VAL:%.*]]) #[[ATTR0]] {70; CHECK-NEXT: [[RMW:%.*]] = atomicrmw xchg ptr addrspace(1) [[UNIFORM_PTR]], i8 [[VAL]] monotonic, align 171; CHECK-NEXT: store i8 [[RMW]], ptr addrspace(1) [[RESULT]], align 172; CHECK-NEXT: ret void73;74 %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 175 store i8 %rmw, ptr addrspace(1) %result76 ret void77}78 79define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i16 %val) {80; CHECK-LABEL: define amdgpu_kernel void @uniform_or_i16(81; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], i16 [[VAL:%.*]]) #[[ATTR0]] {82; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)83; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i3284; CHECK-NEXT: [[TMP3:%.*]] = lshr i64 [[TMP1]], 3285; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i3286; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)87; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])88; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 089; CHECK-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]90; CHECK: [[BB8]]:91; CHECK-NEXT: [[TMP9:%.*]] = atomicrmw or ptr addrspace(1) [[UNIFORM_PTR]], i16 [[VAL]] monotonic, align 292; CHECK-NEXT: br label %[[BB10]]93; CHECK: [[BB10]]:94; CHECK-NEXT: [[TMP11:%.*]] = phi i16 [ poison, [[TMP0:%.*]] ], [ [[TMP9]], %[[BB8]] ]95; CHECK-NEXT: [[TMP16:%.*]] = zext i16 [[TMP11]] to i3296; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP16]])97; CHECK-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP17]] to i1698; CHECK-NEXT: [[TMP13:%.*]] = trunc i32 [[TMP6]] to i1699; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP7]], i16 0, i16 [[VAL]]100; CHECK-NEXT: [[TMP15:%.*]] = or i16 [[TMP12]], [[TMP14]]101; CHECK-NEXT: store i16 [[TMP15]], ptr addrspace(1) [[RESULT]], align 2102; CHECK-NEXT: ret void103;104 %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2105 store i16 %rmw, ptr addrspace(1) %result106 ret void107}108 109define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i16 %val) {110; CHECK-LABEL: define amdgpu_kernel void @uniform_add_i16(111; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], i16 [[VAL:%.*]]) #[[ATTR0]] {112; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)113; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32114; CHECK-NEXT: [[TMP3:%.*]] = lshr i64 [[TMP1]], 32115; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32116; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)117; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])118; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])119; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i16120; CHECK-NEXT: [[TMP9:%.*]] = mul i16 [[VAL]], [[TMP8]]121; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP6]], 0122; CHECK-NEXT: br i1 [[TMP10]], label %[[BB11:.*]], label %[[BB13:.*]]123; CHECK: [[BB11]]:124; CHECK-NEXT: [[TMP12:%.*]] = atomicrmw add ptr addrspace(1) [[UNIFORM_PTR]], i16 [[TMP9]] monotonic, align 2125; CHECK-NEXT: br label %[[BB13]]126; CHECK: [[BB13]]:127; CHECK-NEXT: [[TMP14:%.*]] = phi i16 [ poison, [[TMP0:%.*]] ], [ [[TMP12]], %[[BB11]] ]128; CHECK-NEXT: [[TMP19:%.*]] = zext i16 [[TMP14]] to i32129; CHECK-NEXT: [[TMP20:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP19]])130; CHECK-NEXT: [[TMP15:%.*]] = trunc i32 [[TMP20]] to i16131; CHECK-NEXT: [[TMP16:%.*]] = trunc i32 [[TMP6]] to i16132; CHECK-NEXT: [[TMP17:%.*]] = mul i16 [[VAL]], [[TMP16]]133; CHECK-NEXT: [[TMP18:%.*]] = add i16 [[TMP15]], [[TMP17]]134; CHECK-NEXT: store i16 [[TMP18]], ptr addrspace(1) [[RESULT]], align 2135; CHECK-NEXT: ret void136;137 %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2138 store i16 %rmw, ptr addrspace(1) %result139 ret void140}141 142define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i16 %val) {143; CHECK-LABEL: define amdgpu_kernel void @uniform_xchg_i16(144; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], i16 [[VAL:%.*]]) #[[ATTR0]] {145; CHECK-NEXT: [[RMW:%.*]] = atomicrmw xchg ptr addrspace(1) [[UNIFORM_PTR]], i16 [[VAL]] monotonic, align 2146; CHECK-NEXT: store i16 [[RMW]], ptr addrspace(1) [[RESULT]], align 2147; CHECK-NEXT: ret void148;149 %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2150 store i16 %rmw, ptr addrspace(1) %result151 ret void152}153 154define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, half %val) {155; CHECK-LABEL: define amdgpu_kernel void @uniform_fadd_f16(156; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], half [[VAL:%.*]]) #[[ATTR0]] {157; CHECK-NEXT: [[RMW:%.*]] = atomicrmw fadd ptr addrspace(1) [[UNIFORM_PTR]], half [[VAL]] monotonic, align 2158; CHECK-NEXT: store half [[RMW]], ptr addrspace(1) [[RESULT]], align 2159; CHECK-NEXT: ret void160;161 %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, half %val monotonic, align 2162 store half %rmw, ptr addrspace(1) %result163 ret void164}165 166define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, bfloat %val) {167; CHECK-LABEL: define amdgpu_kernel void @uniform_fadd_bf16(168; CHECK-SAME: ptr addrspace(1) [[RESULT:%.*]], ptr addrspace(1) [[UNIFORM_PTR:%.*]], bfloat [[VAL:%.*]]) #[[ATTR0]] {169; CHECK-NEXT: [[RMW:%.*]] = atomicrmw fadd ptr addrspace(1) [[UNIFORM_PTR]], bfloat [[VAL]] monotonic, align 2170; CHECK-NEXT: store bfloat [[RMW]], ptr addrspace(1) [[RESULT]], align 2171; CHECK-NEXT: ret void172;173 %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, bfloat %val monotonic, align 2174 store bfloat %rmw, ptr addrspace(1) %result175 ret void176}177