124 lines · plain
1// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx90a %s | FileCheck %s --check-prefixes=CHECK,GFX90A2// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx1030 %s | FileCheck %s --check-prefixes=CHECK,GFX103// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx1100 %s | FileCheck %s --check-prefixes=CHECK,GFX114// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx1200 %s | FileCheck %s --check-prefixes=CHECK,GFX125// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx942 %s | FileCheck %s --check-prefixes=CHECK,GFX9426// RUN: mlir-opt -split-input-file -amdgpu-emulate-atomics=chipset=gfx950 %s | FileCheck %s --check-prefixes=CHECK,GFX9507 8// -----9 10func.func @atomic_fmax(%val: f32, %buffer: memref<?xf32>, %idx: i32) {11// CHECK: func @atomic_fmax12// CHECK-SAME: ([[val:%.+]]: f32, [[buffer:%.+]]: memref<?xf32>, [[idx:%.+]]: i32)13// CHECK: gpu.printf "Begin\0A"14// GFX10: amdgpu.raw_buffer_atomic_fmax {foo, indexOffset = 4 : i32} [[val]] -> [[buffer]][[[idx]]]15// GFX11: amdgpu.raw_buffer_atomic_fmax {foo, indexOffset = 4 : i32} [[val]] -> [[buffer]][[[idx]]]16// GFX12: amdgpu.raw_buffer_atomic_fmax {foo, indexOffset = 4 : i32} [[val]] -> [[buffer]][[[idx]]]17// GFX90A: [[ld:%.+]] = amdgpu.raw_buffer_load {foo, indexOffset = 4 : i32} [[buffer]][[[idx]]]18// GFX90A: cf.br [[loop:\^.+]]([[ld]] : f32)19// GFX90A: [[loop]]([[arg:%.+]]: f32):20// GFX90A: [[operated:%.+]] = arith.maximumf [[val]], [[arg]]21// GFX90A: [[atomicRes:%.+]] = amdgpu.raw_buffer_atomic_cmpswap {foo, indexOffset = 4 : i32} [[operated]], [[arg]] -> [[buffer]][[[idx]]]22// GFX90A: [[argCast:%.+]] = arith.bitcast [[arg]] : f32 to i3223// GFX90A: [[resCast:%.+]] = arith.bitcast [[atomicRes]] : f32 to i3224// GFX90A: [[test:%.+]] = arith.cmpi eq, [[resCast]], [[argCast]]25// GFX90A: cf.cond_br [[test]], [[post:\^.+]], [[loop]]([[atomicRes]] : f32)26// GFX90A: [[post]]:27// GFX942: [[ld:%.+]] = amdgpu.raw_buffer_load {foo, indexOffset = 4 : i32} [[buffer]][[[idx]]]28// GFX942: cf.br [[loop:\^.+]]([[ld]] : f32)29// GFX942: [[loop]]([[arg:%.+]]: f32):30// GFX942: [[operated:%.+]] = arith.maximumf [[val]], [[arg]]31// GFX942: [[atomicRes:%.+]] = amdgpu.raw_buffer_atomic_cmpswap {foo, indexOffset = 4 : i32} [[operated]], [[arg]] -> [[buffer]][[[idx]]]32// GFX942: [[argCast:%.+]] = arith.bitcast [[arg]] : f32 to i3233// GFX942: [[resCast:%.+]] = arith.bitcast [[atomicRes]] : f32 to i3234// GFX942: [[test:%.+]] = arith.cmpi eq, [[resCast]], [[argCast]]35// GFX942: cf.cond_br [[test]], [[post:\^.+]], [[loop]]([[atomicRes]] : f32)36// GFX942: [[post]]:37// GFX950: [[ld:%.+]] = amdgpu.raw_buffer_load {foo, indexOffset = 4 : i32} [[buffer]][[[idx]]]38// GFX950: cf.br [[loop:\^.+]]([[ld]] : f32)39// GFX950: [[loop]]([[arg:%.+]]: f32):40// GFX950: [[operated:%.+]] = arith.maximumf [[val]], [[arg]]41// GFX950: [[atomicRes:%.+]] = amdgpu.raw_buffer_atomic_cmpswap {foo, indexOffset = 4 : i32} [[operated]], [[arg]] -> [[buffer]][[[idx]]]42// GFX950: [[argCast:%.+]] = arith.bitcast [[arg]] : f32 to i3243// GFX950: [[resCast:%.+]] = arith.bitcast [[atomicRes]] : f32 to i3244// GFX950: [[test:%.+]] = arith.cmpi eq, [[resCast]], [[argCast]]45// GFX950: cf.cond_br [[test]], [[post:\^.+]], [[loop]]([[atomicRes]] : f32)46// GFX950: [[post]]:47// CHECK-NEXT: gpu.printf "End\0A"48 gpu.printf "Begin\n"49 amdgpu.raw_buffer_atomic_fmax {foo, indexOffset = 4 : i32} %val -> %buffer[%idx] : f32 -> memref<?xf32>, i3250 gpu.printf "End\n"51 func.return52}53 54// -----55 56func.func @atomic_fmax_f64(%val: f64, %buffer: memref<?xf64>, %idx: i32) {57// CHECK: func @atomic_fmax_f6458// CHECK-SAME: ([[val:%.+]]: f64, [[buffer:%.+]]: memref<?xf64>, [[idx:%.+]]: i32)59// CHECK: gpu.printf "Begin\0A"60// GFX90A: amdgpu.raw_buffer_atomic_fmax [[val]] -> [[buffer]][[[idx]]]61// GFX10: amdgpu.raw_buffer_atomic_fmax [[val]] -> [[buffer]][[[idx]]]62// GFX11: amdgpu.raw_buffer_atomic_fmax [[val]] -> [[buffer]][[[idx]]]63// GFX12: amdgpu.raw_buffer_atomic_fmax [[val]] -> [[buffer]][[[idx]]]64// GFX942: amdgpu.raw_buffer_atomic_fmax [[val]] -> [[buffer]][[[idx]]]65// GFX950: amdgpu.raw_buffer_atomic_fmax [[val]] -> [[buffer]][[[idx]]]66// CHECK-NEXT: gpu.printf "End\0A"67 gpu.printf "Begin\n"68 amdgpu.raw_buffer_atomic_fmax %val -> %buffer[%idx] : f64 -> memref<?xf64>, i3269 gpu.printf "End\n"70 func.return71}72 73// -----74 75func.func @atomic_fadd(%val: f32, %buffer: memref<?xf32>, %idx: i32) {76// CHECK: func @atomic_fadd77// GFX90A: amdgpu.raw_buffer_atomic_fadd78// GFX10: amdgpu.raw_buffer_load79// GFX10: amdgpu.raw_buffer_atomic_cmpswap80// GFX11: amdgpu.raw_buffer_atomic_fadd81// GFX12: amdgpu.raw_buffer_atomic_fadd82// GFX942: amdgpu.raw_buffer_atomic_fadd83// GFX950: amdgpu.raw_buffer_atomic_fadd84 amdgpu.raw_buffer_atomic_fadd %val -> %buffer[%idx] : f32 -> memref<?xf32>, i3285 func.return86}87 88// CHECK: func @atomic_fadd_v2f1689func.func @atomic_fadd_v2f16(%val: vector<2xf16>, %buffer: memref<?xf16>, %idx: i32) {90// GFX90A: amdgpu.raw_buffer_atomic_fadd91// GFX10: amdgpu.raw_buffer_load92// GFX10: amdgpu.raw_buffer_atomic_cmpswap93// Note: the atomic operation itself will be done over i32, and then we use bitcasts94// to scalars in order to test for exact bitwise equality instead of float95// equality.96// GFX11: %[[old:.+]] = amdgpu.raw_buffer_atomic_cmpswap97// GFX11: %[[vecCastExpected:.+]] = vector.bitcast %{{.*}} : vector<2xf16> to vector<1xi32>98// GFX11: %[[scalarExpected:.+]] = vector.extract %[[vecCastExpected]][0]99// GFX11: %[[vecCastOld:.+]] = vector.bitcast %[[old]] : vector<2xf16> to vector<1xi32>100// GFX11: %[[scalarOld:.+]] = vector.extract %[[vecCastOld]][0]101// GFX11: arith.cmpi eq, %[[scalarOld]], %[[scalarExpected]]102// GFX942: amdgpu.raw_buffer_atomic_fadd103// GFX12: amdgpu.raw_buffer_atomic_fadd104// GFX950: amdgpu.raw_buffer_atomic_fadd105 amdgpu.raw_buffer_atomic_fadd %val -> %buffer[%idx] : vector<2xf16> -> memref<?xf16>, i32106 func.return107}108 109// CHECK: func @atomic_fadd_v2bf16110func.func @atomic_fadd_v2bf16(%val: vector<2xbf16>, %buffer: memref<?xbf16>, %idx: i32) {111// GFX90A: amdgpu.raw_buffer_load112// GFX90A: amdgpu.raw_buffer_atomic_cmpswap113// GFX10: amdgpu.raw_buffer_load114// GFX10: amdgpu.raw_buffer_atomic_cmpswap115// GFX11: amdgpu.raw_buffer_load116// GFX11: amdgpu.raw_buffer_atomic_cmpswap117// GFX942: amdgpu.raw_buffer_load118// GFX942: amdgpu.raw_buffer_atomic_cmpswap119// GFX12: amdgpu.raw_buffer_atomic_fadd120// GFX950: amdgpu.raw_buffer_atomic_fadd121 amdgpu.raw_buffer_atomic_fadd %val -> %buffer[%idx] : vector<2xbf16> -> memref<?xbf16>, i32122 func.return123}124