577 lines · plain
1// RUN: mlir-opt -pass-pipeline="builtin.module(gpu.module(convert-gpu-to-llvm-spv{use-64bit-index=true}))" -split-input-file -verify-diagnostics %s \2// RUN: | FileCheck --check-prefixes=CHECK-64,CHECK %s3// RUN: mlir-opt -pass-pipeline="builtin.module(gpu.module(convert-gpu-to-llvm-spv))" -split-input-file -verify-diagnostics %s \4// RUN: | FileCheck --check-prefixes=CHECK-32,CHECK %s5// RUN: mlir-opt -pass-pipeline="builtin.module(gpu.module(convert-gpu-to-llvm-spv{use-64bit-index=false}))" -split-input-file -verify-diagnostics %s \6// RUN: | FileCheck --check-prefixes=CHECK-32,CHECK %s7 8gpu.module @builtins {9 // CHECK-64: llvm.func spir_funccc @_Z14get_num_groupsj(i32) -> i64 attributes {10 // CHECK-32: llvm.func spir_funccc @_Z14get_num_groupsj(i32) -> i32 attributes {11 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>12 // CHECK-SAME-DAG: no_unwind13 // CHECK-SAME-DAG: will_return14 // CHECK-NOT: convergent15 // CHECK-SAME: }16 // CHECK-64: llvm.func spir_funccc @_Z12get_local_idj(i32) -> i64 attributes {17 // CHECK-32: llvm.func spir_funccc @_Z12get_local_idj(i32) -> i32 attributes {18 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>19 // CHECK-SAME-DAG: no_unwind20 // CHECK-SAME-DAG: will_return21 // CHECK-NOT: convergent22 // CHECK-SAME: }23 // CHECK-64: llvm.func spir_funccc @_Z14get_local_sizej(i32) -> i64 attributes {24 // CHECK-32: llvm.func spir_funccc @_Z14get_local_sizej(i32) -> i32 attributes {25 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>26 // CHECK-SAME-DAG: no_unwind27 // CHECK-SAME-DAG: will_return28 // CHECK-NOT: convergent29 // CHECK-SAME: }30 // CHECK-64: llvm.func spir_funccc @_Z13get_global_idj(i32) -> i64 attributes {31 // CHECK-32: llvm.func spir_funccc @_Z13get_global_idj(i32) -> i32 attributes {32 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>33 // CHECK-SAME-DAG: no_unwind34 // CHECK-SAME-DAG: will_return35 // CHECK-NOT: convergent36 // CHECK-SAME: }37 // CHECK-64: llvm.func spir_funccc @_Z12get_group_idj(i32) -> i64 attributes {38 // CHECK-32: llvm.func spir_funccc @_Z12get_group_idj(i32) -> i32 attributes {39 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>40 // CHECK-SAME-DAG: no_unwind41 // CHECK-SAME-DAG: will_return42 // CHECK-NOT: convergent43 // CHECK-SAME: }44 45 // CHECK-LABEL: gpu_block_id46 func.func @gpu_block_id() -> (index, index, index) {47 // CHECK: [[C0:%.*]] = llvm.mlir.constant(0 : i32) : i3248 // CHECK: llvm.call spir_funccc @_Z12get_group_idj([[C0]]) {49 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>50 // CHECK-SAME-DAG: no_unwind51 // CHECK-SAME-DAG: will_return52 // CHECK-NOT: convergent53 // CHECK-64-SAME: } : (i32) -> i6454 // CHECK-32-SAME: } : (i32) -> i3255 %block_id_x = gpu.block_id x56 // CHECK: [[C1:%.*]] = llvm.mlir.constant(1 : i32) : i3257 // CHECK: llvm.call spir_funccc @_Z12get_group_idj([[C1]]) {58 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>59 // CHECK-SAME-DAG: no_unwind60 // CHECK-SAME-DAG: will_return61 // CHECK-NOT: convergent62 // CHECK-64-SAME: } : (i32) -> i6463 // CHECK-32-SAME: } : (i32) -> i3264 %block_id_y = gpu.block_id y65 // CHECK: [[C2:%.*]] = llvm.mlir.constant(2 : i32) : i3266 // CHECK: llvm.call spir_funccc @_Z12get_group_idj([[C2]]) {67 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>68 // CHECK-SAME-DAG: no_unwind69 // CHECK-SAME-DAG: will_return70 // CHECK-NOT: convergent71 // CHECK-64-SAME: } : (i32) -> i6472 // CHECK-32-SAME: } : (i32) -> i3273 %block_id_z = gpu.block_id z74 return %block_id_x, %block_id_y, %block_id_z : index, index, index75 }76 77 // CHECK-LABEL: gpu_global_id78 func.func @gpu_global_id() -> (index, index, index) {79 // CHECK: [[C0:%.*]] = llvm.mlir.constant(0 : i32) : i3280 // CHECK: llvm.call spir_funccc @_Z13get_global_idj([[C0]]) {81 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>82 // CHECK-SAME-DAG: no_unwind83 // CHECK-SAME-DAG: will_return84 // CHECK-NOT: convergent85 // CHECK-64-SAME: } : (i32) -> i6486 // CHECK-32-SAME: } : (i32) -> i3287 %global_id_x = gpu.global_id x88 // CHECK: [[C1:%.*]] = llvm.mlir.constant(1 : i32) : i3289 // CHECK: llvm.call spir_funccc @_Z13get_global_idj([[C1]]) {90 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>91 // CHECK-SAME-DAG: no_unwind92 // CHECK-SAME-DAG: will_return93 // CHECK-NOT: convergent94 // CHECK-64-SAME: } : (i32) -> i6495 // CHECK-32-SAME: } : (i32) -> i3296 %global_id_y = gpu.global_id y97 // CHECK: [[C2:%.*]] = llvm.mlir.constant(2 : i32) : i3298 // CHECK: llvm.call spir_funccc @_Z13get_global_idj([[C2]]) {99 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>100 // CHECK-SAME-DAG: no_unwind101 // CHECK-SAME-DAG: will_return102 // CHECK-NOT: convergent103 // CHECK-64-SAME: } : (i32) -> i64104 // CHECK-32-SAME: } : (i32) -> i32105 %global_id_z = gpu.global_id z106 return %global_id_x, %global_id_y, %global_id_z : index, index, index107 }108 109 // CHECK-LABEL: gpu_block_dim110 func.func @gpu_block_dim() -> (index, index, index) {111 // CHECK: [[C0:%.*]] = llvm.mlir.constant(0 : i32) : i32112 // CHECK: llvm.call spir_funccc @_Z14get_local_sizej([[C0]]) {113 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>114 // CHECK-SAME-DAG: no_unwind115 // CHECK-SAME-DAG: will_return116 // CHECK-NOT: convergent117 // CHECK-64-SAME: } : (i32) -> i64118 // CHECK-32-SAME: } : (i32) -> i32119 %block_dim_x = gpu.block_dim x120 // CHECK: [[C1:%.*]] = llvm.mlir.constant(1 : i32) : i32121 // CHECK: llvm.call spir_funccc @_Z14get_local_sizej([[C1]]) {122 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>123 // CHECK-SAME-DAG: no_unwind124 // CHECK-SAME-DAG: will_return125 // CHECK-NOT: convergent126 // CHECK-64-SAME: } : (i32) -> i64127 // CHECK-32-SAME: } : (i32) -> i32128 %block_dim_y = gpu.block_dim y129 // CHECK: [[C2:%.*]] = llvm.mlir.constant(2 : i32) : i32130 // CHECK: llvm.call spir_funccc @_Z14get_local_sizej([[C2]]) {131 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>132 // CHECK-SAME-DAG: no_unwind133 // CHECK-SAME-DAG: will_return134 // CHECK-NOT: convergent135 // CHECK-64-SAME: } : (i32) -> i64136 // CHECK-32-SAME: } : (i32) -> i32137 %block_dim_z = gpu.block_dim z138 return %block_dim_x, %block_dim_y, %block_dim_z : index, index, index139 }140 141 // CHECK-LABEL: gpu_thread_id142 func.func @gpu_thread_id() -> (index, index, index) {143 // CHECK: [[C0:%.*]] = llvm.mlir.constant(0 : i32) : i32144 // CHECK: llvm.call spir_funccc @_Z12get_local_idj([[C0]]) {145 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>146 // CHECK-SAME-DAG: no_unwind147 // CHECK-SAME-DAG: will_return148 // CHECK-NOT: convergent149 // CHECK-64-SAME: } : (i32) -> i64150 // CHECK-32-SAME: } : (i32) -> i32151 %thread_id_x = gpu.thread_id x152 // CHECK: [[C1:%.*]] = llvm.mlir.constant(1 : i32) : i32153 // CHECK: llvm.call spir_funccc @_Z12get_local_idj([[C1]]) {154 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>155 // CHECK-SAME-DAG: no_unwind156 // CHECK-SAME-DAG: will_return157 // CHECK-NOT: convergent158 // CHECK-64-SAME: } : (i32) -> i64159 // CHECK-32-SAME: } : (i32) -> i32160 %thread_id_y = gpu.thread_id y161 // CHECK: [[C2:%.*]] = llvm.mlir.constant(2 : i32) : i32162 // CHECK: llvm.call spir_funccc @_Z12get_local_idj([[C2]]) {163 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>164 // CHECK-SAME-DAG: no_unwind165 // CHECK-SAME-DAG: will_return166 // CHECK-NOT: convergent167 // CHECK-64-SAME: } : (i32) -> i64168 // CHECK-32-SAME: } : (i32) -> i32169 %thread_id_z = gpu.thread_id z170 return %thread_id_x, %thread_id_y, %thread_id_z : index, index, index171 }172 173 // CHECK-LABEL: gpu_grid_dim174 func.func @gpu_grid_dim() -> (index, index, index) {175 // CHECK: [[C0:%.*]] = llvm.mlir.constant(0 : i32) : i32176 // CHECK: llvm.call spir_funccc @_Z14get_num_groupsj([[C0]]) {177 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>178 // CHECK-SAME-DAG: no_unwind179 // CHECK-SAME-DAG: will_return180 // CHECK-NOT: convergent181 // CHECK-64-SAME: } : (i32) -> i64182 // CHECK-32-SAME: } : (i32) -> i32183 %grid_dim_x = gpu.grid_dim x184 // CHECK: [[C1:%.*]] = llvm.mlir.constant(1 : i32) : i32185 // CHECK: llvm.call spir_funccc @_Z14get_num_groupsj([[C1]]) {186 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>187 // CHECK-SAME-DAG: no_unwind188 // CHECK-SAME-DAG: will_return189 // CHECK-NOT: convergent190 // CHECK-64-SAME: } : (i32) -> i64191 // CHECK-32-SAME: } : (i32) -> i32192 %grid_dim_y = gpu.grid_dim y193 // CHECK: [[C2:%.*]] = llvm.mlir.constant(2 : i32) : i32194 // CHECK: llvm.call spir_funccc @_Z14get_num_groupsj([[C2]]) {195 // CHECK-SAME-DAG: memory_effects = #llvm.memory_effects<other = none, argMem = none, inaccessibleMem = none>196 // CHECK-SAME-DAG: no_unwind197 // CHECK-SAME-DAG: will_return198 // CHECK-NOT: convergent199 // CHECK-64-SAME: } : (i32) -> i64200 // CHECK-32-SAME: } : (i32) -> i32201 %grid_dim_z = gpu.grid_dim z202 return %grid_dim_x, %grid_dim_y, %grid_dim_z : index, index, index203 }204}205 206// -----207 208gpu.module @barriers {209 // CHECK: llvm.func spir_funccc @_Z7barrierj(i32) attributes {210 // CHECK-SAME-DAG: no_unwind211 // CHECK-SAME-DAG: convergent212 // CHECK-SAME-DAG: will_return213 // CHECK-NOT: memory_effects = #llvm.memory_effects214 // CHECK-SAME: }215 216 // CHECK-LABEL: gpu_barrier217 func.func @gpu_barrier() {218 // CHECK: [[FLAGS:%.*]] = llvm.mlir.constant(1 : i32) : i32219 // CHECK: llvm.call spir_funccc @_Z7barrierj([[FLAGS]]) {220 // CHECK-SAME-DAG: no_unwind221 // CHECK-SAME-DAG: convergent222 // CHECK-SAME-DAG: will_return223 // CHECK-NOT: memory_effects = #llvm.memory_effects224 // CHECK-SAME: } : (i32) -> ()225 gpu.barrier226 return227 }228}229 230// -----231 232// Check `gpu.shuffle` conversion with explicit subgroup size.233 234gpu.module @shuffles {235 // CHECK: llvm.func spir_funccc @_Z22sub_group_shuffle_downdj(f64, i32) -> f64 attributes {236 // CHECK-SAME-DAG: no_unwind237 // CHECK-SAME-DAG: convergent238 // CHECK-SAME-DAG: will_return239 // CHECK-NOT: memory_effects = #llvm.memory_effects240 // CHECK-SAME: }241 // CHECK: llvm.func spir_funccc @_Z20sub_group_shuffle_upfj(f32, i32) -> f32 attributes {242 // CHECK-SAME-DAG: no_unwind243 // CHECK-SAME-DAG: convergent244 // CHECK-SAME-DAG: will_return245 // CHECK-NOT: memory_effects = #llvm.memory_effects246 // CHECK-SAME: }247 // CHECK: llvm.func spir_funccc @_Z20sub_group_shuffle_upDhj(f16, i32) -> f16 attributes {248 // CHECK-SAME-DAG: no_unwind249 // CHECK-SAME-DAG: convergent250 // CHECK-SAME-DAG: will_return251 // CHECK-NOT: memory_effects = #llvm.memory_effects252 // CHECK-SAME: }253 // CHECK: llvm.func spir_funccc @_Z21sub_group_shuffle_xorlj(i64, i32) -> i64 attributes {254 // CHECK-SAME-DAG: no_unwind255 // CHECK-SAME-DAG: convergent256 // CHECK-SAME-DAG: will_return257 // CHECK-NOT: memory_effects = #llvm.memory_effects258 // CHECK-SAME: }259 // CHECK: llvm.func spir_funccc @_Z17sub_group_shuffleij(i32, i32) -> i32 attributes {260 // CHECK-SAME-DAG: no_unwind261 // CHECK-SAME-DAG: convergent262 // CHECK-SAME-DAG: will_return263 // CHECK-NOT: memory_effects = #llvm.memory_effects264 // CHECK-SAME: }265 // CHECK: llvm.func spir_funccc @_Z21sub_group_shuffle_xorsj(i16, i32) -> i16 attributes {266 // CHECK-SAME-DAG: no_unwind267 // CHECK-SAME-DAG: convergent268 // CHECK-SAME-DAG: will_return269 // CHECK-NOT: memory_effects = #llvm.memory_effects270 // CHECK-SAME: }271 // CHECK: llvm.func spir_funccc @_Z17sub_group_shufflecj(i8, i32) -> i8 attributes {272 // CHECK-SAME-DAG: no_unwind273 // CHECK-SAME-DAG: convergent274 // CHECK-SAME-DAG: will_return275 // CHECK-NOT: memory_effects = #llvm.memory_effects276 // CHECK-SAME: }277 278 // CHECK-LABEL: gpu_shuffles279 // CHECK-SAME: (%[[I8_VAL:.*]]: i8, %[[I16_VAL:.*]]: i16,280 // CHECK-SAME: %[[I32_VAL:.*]]: i32, %[[I64_VAL:.*]]: i64,281 // CHECK-SAME: %[[F16_VAL:.*]]: f16, %[[F32_VAL:.*]]: f32,282 // CHECK-SAME: %[[F64_VAL:.*]]: f64, %[[BF16_VAL:.*]]: bf16,283 // CHECK-SAME: %[[I1_VAL:.*]]: i1, %[[OFFSET:.*]]: i32)284 llvm.func @gpu_shuffles(%i8_val: i8,285 %i16_val: i16,286 %i32_val: i32,287 %i64_val: i64,288 %f16_val: f16,289 %f32_val: f32,290 %f64_val: f64,291 %bf16_val: bf16,292 %i1_val: i1,293 %offset: i32) attributes {intel_reqd_sub_group_size = 16 : i32} {294 %width = arith.constant 16 : i32295 // CHECK: llvm.call spir_funccc @_Z17sub_group_shufflecj(%[[I8_VAL]], %[[OFFSET]])296 // CHECK: llvm.mlir.constant(true) : i1297 // CHECK: llvm.call spir_funccc @_Z21sub_group_shuffle_xorsj(%[[I16_VAL]], %[[OFFSET]])298 // CHECK: llvm.mlir.constant(true) : i1299 // CHECK: llvm.call spir_funccc @_Z17sub_group_shuffleij(%[[I32_VAL]], %[[OFFSET]])300 // CHECK: llvm.mlir.constant(true) : i1301 // CHECK: llvm.call spir_funccc @_Z21sub_group_shuffle_xorlj(%[[I64_VAL]], %[[OFFSET]])302 // CHECK: llvm.mlir.constant(true) : i1303 // CHECK: llvm.call spir_funccc @_Z20sub_group_shuffle_upDhj(%[[F16_VAL]], %[[OFFSET]])304 // CHECK: llvm.mlir.constant(true) : i1305 // CHECK: llvm.call spir_funccc @_Z20sub_group_shuffle_upfj(%[[F32_VAL]], %[[OFFSET]])306 // CHECK: llvm.mlir.constant(true) : i1307 // CHECK: llvm.call spir_funccc @_Z22sub_group_shuffle_downdj(%[[F64_VAL]], %[[OFFSET]])308 // CHECK: llvm.mlir.constant(true) : i1309 // CHECK: %[[BF16_INBC:.*]] = llvm.bitcast %[[BF16_VAL]] : bf16 to i16310 // CHECK: %[[BF16_CALL:.*]] = llvm.call spir_funccc @_Z22sub_group_shuffle_downsj(%[[BF16_INBC]], %[[OFFSET]])311 // CHECK: llvm.bitcast %[[BF16_CALL]] : i16 to bf16312 // CHECK: llvm.mlir.constant(true) : i1313 // CHECK: %[[I1_ZEXT:.*]] = llvm.zext %[[I1_VAL]] : i1 to i8314 // CHECK: %[[I1_CALL:.*]] = llvm.call spir_funccc @_Z21sub_group_shuffle_xorcj(%18, %arg9)315 // CHECK: llvm.trunc %[[I1_CALL:.*]] : i8 to i1316 // CHECK: llvm.mlir.constant(true) : i1317 %shuffleResult0, %valid0 = gpu.shuffle idx %i8_val, %offset, %width : i8318 %shuffleResult1, %valid1 = gpu.shuffle xor %i16_val, %offset, %width : i16319 %shuffleResult2, %valid2 = gpu.shuffle idx %i32_val, %offset, %width : i32320 %shuffleResult3, %valid3 = gpu.shuffle xor %i64_val, %offset, %width : i64321 %shuffleResult4, %valid4 = gpu.shuffle up %f16_val, %offset, %width : f16322 %shuffleResult5, %valid5 = gpu.shuffle up %f32_val, %offset, %width : f32323 %shuffleResult6, %valid6 = gpu.shuffle down %f64_val, %offset, %width : f64324 %shuffleResult7, %valid7 = gpu.shuffle down %bf16_val, %offset, %width : bf16325 %shuffleResult8, %valid8 = gpu.shuffle xor %i1_val, %offset, %width : i1326 llvm.return327 }328}329 330// -----331 332// Cannot convert due to shuffle width and target subgroup size mismatch333 334gpu.module @shuffles_mismatch {335 llvm.func @gpu_shuffles(%val: i32, %id: i32) attributes {intel_reqd_sub_group_size = 32 : i32} {336 %width = arith.constant 16 : i32337 // expected-error@below {{failed to legalize operation 'gpu.shuffle' that was explicitly marked illegal}}338 %shuffleResult, %valid = gpu.shuffle idx %val, %id, %width : i32339 llvm.return340 }341}342 343// -----344 345// Cannot convert due to variable shuffle width346 347gpu.module @shuffles_mismatch {348 llvm.func @gpu_shuffles(%val: i32, %id: i32, %width: i32) attributes {intel_reqd_sub_group_size = 32 : i32} {349 // expected-error@below {{failed to legalize operation 'gpu.shuffle' that was explicitly marked illegal}}350 %shuffleResult, %valid = gpu.shuffle idx %val, %id, %width : i32351 llvm.return352 }353}354 355// -----356 357// Cannot convert due to value type not being supported by the conversion358 359gpu.module @not_supported_lowering {360 llvm.func @gpu_shuffles(%val: f128, %id: i32) attributes {intel_reqd_sub_group_size = 32 : i32} {361 %width = arith.constant 32 : i32362 // expected-error@below {{failed to legalize operation 'gpu.shuffle' that was explicitly marked illegal}}363 %shuffleResult, %valid = gpu.shuffle xor %val, %id, %width : f128364 llvm.return365 }366}367 368 369// -----370 371gpu.module @kernels {372 // CHECK: llvm.func spir_funccc @no_kernel() {373 gpu.func @no_kernel() {374 gpu.return375 }376 377 // CHECK: llvm.func spir_kernelcc @kernel_no_arg() attributes {gpu.kernel} {378 gpu.func @kernel_no_arg() kernel {379 gpu.return380 }381 382 // CHECK: llvm.func spir_kernelcc @kernel_with_args(%{{.*}}: f32, %{{.*}}: i64) attributes {gpu.kernel} {383 gpu.func @kernel_with_args(%arg0: f32, %arg1: i64) kernel {384 gpu.return385 }386 387 // CHECK-64: llvm.func spir_kernelcc @kernel_with_conv_args(%{{.*}}: i64, %{{.*}}: !llvm.ptr<1>, %{{.*}}: !llvm.ptr<1>, %{{.*}}: i64) attributes {gpu.kernel} {388 // CHECK-32: llvm.func spir_kernelcc @kernel_with_conv_args(%{{.*}}: i32, %{{.*}}: !llvm.ptr<1>, %{{.*}}: !llvm.ptr<1>, %{{.*}}: i32) attributes {gpu.kernel} {389 gpu.func @kernel_with_conv_args(%arg0: index, %arg1: memref<index>) kernel {390 gpu.return391 }392 393 // CHECK-64: llvm.func spir_kernelcc @kernel_with_sized_memref(%{{.*}}: !llvm.ptr<1>, %{{.*}}: !llvm.ptr<1>, %{{.*}}: i64, %{{.*}}: i64, %{{.*}}: i64) attributes {gpu.kernel} {394 // CHECK-32: llvm.func spir_kernelcc @kernel_with_sized_memref(%{{.*}}: !llvm.ptr<1>, %{{.*}}: !llvm.ptr<1>, %{{.*}}: i32, %{{.*}}: i32, %{{.*}}: i32) attributes {gpu.kernel} {395 gpu.func @kernel_with_sized_memref(%arg0: memref<1xindex>) kernel {396 gpu.return397 }398 399 // CHECK-64: llvm.func spir_kernelcc @kernel_with_ND_memref(%{{.*}}: !llvm.ptr<1>, %{{.*}}: !llvm.ptr<1>, %{{.*}}: i64, %{{.*}}: i64, %{{.*}}: i64, %{{.*}}: i64, %{{.*}}: i64, %{{.*}}: i64, %{{.*}}: i64) attributes {gpu.kernel} {400 // CHECK-32: llvm.func spir_kernelcc @kernel_with_ND_memref(%{{.*}}: !llvm.ptr<1>, %{{.*}}: !llvm.ptr<1>, %{{.*}}: i32, %{{.*}}: i32, %{{.*}}: i32, %{{.*}}: i32, %{{.*}}: i32, %{{.*}}: i32, %{{.*}}: i32) attributes {gpu.kernel} {401 gpu.func @kernel_with_ND_memref(%arg0: memref<128x128x128xindex>) kernel {402 gpu.return403 }404}405 406// -----407 408gpu.module @kernels {409// CHECK-LABEL: llvm.func spir_kernelcc @kernel_with_private_attributions() attributes {gpu.kernel} {410 411// Private attribution is converted to an llvm.alloca412 413// CHECK: %[[VAL_2:.*]] = llvm.mlir.constant(32 : i64) : i64414// CHECK: %[[VAL_3:.*]] = llvm.alloca %[[VAL_2]] x f32 : (i64) -> !llvm.ptr415 416// MemRef descriptor built from allocated pointer417 418// CHECK-64: %[[VAL_4:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>419// CHECK-32: %[[VAL_4:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i32, array<1 x i32>, array<1 x i32>)>420 421// CHECK: %[[VAL_5:.*]] = llvm.insertvalue %[[VAL_3]], %[[VAL_4]][0]422// CHECK: llvm.insertvalue %[[VAL_3]], %[[VAL_5]][1]423 424// Same code as above425 426// CHECK: %[[VAL_14:.*]] = llvm.mlir.constant(16 : i64) : i64427// CHECK: %[[VAL_15:.*]] = llvm.alloca %[[VAL_14]] x i16 : (i64) -> !llvm.ptr428 429// CHECK-64: %[[VAL_16:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<1 x i64>, array<1 x i64>)>430// CHECK-32: %[[VAL_16:.*]] = llvm.mlir.poison : !llvm.struct<(ptr, ptr, i32, array<1 x i32>, array<1 x i32>)>431 432// CHECK: %[[VAL_17:.*]] = llvm.insertvalue %[[VAL_15]], %[[VAL_16]][0]433// CHECK: llvm.insertvalue %[[VAL_15]], %[[VAL_17]][1]434 gpu.func @kernel_with_private_attributions()435 private(%arg2: memref<32xf32, #gpu.address_space<private>>, %arg3: memref<16xi16, #gpu.address_space<private>>)436 kernel {437 gpu.return438 }439 440// Workgroup attributions are converted to an llvm.ptr<3> argument441 442// CHECK-LABEL: llvm.func spir_kernelcc @kernel_with_workgoup_attributions(443// CHECK-SAME: %[[VAL_29:.*]]: !llvm.ptr<3> {llvm.noalias, llvm.workgroup_attribution = #llvm.mlir.workgroup_attribution<32 : i64, f32>},444// CHECK-SAME: %[[VAL_30:.*]]: !llvm.ptr<3> {llvm.noalias, llvm.workgroup_attribution = #llvm.mlir.workgroup_attribution<16 : i64, i16>}) attributes {gpu.kernel} {445 446// MemRef descriptor built from new argument447 448// CHECK-64: %[[VAL_31:.*]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>449// CHECK-32: %[[VAL_31:.*]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i32, array<1 x i32>, array<1 x i32>)>450 451// CHECK: %[[VAL_32:.*]] = llvm.insertvalue %[[VAL_29]], %[[VAL_31]][0]452// CHECK: llvm.insertvalue %[[VAL_29]], %[[VAL_32]][1]453 454// Same as above455 456// CHECK-64: %[[VAL_41:.*]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i64, array<1 x i64>, array<1 x i64>)>457// CHECK-32: %[[VAL_41:.*]] = llvm.mlir.poison : !llvm.struct<(ptr<3>, ptr<3>, i32, array<1 x i32>, array<1 x i32>)>458 459// CHECK: %[[VAL_42:.*]] = llvm.insertvalue %[[VAL_30]], %[[VAL_41]][0]460// CHECK: llvm.insertvalue %[[VAL_30]], %[[VAL_42]][1]461 gpu.func @kernel_with_workgoup_attributions()462 workgroup(%arg2: memref<32xf32, #gpu.address_space<workgroup>>, %arg3: memref<16xi16, #gpu.address_space<workgroup>>)463 kernel {464 gpu.return465 }466 467// Check with both private and workgroup attributions. Simply check additional468// arguments and a llvm.alloca are present.469 470// CHECK-LABEL: llvm.func spir_kernelcc @kernel_with_both_attributions(471// CHECK-SAME: %{{.*}}: !llvm.ptr<3> {llvm.noalias, llvm.workgroup_attribution = #llvm.mlir.workgroup_attribution<8 : i64, f32>},472// CHECK-64-SAME: %{{.*}}: !llvm.ptr<3> {llvm.noalias, llvm.workgroup_attribution = #llvm.mlir.workgroup_attribution<16 : i64, i64>}) attributes {gpu.kernel} {473// CHECK-32-SAME: %{{.*}}: !llvm.ptr<3> {llvm.noalias, llvm.workgroup_attribution = #llvm.mlir.workgroup_attribution<16 : i64, i32>}) attributes {gpu.kernel} {474 475// CHECK: %[[VAL_79:.*]] = llvm.mlir.constant(32 : i64) : i64476// CHECK: %[[VAL_80:.*]] = llvm.alloca %[[VAL_79]] x i32 : (i64) -> !llvm.ptr477 478// CHECK: %[[VAL_91:.*]] = llvm.mlir.constant(32 : i64) : i64479// CHECK-64: %[[VAL_92:.*]] = llvm.alloca %[[VAL_91]] x i64 : (i64) -> !llvm.ptr480// CHECK-32: %[[VAL_92:.*]] = llvm.alloca %[[VAL_91]] x i32 : (i64) -> !llvm.ptr481 gpu.func @kernel_with_both_attributions()482 workgroup(%arg4: memref<8xf32, #gpu.address_space<workgroup>>, %arg5: memref<16xindex, #gpu.address_space<workgroup>>)483 private(%arg6: memref<32xi32, #gpu.address_space<private>>, %arg7: memref<32xindex, #gpu.address_space<private>>)484 kernel {485 gpu.return486 }487 488// CHECK-LABEL: llvm.func spir_kernelcc @kernel_known_block_size489// CHECK-SAME: reqd_work_group_size = array<i32: 128, 128, 256>490 gpu.func @kernel_known_block_size() kernel attributes {known_block_size = array<i32: 128, 128, 256>} {491 gpu.return492 }493}494 495// -----496 497gpu.module @kernels {498// CHECK-LABEL: llvm.func spir_funccc @address_spaces(499// CHECK-SAME: {{.*}}: !llvm.ptr<1>500// CHECK-SAME: {{.*}}: !llvm.ptr<3>501// CHECK-SAME: {{.*}}: !llvm.ptr502 gpu.func @address_spaces(%arg0: memref<f32, #gpu.address_space<global>>, %arg1: memref<f32, #gpu.address_space<workgroup>>, %arg2: memref<f32, #gpu.address_space<private>>) {503 gpu.return504 }505}506 507// -----508 509gpu.module @kernels {510// CHECK: llvm.func spir_funccc @_Z12get_group_idj(i32)511// CHECK-LABEL: llvm.func spir_funccc @no_address_spaces(512// CHECK-SAME: %{{[a-zA-Z_][a-zA-Z0-9_]*}}: !llvm.ptr<1>513// CHECK-SAME: %{{[a-zA-Z_][a-zA-Z0-9_]*}}: !llvm.ptr<1>514// CHECK-SAME: %{{[a-zA-Z_][a-zA-Z0-9_]*}}: !llvm.ptr<1>515 gpu.func @no_address_spaces(%arg0: memref<f32>, %arg1: memref<f32, #gpu.address_space<global>>, %arg2: memref<f32>) {516 gpu.return517 }518 519// CHECK-LABEL: llvm.func spir_kernelcc @no_address_spaces_complex(520// CHECK-SAME: %{{[a-zA-Z_][a-zA-Z0-9_]*}}: !llvm.ptr<1>521// CHECK-SAME: %{{[a-zA-Z_][a-zA-Z0-9_]*}}: !llvm.ptr<1>522// CHECK: func.call @no_address_spaces_callee(%{{[0-9]+}}, %{{[0-9]+}})523// CHECK-SAME: : (memref<2x2xf32, 1>, memref<4xf32, 1>)524 gpu.func @no_address_spaces_complex(%arg0: memref<2x2xf32>, %arg1: memref<4xf32>) kernel {525 func.call @no_address_spaces_callee(%arg0, %arg1) : (memref<2x2xf32>, memref<4xf32>) -> ()526 gpu.return527 }528// CHECK-LABEL: func.func @no_address_spaces_callee(529// CHECK-SAME: [[ARG0:%.*]]: memref<2x2xf32, 1>530// CHECK-SAME: [[ARG1:%.*]]: memref<4xf32, 1>531// CHECK: [[C0:%.*]] = llvm.mlir.constant(0 : i32) : i32532// CHECK: [[I0:%.*]] = llvm.call spir_funccc @_Z12get_group_idj([[C0]]) {533// CHECK-32: [[I1:%.*]] = builtin.unrealized_conversion_cast [[I0]] : i32 to index534// CHECK-64: [[I1:%.*]] = builtin.unrealized_conversion_cast [[I0]] : i64 to index535// CHECK: [[LD:%.*]] = memref.load [[ARG0]]{{\[}}[[I1]], [[I1]]{{\]}} : memref<2x2xf32, 1>536// CHECK: memref.store [[LD]], [[ARG1]]{{\[}}[[I1]]{{\]}} : memref<4xf32, 1>537 func.func @no_address_spaces_callee(%arg0: memref<2x2xf32>, %arg1: memref<4xf32>) {538 %block_id = gpu.block_id x539 %0 = memref.load %arg0[%block_id, %block_id] : memref<2x2xf32>540 memref.store %0, %arg1[%block_id] : memref<4xf32>541 func.return542 }543}544 545// -----546 547// Lowering of subgroup query operations548 549// CHECK-DAG: llvm.func spir_funccc @_Z18get_sub_group_size() -> i32 attributes {no_unwind, will_return}550// CHECK-DAG: llvm.func spir_funccc @_Z18get_num_sub_groups() -> i32 attributes {no_unwind, will_return}551// CHECK-DAG: llvm.func spir_funccc @_Z22get_sub_group_local_id() -> i32 attributes {no_unwind, will_return}552// CHECK-DAG: llvm.func spir_funccc @_Z16get_sub_group_id() -> i32 attributes {no_unwind, will_return}553 554 555gpu.module @subgroup_operations {556// CHECK-LABEL: @gpu_subgroup557 func.func @gpu_subgroup() {558 // CHECK: %[[SG_ID:.*]] = llvm.call spir_funccc @_Z16get_sub_group_id() {no_unwind, will_return} : () -> i32559 // CHECK-32-NOT: llvm.zext560 // CHECK-64 %{{.*}} = llvm.zext %[[SG_ID]] : i32 to i64561 %0 = gpu.subgroup_id : index562 // CHECK: %[[SG_LOCAL_ID:.*]] = llvm.call spir_funccc @_Z22get_sub_group_local_id() {no_unwind, will_return} : () -> i32563 // CHECK-32-NOT: llvm.zext564 // CHECK-64: %{{.*}} = llvm.zext %[[SG_LOCAL_ID]] : i32 to i64565 %1 = gpu.lane_id566 // CHECK: %[[NUM_SGS:.*]] = llvm.call spir_funccc @_Z18get_num_sub_groups() {no_unwind, will_return} : () -> i32567 // CHECK-32-NOT: llvm.zext568 // CHECK-64: %{{.*}} = llvm.zext %[[NUM_SGS]] : i32 to i64569 %2 = gpu.num_subgroups : index570 // CHECK: %[[SG_SIZE:.*]] = llvm.call spir_funccc @_Z18get_sub_group_size() {no_unwind, will_return} : () -> i32571 // CHECK-32-NOT: llvm.zext572 // CHECK-64: %{{.*}} = llvm.zext %[[SG_SIZE]] : i32 to i64573 %3 = gpu.subgroup_size : index574 return575 }576}577