153 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -S %s -passes=scalarize-masked-mem-intrin -mtriple=amdgcn-amd-amdhsa | FileCheck %s3 4; COM: Test that, unlike on CPU targets, the mask doesn't get bitcast to a scalar,5; COM: since, on GPUs, each i1 takes up at least one register and so they should6; COM: be treated separately.7 8define void @scalarize_v2i32(ptr %p, <2 x i1> %mask, <2 x i32> %data) {9; CHECK-LABEL: define void @scalarize_v2i32(10; CHECK-SAME: ptr [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i32> [[DATA:%.*]]) {11; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i1> [[MASK]], i64 012; CHECK-NEXT: br i1 [[TMP1]], label %[[COND_STORE:.*]], label %[[ELSE:.*]]13; CHECK: [[COND_STORE]]:14; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[DATA]], i64 015; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 016; CHECK-NEXT: store i32 [[TMP3]], ptr [[TMP4]], align 417; CHECK-NEXT: br label %[[ELSE]]18; CHECK: [[ELSE]]:19; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[MASK]], i64 120; CHECK-NEXT: br i1 [[TMP5]], label %[[COND_STORE1:.*]], label %[[ELSE2:.*]]21; CHECK: [[COND_STORE1]]:22; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[DATA]], i64 123; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 124; CHECK-NEXT: store i32 [[TMP7]], ptr [[TMP8]], align 425; CHECK-NEXT: br label %[[ELSE2]]26; CHECK: [[ELSE2]]:27; CHECK-NEXT: ret void28;29 call void @llvm.masked.store.v2i32.p0(<2 x i32> %data, ptr %p, i32 128, <2 x i1> %mask)30 ret void31}32 33define void @scalarize_v2i32_splat_mask(ptr %p, <2 x i32> %data, i1 %mask) {34; CHECK-LABEL: define void @scalarize_v2i32_splat_mask(35; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[DATA:%.*]], i1 [[MASK:%.*]]) {36; CHECK-NEXT: [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 037; CHECK-NEXT: [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer38; CHECK-NEXT: [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 039; CHECK-NEXT: br i1 [[MASK_SPLAT_FIRST]], label %[[COND_STORE:.*]], label %[[BB1:.*]]40; CHECK: [[COND_STORE]]:41; CHECK-NEXT: store <2 x i32> [[DATA]], ptr [[P]], align 842; CHECK-NEXT: br label %[[BB1]]43; CHECK: [[BB1]]:44; CHECK-NEXT: ret void45;46 %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 047 %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer48 call void @llvm.masked.store.v2i32.p0(<2 x i32> %data, ptr %p, i32 8, <2 x i1> %mask.splat)49 ret void50}51 52define void @scalarize_v2f16(ptr %p, <2 x i1> %mask, <2 x half> %data) {53; CHECK-LABEL: define void @scalarize_v2f16(54; CHECK-SAME: ptr [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x half> [[DATA:%.*]]) {55; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i1> [[MASK]], i64 056; CHECK-NEXT: br i1 [[TMP1]], label %[[COND_STORE:.*]], label %[[ELSE:.*]]57; CHECK: [[COND_STORE]]:58; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x half> [[DATA]], i64 059; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds half, ptr [[P]], i32 060; CHECK-NEXT: store half [[TMP3]], ptr [[TMP4]], align 261; CHECK-NEXT: br label %[[ELSE]]62; CHECK: [[ELSE]]:63; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[MASK]], i64 164; CHECK-NEXT: br i1 [[TMP5]], label %[[COND_STORE1:.*]], label %[[ELSE2:.*]]65; CHECK: [[COND_STORE1]]:66; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x half> [[DATA]], i64 167; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds half, ptr [[P]], i32 168; CHECK-NEXT: store half [[TMP7]], ptr [[TMP8]], align 269; CHECK-NEXT: br label %[[ELSE2]]70; CHECK: [[ELSE2]]:71; CHECK-NEXT: ret void72;73 call void @llvm.masked.store.v2f16.p0(<2 x half> %data, ptr %p, i32 128, <2 x i1> %mask)74 ret void75}76 77define void @scalarize_v2i32_p3(ptr addrspace(3) %p, <2 x i1> %mask, <2 x i32> %data) {78; CHECK-LABEL: define void @scalarize_v2i32_p3(79; CHECK-SAME: ptr addrspace(3) [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i32> [[DATA:%.*]]) {80; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i1> [[MASK]], i64 081; CHECK-NEXT: br i1 [[TMP1]], label %[[COND_STORE:.*]], label %[[ELSE:.*]]82; CHECK: [[COND_STORE]]:83; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[DATA]], i64 084; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[P]], i32 085; CHECK-NEXT: store i32 [[TMP3]], ptr addrspace(3) [[TMP4]], align 486; CHECK-NEXT: br label %[[ELSE]]87; CHECK: [[ELSE]]:88; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[MASK]], i64 189; CHECK-NEXT: br i1 [[TMP5]], label %[[COND_STORE1:.*]], label %[[ELSE2:.*]]90; CHECK: [[COND_STORE1]]:91; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[DATA]], i64 192; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[P]], i32 193; CHECK-NEXT: store i32 [[TMP7]], ptr addrspace(3) [[TMP8]], align 494; CHECK-NEXT: br label %[[ELSE2]]95; CHECK: [[ELSE2]]:96; CHECK-NEXT: ret void97;98 call void @llvm.masked.store.v2i32.p3(<2 x i32> %data, ptr addrspace(3) %p, i32 128, <2 x i1> %mask)99 ret void100}101 102define void @scalarize_v2i32_lane_mask(ptr %p, <2 x i32> %data) {103; CHECK-LABEL: define void @scalarize_v2i32_lane_mask(104; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[DATA:%.*]]) {105; CHECK-NEXT: [[ITEM_ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()106; CHECK-NEXT: [[MASK:%.*]] = icmp ult i32 [[ITEM_ID]], 16107; CHECK-NEXT: [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 0108; CHECK-NEXT: [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer109; CHECK-NEXT: [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0110; CHECK-NEXT: br i1 [[MASK_SPLAT_FIRST]], label %[[COND_STORE:.*]], label %[[BB1:.*]]111; CHECK: [[COND_STORE]]:112; CHECK-NEXT: store <2 x i32> [[DATA]], ptr [[P]], align 8113; CHECK-NEXT: br label %[[BB1]]114; CHECK: [[BB1]]:115; CHECK-NEXT: ret void116;117 %item.id = call i32 @llvm.amdgcn.workitem.id.x()118 %mask = icmp ult i32 %item.id, 16119 %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0120 %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer121 call void @llvm.masked.store.v2i32.p0(<2 x i32> %data, ptr %p, i32 8, <2 x i1> %mask.splat)122 ret void123}124 125define void @scalarize_v2i32_group_mask(ptr %p, <2 x i32> %data) {126; CHECK-LABEL: define void @scalarize_v2i32_group_mask(127; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[DATA:%.*]]) {128; CHECK-NEXT: [[ITEM_ID:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()129; CHECK-NEXT: [[MASK:%.*]] = icmp ult i32 [[ITEM_ID]], 4130; CHECK-NEXT: [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 0131; CHECK-NEXT: [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer132; CHECK-NEXT: [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0133; CHECK-NEXT: br i1 [[MASK_SPLAT_FIRST]], label %[[COND_STORE:.*]], label %[[BB1:.*]]134; CHECK: [[COND_STORE]]:135; CHECK-NEXT: store <2 x i32> [[DATA]], ptr [[P]], align 8136; CHECK-NEXT: br label %[[BB1]]137; CHECK: [[BB1]]:138; CHECK-NEXT: ret void139;140 %group.id = call i32 @llvm.amdgcn.workgroup.id.x()141 %mask = icmp ult i32 %group.id, 4142 %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0143 %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer144 call void @llvm.masked.store.v2i32.p0(<2 x i32> %data, ptr %p, i32 8, <2 x i1> %mask.splat)145 ret void146}147 148declare void @llvm.masked.store.v2i32.p0(<2 x i32>, ptr, i32, <2 x i1>)149declare void @llvm.masked.store.v2f16.p0(<2 x half>, ptr, i32, <2 x i1>)150declare void @llvm.masked.store.v2i32.p3(<2 x i32>, ptr addrspace(3), i32, <2 x i1>)151declare noundef i32 @llvm.amdgcn.workitem.id.x()152declare noundef i32 @llvm.amdgcn.workgroup.id.x()153