173 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt --codegen-opt-level=2 -mtriple=x86_64 -x86-lower-amx-type %s -S | FileCheck %s3; RUN: opt --codegen-opt-level=2 -mtriple=x86_64 -passes=x86-lower-amx-type %s -S | FileCheck %s4 5define void @combine_store(ptr%p) {6; CHECK-LABEL: @combine_store(7; CHECK-NEXT: [[T1:%.*]] = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)8; CHECK-NEXT: call void @llvm.x86.tilestored64.internal(i16 16, i16 64, ptr [[P:%.*]], i64 64, x86_amx [[T1]])9; CHECK-NEXT: ret void10;11 %t1 = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)12 %t2 = call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %t1)13 store <256 x i32> %t2, ptr %p, align 6414 ret void15}16 17define <256 x i32> @combine_store_2user(ptr%p) {18; CHECK-LABEL: @combine_store_2user(19; CHECK-NEXT: [[TMP1:%.*]] = alloca <256 x i32>, align 6420; CHECK-NEXT: [[T1:%.*]] = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)21; CHECK-NEXT: call void @llvm.x86.tilestored64.internal(i16 16, i16 64, ptr [[TMP1]], i64 64, x86_amx [[T1]])22; CHECK-NEXT: [[TMP2:%.*]] = load <256 x i32>, ptr [[TMP1]], align 102423; CHECK-NEXT: call void @llvm.x86.tilestored64.internal(i16 16, i16 64, ptr [[P:%.*]], i64 64, x86_amx [[T1]])24; CHECK-NEXT: ret <256 x i32> [[TMP2]]25;26 %t1 = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)27 %t2 = call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %t1)28 store <256 x i32> %t2, ptr %p, align 6429 ret <256 x i32> %t230}31 32define void @combine_load(ptr%p, ptr%p2) {33; CHECK-LABEL: @combine_load(34; CHECK-NEXT: [[TMP1:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 16, i16 64, ptr [[P:%.*]], i64 64)35; CHECK-NEXT: call void @llvm.x86.tilestored64.internal(i16 16, i16 64, ptr [[P2:%.*]], i64 64, x86_amx [[TMP1]])36; CHECK-NEXT: ret void37;38 %t1 = load <256 x i32>, ptr %p, align 6439 %t2 = call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %t1)40 call void @llvm.x86.tilestored64.internal(i16 16, i16 64, ptr %p2, i64 64, x86_amx %t2)41 ret void42}43 44define void @combine_cast_across_store(ptr%p, ptr%p2) {45; CHECK-LABEL: @combine_cast_across_store(46; CHECK-NEXT: [[TMP1:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 16, i16 64, ptr [[P:%.*]], i64 64)47; CHECK-NEXT: store <256 x i32> zeroinitializer, ptr [[P]], align 6448; CHECK-NEXT: call void @llvm.x86.tilestored64.internal(i16 16, i16 64, ptr [[P2:%.*]], i64 64, x86_amx [[TMP1]])49; CHECK-NEXT: ret void50;51 %t1 = load <256 x i32>, ptr %p, align 6452 store <256 x i32> zeroinitializer, ptr %p, align 6453 %t2 = call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %t1)54 call void @llvm.x86.tilestored64.internal(i16 16, i16 64, ptr %p2, i64 64, x86_amx %t2)55 ret void56}57 58define <256 x i32> @combine_load_2user(ptr%p, ptr%p2) {59; CHECK-LABEL: @combine_load_2user(60; CHECK-NEXT: [[TMP1:%.*]] = alloca <256 x i32>, align 6461; CHECK-NEXT: [[T1:%.*]] = load <256 x i32>, ptr [[P:%.*]], align 6462; CHECK-NEXT: store <256 x i32> [[T1]], ptr [[TMP1]], align 102463; CHECK-NEXT: [[TMP2:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 16, i16 64, ptr [[TMP1]], i64 64)64; CHECK-NEXT: call void @llvm.x86.tilestored64.internal(i16 16, i16 64, ptr [[P2:%.*]], i64 64, x86_amx [[TMP2]])65; CHECK-NEXT: ret <256 x i32> [[T1]]66;67 %t1 = load <256 x i32>, ptr %p, align 6468 %t2 = call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %t1)69 call void @llvm.x86.tilestored64.internal(i16 16, i16 64, ptr %p2, i64 64, x86_amx %t2)70 %t3 = call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %t2)71 ret <256 x i32> %t372}73 74define <256 x i32> @combine_load_3user(ptr%p, ptr%p2) {75; CHECK-LABEL: @combine_load_3user(76; CHECK-NEXT: [[TMP1:%.*]] = alloca <256 x i32>, align 6477; CHECK-NEXT: [[T1:%.*]] = load <256 x i32>, ptr [[P:%.*]], align 6478; CHECK-NEXT: store <256 x i32> [[T1]], ptr [[TMP1]], align 102479; CHECK-NEXT: [[TMP2:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 16, i16 16, ptr [[TMP1]], i64 16)80; CHECK-NEXT: call void @llvm.x86.tilestored64.internal(i16 16, i16 64, ptr [[P2:%.*]], i64 64, x86_amx [[TMP2]])81; CHECK-NEXT: [[TMP3:%.*]] = call x86_amx @llvm.x86.tdpbssd.internal(i16 16, i16 16, i16 64, x86_amx [[TMP2]], x86_amx [[TMP2]], x86_amx [[TMP2]])82; CHECK-NEXT: ret <256 x i32> [[T1]]83;84 %t1 = load <256 x i32>, ptr %p, align 6485 %t2 = call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %t1)86 call void @llvm.x86.tilestored64.internal(i16 16, i16 64, ptr %p2, i64 64, x86_amx %t2)87 %t3 = call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %t2)88 call x86_amx @llvm.x86.tdpbssd.internal(i16 16, i16 16, i16 64, x86_amx %t2, x86_amx %t2, x86_amx %t2)89 ret <256 x i32> %t390}91 92; the shape is loaded after tile.93%struct.__tile1024i_str = type <{ i16, i16, [60 x i8], <256 x i32> }>94define void @test_tile_dpbssd(ptr byval(%struct.__tile1024i_str) align 64 %a, ptr byval(%struct.__tile1024i_str) align 64 %b, ptr byval(%struct.__tile1024i_str) align 64 %c) {95; CHECK-LABEL: @test_tile_dpbssd(96; CHECK-NEXT: entry:97; CHECK-NEXT: [[TMP0:%.*]] = alloca <256 x i32>, align 6498; CHECK-NEXT: [[B_ROW_PTR:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 299; CHECK-NEXT: [[B_ROW:%.*]] = load i16, ptr [[B_ROW_PTR]], align 2100; CHECK-NEXT: [[B_TILE_PTR:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64101; CHECK-NEXT: [[TMP1:%.*]] = sext i16 [[B_ROW]] to i64102; CHECK-NEXT: [[B_TILE:%.*]] = load <256 x i32>, ptr [[B_TILE_PTR]], align 64103; CHECK-NEXT: store <256 x i32> [[B_TILE]], ptr [[TMP0]], align 1024104; CHECK-NEXT: [[A_ROW:%.*]] = load i16, ptr [[A:%.*]], align 64105; CHECK-NEXT: [[A_COL_PTR:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 2106; CHECK-NEXT: [[A_COL:%.*]] = load i16, ptr [[A_COL_PTR]], align 2107; CHECK-NEXT: [[TMP2:%.*]] = udiv i16 [[A_COL]], 4108; CHECK-NEXT: [[A_TILE_PTR:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64109; CHECK-NEXT: [[TMP3:%.*]] = sext i16 [[A_COL]] to i64110; CHECK-NEXT: [[TMP4:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[A_ROW]], i16 [[A_COL]], ptr [[A_TILE_PTR]], i64 [[TMP3]])111; CHECK-NEXT: [[C_TILE_PTR:%.*]] = getelementptr inbounds [[STRUCT___TILE1024I_STR:%.*]], ptr [[C:%.*]], i64 0, i32 3112; CHECK-NEXT: [[TMP5:%.*]] = sext i16 [[B_ROW]] to i64113; CHECK-NEXT: [[TMP6:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[A_ROW]], i16 [[B_ROW]], ptr [[C_TILE_PTR]], i64 [[TMP5]])114; CHECK-NEXT: [[TMP7:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP2]], i16 [[B_ROW]], ptr [[TMP0]], i64 [[TMP1]])115; CHECK-NEXT: [[RES:%.*]] = tail call x86_amx @llvm.x86.tdpbssd.internal(i16 [[A_ROW]], i16 [[B_ROW]], i16 [[A_COL]], x86_amx [[TMP6]], x86_amx [[TMP4]], x86_amx [[TMP7]])116; CHECK-NEXT: ret void117;118entry:119 %b.row.ptr= getelementptr inbounds i8, ptr %b, i64 2120 %b.row = load i16, ptr %b.row.ptr, align 2121 %b.tile.ptr = getelementptr inbounds i8, ptr %b, i64 64122 %b.tile = load <256 x i32>, ptr %b.tile.ptr, align 64123 %a.row = load i16, ptr %a, align 64124 %a.col.ptr = getelementptr inbounds i8, ptr %a, i64 2125 %a.col = load i16, ptr %a.col.ptr, align 2126 %a.tile.ptr = getelementptr inbounds i8, ptr %a, i64 64127 %a.tile = load <256 x i32>, ptr %a.tile.ptr, align 64128 %c.tile.ptr = getelementptr inbounds %struct.__tile1024i_str, ptr %c, i64 0, i32 3129 %c.tile = load <256 x i32>, ptr %c.tile.ptr, align 64130 %c.amx = tail call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %c.tile)131 %a.amx = tail call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %a.tile)132 %b.amx = tail call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %b.tile)133 %res = tail call x86_amx @llvm.x86.tdpbssd.internal(i16 %a.row, i16 %b.row, i16 %a.col, x86_amx %c.amx, x86_amx %a.amx, x86_amx %b.amx)134 ret void135}136 137define void @combine_v256i8amcast_with_store(ptr %src_ptr, ptr %dst_ptr) {138; CHECK-LABEL: @combine_v256i8amcast_with_store(139; CHECK-NEXT: entry:140; CHECK-NEXT: [[TILE:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 32, ptr [[SRC_PTR:%.*]], i64 64)141; CHECK-NEXT: call void @llvm.x86.tilestored64.internal(i16 8, i16 32, ptr [[DST_PTR:%.*]], i64 32, x86_amx [[TILE]])142; CHECK-NEXT: ret void143;144entry:145 %tile = call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 32, ptr %src_ptr, i64 64)146 %vec = call <256 x i8> @llvm.x86.cast.tile.to.vector.v256i8(x86_amx %tile)147 store <256 x i8> %vec, ptr %dst_ptr, align 256148 ret void149}150 151define void @combine_v256i8amcast_with_load(ptr %src_ptr, ptr %dst_ptr) {152; CHECK-LABEL: @combine_v256i8amcast_with_load(153; CHECK-NEXT: entry:154; CHECK-NEXT: [[TMP0:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 32, ptr [[SRC_PTR:%.*]], i64 32)155; CHECK-NEXT: call void @llvm.x86.tilestored64.internal(i16 8, i16 32, ptr [[DST_PTR:%.*]], i64 32, x86_amx [[TMP0]])156; CHECK-NEXT: ret void157;158entry:159 %vec = load <256 x i8>, ptr %src_ptr, align 256160 %tile = call x86_amx @llvm.x86.cast.vector.to.tile.v256i8(<256 x i8> %vec)161 call void @llvm.x86.tilestored64.internal(i16 8, i16 32, ptr %dst_ptr, i64 32, x86_amx %tile)162 ret void163}164 165declare x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32>)166declare <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx)167declare x86_amx @llvm.x86.cast.vector.to.tile.v256i8(<256 x i8>)168declare <256 x i8> @llvm.x86.cast.tile.to.vector.v256i8(x86_amx)169declare x86_amx @llvm.x86.tilezero.internal(i16, i16)170declare x86_amx @llvm.x86.tileloadd64.internal(i16, i16, ptr, i64)171declare void @llvm.x86.tilestored64.internal(i16, i16, ptr, i64, x86_amx)172declare x86_amx @llvm.x86.tdpbssd.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)173