172 lines · plain
1; RUN: opt -p lower-matrix-intrinsics -matrix-print-after-transpose-opt -disable-output -S %s 2>&1 | FileCheck %s2 3; REQUIRES: asserts4 5target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"6 7; FIXME: Lifted transpose dimensions are incorrect.8define <6 x double> @lift_through_add_matching_transpose_dimensions(<6 x double> %a, <6 x double> %b) {9; CHECK-LABEL: define <6 x double> @lift_through_add_matching_transpose_dimensions(<6 x double> %a, <6 x double> %b) {10; CHECK-NEXT: entry:11; CHECK-NEXT: [[A:%.+]] = fadd <6 x double> %a, %b12; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[A]], i32 3, i32 2)13; CHECK-NEXT: ret <6 x double> [[T]]14;15entry:16 %a.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %a, i32 3, i32 2)17 %b.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %b, i32 3, i32 2)18 %add = fadd <6 x double> %a.t, %b.t19 ret <6 x double> %add20}21 22define <6 x double> @lift_through_add_matching_transpose_dimensions_ops_also_have_shape_info(ptr %a.ptr, ptr %b.ptr) {23; CHECK-LABEL: define <6 x double> @lift_through_add_matching_transpose_dimensions_ops_also_have_shape_info(ptr %a.ptr, ptr %b.ptr)24; CHECK-NEXT: entry:25; CHECK-NEXT: [[A:%.+]] = load <6 x double>, ptr %a.ptr26; CHECK-NEXT: [[B:%.+]] = load <6 x double>, ptr %b.ptr27; CHECK-NEXT: [[ADD:%.+]] = fadd <6 x double> [[A]], [[B]]28; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[ADD]], i32 3, i32 2)29; CHECK-NEXT: ret <6 x double> [[T]]30;31entry:32 %a = load <6 x double>, ptr %a.ptr33 %b = load <6 x double>, ptr %b.ptr34 %a.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %a, i32 3, i32 2)35 %b.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %b, i32 3, i32 2)36 %add = fadd <6 x double> %a.t, %b.t37 ret <6 x double> %add38}39 40define <6 x double> @lift_through_add_mismatching_dimensions_1(<6 x double> %a, <6 x double> %b) {41; CHECK-LABEL: define <6 x double> @lift_through_add_mismatching_dimensions_1(<6 x double> %a, <6 x double> %b) {42; CHECK-NEXT: entry:43; CHECK-NEXT: [[A:%.+]] = fadd <6 x double> %a, %b44; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[A]], i32 1, i32 6)45; CHECK-NEXT: ret <6 x double> [[T]]46;47entry:48 %a.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %a, i32 1, i32 6)49 %b.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %b, i32 3, i32 2)50 %add = fadd <6 x double> %a.t, %b.t51 ret <6 x double> %add52}53 54define <6 x double> @lift_through_add_mismatching_dimensions_1_transpose_dimensions_ops_also_have_shape_info(ptr %a.ptr, ptr %b.ptr) {55; CHECK-LABEL: define <6 x double> @lift_through_add_mismatching_dimensions_1_transpose_dimensions_ops_also_have_shape_info(ptr %a.ptr, ptr %b.ptr)56; CHECK-NEXT: entry:57; CHECK-NEXT: [[A:%.+]] = load <6 x double>, ptr %a.ptr58; CHECK-NEXT: [[B:%.+]] = load <6 x double>, ptr %b.ptr59; CHECK-NEXT: [[ADD:%.+]] = fadd <6 x double> [[A]], [[B]]60; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[ADD]], i32 1, i32 6)61; CHECK-NEXT: ret <6 x double> [[T]]62;63entry:64 %a = load <6 x double>, ptr %a.ptr65 %b = load <6 x double>, ptr %b.ptr66 %a.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %a, i32 1, i32 6)67 %b.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %b, i32 3, i32 2)68 %add = fadd <6 x double> %a.t, %b.t69 ret <6 x double> %add70}71 72define <6 x double> @lift_through_add_mismatching_dimensions_2(<6 x double> %a, <6 x double> %b) {73; CHECK-LABEL: define <6 x double> @lift_through_add_mismatching_dimensions_2(<6 x double> %a, <6 x double> %b) {74; CHECK-NEXT: entry:75; CHECK-NEXT: [[A:%.+]] = fadd <6 x double> %a, %b76; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[A]], i32 3, i32 2)77; CHECK-NEXT: ret <6 x double> [[T]]78;79 80entry:81 %a.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %a, i32 3, i32 2)82 %b.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %b, i32 6, i32 1)83 %add = fadd <6 x double> %a.t, %b.t84 ret <6 x double> %add85}86 87define <6 x double> @lift_through_add_mismatching_dimensions_2_transpose_dimensions_ops_also_have_shape_info(ptr %a.ptr, ptr %b.ptr) {88; CHECK-LABEL: define <6 x double> @lift_through_add_mismatching_dimensions_2_transpose_dimensions_ops_also_have_shape_info(ptr %a.ptr, ptr %b.ptr)89; CHECK-NEXT: entry:90; CHECK-NEXT: [[A:%.+]] = load <6 x double>, ptr %a.ptr91; CHECK-NEXT: [[B:%.+]] = load <6 x double>, ptr %b.ptr92; CHECK-NEXT: [[ADD:%.+]] = fadd <6 x double> [[A]], [[B]]93; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[ADD]], i32 3, i32 2)94; CHECK-NEXT: ret <6 x double> [[T]]95;96entry:97 %a = load <6 x double>, ptr %a.ptr98 %b = load <6 x double>, ptr %b.ptr99 %a.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %a, i32 3, i32 2)100 %b.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %b, i32 6, i32 1)101 %add = fadd <6 x double> %a.t, %b.t102 ret <6 x double> %add103}104 105define <9 x double> @lift_through_multiply(<6 x double> %a, <6 x double> %b) {106; CHECK-LABEL: define <9 x double> @lift_through_multiply(<6 x double> %a, <6 x double> %b) {107; CHECK-NEXT: entry:108; CHECK-NEXT: [[MUL:%.+]] = call <9 x double> @llvm.matrix.multiply.v9f64.v6f64.v6f64(<6 x double> %b, <6 x double> %a, i32 3, i32 2, i32 3)109; CHECK-NEXT: [[T:%.+]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[MUL]], i32 3, i32 3)110; CHECK-NEXT: ret <9 x double> [[T]]111;112entry:113 %a.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %a, i32 3, i32 2)114 %b.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %b, i32 2, i32 3)115 %mul = call <9 x double> @llvm.matrix.multiply.v9f64.v6f64(<6 x double> %a.t, <6 x double> %b.t, i32 3, i32 2 , i32 3)116 ret <9 x double> %mul117}118 119define <6 x double> @lift_through_multiply_2(<6 x double> %a, <4 x double> %b) {120; CHECK-LABEL: define <6 x double> @lift_through_multiply_2(<6 x double> %a, <4 x double> %b) {121; CHECK-NEXT: entry:122; CHECK-NEXT: [[MUL:%.+]] = call <6 x double> @llvm.matrix.multiply.v6f64.v4f64.v6f64(<4 x double> %b, <6 x double> %a, i32 2, i32 2, i32 3)123; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[MUL]], i32 2, i32 3)124; CHECK-NEXT: ret <6 x double> [[T]]125;126entry:127 %a.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %a, i32 3, i32 2)128 %b.t = call <4 x double> @llvm.matrix.transpose.v4f64(<4 x double> %b, i32 2, i32 2)129 %mul = call <6 x double> @llvm.matrix.multiply.v6f64.v6f64.v4f64(<6 x double> %a.t, <4 x double> %b.t, i32 3, i32 2 , i32 2)130 ret <6 x double> %mul131}132 133define <6 x double> @lift_through_multiply_3(<4 x double> %a, <6 x double> %b) {134; CHECK-LABEL: define <6 x double> @lift_through_multiply_3(<4 x double> %a, <6 x double> %b) {135; CHECK-NEXT: entry:136; CHECK-NEXT: [[MUL:%.+]] = call <6 x double> @llvm.matrix.multiply.v6f64.v6f64.v4f64(<6 x double> %b, <4 x double> %a, i32 3, i32 2, i32 2)137; CHECK-NEXT: [[T:%.+]] = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> [[MUL]], i32 3, i32 2)138; CHECK-NEXT: ret <6 x double> [[T]]139;140entry:141 %a.t = call <4 x double> @llvm.matrix.transpose.v4f64(<4 x double> %a, i32 2, i32 2)142 %b.t = call <6 x double> @llvm.matrix.transpose.v6f64(<6 x double> %b, i32 2, i32 3)143 %mul = call <6 x double> @llvm.matrix.multiply.v6f64.v4f64.v6f64(<4 x double> %a.t, <6 x double> %b.t, i32 2, i32 2 , i32 3)144 ret <6 x double> %mul145}146 147define void @test_remove_entries_from_shape_map(<3 x float> %a, <2 x float> %b, <6 x float> %c, ptr %dst) {148; CHECK-LABEL: define void @test_remove_entries_from_shape_map(149; CHECK-SAME: <3 x float> [[A:%.*]], <2 x float> [[B:%.*]], <6 x float> [[C:%.*]], ptr [[DST:%.*]]) {150; CHECK-NEXT: [[ENTRY:.*:]]151; CHECK-NEXT: [[TMP0:%.*]] = call <6 x float> @llvm.matrix.multiply.v6f32.v3f32.v2f32(<3 x float> [[A]], <2 x float> [[B]], i32 3, i32 1, i32 2)152; CHECK-NEXT: [[MFADD:%.*]] = fadd <6 x float> [[C]], [[TMP0]]153; CHECK-NEXT: [[MFADD_T:%.*]] = call <6 x float> @llvm.matrix.transpose.v6f32(<6 x float> [[MFADD]], i32 3, i32 2)154; CHECK-NEXT: store <6 x float> [[MFADD_T]], ptr [[DST]], align 4155; CHECK-NEXT: ret void156;157entry:158 %m = tail call <6 x float> @llvm.matrix.multiply.v6f32.v3f32.v2f32(<3 x float> %a, <2 x float> %b, i32 3, i32 1, i32 2)159 %add = fadd <6 x float> %c, %m160 %t = tail call <6 x float> @llvm.matrix.transpose.v6f32(<6 x float> %add, i32 3, i32 2)161 store <6 x float> %t, ptr %dst, align 4162 ret void163}164 165declare <6 x double> @llvm.matrix.transpose.v6f64.v6f64(<6 x double>, i32, i32)166declare <4 x double> @llvm.matrix.transpose.v4f64.v4f64(<4 x double>, i32, i32)167declare <9 x double> @llvm.matrix.multiply.v9f64.v6f64(<6 x double>, <6 x double>, i32, i32, i32)168declare <6 x double> @llvm.matrix.multiply.v6f64.v6f64.v4f64(<6 x double>, <4 x double>, i32, i32, i32)169declare <6 x double> @llvm.matrix.multiply.v6f64.v6f64.v6f64(<6 x double>, <4 x double>, i32, i32, i32)170declare <6 x float> @llvm.matrix.transpose.v6f32(<6 x float>, i32 immarg, i32 immarg)171declare <6 x float> @llvm.matrix.multiply.v6f32.v3f32.v2f32(<3 x float>, <2 x float>, i32 immarg, i32 immarg, i32 immarg)172