282 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; REQUIRES: aarch64-registered-target3 4; RUN: opt -passes='lower-matrix-intrinsics' -matrix-print-after-transpose-opt -disable-output %s 2>&1 | FileCheck %s5 6target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"7target triple = "aarch64-apple-ios"8 9; k * A^T10define void @kat(ptr %Aptr, double %k, ptr %C) {11; CHECK-LABEL: @kat(12; CHECK-NEXT: entry:13; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 12814; CHECK-NEXT: [[VECK:%.*]] = insertelement <9 x double> poison, double [[K:%.*]], i64 015; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <9 x double> [[VECK]], <9 x double> poison, <9 x i32> zeroinitializer16; CHECK-NEXT: [[AT:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[A]], i32 3, i32 3)17; CHECK-NEXT: [[MUL:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[SPLAT]], <9 x double> [[AT]], i32 3, i32 3, i32 3)18; CHECK-NEXT: store <9 x double> [[MUL]], ptr [[C:%.*]], align 12819; CHECK-NEXT: ret void20;21entry:22 %a = load <9 x double>, ptr %Aptr23 %veck = insertelement <9 x double> poison, double %k, i64 024 %splat = shufflevector <9 x double> %veck, <9 x double> poison, <9 x i32> zeroinitializer25 %at = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %a, i32 3, i32 3)26 %mul = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %splat, <9 x double> %at, i32 3, i32 3, i32 3)27 store <9 x double> %mul, ptr %C28 ret void29}30 31; (k * A)^T -> A^T * k32define void @ka_t(ptr %Aptr, double %k, ptr %C) {33; CHECK-LABEL: @ka_t(34; CHECK-NEXT: entry:35; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 12836; CHECK-NEXT: [[VECK:%.*]] = insertelement <9 x double> poison, double [[K:%.*]], i64 037; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <9 x double> [[VECK]], <9 x double> poison, <9 x i32> zeroinitializer38; CHECK-NEXT: [[A_T:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[A]], i32 3, i32 3)39; CHECK-NEXT: [[MMUL:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[A_T]], <9 x double> [[SPLAT]], i32 3, i32 3, i32 3)40; CHECK-NEXT: store <9 x double> [[MMUL]], ptr [[C:%.*]], align 12841; CHECK-NEXT: ret void42;43entry:44 %a = load <9 x double>, ptr %Aptr45 %veck = insertelement <9 x double> poison, double %k, i64 046 %splat = shufflevector <9 x double> %veck, <9 x double> poison, <9 x i32> zeroinitializer47 %mul = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %splat, <9 x double> %a, i32 3, i32 3, i32 3)48 %t = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %mul, i32 3, i32 3)49 store <9 x double> %t, ptr %C50 ret void51}52 53; (k * A)^T -> A^T * k with fmul54define void @ka_t_fmul(ptr %Aptr, double %k, ptr %C) {55; CHECK-LABEL: @ka_t_fmul(56; CHECK-NEXT: entry:57; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 12858; CHECK-NEXT: [[VECK:%.*]] = insertelement <9 x double> poison, double [[K:%.*]], i64 059; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <9 x double> [[VECK]], <9 x double> poison, <9 x i32> zeroinitializer60; CHECK-NEXT: [[A_T:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[A]], i32 3, i32 3)61; CHECK-NEXT: [[MMUL:%.*]] = fmul <9 x double> [[SPLAT]], [[A_T]]62; CHECK-NEXT: store <9 x double> [[MMUL]], ptr [[C:%.*]], align 12863; CHECK-NEXT: ret void64;65entry:66 %a = load <9 x double>, ptr %Aptr67 %veck = insertelement <9 x double> poison, double %k, i64 068 %splat = shufflevector <9 x double> %veck, <9 x double> poison, <9 x i32> zeroinitializer69 %mul = fmul <9 x double> %splat, %a70 %t = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %mul, i32 3, i32 3)71 store <9 x double> %t, ptr %C72 ret void73}74 75; (k * A)^T -> A^T * k with mul (non-fp types)76define void @ka_t_mul(ptr %Aptr, i32 %k, ptr %C) {77; CHECK-LABEL: @ka_t_mul(78; CHECK-NEXT: entry:79; CHECK-NEXT: [[A:%.*]] = load <9 x i32>, ptr [[APTR:%.*]], align 6480; CHECK-NEXT: [[VECK:%.*]] = insertelement <9 x i32> poison, i32 [[K:%.*]], i64 081; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <9 x i32> [[VECK]], <9 x i32> poison, <9 x i32> zeroinitializer82; CHECK-NEXT: [[A_T:%.*]] = call <9 x i32> @llvm.matrix.transpose.v9i32(<9 x i32> [[A]], i32 3, i32 3)83; CHECK-NEXT: [[MMUL:%.*]] = mul <9 x i32> [[SPLAT]], [[A_T]]84; CHECK-NEXT: store <9 x i32> [[MMUL]], ptr [[C:%.*]], align 6485; CHECK-NEXT: ret void86;87entry:88 %a = load <9 x i32>, ptr %Aptr89 %veck = insertelement <9 x i32> poison, i32 %k, i64 090 %splat = shufflevector <9 x i32> %veck, <9 x i32> poison, <9 x i32> zeroinitializer91 %mul = mul <9 x i32> %splat, %a92 %t = call <9 x i32> @llvm.matrix.transpose.v9i32.v9i32(<9 x i32> %mul, i32 3, i32 3)93 store <9 x i32> %t, ptr %C94 ret void95}96 97; A^T + B^T -> (A + B)^T98define void @at_plus_bt(ptr %Aptr, ptr %Bptr, ptr %C) {99; CHECK-LABEL: @at_plus_bt(100; CHECK-NEXT: entry:101; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128102; CHECK-NEXT: [[B:%.*]] = load <9 x double>, ptr [[BPTR:%.*]], align 128103; CHECK-NEXT: [[MFADD:%.*]] = fadd <9 x double> [[A]], [[B]]104; CHECK-NEXT: [[MFADD_T:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[MFADD]], i32 3, i32 3)105; CHECK-NEXT: store <9 x double> [[MFADD_T]], ptr [[C:%.*]], align 128106; CHECK-NEXT: ret void107;108entry:109 %a = load <9 x double>, ptr %Aptr110 %b = load <9 x double>, ptr %Bptr111 %at = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %a, i32 3, i32 3)112 %bt = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %b, i32 3, i32 3)113 %fadd = fadd <9 x double> %at, %bt114 store <9 x double> %fadd, ptr %C115 ret void116}117 118; (A + B)^T -> A^T + B^T -> (A + B)^T119define void @a_plus_b_t(ptr %Aptr, ptr %Bptr, ptr %C) {120; CHECK-LABEL: @a_plus_b_t(121; CHECK-NEXT: entry:122; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128123; CHECK-NEXT: [[B:%.*]] = load <9 x double>, ptr [[BPTR:%.*]], align 128124; CHECK-NEXT: [[MFADD:%.*]] = fadd <9 x double> [[A]], [[B]]125; CHECK-NEXT: [[MFADD_T:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[MFADD]], i32 3, i32 3)126; CHECK-NEXT: store <9 x double> [[MFADD_T]], ptr [[C:%.*]], align 128127; CHECK-NEXT: ret void128;129entry:130 %a = load <9 x double>, ptr %Aptr131 %b = load <9 x double>, ptr %Bptr132 %fadd = fadd <9 x double> %a, %b133 %t = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %fadd, i32 3, i32 3)134 store <9 x double> %t, ptr %C135 ret void136}137 138; A^T * B^T + C^T * D^T -> (B * A + D * C)^T139define void @atbt_plus_ctdt(ptr %Aptr, ptr %Bptr, ptr %Cptr, ptr %Dptr, ptr %E) {140; CHECK-LABEL: @atbt_plus_ctdt(141; CHECK-NEXT: entry:142; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128143; CHECK-NEXT: [[B:%.*]] = load <9 x double>, ptr [[BPTR:%.*]], align 128144; CHECK-NEXT: [[C:%.*]] = load <9 x double>, ptr [[CPTR:%.*]], align 128145; CHECK-NEXT: [[D:%.*]] = load <9 x double>, ptr [[DPTR:%.*]], align 128146; CHECK-NEXT: [[TMP0:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[B]], <9 x double> [[A]], i32 3, i32 3, i32 3)147; CHECK-NEXT: [[TMP1:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[D]], <9 x double> [[C]], i32 3, i32 3, i32 3)148; CHECK-NEXT: [[MFADD:%.*]] = fadd <9 x double> [[TMP0]], [[TMP1]]149; CHECK-NEXT: [[MFADD_T:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[MFADD]], i32 3, i32 3)150; CHECK-NEXT: store <9 x double> [[MFADD_T]], ptr [[E:%.*]], align 128151; CHECK-NEXT: ret void152;153entry:154 %a = load <9 x double>, ptr %Aptr155 %b = load <9 x double>, ptr %Bptr156 %c = load <9 x double>, ptr %Cptr157 %d = load <9 x double>, ptr %Dptr158 %at = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %a, i32 3, i32 3)159 %bt = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %b, i32 3, i32 3)160 %ct = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %c, i32 3, i32 3)161 %dt = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %d, i32 3, i32 3)162 %atbt = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %at, <9 x double> %bt, i32 3, i32 3, i32 3)163 %ctdt = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %ct, <9 x double> %dt, i32 3, i32 3, i32 3)164 %fadd = fadd <9 x double> %atbt, %ctdt165 store <9 x double> %fadd, ptr %E166 ret void167}168 169; -(A^T) + B^T170define void @negat_plus_bt(ptr %Aptr, ptr %Bptr, ptr %C) {171; CHECK-LABEL: @negat_plus_bt(172; CHECK-NEXT: entry:173; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128174; CHECK-NEXT: [[B:%.*]] = load <9 x double>, ptr [[BPTR:%.*]], align 128175; CHECK-NEXT: [[AT:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[A]], i32 3, i32 3)176; CHECK-NEXT: [[NEGAT:%.*]] = fneg <9 x double> [[AT]]177; CHECK-NEXT: [[BT:%.*]] = call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> [[B]], i32 3, i32 3)178; CHECK-NEXT: [[FADD:%.*]] = fadd <9 x double> [[NEGAT]], [[BT]]179; CHECK-NEXT: store <9 x double> [[FADD]], ptr [[C:%.*]], align 128180; CHECK-NEXT: ret void181;182entry:183 %a = load <9 x double>, ptr %Aptr184 %b = load <9 x double>, ptr %Bptr185 %at = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %a, i32 3, i32 3)186 %negat = fneg <9 x double> %at187 %bt = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %b, i32 3, i32 3)188 %fadd = fadd <9 x double> %negat, %bt189 store <9 x double> %fadd, ptr %C190 ret void191}192 193; (A^T * B^T + k * C^T * D^T)^T -> (B * A) + (D * C * k)194define void @atbt_plus_kctdt_t(ptr %Aptr, ptr %Bptr, ptr %Cptr, ptr %Dptr, double %k, ptr %E) {195; CHECK-LABEL: @atbt_plus_kctdt_t(196; CHECK-NEXT: entry:197; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128198; CHECK-NEXT: [[B:%.*]] = load <9 x double>, ptr [[BPTR:%.*]], align 128199; CHECK-NEXT: [[C:%.*]] = load <9 x double>, ptr [[CPTR:%.*]], align 128200; CHECK-NEXT: [[D:%.*]] = load <9 x double>, ptr [[DPTR:%.*]], align 128201; CHECK-NEXT: [[VECK:%.*]] = insertelement <9 x double> poison, double [[K:%.*]], i64 0202; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <9 x double> [[VECK]], <9 x double> poison, <9 x i32> zeroinitializer203; CHECK-NEXT: [[MMUL2:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[B]], <9 x double> [[A]], i32 3, i32 3, i32 3)204; CHECK-NEXT: [[MMUL1:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[C]], <9 x double> [[SPLAT]], i32 3, i32 3, i32 3)205; CHECK-NEXT: [[MMUL:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[D]], <9 x double> [[MMUL1]], i32 3, i32 3, i32 3)206; CHECK-NEXT: [[MADD:%.*]] = fadd <9 x double> [[MMUL2]], [[MMUL]]207; CHECK-NEXT: store <9 x double> [[MADD]], ptr [[E:%.*]], align 128208; CHECK-NEXT: ret void209;210entry:211 %a = load <9 x double>, ptr %Aptr212 %b = load <9 x double>, ptr %Bptr213 %c = load <9 x double>, ptr %Cptr214 %d = load <9 x double>, ptr %Dptr215 %at = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %a, i32 3, i32 3)216 %bt = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %b, i32 3, i32 3)217 %ct = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %c, i32 3, i32 3)218 %dt = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %d, i32 3, i32 3)219 %atbt = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %at, <9 x double> %bt, i32 3, i32 3, i32 3)220 %veck = insertelement <9 x double> poison, double %k, i64 0221 %splat = shufflevector <9 x double> %veck, <9 x double> poison, <9 x i32> zeroinitializer222 %kct = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %splat, <9 x double> %ct, i32 3, i32 3, i32 3)223 %kctdt = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %kct, <9 x double> %dt, i32 3, i32 3, i32 3)224 %fadd = fadd <9 x double> %atbt, %kctdt225 %t = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %fadd, i32 3, i32 3)226 store <9 x double> %t, ptr %E227 ret void228}229 230; (A^T * (k * B^T))^T => (B * k) * A231define void @atkbt_t(ptr %Aptr, ptr %Bptr, double %k, ptr %C) {232; CHECK-LABEL: @atkbt_t(233; CHECK-NEXT: entry:234; CHECK-NEXT: [[A:%.*]] = load <9 x double>, ptr [[APTR:%.*]], align 128235; CHECK-NEXT: [[B:%.*]] = load <9 x double>, ptr [[BPTR:%.*]], align 128236; CHECK-NEXT: [[VECK:%.*]] = insertelement <9 x double> poison, double [[K:%.*]], i64 0237; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <9 x double> [[VECK]], <9 x double> poison, <9 x i32> zeroinitializer238; CHECK-NEXT: [[MMUL1:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[B]], <9 x double> [[SPLAT]], i32 3, i32 3, i32 3)239; CHECK-NEXT: [[MMUL:%.*]] = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> [[MMUL1]], <9 x double> [[A]], i32 3, i32 3, i32 3)240; CHECK-NEXT: store <9 x double> [[MMUL]], ptr [[C:%.*]], align 128241; CHECK-NEXT: ret void242;243entry:244 %a = load <9 x double>, ptr %Aptr245 %b = load <9 x double>, ptr %Bptr246 %at = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %a, i32 3, i32 3)247 %bt = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %b, i32 3, i32 3)248 %veck = insertelement <9 x double> poison, double %k, i64 0249 %splat = shufflevector <9 x double> %veck, <9 x double> poison, <9 x i32> zeroinitializer250 %kbt = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %splat, <9 x double> %bt, i32 3, i32 3, i32 3)251 %atkbt = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %at, <9 x double> %kbt, i32 3, i32 3, i32 3)252 %t = call <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double> %atkbt, i32 3, i32 3)253 store <9 x double> %t, ptr %C254 ret void255}256 257declare <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double>, <9 x double>, i32, i32, i32)258declare <9 x double> @llvm.matrix.transpose.v9f64.v9f64(<9 x double>, i32, i32)259declare <9 x i32> @llvm.matrix.transpose.v9i32.v9i32(<9 x i32>, i32, i32)260 261 262; (a * b + c)^T -> (a * b)^T + b^T with integer types.263define noundef <4 x i32> @mul_add_transpose_int(<4 x i32> noundef %a, <4 x i32> noundef %b, <4 x i32> noundef %c) {264; CHECK-LABEL: @mul_add_transpose_int(265; CHECK-NEXT: entry:266; CHECK-NEXT: [[TMP0:%.*]] = call <4 x i32> @llvm.matrix.multiply.v4i32.v4i32.v4i32(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 2, i32 2, i32 2)267; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.matrix.transpose.v4i32(<4 x i32> [[TMP0]], i32 2, i32 2)268; CHECK-NEXT: [[C_T:%.*]] = call <4 x i32> @llvm.matrix.transpose.v4i32(<4 x i32> [[C:%.*]], i32 2, i32 2)269; CHECK-NEXT: [[MADD:%.*]] = add <4 x i32> [[TMP1]], [[C_T]]270; CHECK-NEXT: ret <4 x i32> [[MADD]]271;272entry:273 %mul = tail call <4 x i32> @llvm.matrix.multiply.v4i32.v4i32.v4i32(<4 x i32> %a, <4 x i32> %b, i32 2, i32 2, i32 2)274 %add = add <4 x i32> %mul, %c275 %t = tail call <4 x i32> @llvm.matrix.transpose.v4i32(<4 x i32> %add, i32 2, i32 2)276 ret <4 x i32> %t277}278 279declare <4 x i32> @llvm.matrix.multiply.v4i32.v4i32.v4i32(<4 x i32>, <4 x i32>, i32 immarg, i32 immarg, i32 immarg)280 281declare <4 x i32> @llvm.matrix.transpose.v4i32(<4 x i32>, i32 immarg, i32 immarg)282