435 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s3 4define void @add_2x2(ptr %lhs, ptr %rhs, ptr %out) {5; CHECK-LABEL: @add_2x2(6; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[LHS:%.*]], align 167; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[LHS]], i64 28; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 89; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i32>, ptr [[RHS:%.*]], align 1610; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[RHS]], i64 211; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 812; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]13; CHECK-NEXT: [[TMP2:%.*]] = add <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]14; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 415; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 216; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 417; CHECK-NEXT: ret void18;19 %lhsv = load <4 x i32>, ptr %lhs20 %rhsv = load <4 x i32>, ptr %rhs21 %op = add <4 x i32> %lhsv, %rhsv22 call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)23 ret void24}25 26define void @fadd_2x2(ptr %lhs, ptr %rhs, ptr %out) {27; CHECK-LABEL: @fadd_2x2(28; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x float>, ptr [[LHS:%.*]], align 1629; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, ptr [[LHS]], i64 230; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 831; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x float>, ptr [[RHS:%.*]], align 1632; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr float, ptr [[RHS]], i64 233; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x float>, ptr [[VEC_GEP3]], align 834; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x float> [[COL_LOAD]], [[COL_LOAD2]]35; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x float> [[COL_LOAD1]], [[COL_LOAD4]]36; CHECK-NEXT: store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 437; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr float, ptr [[OUT]], i64 238; CHECK-NEXT: store <2 x float> [[TMP2]], ptr [[VEC_GEP5]], align 439; CHECK-NEXT: ret void40;41 %lhsv = load <4 x float>, ptr %lhs42 %rhsv = load <4 x float>, ptr %rhs43 %op = fadd <4 x float> %lhsv, %rhsv44 call void @llvm.matrix.column.major.store(<4 x float> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)45 ret void46}47 48define void @sub_2x2(ptr %lhs, ptr %rhs, ptr %out) {49; CHECK-LABEL: @sub_2x2(50; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[LHS:%.*]], align 1651; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[LHS]], i64 252; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 853; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i32>, ptr [[RHS:%.*]], align 1654; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[RHS]], i64 255; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 856; CHECK-NEXT: [[TMP1:%.*]] = sub <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]57; CHECK-NEXT: [[TMP2:%.*]] = sub <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]58; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 459; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 260; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 461; CHECK-NEXT: ret void62;63 %lhsv = load <4 x i32>, ptr %lhs64 %rhsv = load <4 x i32>, ptr %rhs65 %op = sub <4 x i32> %lhsv, %rhsv66 call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)67 ret void68}69 70define void @fsub_2x2(ptr %lhs, ptr %rhs, ptr %out) {71; CHECK-LABEL: @fsub_2x2(72; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x float>, ptr [[LHS:%.*]], align 1673; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, ptr [[LHS]], i64 274; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 875; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x float>, ptr [[RHS:%.*]], align 1676; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr float, ptr [[RHS]], i64 277; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x float>, ptr [[VEC_GEP3]], align 878; CHECK-NEXT: [[TMP1:%.*]] = fsub nnan <2 x float> [[COL_LOAD]], [[COL_LOAD2]]79; CHECK-NEXT: [[TMP2:%.*]] = fsub nnan <2 x float> [[COL_LOAD1]], [[COL_LOAD4]]80; CHECK-NEXT: store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 481; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr float, ptr [[OUT]], i64 282; CHECK-NEXT: store <2 x float> [[TMP2]], ptr [[VEC_GEP5]], align 483; CHECK-NEXT: ret void84;85 %lhsv = load <4 x float>, ptr %lhs86 %rhsv = load <4 x float>, ptr %rhs87 %op = fsub nnan <4 x float> %lhsv, %rhsv88 call void @llvm.matrix.column.major.store(<4 x float> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)89 ret void90}91 92define void @mul_2x2(ptr %lhs, ptr %rhs, ptr %out) {93; CHECK-LABEL: @mul_2x2(94; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[LHS:%.*]], align 1695; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[LHS]], i64 296; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 897; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i32>, ptr [[RHS:%.*]], align 1698; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[RHS]], i64 299; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8100; CHECK-NEXT: [[TMP1:%.*]] = mul <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]101; CHECK-NEXT: [[TMP2:%.*]] = mul <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]102; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4103; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2104; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4105; CHECK-NEXT: ret void106;107 %lhsv = load <4 x i32>, ptr %lhs108 %rhsv = load <4 x i32>, ptr %rhs109 %op = mul <4 x i32> %lhsv, %rhsv110 call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)111 ret void112}113 114define void @fmul_2x2(ptr %lhs, ptr %rhs, ptr %out) {115; CHECK-LABEL: @fmul_2x2(116; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x float>, ptr [[LHS:%.*]], align 16117; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, ptr [[LHS]], i64 2118; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 8119; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x float>, ptr [[RHS:%.*]], align 16120; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr float, ptr [[RHS]], i64 2121; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x float>, ptr [[VEC_GEP3]], align 8122; CHECK-NEXT: [[TMP1:%.*]] = fmul contract <2 x float> [[COL_LOAD]], [[COL_LOAD2]]123; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x float> [[COL_LOAD1]], [[COL_LOAD4]]124; CHECK-NEXT: store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 4125; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr float, ptr [[OUT]], i64 2126; CHECK-NEXT: store <2 x float> [[TMP2]], ptr [[VEC_GEP5]], align 4127; CHECK-NEXT: ret void128;129 %lhsv = load <4 x float>, ptr %lhs130 %rhsv = load <4 x float>, ptr %rhs131 %op = fmul contract <4 x float> %lhsv, %rhsv132 call void @llvm.matrix.column.major.store(<4 x float> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)133 ret void134}135 136define void @udiv_2x2(ptr %lhs, ptr %rhs, ptr %out) {137; CHECK-LABEL: @udiv_2x2(138; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[LHS:%.*]], align 16139; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[LHS]], i64 2140; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 8141; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i32>, ptr [[RHS:%.*]], align 16142; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[RHS]], i64 2143; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8144; CHECK-NEXT: [[TMP1:%.*]] = udiv <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]145; CHECK-NEXT: [[TMP2:%.*]] = udiv <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]146; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4147; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2148; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4149; CHECK-NEXT: ret void150;151 %lhsv = load <4 x i32>, ptr %lhs152 %rhsv = load <4 x i32>, ptr %rhs153 %op = udiv <4 x i32> %lhsv, %rhsv154 call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)155 ret void156}157 158define void @sdiv_2x2(ptr %lhs, ptr %rhs, ptr %out) {159; CHECK-LABEL: @sdiv_2x2(160; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[LHS:%.*]], align 16161; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[LHS]], i64 2162; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 8163; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i32>, ptr [[RHS:%.*]], align 16164; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[RHS]], i64 2165; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8166; CHECK-NEXT: [[TMP1:%.*]] = sdiv <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]167; CHECK-NEXT: [[TMP2:%.*]] = sdiv <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]168; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4169; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2170; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4171; CHECK-NEXT: ret void172;173 %lhsv = load <4 x i32>, ptr %lhs174 %rhsv = load <4 x i32>, ptr %rhs175 %op = sdiv <4 x i32> %lhsv, %rhsv176 call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)177 ret void178}179 180define void @fdiv_2x2(ptr %num, ptr %denom, ptr %out) {181; CHECK-LABEL: @fdiv_2x2(182; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[NUM:%.*]], align 32183; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[NUM]], i64 2184; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 16185; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[DENOM:%.*]], align 32186; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[DENOM]], i64 2187; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 16188; CHECK-NEXT: [[TMP1:%.*]] = fdiv nnan <2 x double> [[COL_LOAD]], [[COL_LOAD2]]189; CHECK-NEXT: [[TMP2:%.*]] = fdiv nnan <2 x double> [[COL_LOAD1]], [[COL_LOAD4]]190; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[OUT:%.*]], align 8191; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr double, ptr [[OUT]], i64 2192; CHECK-NEXT: store <2 x double> [[TMP2]], ptr [[VEC_GEP5]], align 8193; CHECK-NEXT: ret void194;195 %numv = load <4 x double>, ptr %num196 %denomv = load <4 x double>, ptr %denom197 %op = fdiv nnan <4 x double> %numv, %denomv198 call void @llvm.matrix.column.major.store(<4 x double> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)199 ret void200}201 202define void @urem_2x2(ptr %lhs, ptr %rhs, ptr %out) {203; CHECK-LABEL: @urem_2x2(204; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[LHS:%.*]], align 16205; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[LHS]], i64 2206; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 8207; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i32>, ptr [[RHS:%.*]], align 16208; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[RHS]], i64 2209; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8210; CHECK-NEXT: [[TMP1:%.*]] = urem <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]211; CHECK-NEXT: [[TMP2:%.*]] = urem <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]212; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4213; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2214; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4215; CHECK-NEXT: ret void216;217 %lhsv = load <4 x i32>, ptr %lhs218 %rhsv = load <4 x i32>, ptr %rhs219 %op = urem <4 x i32> %lhsv, %rhsv220 call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)221 ret void222}223 224define void @srem_2x2(ptr %lhs, ptr %rhs, ptr %out) {225; CHECK-LABEL: @srem_2x2(226; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[LHS:%.*]], align 16227; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[LHS]], i64 2228; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 8229; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i32>, ptr [[RHS:%.*]], align 16230; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[RHS]], i64 2231; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8232; CHECK-NEXT: [[TMP1:%.*]] = srem <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]233; CHECK-NEXT: [[TMP2:%.*]] = srem <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]234; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4235; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2236; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4237; CHECK-NEXT: ret void238;239 %lhsv = load <4 x i32>, ptr %lhs240 %rhsv = load <4 x i32>, ptr %rhs241 %op = srem <4 x i32> %lhsv, %rhsv242 call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)243 ret void244}245 246define void @frem_2x2(ptr %lhs, ptr %rhs, ptr %out) {247; CHECK-LABEL: @frem_2x2(248; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x float>, ptr [[LHS:%.*]], align 16249; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr float, ptr [[LHS]], i64 2250; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 8251; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x float>, ptr [[RHS:%.*]], align 16252; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr float, ptr [[RHS]], i64 2253; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x float>, ptr [[VEC_GEP3]], align 8254; CHECK-NEXT: [[TMP1:%.*]] = frem fast <2 x float> [[COL_LOAD]], [[COL_LOAD2]]255; CHECK-NEXT: [[TMP2:%.*]] = frem fast <2 x float> [[COL_LOAD1]], [[COL_LOAD4]]256; CHECK-NEXT: store <2 x float> [[TMP1]], ptr [[OUT:%.*]], align 4257; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr float, ptr [[OUT]], i64 2258; CHECK-NEXT: store <2 x float> [[TMP2]], ptr [[VEC_GEP5]], align 4259; CHECK-NEXT: ret void260;261 %lhsv = load <4 x float>, ptr %lhs262 %rhsv = load <4 x float>, ptr %rhs263 %op = frem fast <4 x float> %lhsv, %rhsv264 call void @llvm.matrix.column.major.store(<4 x float> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)265 ret void266}267 268define void @shl_2x2(ptr %lhs, ptr %rhs, ptr %out) {269; CHECK-LABEL: @shl_2x2(270; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[LHS:%.*]], align 16271; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[LHS]], i64 2272; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 8273; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i32>, ptr [[RHS:%.*]], align 16274; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[RHS]], i64 2275; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8276; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]277; CHECK-NEXT: [[TMP2:%.*]] = shl <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]278; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4279; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2280; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4281; CHECK-NEXT: ret void282;283 %lhsv = load <4 x i32>, ptr %lhs284 %rhsv = load <4 x i32>, ptr %rhs285 %op = shl <4 x i32> %lhsv, %rhsv286 call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)287 ret void288}289 290define void @lshr_2x2(ptr %lhs, ptr %rhs, ptr %out) {291; CHECK-LABEL: @lshr_2x2(292; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[LHS:%.*]], align 16293; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[LHS]], i64 2294; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 8295; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i32>, ptr [[RHS:%.*]], align 16296; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[RHS]], i64 2297; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8298; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]299; CHECK-NEXT: [[TMP2:%.*]] = lshr <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]300; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4301; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2302; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4303; CHECK-NEXT: ret void304;305 %lhsv = load <4 x i32>, ptr %lhs306 %rhsv = load <4 x i32>, ptr %rhs307 %op = lshr <4 x i32> %lhsv, %rhsv308 call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)309 ret void310}311 312define void @ashr_2x2(ptr %lhs, ptr %rhs, ptr %out) {313; CHECK-LABEL: @ashr_2x2(314; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[LHS:%.*]], align 16315; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[LHS]], i64 2316; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 8317; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i32>, ptr [[RHS:%.*]], align 16318; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[RHS]], i64 2319; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8320; CHECK-NEXT: [[TMP1:%.*]] = ashr <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]321; CHECK-NEXT: [[TMP2:%.*]] = ashr <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]322; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4323; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2324; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4325; CHECK-NEXT: ret void326;327 %lhsv = load <4 x i32>, ptr %lhs328 %rhsv = load <4 x i32>, ptr %rhs329 %op = ashr <4 x i32> %lhsv, %rhsv330 call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)331 ret void332}333 334define void @and_2x2(ptr %lhs, ptr %rhs, ptr %out) {335; CHECK-LABEL: @and_2x2(336; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[LHS:%.*]], align 16337; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[LHS]], i64 2338; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 8339; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i32>, ptr [[RHS:%.*]], align 16340; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[RHS]], i64 2341; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8342; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]343; CHECK-NEXT: [[TMP2:%.*]] = and <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]344; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4345; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2346; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4347; CHECK-NEXT: ret void348;349 %lhsv = load <4 x i32>, ptr %lhs350 %rhsv = load <4 x i32>, ptr %rhs351 %op = and <4 x i32> %lhsv, %rhsv352 call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)353 ret void354}355 356define void @or_2x2(ptr %lhs, ptr %rhs, ptr %out) {357; CHECK-LABEL: @or_2x2(358; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[LHS:%.*]], align 16359; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[LHS]], i64 2360; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 8361; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i32>, ptr [[RHS:%.*]], align 16362; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[RHS]], i64 2363; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8364; CHECK-NEXT: [[TMP1:%.*]] = or <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]365; CHECK-NEXT: [[TMP2:%.*]] = or <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]366; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4367; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2368; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4369; CHECK-NEXT: ret void370;371 %lhsv = load <4 x i32>, ptr %lhs372 %rhsv = load <4 x i32>, ptr %rhs373 %op = or <4 x i32> %lhsv, %rhsv374 call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)375 ret void376}377 378define void @xor_2x2(ptr %lhs, ptr %rhs, ptr %out) {379; CHECK-LABEL: @xor_2x2(380; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[LHS:%.*]], align 16381; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[LHS]], i64 2382; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 8383; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i32>, ptr [[RHS:%.*]], align 16384; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i32, ptr [[RHS]], i64 2385; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i32>, ptr [[VEC_GEP3]], align 8386; CHECK-NEXT: [[TMP1:%.*]] = xor <2 x i32> [[COL_LOAD]], [[COL_LOAD2]]387; CHECK-NEXT: [[TMP2:%.*]] = xor <2 x i32> [[COL_LOAD1]], [[COL_LOAD4]]388; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4389; CHECK-NEXT: [[VEC_GEP5:%.*]] = getelementptr i32, ptr [[OUT]], i64 2390; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[VEC_GEP5]], align 4391; CHECK-NEXT: ret void392;393 %lhsv = load <4 x i32>, ptr %lhs394 %rhsv = load <4 x i32>, ptr %rhs395 %op = xor <4 x i32> %lhsv, %rhsv396 call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)397 ret void398}399 400define void @fabs_2x2f64(ptr %in, ptr %out) {401; CHECK-LABEL: @fabs_2x2f64(402; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[IN:%.*]], align 32403; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN]], i64 2404; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 16405; CHECK-NEXT: [[TMP1:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[COL_LOAD]])406; CHECK-NEXT: [[TMP2:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[COL_LOAD1]])407; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[OUT:%.*]], align 8408; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr double, ptr [[OUT]], i64 2409; CHECK-NEXT: store <2 x double> [[TMP2]], ptr [[VEC_GEP2]], align 8410; CHECK-NEXT: ret void411;412 %load = load <4 x double>, ptr %in413 %op = call <4 x double> @llvm.fabs.v4f64(<4 x double> %load)414 call void @llvm.matrix.column.major.store(<4 x double> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)415 ret void416}417 418define void @abs_2x2i32(ptr %in, ptr %out) {419; CHECK-LABEL: @abs_2x2i32(420; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i32>, ptr [[IN:%.*]], align 16421; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i32, ptr [[IN]], i64 2422; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i32>, ptr [[VEC_GEP]], align 8423; CHECK-NEXT: [[TMP1:%.*]] = call <2 x i32> @llvm.abs.v2i32(<2 x i32> [[COL_LOAD]], i1 false)424; CHECK-NEXT: [[TMP2:%.*]] = call <2 x i32> @llvm.abs.v2i32(<2 x i32> [[COL_LOAD1]], i1 false)425; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4426; CHECK-NEXT: [[VEC_GEP2:%.*]] = getelementptr i32, ptr [[OUT]], i64 2427; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[VEC_GEP2]], align 4428; CHECK-NEXT: ret void429;430 %load = load <4 x i32>, ptr %in431 %op = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %load, i1 false)432 call void @llvm.matrix.column.major.store(<4 x i32> %op, ptr %out, i64 2, i1 false, i32 2, i32 2)433 ret void434}435