370 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -passes=lower-matrix-intrinsics,instcombine -fuse-matrix-use-loops -fuse-matrix-tile-size=2 -matrix-allow-contract -force-fuse-matrix -verify-dom-info %s -S | FileCheck %s3 4; REQUIRES: aarch64-registered-target5 6target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"7target triple = "aarch64-apple-ios"8 9define void @multiply_noalias_4x4(ptr noalias %A, ptr noalias %B, ptr noalias %C) {10; CHECK-LABEL: @multiply_noalias_4x4(11; CHECK-NEXT: entry:12; CHECK-NEXT: br label [[COLS_HEADER:%.*]]13; CHECK: cols.header:14; CHECK-NEXT: [[COLS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ]15; CHECK-NEXT: br label [[COLS_BODY:%.*]]16; CHECK: cols.body:17; CHECK-NEXT: br label [[ROWS_HEADER:%.*]]18; CHECK: rows.header:19; CHECK-NEXT: [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ]20; CHECK-NEXT: br label [[ROWS_BODY:%.*]]21; CHECK: rows.body:22; CHECK-NEXT: br label [[INNER_HEADER:%.*]]23; CHECK: inner.header:24; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]25; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP5:%.*]], [[INNER_LATCH]] ]26; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP7:%.*]], [[INNER_LATCH]] ]27; CHECK-NEXT: br label [[INNER_BODY:%.*]]28; CHECK: inner.body:29; CHECK-NEXT: [[DOTIDX:%.*]] = shl i64 [[INNER_IV]], 530; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[DOTIDX]]31; CHECK-NEXT: [[TMP1:%.*]] = getelementptr double, ptr [[TMP0]], i64 [[ROWS_IV]]32; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[TMP1]], align 833; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i8, ptr [[TMP1]], i64 3234; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 835; CHECK-NEXT: [[DOTIDX17:%.*]] = shl i64 [[COLS_IV]], 536; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[DOTIDX17]]37; CHECK-NEXT: [[TMP3:%.*]] = getelementptr double, ptr [[TMP2]], i64 [[INNER_IV]]38; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[TMP3]], align 839; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i8, ptr [[TMP3]], i64 3240; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 841; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer42; CHECK-NEXT: [[TMP4:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[RESULT_VEC_0]])43; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> <i32 1, i32 1>44; CHECK-NEXT: [[TMP5]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT8]], <2 x double> [[TMP4]])45; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer46; CHECK-NEXT: [[TMP6:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[RESULT_VEC_1]])47; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> <i32 1, i32 1>48; CHECK-NEXT: [[TMP7]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT15]], <2 x double> [[TMP6]])49; CHECK-NEXT: br label [[INNER_LATCH]]50; CHECK: inner.latch:51; CHECK-NEXT: [[INNER_STEP]] = add i64 [[INNER_IV]], 252; CHECK-NEXT: [[INNER_COND_NOT:%.*]] = icmp eq i64 [[INNER_STEP]], 453; CHECK-NEXT: br i1 [[INNER_COND_NOT]], label [[ROWS_LATCH]], label [[INNER_HEADER]], !llvm.loop [[LOOP0:![0-9]+]]54; CHECK: rows.latch:55; CHECK-NEXT: [[ROWS_STEP]] = add i64 [[ROWS_IV]], 256; CHECK-NEXT: [[ROWS_COND_NOT:%.*]] = icmp eq i64 [[ROWS_STEP]], 457; CHECK-NEXT: [[DOTIDX18:%.*]] = shl i64 [[COLS_IV]], 558; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 [[DOTIDX18]]59; CHECK-NEXT: [[TMP9:%.*]] = getelementptr double, ptr [[TMP8]], i64 [[ROWS_IV]]60; CHECK-NEXT: store <2 x double> [[TMP5]], ptr [[TMP9]], align 861; CHECK-NEXT: [[VEC_GEP16:%.*]] = getelementptr i8, ptr [[TMP9]], i64 3262; CHECK-NEXT: store <2 x double> [[TMP7]], ptr [[VEC_GEP16]], align 863; CHECK-NEXT: br i1 [[ROWS_COND_NOT]], label [[COLS_LATCH]], label [[ROWS_HEADER]]64; CHECK: cols.latch:65; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 266; CHECK-NEXT: [[COLS_COND_NOT:%.*]] = icmp eq i64 [[COLS_STEP]], 467; CHECK-NEXT: br i1 [[COLS_COND_NOT]], label [[CONTINUE:%.*]], label [[COLS_HEADER]]68; CHECK: continue:69; CHECK-NEXT: ret void70;71 72entry:73 %a = load <16 x double>, ptr %A, align 874 %b = load <16 x double>, ptr %B, align 875 76 %c = call <16 x double> @llvm.matrix.multiply.v16f64.v16f64.v16f64(<16 x double> %a, <16 x double> %b, i32 4, i32 4, i32 4)77 78 store <16 x double> %c, ptr %C, align 879 ret void80}81 82 83declare <16 x double> @llvm.matrix.multiply.v16f64.v16f64.v16f64(<16 x double>, <16 x double>, i32, i32, i32)84 85define void @multiply_noalias_2x4(ptr noalias %A, ptr noalias %B, ptr noalias %C) {86; CHECK-LABEL: @multiply_noalias_2x4(87; CHECK-NEXT: entry:88; CHECK-NEXT: br label [[COLS_HEADER:%.*]]89; CHECK: cols.header:90; CHECK-NEXT: [[COLS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ]91; CHECK-NEXT: br label [[COLS_BODY:%.*]]92; CHECK: cols.body:93; CHECK-NEXT: br label [[ROWS_HEADER:%.*]]94; CHECK: rows.header:95; CHECK-NEXT: [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ]96; CHECK-NEXT: br label [[ROWS_BODY:%.*]]97; CHECK: rows.body:98; CHECK-NEXT: br label [[INNER_HEADER:%.*]]99; CHECK: inner.header:100; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]101; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x i64> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP7:%.*]], [[INNER_LATCH]] ]102; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x i64> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP11:%.*]], [[INNER_LATCH]] ]103; CHECK-NEXT: br label [[INNER_BODY:%.*]]104; CHECK: inner.body:105; CHECK-NEXT: [[DOTIDX:%.*]] = shl i64 [[INNER_IV]], 4106; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[DOTIDX]]107; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[ROWS_IV]]108; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8109; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i8, ptr [[TMP1]], i64 16110; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i64>, ptr [[VEC_GEP]], align 8111; CHECK-NEXT: [[DOTIDX17:%.*]] = shl i64 [[COLS_IV]], 5112; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[DOTIDX17]]113; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[TMP2]], i64 [[INNER_IV]]114; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP3]], align 8115; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i8, ptr [[TMP3]], i64 32116; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i64>, ptr [[VEC_GEP3]], align 8117; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x i64> [[COL_LOAD2]], <2 x i64> poison, <2 x i32> zeroinitializer118; CHECK-NEXT: [[TMP4:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT]]119; CHECK-NEXT: [[TMP5:%.*]] = add <2 x i64> [[RESULT_VEC_0]], [[TMP4]]120; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x i64> [[COL_LOAD2]], <2 x i64> poison, <2 x i32> <i32 1, i32 1>121; CHECK-NEXT: [[TMP6:%.*]] = mul <2 x i64> [[COL_LOAD1]], [[SPLAT_SPLAT8]]122; CHECK-NEXT: [[TMP7]] = add <2 x i64> [[TMP5]], [[TMP6]]123; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x i64> [[COL_LOAD4]], <2 x i64> poison, <2 x i32> zeroinitializer124; CHECK-NEXT: [[TMP8:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT12]]125; CHECK-NEXT: [[TMP9:%.*]] = add <2 x i64> [[RESULT_VEC_1]], [[TMP8]]126; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x i64> [[COL_LOAD4]], <2 x i64> poison, <2 x i32> <i32 1, i32 1>127; CHECK-NEXT: [[TMP10:%.*]] = mul <2 x i64> [[COL_LOAD1]], [[SPLAT_SPLAT15]]128; CHECK-NEXT: [[TMP11]] = add <2 x i64> [[TMP9]], [[TMP10]]129; CHECK-NEXT: br label [[INNER_LATCH]]130; CHECK: inner.latch:131; CHECK-NEXT: [[INNER_STEP]] = add i64 [[INNER_IV]], 2132; CHECK-NEXT: [[INNER_COND_NOT:%.*]] = icmp eq i64 [[INNER_STEP]], 4133; CHECK-NEXT: br i1 [[INNER_COND_NOT]], label [[ROWS_LATCH]], label [[INNER_HEADER]], !llvm.loop [[LOOP2:![0-9]+]]134; CHECK: rows.latch:135; CHECK-NEXT: [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2136; CHECK-NEXT: [[ROWS_COND_NOT:%.*]] = icmp eq i64 [[ROWS_IV]], 0137; CHECK-NEXT: [[DOTIDX18:%.*]] = shl i64 [[COLS_IV]], 4138; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 [[DOTIDX18]]139; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[TMP12]], i64 [[ROWS_IV]]140; CHECK-NEXT: store <2 x i64> [[TMP7]], ptr [[TMP13]], align 8141; CHECK-NEXT: [[VEC_GEP16:%.*]] = getelementptr i8, ptr [[TMP13]], i64 16142; CHECK-NEXT: store <2 x i64> [[TMP11]], ptr [[VEC_GEP16]], align 8143; CHECK-NEXT: br i1 [[ROWS_COND_NOT]], label [[COLS_LATCH]], label [[ROWS_HEADER]]144; CHECK: cols.latch:145; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2146; CHECK-NEXT: [[COLS_COND_NOT:%.*]] = icmp eq i64 [[COLS_IV]], 0147; CHECK-NEXT: br i1 [[COLS_COND_NOT]], label [[CONTINUE:%.*]], label [[COLS_HEADER]]148; CHECK: continue:149; CHECK-NEXT: ret void150;151 152; In the inner loop, compute153; Result += Load(A, ROWS_IV, INNER_IV) * Load(B, INNER_IV, COLS_IV)154 155 156; Store the current 2x2 tile.157 158entry:159 %a = load <8 x i64>, ptr %A, align 8160 %b = load <8 x i64>, ptr %B, align 8161 162 %c = call <4 x i64> @llvm.matrix.multiply.v4i64.v8i64.v8i64(<8 x i64> %a, <8 x i64> %b, i32 2, i32 4, i32 2)163 164 store <4 x i64> %c, ptr %C, align 8165 ret void166}167 168 169declare <4 x i64> @llvm.matrix.multiply.v4i64.v8i64.v8i64(<8 x i64>, <8 x i64>, i32, i32, i32)170 171define void @multiply_noalias_4x2_2x8(ptr noalias %A, ptr noalias %B, ptr noalias %C) {172; CHECK-LABEL: @multiply_noalias_4x2_2x8(173; CHECK-NEXT: entry:174; CHECK-NEXT: br label [[COLS_HEADER:%.*]]175; CHECK: cols.header:176; CHECK-NEXT: [[COLS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ]177; CHECK-NEXT: br label [[COLS_BODY:%.*]]178; CHECK: cols.body:179; CHECK-NEXT: br label [[ROWS_HEADER:%.*]]180; CHECK: rows.header:181; CHECK-NEXT: [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ]182; CHECK-NEXT: br label [[ROWS_BODY:%.*]]183; CHECK: rows.body:184; CHECK-NEXT: br label [[INNER_HEADER:%.*]]185; CHECK: inner.header:186; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]187; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x i64> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP7:%.*]], [[INNER_LATCH]] ]188; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x i64> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP11:%.*]], [[INNER_LATCH]] ]189; CHECK-NEXT: br label [[INNER_BODY:%.*]]190; CHECK: inner.body:191; CHECK-NEXT: [[DOTIDX:%.*]] = shl i64 [[INNER_IV]], 5192; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[DOTIDX]]193; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[ROWS_IV]]194; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x i64>, ptr [[TMP1]], align 8195; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i8, ptr [[TMP1]], i64 32196; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x i64>, ptr [[VEC_GEP]], align 8197; CHECK-NEXT: [[DOTIDX17:%.*]] = shl i64 [[COLS_IV]], 4198; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[DOTIDX17]]199; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[TMP2]], i64 [[INNER_IV]]200; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x i64>, ptr [[TMP3]], align 8201; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i8, ptr [[TMP3]], i64 16202; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x i64>, ptr [[VEC_GEP3]], align 8203; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x i64> [[COL_LOAD2]], <2 x i64> poison, <2 x i32> zeroinitializer204; CHECK-NEXT: [[TMP4:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT]]205; CHECK-NEXT: [[TMP5:%.*]] = add <2 x i64> [[RESULT_VEC_0]], [[TMP4]]206; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x i64> [[COL_LOAD2]], <2 x i64> poison, <2 x i32> <i32 1, i32 1>207; CHECK-NEXT: [[TMP6:%.*]] = mul <2 x i64> [[COL_LOAD1]], [[SPLAT_SPLAT8]]208; CHECK-NEXT: [[TMP7]] = add <2 x i64> [[TMP5]], [[TMP6]]209; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x i64> [[COL_LOAD4]], <2 x i64> poison, <2 x i32> zeroinitializer210; CHECK-NEXT: [[TMP8:%.*]] = mul <2 x i64> [[COL_LOAD]], [[SPLAT_SPLAT12]]211; CHECK-NEXT: [[TMP9:%.*]] = add <2 x i64> [[RESULT_VEC_1]], [[TMP8]]212; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x i64> [[COL_LOAD4]], <2 x i64> poison, <2 x i32> <i32 1, i32 1>213; CHECK-NEXT: [[TMP10:%.*]] = mul <2 x i64> [[COL_LOAD1]], [[SPLAT_SPLAT15]]214; CHECK-NEXT: [[TMP11]] = add <2 x i64> [[TMP9]], [[TMP10]]215; CHECK-NEXT: br label [[INNER_LATCH]]216; CHECK: inner.latch:217; CHECK-NEXT: [[INNER_STEP]] = add i64 [[INNER_IV]], 2218; CHECK-NEXT: [[INNER_COND_NOT:%.*]] = icmp eq i64 [[INNER_IV]], 0219; CHECK-NEXT: br i1 [[INNER_COND_NOT]], label [[ROWS_LATCH]], label [[INNER_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]220; CHECK: rows.latch:221; CHECK-NEXT: [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2222; CHECK-NEXT: [[ROWS_COND_NOT:%.*]] = icmp eq i64 [[ROWS_STEP]], 4223; CHECK-NEXT: [[DOTIDX18:%.*]] = shl i64 [[COLS_IV]], 5224; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 [[DOTIDX18]]225; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[TMP12]], i64 [[ROWS_IV]]226; CHECK-NEXT: store <2 x i64> [[TMP7]], ptr [[TMP13]], align 8227; CHECK-NEXT: [[VEC_GEP16:%.*]] = getelementptr i8, ptr [[TMP13]], i64 32228; CHECK-NEXT: store <2 x i64> [[TMP11]], ptr [[VEC_GEP16]], align 8229; CHECK-NEXT: br i1 [[ROWS_COND_NOT]], label [[COLS_LATCH]], label [[ROWS_HEADER]]230; CHECK: cols.latch:231; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2232; CHECK-NEXT: [[COLS_COND_NOT:%.*]] = icmp eq i64 [[COLS_STEP]], 8233; CHECK-NEXT: br i1 [[COLS_COND_NOT]], label [[CONTINUE:%.*]], label [[COLS_HEADER]]234; CHECK: continue:235; CHECK-NEXT: ret void236;237 238; In the inner loop, compute239; Result += Load(A, ROWS_IV, INNER_IV) * Load(B, INNER_IV, COLS_IV)240 241 242; Store the current 2x2 tile.243 244entry:245 %a = load <8 x i64>, ptr %A, align 8246 %b = load <16 x i64>, ptr %B, align 8247 248 %c = call <32 x i64> @llvm.matrix.multiply.v32i64.v8i64.v16i64(<8 x i64> %a, <16 x i64> %b, i32 4, i32 2, i32 8)249 250 store <32 x i64> %c, ptr %C, align 8251 ret void252}253 254declare <32 x i64> @llvm.matrix.multiply.v32i64.v8i64.v16i64(<8 x i64>, <16 x i64>, i32, i32, i32)255 256 257; Check the runtime aliasing checks.258define void @multiply_alias_2x2(ptr %A, ptr %B, ptr %C) {259; CHECK-LABEL: @multiply_alias_2x2(260; CHECK-NEXT: entry:261; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint ptr [[C:%.*]] to i64262; CHECK-NEXT: [[STORE_END:%.*]] = add nuw nsw i64 [[STORE_BEGIN]], 16263; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint ptr [[A:%.*]] to i64264; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i64 [[STORE_END]], [[LOAD_BEGIN]]265; CHECK-NEXT: br i1 [[TMP0]], label [[ALIAS_CONT:%.*]], label [[NO_ALIAS:%.*]]266; CHECK: alias_cont:267; CHECK-NEXT: [[LOAD_END:%.*]] = add nuw nsw i64 [[LOAD_BEGIN]], 16268; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i64 [[LOAD_END]], [[STORE_BEGIN]]269; CHECK-NEXT: br i1 [[TMP1]], label [[COPY:%.*]], label [[NO_ALIAS]]270; CHECK: copy:271; CHECK-NEXT: [[TMP2:%.*]] = alloca [4 x float], align 4272; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(16) [[TMP2]], ptr noundef nonnull align 8 dereferenceable(16) [[A]], i64 16, i1 false)273; CHECK-NEXT: br label [[NO_ALIAS]]274; CHECK: no_alias:275; CHECK-NEXT: [[TMP3:%.*]] = phi ptr [ [[A]], [[ENTRY:%.*]] ], [ [[A]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]276; CHECK-NEXT: [[STORE_BEGIN4:%.*]] = ptrtoint ptr [[C]] to i64277; CHECK-NEXT: [[STORE_END5:%.*]] = add nuw nsw i64 [[STORE_BEGIN4]], 16278; CHECK-NEXT: [[LOAD_BEGIN6:%.*]] = ptrtoint ptr [[B:%.*]] to i64279; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt i64 [[STORE_END5]], [[LOAD_BEGIN6]]280; CHECK-NEXT: br i1 [[TMP4]], label [[ALIAS_CONT1:%.*]], label [[NO_ALIAS3:%.*]]281; CHECK: alias_cont1:282; CHECK-NEXT: [[LOAD_END7:%.*]] = add nuw nsw i64 [[LOAD_BEGIN6]], 16283; CHECK-NEXT: [[TMP5:%.*]] = icmp ugt i64 [[LOAD_END7]], [[STORE_BEGIN4]]284; CHECK-NEXT: br i1 [[TMP5]], label [[COPY2:%.*]], label [[NO_ALIAS3]]285; CHECK: copy2:286; CHECK-NEXT: [[TMP6:%.*]] = alloca [4 x float], align 4287; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(16) [[TMP6]], ptr noundef nonnull align 8 dereferenceable(16) [[B]], i64 16, i1 false)288; CHECK-NEXT: br label [[NO_ALIAS3]]289; CHECK: no_alias3:290; CHECK-NEXT: [[TMP7:%.*]] = phi ptr [ [[B]], [[NO_ALIAS]] ], [ [[B]], [[ALIAS_CONT1]] ], [ [[TMP6]], [[COPY2]] ]291; CHECK-NEXT: br label [[COLS_HEADER:%.*]]292; CHECK: cols.header:293; CHECK-NEXT: [[COLS_IV:%.*]] = phi i64 [ 0, [[NO_ALIAS3]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ]294; CHECK-NEXT: br label [[COLS_BODY:%.*]]295; CHECK: cols.body:296; CHECK-NEXT: br label [[ROWS_HEADER:%.*]]297; CHECK: rows.header:298; CHECK-NEXT: [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ]299; CHECK-NEXT: br label [[ROWS_BODY:%.*]]300; CHECK: rows.body:301; CHECK-NEXT: br label [[INNER_HEADER:%.*]]302; CHECK: inner.header:303; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ]304; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x float> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP15:%.*]], [[INNER_LATCH]] ]305; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x float> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ]306; CHECK-NEXT: br label [[INNER_BODY:%.*]]307; CHECK: inner.body:308; CHECK-NEXT: [[DOTIDX1:%.*]] = shl i64 [[INNER_IV]], 3309; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[DOTIDX1]]310; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, ptr [[TMP8]], i64 [[ROWS_IV]]311; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x float>, ptr [[TMP10]], align 4312; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i8, ptr [[TMP10]], i64 8313; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <2 x float>, ptr [[VEC_GEP]], align 4314; CHECK-NEXT: [[DOTIDX24:%.*]] = shl i64 [[COLS_IV]], 3315; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP7]], i64 [[DOTIDX24]]316; CHECK-NEXT: [[TMP13:%.*]] = getelementptr float, ptr [[TMP11]], i64 [[INNER_IV]]317; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <2 x float>, ptr [[TMP13]], align 4318; CHECK-NEXT: [[VEC_GEP10:%.*]] = getelementptr i8, ptr [[TMP13]], i64 8319; CHECK-NEXT: [[COL_LOAD11:%.*]] = load <2 x float>, ptr [[VEC_GEP10]], align 4320; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x float> [[COL_LOAD9]], <2 x float> poison, <2 x i32> zeroinitializer321; CHECK-NEXT: [[TMP14:%.*]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD]], <2 x float> [[SPLAT_SPLAT]], <2 x float> [[RESULT_VEC_0]])322; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x float> [[COL_LOAD9]], <2 x float> poison, <2 x i32> <i32 1, i32 1>323; CHECK-NEXT: [[TMP15]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD8]], <2 x float> [[SPLAT_SPLAT15]], <2 x float> [[TMP14]])324; CHECK-NEXT: [[SPLAT_SPLAT19:%.*]] = shufflevector <2 x float> [[COL_LOAD11]], <2 x float> poison, <2 x i32> zeroinitializer325; CHECK-NEXT: [[TMP16:%.*]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD]], <2 x float> [[SPLAT_SPLAT19]], <2 x float> [[RESULT_VEC_1]])326; CHECK-NEXT: [[SPLAT_SPLAT22:%.*]] = shufflevector <2 x float> [[COL_LOAD11]], <2 x float> poison, <2 x i32> <i32 1, i32 1>327; CHECK-NEXT: [[TMP17]] = call contract <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[COL_LOAD8]], <2 x float> [[SPLAT_SPLAT22]], <2 x float> [[TMP16]])328; CHECK-NEXT: br label [[INNER_LATCH]]329; CHECK: inner.latch:330; CHECK-NEXT: [[INNER_STEP]] = add i64 [[INNER_IV]], 2331; CHECK-NEXT: [[INNER_COND_NOT:%.*]] = icmp eq i64 [[INNER_IV]], 0332; CHECK-NEXT: br i1 [[INNER_COND_NOT]], label [[ROWS_LATCH]], label [[INNER_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]333; CHECK: rows.latch:334; CHECK-NEXT: [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2335; CHECK-NEXT: [[ROWS_COND_NOT:%.*]] = icmp eq i64 [[ROWS_IV]], 0336; CHECK-NEXT: [[DOTIDX:%.*]] = shl i64 [[COLS_IV]], 3337; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[C]], i64 [[DOTIDX]]338; CHECK-NEXT: [[TMP19:%.*]] = getelementptr float, ptr [[TMP18]], i64 [[ROWS_IV]]339; CHECK-NEXT: store <2 x float> [[TMP15]], ptr [[TMP19]], align 8340; CHECK-NEXT: [[VEC_GEP23:%.*]] = getelementptr i8, ptr [[TMP19]], i64 8341; CHECK-NEXT: store <2 x float> [[TMP17]], ptr [[VEC_GEP23]], align 8342; CHECK-NEXT: br i1 [[ROWS_COND_NOT]], label [[COLS_LATCH]], label [[ROWS_HEADER]]343; CHECK: cols.latch:344; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2345; CHECK-NEXT: [[COLS_COND_NOT:%.*]] = icmp eq i64 [[COLS_IV]], 0346; CHECK-NEXT: br i1 [[COLS_COND_NOT]], label [[CONTINUE:%.*]], label [[COLS_HEADER]]347; CHECK: continue:348; CHECK-NEXT: ret void349;350 351; First, check for aliasing at runtime, create non-aliasing copies if required.352entry:353 %a = load <4 x float>, ptr %A, align 8354 %b = load <4 x float>, ptr %B, align 8355 356 %c = call <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float> %a, <4 x float> %b, i32 2, i32 2, i32 2)357 358 store <4 x float> %c, ptr %C, align 8359 ret void360}361 362declare <4 x float> @llvm.matrix.multiply.v4f32.v4f32.v4f32(<4 x float>, <4 x float>, i32, i32, i32)363 364; CHECK: !0 = distinct !{!0, !1}365; CHECK-NEXT: !1 = !{!"llvm.loop.unroll.count", i32 2}366; CHECK-NEXT: !2 = distinct !{!2, !1}367; CHECK-NEXT: !3 = distinct !{!3, !4}368; CHECK-NEXT: !4 = !{!"llvm.loop.unroll.count", i32 1}369; CHECK-NEXT: !5 = distinct !{!5, !4}370