374 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -passes=lower-matrix-intrinsics,instcombine -fuse-matrix-use-loops=false -fuse-matrix-tile-size=1 -matrix-allow-contract -force-fuse-matrix -verify-dom-info %s -S | FileCheck %s3 4; REQUIRES: aarch64-registered-target5 6target datalayout = "e-m:o-i64:64-f80:128-n8:4:32:64-S128"7target triple = "aarch64-apple-ios"8 9define void @multiply_can_hoist_cast(ptr noalias %A, ptr %B, ptr %C) {10; CHECK-LABEL: @multiply_can_hoist_cast(11; CHECK-NEXT: entry:12; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint ptr [[C:%.*]] to i6413; CHECK-NEXT: [[STORE_END:%.*]] = add nuw nsw i64 [[STORE_BEGIN]], 3214; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint ptr [[B:%.*]] to i6415; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i64 [[STORE_END]], [[LOAD_BEGIN]]16; CHECK-NEXT: br i1 [[TMP0]], label [[ALIAS_CONT:%.*]], label [[NO_ALIAS:%.*]]17; CHECK: alias_cont:18; CHECK-NEXT: [[LOAD_END:%.*]] = add nuw nsw i64 [[LOAD_BEGIN]], 3219; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i64 [[LOAD_END]], [[STORE_BEGIN]]20; CHECK-NEXT: br i1 [[TMP1]], label [[COPY:%.*]], label [[NO_ALIAS]]21; CHECK: copy:22; CHECK-NEXT: [[TMP2:%.*]] = alloca [4 x double], align 823; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[TMP2]], ptr noundef nonnull align 8 dereferenceable(32) [[B]], i64 32, i1 false)24; CHECK-NEXT: br label [[NO_ALIAS]]25; CHECK: no_alias:26; CHECK-NEXT: [[TMP3:%.*]] = phi ptr [ [[B]], [[ENTRY:%.*]] ], [ [[B]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]27; CHECK-NEXT: [[COL_LOAD:%.*]] = load <1 x double>, ptr [[A:%.*]], align 828; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <1 x double>, ptr [[TMP3]], align 829; CHECK-NEXT: [[TMP4:%.*]] = fmul contract <1 x double> [[COL_LOAD]], [[COL_LOAD1]]30; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 1631; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <1 x double>, ptr [[TMP5]], align 832; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP3]], i64 833; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <1 x double>, ptr [[TMP6]], align 834; CHECK-NEXT: [[TMP7:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD2]], <1 x double> [[COL_LOAD3]], <1 x double> [[TMP4]])35; CHECK-NEXT: store <1 x double> [[TMP7]], ptr [[C]], align 836; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 837; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <1 x double>, ptr [[TMP8]], align 838; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <1 x double>, ptr [[TMP3]], align 839; CHECK-NEXT: [[TMP9:%.*]] = fmul contract <1 x double> [[COL_LOAD8]], [[COL_LOAD9]]40; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 2441; CHECK-NEXT: [[COL_LOAD13:%.*]] = load <1 x double>, ptr [[TMP10]], align 842; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP3]], i64 843; CHECK-NEXT: [[COL_LOAD14:%.*]] = load <1 x double>, ptr [[TMP11]], align 844; CHECK-NEXT: [[TMP12:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD13]], <1 x double> [[COL_LOAD14]], <1 x double> [[TMP9]])45; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[C]], i64 846; CHECK-NEXT: store <1 x double> [[TMP12]], ptr [[TMP13]], align 847; CHECK-NEXT: [[COL_LOAD19:%.*]] = load <1 x double>, ptr [[A]], align 848; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP3]], i64 1649; CHECK-NEXT: [[COL_LOAD20:%.*]] = load <1 x double>, ptr [[TMP14]], align 850; CHECK-NEXT: [[TMP15:%.*]] = fmul contract <1 x double> [[COL_LOAD19]], [[COL_LOAD20]]51; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[A]], i64 1652; CHECK-NEXT: [[COL_LOAD24:%.*]] = load <1 x double>, ptr [[TMP16]], align 853; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[TMP3]], i64 2454; CHECK-NEXT: [[COL_LOAD25:%.*]] = load <1 x double>, ptr [[TMP17]], align 855; CHECK-NEXT: [[TMP18:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[COL_LOAD25]], <1 x double> [[TMP15]])56; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[C]], i64 1657; CHECK-NEXT: store <1 x double> [[TMP18]], ptr [[TMP19]], align 858; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[A]], i64 859; CHECK-NEXT: [[COL_LOAD30:%.*]] = load <1 x double>, ptr [[TMP20]], align 860; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP3]], i64 1661; CHECK-NEXT: [[COL_LOAD31:%.*]] = load <1 x double>, ptr [[TMP21]], align 862; CHECK-NEXT: [[TMP22:%.*]] = fmul contract <1 x double> [[COL_LOAD30]], [[COL_LOAD31]]63; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[A]], i64 2464; CHECK-NEXT: [[COL_LOAD35:%.*]] = load <1 x double>, ptr [[TMP23]], align 865; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP3]], i64 2466; CHECK-NEXT: [[COL_LOAD36:%.*]] = load <1 x double>, ptr [[TMP24]], align 867; CHECK-NEXT: [[TMP25:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD35]], <1 x double> [[COL_LOAD36]], <1 x double> [[TMP22]])68; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[C]], i64 2469; CHECK-NEXT: store <1 x double> [[TMP25]], ptr [[TMP26]], align 870; CHECK-NEXT: ret void71;72entry:73 %a = load <4 x double>, ptr %A, align 874 %b = load <4 x double>, ptr %B, align 875 %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)76 store <4 x double> %c, ptr %C, align 877 ret void78}79 80define void @multiply_can_hoist_multiple_insts(ptr noalias %A, ptr %B, ptr %C) {81; CHECK-LABEL: @multiply_can_hoist_multiple_insts(82; CHECK-NEXT: entry:83; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 6484; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint ptr [[GEP]] to i6485; CHECK-NEXT: [[STORE_END:%.*]] = add nuw nsw i64 [[STORE_BEGIN]], 3286; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint ptr [[B:%.*]] to i6487; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i64 [[STORE_END]], [[LOAD_BEGIN]]88; CHECK-NEXT: br i1 [[TMP0]], label [[ALIAS_CONT:%.*]], label [[NO_ALIAS:%.*]]89; CHECK: alias_cont:90; CHECK-NEXT: [[LOAD_END:%.*]] = add nuw nsw i64 [[LOAD_BEGIN]], 3291; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i64 [[LOAD_END]], [[STORE_BEGIN]]92; CHECK-NEXT: br i1 [[TMP1]], label [[COPY:%.*]], label [[NO_ALIAS]]93; CHECK: copy:94; CHECK-NEXT: [[TMP2:%.*]] = alloca [4 x double], align 895; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[TMP2]], ptr noundef nonnull align 8 dereferenceable(32) [[B]], i64 32, i1 false)96; CHECK-NEXT: br label [[NO_ALIAS]]97; CHECK: no_alias:98; CHECK-NEXT: [[TMP3:%.*]] = phi ptr [ [[B]], [[ENTRY:%.*]] ], [ [[B]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]99; CHECK-NEXT: [[COL_LOAD:%.*]] = load <1 x double>, ptr [[A:%.*]], align 8100; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <1 x double>, ptr [[TMP3]], align 8101; CHECK-NEXT: [[TMP4:%.*]] = fmul contract <1 x double> [[COL_LOAD]], [[COL_LOAD1]]102; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 16103; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <1 x double>, ptr [[TMP5]], align 8104; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8105; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <1 x double>, ptr [[TMP6]], align 8106; CHECK-NEXT: [[TMP7:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD2]], <1 x double> [[COL_LOAD3]], <1 x double> [[TMP4]])107; CHECK-NEXT: store <1 x double> [[TMP7]], ptr [[GEP]], align 8108; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 8109; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <1 x double>, ptr [[TMP8]], align 8110; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <1 x double>, ptr [[TMP3]], align 8111; CHECK-NEXT: [[TMP9:%.*]] = fmul contract <1 x double> [[COL_LOAD8]], [[COL_LOAD9]]112; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 24113; CHECK-NEXT: [[COL_LOAD13:%.*]] = load <1 x double>, ptr [[TMP10]], align 8114; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8115; CHECK-NEXT: [[COL_LOAD14:%.*]] = load <1 x double>, ptr [[TMP11]], align 8116; CHECK-NEXT: [[TMP12:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD13]], <1 x double> [[COL_LOAD14]], <1 x double> [[TMP9]])117; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[C]], i64 72118; CHECK-NEXT: store <1 x double> [[TMP12]], ptr [[TMP13]], align 8119; CHECK-NEXT: [[COL_LOAD19:%.*]] = load <1 x double>, ptr [[A]], align 8120; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP3]], i64 16121; CHECK-NEXT: [[COL_LOAD20:%.*]] = load <1 x double>, ptr [[TMP14]], align 8122; CHECK-NEXT: [[TMP15:%.*]] = fmul contract <1 x double> [[COL_LOAD19]], [[COL_LOAD20]]123; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[A]], i64 16124; CHECK-NEXT: [[COL_LOAD24:%.*]] = load <1 x double>, ptr [[TMP16]], align 8125; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[TMP3]], i64 24126; CHECK-NEXT: [[COL_LOAD25:%.*]] = load <1 x double>, ptr [[TMP17]], align 8127; CHECK-NEXT: [[TMP18:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[COL_LOAD25]], <1 x double> [[TMP15]])128; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[C]], i64 80129; CHECK-NEXT: store <1 x double> [[TMP18]], ptr [[TMP19]], align 8130; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[A]], i64 8131; CHECK-NEXT: [[COL_LOAD30:%.*]] = load <1 x double>, ptr [[TMP20]], align 8132; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP3]], i64 16133; CHECK-NEXT: [[COL_LOAD31:%.*]] = load <1 x double>, ptr [[TMP21]], align 8134; CHECK-NEXT: [[TMP22:%.*]] = fmul contract <1 x double> [[COL_LOAD30]], [[COL_LOAD31]]135; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[A]], i64 24136; CHECK-NEXT: [[COL_LOAD35:%.*]] = load <1 x double>, ptr [[TMP23]], align 8137; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP3]], i64 24138; CHECK-NEXT: [[COL_LOAD36:%.*]] = load <1 x double>, ptr [[TMP24]], align 8139; CHECK-NEXT: [[TMP25:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD35]], <1 x double> [[COL_LOAD36]], <1 x double> [[TMP22]])140; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[C]], i64 88141; CHECK-NEXT: store <1 x double> [[TMP25]], ptr [[TMP26]], align 8142; CHECK-NEXT: ret void143;144entry:145 %a = load <4 x double>, ptr %A, align 8146 %b = load <4 x double>, ptr %B, align 8147 %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)148 %gep = getelementptr [4 x double], ptr %C, i32 2149 store <4 x double> %c, ptr %gep, align 8150 ret void151}152 153; Make sure the correct instruction order is preserved when hoisting.154define void @multiply_can_hoist_multiple_insts2(ptr noalias %A, ptr %B, ptr %C) {155; CHECK-LABEL: @multiply_can_hoist_multiple_insts2(156; CHECK-NEXT: entry:157; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 1344158; CHECK-NEXT: [[STORE_BEGIN:%.*]] = ptrtoint ptr [[GEP_1]] to i64159; CHECK-NEXT: [[STORE_END:%.*]] = add nuw nsw i64 [[STORE_BEGIN]], 32160; CHECK-NEXT: [[LOAD_BEGIN:%.*]] = ptrtoint ptr [[B:%.*]] to i64161; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt i64 [[STORE_END]], [[LOAD_BEGIN]]162; CHECK-NEXT: br i1 [[TMP0]], label [[ALIAS_CONT:%.*]], label [[NO_ALIAS:%.*]]163; CHECK: alias_cont:164; CHECK-NEXT: [[LOAD_END:%.*]] = add nuw nsw i64 [[LOAD_BEGIN]], 32165; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt i64 [[LOAD_END]], [[STORE_BEGIN]]166; CHECK-NEXT: br i1 [[TMP1]], label [[COPY:%.*]], label [[NO_ALIAS]]167; CHECK: copy:168; CHECK-NEXT: [[TMP2:%.*]] = alloca [4 x double], align 8169; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[TMP2]], ptr noundef nonnull align 8 dereferenceable(32) [[B]], i64 32, i1 false)170; CHECK-NEXT: br label [[NO_ALIAS]]171; CHECK: no_alias:172; CHECK-NEXT: [[TMP3:%.*]] = phi ptr [ [[B]], [[ENTRY:%.*]] ], [ [[B]], [[ALIAS_CONT]] ], [ [[TMP2]], [[COPY]] ]173; CHECK-NEXT: [[COL_LOAD:%.*]] = load <1 x double>, ptr [[A:%.*]], align 8174; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <1 x double>, ptr [[TMP3]], align 8175; CHECK-NEXT: [[TMP4:%.*]] = fmul contract <1 x double> [[COL_LOAD]], [[COL_LOAD1]]176; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 16177; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <1 x double>, ptr [[TMP5]], align 8178; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8179; CHECK-NEXT: [[COL_LOAD3:%.*]] = load <1 x double>, ptr [[TMP6]], align 8180; CHECK-NEXT: [[TMP7:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD2]], <1 x double> [[COL_LOAD3]], <1 x double> [[TMP4]])181; CHECK-NEXT: store <1 x double> [[TMP7]], ptr [[GEP_1]], align 8182; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 8183; CHECK-NEXT: [[COL_LOAD8:%.*]] = load <1 x double>, ptr [[TMP8]], align 8184; CHECK-NEXT: [[COL_LOAD9:%.*]] = load <1 x double>, ptr [[TMP3]], align 8185; CHECK-NEXT: [[TMP9:%.*]] = fmul contract <1 x double> [[COL_LOAD8]], [[COL_LOAD9]]186; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 24187; CHECK-NEXT: [[COL_LOAD13:%.*]] = load <1 x double>, ptr [[TMP10]], align 8188; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8189; CHECK-NEXT: [[COL_LOAD14:%.*]] = load <1 x double>, ptr [[TMP11]], align 8190; CHECK-NEXT: [[TMP12:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD13]], <1 x double> [[COL_LOAD14]], <1 x double> [[TMP9]])191; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[C]], i64 1352192; CHECK-NEXT: store <1 x double> [[TMP12]], ptr [[TMP13]], align 8193; CHECK-NEXT: [[COL_LOAD19:%.*]] = load <1 x double>, ptr [[A]], align 8194; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[TMP3]], i64 16195; CHECK-NEXT: [[COL_LOAD20:%.*]] = load <1 x double>, ptr [[TMP14]], align 8196; CHECK-NEXT: [[TMP15:%.*]] = fmul contract <1 x double> [[COL_LOAD19]], [[COL_LOAD20]]197; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[A]], i64 16198; CHECK-NEXT: [[COL_LOAD24:%.*]] = load <1 x double>, ptr [[TMP16]], align 8199; CHECK-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[TMP3]], i64 24200; CHECK-NEXT: [[COL_LOAD25:%.*]] = load <1 x double>, ptr [[TMP17]], align 8201; CHECK-NEXT: [[TMP18:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD24]], <1 x double> [[COL_LOAD25]], <1 x double> [[TMP15]])202; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[C]], i64 1360203; CHECK-NEXT: store <1 x double> [[TMP18]], ptr [[TMP19]], align 8204; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr [[A]], i64 8205; CHECK-NEXT: [[COL_LOAD30:%.*]] = load <1 x double>, ptr [[TMP20]], align 8206; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP3]], i64 16207; CHECK-NEXT: [[COL_LOAD31:%.*]] = load <1 x double>, ptr [[TMP21]], align 8208; CHECK-NEXT: [[TMP22:%.*]] = fmul contract <1 x double> [[COL_LOAD30]], [[COL_LOAD31]]209; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[A]], i64 24210; CHECK-NEXT: [[COL_LOAD35:%.*]] = load <1 x double>, ptr [[TMP23]], align 8211; CHECK-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP3]], i64 24212; CHECK-NEXT: [[COL_LOAD36:%.*]] = load <1 x double>, ptr [[TMP24]], align 8213; CHECK-NEXT: [[TMP25:%.*]] = call contract <1 x double> @llvm.fmuladd.v1f64(<1 x double> [[COL_LOAD35]], <1 x double> [[COL_LOAD36]], <1 x double> [[TMP22]])214; CHECK-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[C]], i64 1368215; CHECK-NEXT: store <1 x double> [[TMP25]], ptr [[TMP26]], align 8216; CHECK-NEXT: ret void217;218entry:219 %a = load <4 x double>, ptr %A, align 8220 %b = load <4 x double>, ptr %B, align 8221 %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)222 %off.0 = add i32 10, 10223 %off.1 = add i32 %off.0, 2224 %off.2 = add i32 %off.0, %off.1225 %gep.1 = getelementptr <4 x double>, ptr %C, i32 %off.2226 store <4 x double> %c, ptr %gep.1, align 8227 ret void228}229 230define void @multiply_dont_hoist_phi(ptr noalias %A, ptr %B, ptr %C) {231; CHECK-LABEL: @multiply_dont_hoist_phi(232; CHECK-NEXT: entry:233; CHECK-NEXT: br label [[NEXT:%.*]]234; CHECK: next:235; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 16236; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8237; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 16238; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8239; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> <i32 1, i32 1>240; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A]], align 8241; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer242; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]243; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP0]])244; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[B]], align 8245; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> <i32 1, i32 1>246; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer247; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]248; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP2]])249; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 832250; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[GEP_1]], align 8251; CHECK-NEXT: [[VEC_GEP14:%.*]] = getelementptr i8, ptr [[C]], i64 848252; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[VEC_GEP14]], align 8253; CHECK-NEXT: ret void254;255entry:256 %a = load <4 x double>, ptr %A, align 8257 %b = load <4 x double>, ptr %B, align 8258 %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)259 br label %next260 261next:262 %p = phi i32 [ 2, %entry ]263 %off.0 = add i32 10, %p264 %off.1 = add i32 %off.0, 2265 %off.2 = add i32 %off.0, %off.1266 %gep.1 = getelementptr <4 x double>, ptr %C, i32 %off.2267 store <4 x double> %c, ptr %gep.1, align 8268 ret void269}270 271; The address load may alias, so avoid moving it for now.272define void @multiply_dont_hoist_cast_due_to_operand(ptr noalias %A, ptr %B, ptr %C.ptr) {273; CHECK-LABEL: @multiply_dont_hoist_cast_due_to_operand(274; CHECK-NEXT: entry:275; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8276; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 16277; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8278; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[B:%.*]], align 8279; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i8, ptr [[B]], i64 16280; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8281; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer282; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]283; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> <i32 1, i32 1>284; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]])285; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer286; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]287; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> <i32 1, i32 1>288; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]])289; CHECK-NEXT: [[C:%.*]] = load ptr, ptr [[C_PTR:%.*]], align 8290; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[C]], align 8291; CHECK-NEXT: [[VEC_GEP14:%.*]] = getelementptr i8, ptr [[C]], i64 16292; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[VEC_GEP14]], align 8293; CHECK-NEXT: ret void294;295entry:296 %a = load <4 x double>, ptr %A, align 8297 %b = load <4 x double>, ptr %B, align 8298 %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)299 %C = load ptr, ptr %C.ptr300 store <4 x double> %c, ptr %C, align 8301 ret void302}303 304; The address load may alias, so avoid moving it for now.305define void @multiply_dont_hoist_load(ptr noalias %A, ptr %B, ptr %C.ptr) {306; CHECK-LABEL: @multiply_dont_hoist_load(307; CHECK-NEXT: entry:308; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8309; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 16310; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8311; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[B:%.*]], align 8312; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i8, ptr [[B]], i64 16313; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8314; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer315; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]316; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> <i32 1, i32 1>317; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]])318; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer319; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]320; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> <i32 1, i32 1>321; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]])322; CHECK-NEXT: [[C:%.*]] = load ptr, ptr [[C_PTR:%.*]], align 8323; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[C]], align 8324; CHECK-NEXT: [[VEC_GEP14:%.*]] = getelementptr i8, ptr [[C]], i64 16325; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[VEC_GEP14]], align 8326; CHECK-NEXT: ret void327;328entry:329 %a = load <4 x double>, ptr %A, align 8330 %b = load <4 x double>, ptr %B, align 8331 %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)332 %C = load ptr, ptr %C.ptr333 store <4 x double> %c, ptr %C, align 8334 ret void335}336 337; The call to @get_adress may clobber memory, avoid moving it for now.338define void @multiply_dont_hoist_call(ptr noalias %A, ptr %B) {339; CHECK-LABEL: @multiply_dont_hoist_call(340; CHECK-NEXT: entry:341; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[A:%.*]], align 8342; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 16343; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8344; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[B:%.*]], align 8345; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr i8, ptr [[B]], i64 16346; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8347; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> zeroinitializer348; CHECK-NEXT: [[TMP0:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT]]349; CHECK-NEXT: [[SPLAT_SPLAT7:%.*]] = shufflevector <2 x double> [[COL_LOAD2]], <2 x double> poison, <2 x i32> <i32 1, i32 1>350; CHECK-NEXT: [[TMP1:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT7]], <2 x double> [[TMP0]])351; CHECK-NEXT: [[SPLAT_SPLAT10:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> zeroinitializer352; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[COL_LOAD]], [[SPLAT_SPLAT10]]353; CHECK-NEXT: [[SPLAT_SPLAT13:%.*]] = shufflevector <2 x double> [[COL_LOAD4]], <2 x double> poison, <2 x i32> <i32 1, i32 1>354; CHECK-NEXT: [[TMP3:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[COL_LOAD1]], <2 x double> [[SPLAT_SPLAT13]], <2 x double> [[TMP2]])355; CHECK-NEXT: [[C:%.*]] = call ptr @get_address()356; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[C]], align 8357; CHECK-NEXT: [[VEC_GEP14:%.*]] = getelementptr i8, ptr [[C]], i64 16358; CHECK-NEXT: store <2 x double> [[TMP3]], ptr [[VEC_GEP14]], align 8359; CHECK-NEXT: ret void360;361entry:362 %a = load <4 x double>, ptr %A, align 8363 %b = load <4 x double>, ptr %B, align 8364 %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)365 %C = call ptr @get_address()366 store <4 x double> %c, ptr %C, align 8367 ret void368}369 370declare ptr @get_address()371 372 373declare <4 x double> @llvm.matrix.multiply(<4 x double>, <4 x double>, i32, i32, i32)374