574 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -passes=slp-vectorizer -S %s | FileCheck %s3 4target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"5target triple = "arm64-apple-darwin"6 7declare void @use(double)8 9; The extracts %v1.lane.0 and %v1.lane.1 should be considered free during SLP,10; because they will be directly in a vector register on AArch64.11define void @noop_extracts_first_2_lanes(ptr %ptr.1, ptr %ptr.2) {12; CHECK-LABEL: @noop_extracts_first_2_lanes(13; CHECK-NEXT: bb:14; CHECK-NEXT: [[V_1:%.*]] = load <2 x double>, ptr [[PTR_1:%.*]], align 815; CHECK-NEXT: [[V1_LANE_0:%.*]] = extractelement <2 x double> [[V_1]], i32 016; CHECK-NEXT: [[V1_LANE_1:%.*]] = extractelement <2 x double> [[V_1]], i32 117; CHECK-NEXT: [[V_2:%.*]] = load <4 x double>, ptr [[PTR_2:%.*]], align 1618; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <2 x i32> <i32 2, i32 3>19; CHECK-NEXT: [[TMP1:%.*]] = fmul <2 x double> [[V_1]], [[TMP0]]20; CHECK-NEXT: call void @use(double [[V1_LANE_0]])21; CHECK-NEXT: call void @use(double [[V1_LANE_1]])22; CHECK-NEXT: store <2 x double> [[TMP1]], ptr [[PTR_1]], align 823; CHECK-NEXT: ret void24;25bb:26 %v.1 = load <2 x double>, ptr %ptr.1, align 827 %v1.lane.0 = extractelement <2 x double> %v.1, i32 028 %v1.lane.1 = extractelement <2 x double> %v.1, i32 129 30 %v.2 = load <4 x double>, ptr %ptr.2, align 1631 %v2.lane.2 = extractelement <4 x double> %v.2, i32 232 %v2.lane.3 = extractelement <4 x double> %v.2, i32 333 34 %a.lane.0 = fmul double %v1.lane.0, %v2.lane.235 %a.lane.1 = fmul double %v1.lane.1, %v2.lane.336 37 %a.ins.0 = insertelement <2 x double> zeroinitializer, double %a.lane.0, i32 038 %a.ins.1 = insertelement <2 x double> %a.ins.0, double %a.lane.1, i32 139 40 call void @use(double %v1.lane.0)41 call void @use(double %v1.lane.1)42 43 store <2 x double> %a.ins.1, ptr %ptr.1, align 844 ret void45}46 47; Extracts of consecutive indices, but different vector operand.48define void @extracts_first_2_lanes_different_vectors(ptr %ptr.1, ptr %ptr.2, ptr %ptr.3) {49; CHECK-LABEL: @extracts_first_2_lanes_different_vectors(50; CHECK-NEXT: bb:51; CHECK-NEXT: [[V_1:%.*]] = load <2 x double>, ptr [[PTR_1:%.*]], align 852; CHECK-NEXT: [[V1_LANE_0:%.*]] = extractelement <2 x double> [[V_1]], i32 053; CHECK-NEXT: [[V_3:%.*]] = load <2 x double>, ptr [[PTR_3:%.*]], align 854; CHECK-NEXT: [[V3_LANE_1:%.*]] = extractelement <2 x double> [[V_3]], i32 155; CHECK-NEXT: [[V_2:%.*]] = load <4 x double>, ptr [[PTR_2:%.*]], align 1656; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <2 x double> [[V_1]], <2 x double> [[V_3]], <2 x i32> <i32 0, i32 3>57; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <2 x i32> <i32 2, i32 2>58; CHECK-NEXT: [[TMP2:%.*]] = fmul <2 x double> [[TMP0]], [[TMP1]]59; CHECK-NEXT: call void @use(double [[V1_LANE_0]])60; CHECK-NEXT: call void @use(double [[V3_LANE_1]])61; CHECK-NEXT: store <2 x double> [[TMP2]], ptr [[PTR_1]], align 862; CHECK-NEXT: ret void63;64bb:65 %v.1 = load <2 x double>, ptr %ptr.1, align 866 %v1.lane.0 = extractelement <2 x double> %v.1, i32 067 %v.3 = load <2 x double>, ptr %ptr.3, align 868 %v3.lane.1 = extractelement <2 x double> %v.3, i32 169 70 %v.2 = load <4 x double>, ptr %ptr.2, align 1671 %v2.lane.2 = extractelement <4 x double> %v.2, i32 272 73 %a.lane.0 = fmul double %v1.lane.0, %v2.lane.274 %a.lane.1 = fmul double %v3.lane.1, %v2.lane.275 76 %a.ins.0 = insertelement <2 x double> zeroinitializer, double %a.lane.0, i32 077 %a.ins.1 = insertelement <2 x double> %a.ins.0, double %a.lane.1, i32 178 79 call void @use(double %v1.lane.0)80 call void @use(double %v3.lane.1)81 82 store <2 x double> %a.ins.1, ptr %ptr.1, align 883 ret void84}85 86; The extracts %v1.lane.2 and %v1.lane.3 should be considered free during SLP,87; because they will be directly in a vector register on AArch64.88define void @noop_extract_second_2_lanes(ptr %ptr.1, ptr %ptr.2) {89; CHECK-LABEL: @noop_extract_second_2_lanes(90; CHECK-NEXT: bb:91; CHECK-NEXT: [[V_1:%.*]] = load <4 x double>, ptr [[PTR_1:%.*]], align 892; CHECK-NEXT: [[V1_LANE_2:%.*]] = extractelement <4 x double> [[V_1]], i32 293; CHECK-NEXT: [[V1_LANE_3:%.*]] = extractelement <4 x double> [[V_1]], i32 394; CHECK-NEXT: [[V_2:%.*]] = load <4 x double>, ptr [[PTR_2:%.*]], align 1695; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x double> [[V_1]], <4 x double> poison, <2 x i32> <i32 2, i32 3>96; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <2 x i32> <i32 2, i32 2>97; CHECK-NEXT: [[TMP2:%.*]] = fmul <2 x double> [[TMP0]], [[TMP1]]98; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>99; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x double> zeroinitializer, <4 x double> [[TMP4]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>100; CHECK-NEXT: call void @use(double [[V1_LANE_2]])101; CHECK-NEXT: call void @use(double [[V1_LANE_3]])102; CHECK-NEXT: store <4 x double> [[TMP3]], ptr [[PTR_1]], align 8103; CHECK-NEXT: ret void104;105bb:106 %v.1 = load <4 x double>, ptr %ptr.1, align 8107 %v1.lane.2 = extractelement <4 x double> %v.1, i32 2108 %v1.lane.3 = extractelement <4 x double> %v.1, i32 3109 110 %v.2 = load <4 x double>, ptr %ptr.2, align 16111 %v2.lane.2 = extractelement <4 x double> %v.2, i32 2112 113 %a.lane.0 = fmul double %v1.lane.2, %v2.lane.2114 %a.lane.1 = fmul double %v1.lane.3, %v2.lane.2115 116 %a.ins.0 = insertelement <4 x double> zeroinitializer, double %a.lane.0, i32 0117 %a.ins.1 = insertelement <4 x double> %a.ins.0, double %a.lane.1, i32 1118 119 call void @use(double %v1.lane.2)120 call void @use(double %v1.lane.3)121 store <4 x double> %a.ins.1, ptr %ptr.1, align 8122 ret void123}124 125; %v1.lane.0 and %v1.lane.1 are used in reverse-order, so they won't be126; directly in a vector register on AArch64.127define void @extract_reverse_order(ptr %ptr.1, ptr %ptr.2) {128; CHECK-LABEL: @extract_reverse_order(129; CHECK-NEXT: bb:130; CHECK-NEXT: [[V_1:%.*]] = load <2 x double>, ptr [[PTR_1:%.*]], align 8131; CHECK-NEXT: [[V1_LANE_0:%.*]] = extractelement <2 x double> [[V_1]], i32 0132; CHECK-NEXT: [[V1_LANE_1:%.*]] = extractelement <2 x double> [[V_1]], i32 1133; CHECK-NEXT: [[V_2:%.*]] = load <4 x double>, ptr [[PTR_2:%.*]], align 16134; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <2 x i32> <i32 2, i32 2>135; CHECK-NEXT: [[TMP1:%.*]] = fmul <2 x double> [[V_1]], [[TMP0]]136; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 1, i32 0>137; CHECK-NEXT: call void @use(double [[V1_LANE_0]])138; CHECK-NEXT: call void @use(double [[V1_LANE_1]])139; CHECK-NEXT: store <2 x double> [[TMP2]], ptr [[PTR_1]], align 8140; CHECK-NEXT: ret void141;142bb:143 %v.1 = load <2 x double>, ptr %ptr.1, align 8144 %v1.lane.0 = extractelement <2 x double> %v.1, i32 0145 %v1.lane.1 = extractelement <2 x double> %v.1, i32 1146 147 %v.2 = load <4 x double>, ptr %ptr.2, align 16148 %v2.lane.2 = extractelement <4 x double> %v.2, i32 2149 150 %a.lane.0 = fmul double %v1.lane.1, %v2.lane.2151 %a.lane.1 = fmul double %v1.lane.0, %v2.lane.2152 153 %a.ins.0 = insertelement <2 x double> zeroinitializer, double %a.lane.0, i32 0154 %a.ins.1 = insertelement <2 x double> %a.ins.0, double %a.lane.1, i32 1155 156 call void @use(double %v1.lane.0)157 call void @use(double %v1.lane.1)158 159 store <2 x double> %a.ins.1, ptr %ptr.1, align 8160 ret void161}162 163; %v1.lane.1 and %v1.lane.2 are extracted from different vector registers on AArch64.164define void @extract_lanes_1_and_2(ptr %ptr.1, ptr %ptr.2) {165; CHECK-LABEL: @extract_lanes_1_and_2(166; CHECK-NEXT: bb:167; CHECK-NEXT: [[V_1:%.*]] = load <4 x double>, ptr [[PTR_1:%.*]], align 8168; CHECK-NEXT: [[V1_LANE_1:%.*]] = extractelement <4 x double> [[V_1]], i32 1169; CHECK-NEXT: [[V1_LANE_2:%.*]] = extractelement <4 x double> [[V_1]], i32 2170; CHECK-NEXT: [[V_2:%.*]] = load <4 x double>, ptr [[PTR_2:%.*]], align 16171; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x double> [[V_1]], <4 x double> poison, <2 x i32> <i32 1, i32 2>172; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <2 x i32> <i32 2, i32 2>173; CHECK-NEXT: [[TMP2:%.*]] = fmul <2 x double> [[TMP0]], [[TMP1]]174; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>175; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x double> zeroinitializer, <4 x double> [[TMP4]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>176; CHECK-NEXT: call void @use(double [[V1_LANE_1]])177; CHECK-NEXT: call void @use(double [[V1_LANE_2]])178; CHECK-NEXT: store <4 x double> [[TMP3]], ptr [[PTR_1]], align 8179; CHECK-NEXT: ret void180;181bb:182 %v.1 = load <4 x double>, ptr %ptr.1, align 8183 %v1.lane.1 = extractelement <4 x double> %v.1, i32 1184 %v1.lane.2 = extractelement <4 x double> %v.1, i32 2185 186 %v.2 = load <4 x double>, ptr %ptr.2, align 16187 %v2.lane.2 = extractelement <4 x double> %v.2, i32 2188 189 %a.lane.0 = fmul double %v1.lane.1, %v2.lane.2190 %a.lane.1 = fmul double %v1.lane.2, %v2.lane.2191 192 %a.ins.0 = insertelement <4 x double> zeroinitializer, double %a.lane.0, i32 0193 %a.ins.1 = insertelement <4 x double> %a.ins.0, double %a.lane.1, i32 1194 195 call void @use(double %v1.lane.1)196 call void @use(double %v1.lane.2)197 198 store <4 x double> %a.ins.1, ptr %ptr.1, align 8199 ret void200}201 202; More complex case where the extracted lanes are directly from a vector203; register on AArch64 and should be considered free, because we can204; directly use the source vector register.205define void @noop_extracts_existing_vector_4_lanes(ptr %ptr.1, ptr %ptr.2) {206; CHECK-LABEL: @noop_extracts_existing_vector_4_lanes(207; CHECK-NEXT: bb:208; CHECK-NEXT: [[V_1:%.*]] = load <9 x double>, ptr [[PTR_1:%.*]], align 8209; CHECK-NEXT: [[V1_LANE_0:%.*]] = extractelement <9 x double> [[V_1]], i32 0210; CHECK-NEXT: [[V1_LANE_1:%.*]] = extractelement <9 x double> [[V_1]], i32 1211; CHECK-NEXT: [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2212; CHECK-NEXT: [[V1_LANE_3:%.*]] = extractelement <9 x double> [[V_1]], i32 3213; CHECK-NEXT: [[V_2:%.*]] = load <4 x double>, ptr [[PTR_2:%.*]], align 16214; CHECK-NEXT: [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1215; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>216; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <4 x i32> <i32 2, i32 0, i32 2, i32 2>217; CHECK-NEXT: [[TMP2:%.*]] = fmul <4 x double> [[TMP0]], [[TMP1]]218; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> poison, <9 x i32> <i32 2, i32 3, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>219; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <9 x double> zeroinitializer, <9 x double> [[TMP4]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 4, i32 5, i32 6, i32 7, i32 8>220; CHECK-NEXT: call void @use(double [[V1_LANE_0]])221; CHECK-NEXT: call void @use(double [[V1_LANE_1]])222; CHECK-NEXT: call void @use(double [[V1_LANE_2]])223; CHECK-NEXT: call void @use(double [[V1_LANE_3]])224; CHECK-NEXT: store <9 x double> [[TMP3]], ptr [[PTR_1]], align 8225; CHECK-NEXT: ret void226;227bb:228 %v.1 = load <9 x double>, ptr %ptr.1, align 8229 %v1.lane.0 = extractelement <9 x double> %v.1, i32 0230 %v1.lane.1 = extractelement <9 x double> %v.1, i32 1231 %v1.lane.2 = extractelement <9 x double> %v.1, i32 2232 %v1.lane.3 = extractelement <9 x double> %v.1, i32 3233 %v.2 = load <4 x double>, ptr %ptr.2, align 16234 %v2.lane.0 = extractelement <4 x double> %v.2, i32 0235 %v2.lane.1 = extractelement <4 x double> %v.2, i32 1236 %v2.lane.2 = extractelement <4 x double> %v.2, i32 2237 %a.lane.0 = fmul double %v1.lane.2, %v2.lane.2238 %a.lane.1 = fmul double %v1.lane.3, %v2.lane.2239 %a.lane.2 = fmul double %v1.lane.0, %v2.lane.2240 %a.lane.3 = fmul double %v1.lane.1, %v2.lane.0241 %a.ins.0 = insertelement <9 x double> zeroinitializer, double %a.lane.0, i32 0242 %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1243 %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2244 %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3245 call void @use(double %v1.lane.0)246 call void @use(double %v1.lane.1)247 call void @use(double %v1.lane.2)248 call void @use(double %v1.lane.3)249 store <9 x double> %a.ins.3, ptr %ptr.1, align 8250 ret void251}252 253; Extracted lanes are not used in the right order, so we cannot reuse the254; source vector registers directly.255define void @extracts_jumbled_4_lanes(ptr %ptr.1, ptr %ptr.2) {256; CHECK-LABEL: @extracts_jumbled_4_lanes(257; CHECK-NEXT: bb:258; CHECK-NEXT: [[V_1:%.*]] = load <9 x double>, ptr [[PTR_1:%.*]], align 8259; CHECK-NEXT: [[V1_LANE_0:%.*]] = extractelement <9 x double> [[V_1]], i32 0260; CHECK-NEXT: [[V1_LANE_1:%.*]] = extractelement <9 x double> [[V_1]], i32 1261; CHECK-NEXT: [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2262; CHECK-NEXT: [[V1_LANE_3:%.*]] = extractelement <9 x double> [[V_1]], i32 3263; CHECK-NEXT: [[V_2:%.*]] = load <4 x double>, ptr [[PTR_2:%.*]], align 16264; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>265; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <4 x i32> <i32 2, i32 2, i32 1, i32 0>266; CHECK-NEXT: [[TMP2:%.*]] = fmul <4 x double> [[TMP0]], [[TMP1]]267; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> poison, <9 x i32> <i32 0, i32 2, i32 1, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>268; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <9 x double> zeroinitializer, <9 x double> [[TMP4]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 4, i32 5, i32 6, i32 7, i32 8>269; CHECK-NEXT: call void @use(double [[V1_LANE_0]])270; CHECK-NEXT: call void @use(double [[V1_LANE_1]])271; CHECK-NEXT: call void @use(double [[V1_LANE_2]])272; CHECK-NEXT: call void @use(double [[V1_LANE_3]])273; CHECK-NEXT: store <9 x double> [[TMP3]], ptr [[PTR_1]], align 8274; CHECK-NEXT: ret void275;276bb:277 %v.1 = load <9 x double>, ptr %ptr.1, align 8278 %v1.lane.0 = extractelement <9 x double> %v.1, i32 0279 %v1.lane.1 = extractelement <9 x double> %v.1, i32 1280 %v1.lane.2 = extractelement <9 x double> %v.1, i32 2281 %v1.lane.3 = extractelement <9 x double> %v.1, i32 3282 %v.2 = load <4 x double>, ptr %ptr.2, align 16283 %v2.lane.0 = extractelement <4 x double> %v.2, i32 0284 %v2.lane.1 = extractelement <4 x double> %v.2, i32 1285 %v2.lane.2 = extractelement <4 x double> %v.2, i32 2286 %a.lane.0 = fmul double %v1.lane.0, %v2.lane.2287 %a.lane.1 = fmul double %v1.lane.2, %v2.lane.1288 %a.lane.2 = fmul double %v1.lane.1, %v2.lane.2289 %a.lane.3 = fmul double %v1.lane.3, %v2.lane.0290 %a.ins.0 = insertelement <9 x double> zeroinitializer, double %a.lane.0, i32 0291 %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1292 %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2293 %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3294 call void @use(double %v1.lane.0)295 call void @use(double %v1.lane.1)296 call void @use(double %v1.lane.2)297 call void @use(double %v1.lane.3)298 store <9 x double> %a.ins.3, ptr %ptr.1, align 8299 ret void300}301 302 303; Even more complex case where the extracted lanes are directly from a vector304; register on AArch64 and should be considered free, because we can305; directly use the source vector register.306define void @noop_extracts_9_lanes(ptr %ptr.1, ptr %ptr.2) {307; CHECK-LABEL: @noop_extracts_9_lanes(308; CHECK-NEXT: bb:309; CHECK-NEXT: [[V_1:%.*]] = load <9 x double>, ptr [[PTR_1:%.*]], align 8310; CHECK-NEXT: [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2311; CHECK-NEXT: [[V1_LANE_5:%.*]] = extractelement <9 x double> [[V_1]], i32 5312; CHECK-NEXT: [[V_2:%.*]] = load <4 x double>, ptr [[PTR_2:%.*]], align 16313; CHECK-NEXT: [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0314; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> <i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 0, i32 1>315; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> <i32 0, i32 2, i32 1, i32 0, i32 2, i32 0, i32 2, i32 1>316; CHECK-NEXT: [[TMP2:%.*]] = fmul <8 x double> [[TMP0]], [[TMP1]]317; CHECK-NEXT: [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_0]]318; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>319; CHECK-NEXT: [[A_INS_72:%.*]] = shufflevector <9 x double> zeroinitializer, <9 x double> [[TMP3]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>320; CHECK-NEXT: [[A_INS_8:%.*]] = insertelement <9 x double> [[A_INS_72]], double [[A_LANE_8]], i32 8321; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> <i32 6, i32 7, i32 8, i32 0, i32 1, i32 2, i32 3, i32 4>322; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> <i32 2, i32 1, i32 0, i32 2, i32 1, i32 0, i32 2, i32 1>323; CHECK-NEXT: [[TMP6:%.*]] = fmul <8 x double> [[TMP4]], [[TMP5]]324; CHECK-NEXT: [[B_LANE_8:%.*]] = fmul double [[V1_LANE_5]], [[V2_LANE_0]]325; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>326; CHECK-NEXT: [[B_INS_71:%.*]] = shufflevector <9 x double> zeroinitializer, <9 x double> [[TMP7]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>327; CHECK-NEXT: [[B_INS_8:%.*]] = insertelement <9 x double> [[B_INS_71]], double [[B_LANE_8]], i32 8328; CHECK-NEXT: [[RES:%.*]] = fsub <9 x double> [[A_INS_8]], [[B_INS_8]]329; CHECK-NEXT: store <9 x double> [[RES]], ptr [[PTR_1]], align 8330; CHECK-NEXT: ret void331;332bb:333 %v.1 = load <9 x double>, ptr %ptr.1, align 8334 %v1.lane.0 = extractelement <9 x double> %v.1, i32 0335 %v1.lane.1 = extractelement <9 x double> %v.1, i32 1336 %v1.lane.2 = extractelement <9 x double> %v.1, i32 2337 %v1.lane.3 = extractelement <9 x double> %v.1, i32 3338 %v1.lane.4 = extractelement <9 x double> %v.1, i32 4339 %v1.lane.5 = extractelement <9 x double> %v.1, i32 5340 %v1.lane.6 = extractelement <9 x double> %v.1, i32 6341 %v1.lane.7 = extractelement <9 x double> %v.1, i32 7342 %v1.lane.8 = extractelement <9 x double> %v.1, i32 8343 344 %v.2 = load <4 x double>, ptr %ptr.2, align 16345 %v2.lane.0 = extractelement <4 x double> %v.2, i32 0346 %v2.lane.1 = extractelement <4 x double> %v.2, i32 1347 %v2.lane.2 = extractelement <4 x double> %v.2, i32 2348 349 %a.lane.0 = fmul double %v1.lane.3, %v2.lane.0350 %a.lane.1 = fmul double %v1.lane.4, %v2.lane.2351 %a.lane.2 = fmul double %v1.lane.5, %v2.lane.1352 %a.lane.3 = fmul double %v1.lane.6, %v2.lane.0353 %a.lane.4 = fmul double %v1.lane.7, %v2.lane.2354 %a.lane.5 = fmul double %v1.lane.8, %v2.lane.0355 %a.lane.6 = fmul double %v1.lane.0, %v2.lane.2356 %a.lane.7 = fmul double %v1.lane.1, %v2.lane.1357 %a.lane.8 = fmul double %v1.lane.2, %v2.lane.0358 359 %a.ins.0 = insertelement <9 x double> zeroinitializer, double %a.lane.0, i32 0360 %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1361 %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2362 %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3363 %a.ins.4 = insertelement <9 x double> %a.ins.3, double %a.lane.4, i32 4364 %a.ins.5 = insertelement <9 x double> %a.ins.4, double %a.lane.5, i32 5365 %a.ins.6 = insertelement <9 x double> %a.ins.5, double %a.lane.6, i32 6366 %a.ins.7 = insertelement <9 x double> %a.ins.6, double %a.lane.7, i32 7367 %a.ins.8 = insertelement <9 x double> %a.ins.7, double %a.lane.8, i32 8368 369 %b.lane.0 = fmul double %v1.lane.6, %v2.lane.2370 %b.lane.1 = fmul double %v1.lane.7, %v2.lane.1371 %b.lane.2 = fmul double %v1.lane.8, %v2.lane.0372 %b.lane.3 = fmul double %v1.lane.0, %v2.lane.2373 %b.lane.4 = fmul double %v1.lane.1, %v2.lane.1374 %b.lane.5 = fmul double %v1.lane.2, %v2.lane.0375 %b.lane.6 = fmul double %v1.lane.3, %v2.lane.2376 %b.lane.7 = fmul double %v1.lane.4, %v2.lane.1377 %b.lane.8 = fmul double %v1.lane.5, %v2.lane.0378 379 %b.ins.0 = insertelement <9 x double> zeroinitializer, double %b.lane.0, i32 0380 %b.ins.1 = insertelement <9 x double> %b.ins.0, double %b.lane.1, i32 1381 %b.ins.2 = insertelement <9 x double> %b.ins.1, double %b.lane.2, i32 2382 %b.ins.3 = insertelement <9 x double> %b.ins.2, double %b.lane.3, i32 3383 %b.ins.4 = insertelement <9 x double> %b.ins.3, double %b.lane.4, i32 4384 %b.ins.5 = insertelement <9 x double> %b.ins.4, double %b.lane.5, i32 5385 %b.ins.6 = insertelement <9 x double> %b.ins.5, double %b.lane.6, i32 6386 %b.ins.7 = insertelement <9 x double> %b.ins.6, double %b.lane.7, i32 7387 %b.ins.8 = insertelement <9 x double> %b.ins.7, double %b.lane.8, i32 8388 389 %res = fsub <9 x double> %a.ins.8, %b.ins.8390 store <9 x double> %res, ptr %ptr.1, align 8391 ret void392}393 394; Extracted lanes used in first fmul chain are not used in the right order, so395; we cannot reuse the source vector registers directly.396define void @first_mul_chain_jumbled(ptr %ptr.1, ptr %ptr.2) {397; CHECK-LABEL: @first_mul_chain_jumbled(398; CHECK-NEXT: bb:399; CHECK-NEXT: [[V_1:%.*]] = load <9 x double>, ptr [[PTR_1:%.*]], align 8400; CHECK-NEXT: [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2401; CHECK-NEXT: [[V1_LANE_5:%.*]] = extractelement <9 x double> [[V_1]], i32 5402; CHECK-NEXT: [[V_2:%.*]] = load <4 x double>, ptr [[PTR_2:%.*]], align 16403; CHECK-NEXT: [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0404; CHECK-NEXT: [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1405; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> <i32 4, i32 3, i32 6, i32 5, i32 8, i32 7, i32 1, i32 0>406; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> <i32 1, i32 0, i32 2, i32 0, i32 2, i32 1, i32 0, i32 2>407; CHECK-NEXT: [[TMP2:%.*]] = fmul <8 x double> [[TMP0]], [[TMP1]]408; CHECK-NEXT: [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_1]]409; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>410; CHECK-NEXT: [[A_INS_72:%.*]] = shufflevector <9 x double> zeroinitializer, <9 x double> [[TMP3]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>411; CHECK-NEXT: [[A_INS_8:%.*]] = insertelement <9 x double> [[A_INS_72]], double [[A_LANE_8]], i32 8412; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> <i32 6, i32 7, i32 8, i32 0, i32 1, i32 2, i32 3, i32 4>413; CHECK-NEXT: [[TMP5:%.*]] = fmul <8 x double> [[TMP4]], [[TMP1]]414; CHECK-NEXT: [[B_LANE_8:%.*]] = fmul double [[V1_LANE_5]], [[V2_LANE_0]]415; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x double> [[TMP5]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>416; CHECK-NEXT: [[B_INS_71:%.*]] = shufflevector <9 x double> zeroinitializer, <9 x double> [[TMP6]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>417; CHECK-NEXT: [[B_INS_8:%.*]] = insertelement <9 x double> [[B_INS_71]], double [[B_LANE_8]], i32 8418; CHECK-NEXT: [[RES:%.*]] = fsub <9 x double> [[A_INS_8]], [[B_INS_8]]419; CHECK-NEXT: store <9 x double> [[RES]], ptr [[PTR_1]], align 8420; CHECK-NEXT: ret void421;422bb:423 %v.1 = load <9 x double>, ptr %ptr.1, align 8424 %v1.lane.0 = extractelement <9 x double> %v.1, i32 0425 %v1.lane.1 = extractelement <9 x double> %v.1, i32 1426 %v1.lane.2 = extractelement <9 x double> %v.1, i32 2427 %v1.lane.3 = extractelement <9 x double> %v.1, i32 3428 %v1.lane.4 = extractelement <9 x double> %v.1, i32 4429 %v1.lane.5 = extractelement <9 x double> %v.1, i32 5430 %v1.lane.6 = extractelement <9 x double> %v.1, i32 6431 %v1.lane.7 = extractelement <9 x double> %v.1, i32 7432 %v1.lane.8 = extractelement <9 x double> %v.1, i32 8433 434 %v.2 = load <4 x double>, ptr %ptr.2, align 16435 %v2.lane.0 = extractelement <4 x double> %v.2, i32 0436 %v2.lane.1 = extractelement <4 x double> %v.2, i32 1437 %v2.lane.2 = extractelement <4 x double> %v.2, i32 2438 439 %a.lane.0 = fmul double %v1.lane.4, %v2.lane.1440 %a.lane.1 = fmul double %v1.lane.3, %v2.lane.0441 %a.lane.2 = fmul double %v1.lane.6, %v2.lane.2442 %a.lane.3 = fmul double %v1.lane.5, %v2.lane.0443 %a.lane.4 = fmul double %v1.lane.8, %v2.lane.2444 %a.lane.5 = fmul double %v1.lane.7, %v2.lane.1445 %a.lane.6 = fmul double %v1.lane.1, %v2.lane.0446 %a.lane.7 = fmul double %v1.lane.0, %v2.lane.2447 %a.lane.8 = fmul double %v1.lane.2, %v2.lane.1448 449 %a.ins.0 = insertelement <9 x double> zeroinitializer, double %a.lane.0, i32 0450 %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1451 %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2452 %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3453 %a.ins.4 = insertelement <9 x double> %a.ins.3, double %a.lane.4, i32 4454 %a.ins.5 = insertelement <9 x double> %a.ins.4, double %a.lane.5, i32 5455 %a.ins.6 = insertelement <9 x double> %a.ins.5, double %a.lane.6, i32 6456 %a.ins.7 = insertelement <9 x double> %a.ins.6, double %a.lane.7, i32 7457 %a.ins.8 = insertelement <9 x double> %a.ins.7, double %a.lane.8, i32 8458 459 %b.lane.0 = fmul double %v1.lane.6, %v2.lane.1460 %b.lane.1 = fmul double %v1.lane.7, %v2.lane.0461 %b.lane.2 = fmul double %v1.lane.8, %v2.lane.2462 %b.lane.3 = fmul double %v1.lane.0, %v2.lane.0463 %b.lane.4 = fmul double %v1.lane.1, %v2.lane.2464 %b.lane.5 = fmul double %v1.lane.2, %v2.lane.1465 %b.lane.6 = fmul double %v1.lane.3, %v2.lane.0466 %b.lane.7 = fmul double %v1.lane.4, %v2.lane.2467 %b.lane.8 = fmul double %v1.lane.5, %v2.lane.0468 469 %b.ins.0 = insertelement <9 x double> zeroinitializer, double %b.lane.0, i32 0470 %b.ins.1 = insertelement <9 x double> %b.ins.0, double %b.lane.1, i32 1471 %b.ins.2 = insertelement <9 x double> %b.ins.1, double %b.lane.2, i32 2472 %b.ins.3 = insertelement <9 x double> %b.ins.2, double %b.lane.3, i32 3473 %b.ins.4 = insertelement <9 x double> %b.ins.3, double %b.lane.4, i32 4474 %b.ins.5 = insertelement <9 x double> %b.ins.4, double %b.lane.5, i32 5475 %b.ins.6 = insertelement <9 x double> %b.ins.5, double %b.lane.6, i32 6476 %b.ins.7 = insertelement <9 x double> %b.ins.6, double %b.lane.7, i32 7477 %b.ins.8 = insertelement <9 x double> %b.ins.7, double %b.lane.8, i32 8478 479 %res = fsub <9 x double> %a.ins.8, %b.ins.8480 store <9 x double> %res, ptr %ptr.1, align 8481 ret void482}483 484; Extracted lanes used in both fmul chain are not used in the right order, so485; we cannot reuse the source vector registers directly.486define void @first_and_second_mul_chain_jumbled(ptr %ptr.1, ptr %ptr.2) {487; CHECK-LABEL: @first_and_second_mul_chain_jumbled(488; CHECK-NEXT: bb:489; CHECK-NEXT: [[V_1:%.*]] = load <9 x double>, ptr [[PTR_1:%.*]], align 8490; CHECK-NEXT: [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2491; CHECK-NEXT: [[V1_LANE_4:%.*]] = extractelement <9 x double> [[V_1]], i32 4492; CHECK-NEXT: [[V_2:%.*]] = load <4 x double>, ptr [[PTR_2:%.*]], align 16493; CHECK-NEXT: [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0494; CHECK-NEXT: [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2495; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> <i32 4, i32 3, i32 5, i32 6, i32 8, i32 7, i32 1, i32 0>496; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> <i32 0, i32 2, i32 1, i32 2, i32 1, i32 0, i32 2, i32 1>497; CHECK-NEXT: [[TMP2:%.*]] = fmul <8 x double> [[TMP0]], [[TMP1]]498; CHECK-NEXT: [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_0]]499; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>500; CHECK-NEXT: [[A_INS_72:%.*]] = shufflevector <9 x double> zeroinitializer, <9 x double> [[TMP3]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>501; CHECK-NEXT: [[A_INS_8:%.*]] = insertelement <9 x double> [[A_INS_72]], double [[A_LANE_8]], i32 8502; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> <i32 7, i32 6, i32 8, i32 1, i32 0, i32 3, i32 2, i32 5>503; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> <i32 2, i32 1, i32 0, i32 2, i32 0, i32 2, i32 1, i32 0>504; CHECK-NEXT: [[TMP6:%.*]] = fmul <8 x double> [[TMP4]], [[TMP5]]505; CHECK-NEXT: [[B_LANE_8:%.*]] = fmul double [[V1_LANE_4]], [[V2_LANE_2]]506; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>507; CHECK-NEXT: [[B_INS_71:%.*]] = shufflevector <9 x double> zeroinitializer, <9 x double> [[TMP7]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>508; CHECK-NEXT: [[B_INS_8:%.*]] = insertelement <9 x double> [[B_INS_71]], double [[B_LANE_8]], i32 8509; CHECK-NEXT: [[RES:%.*]] = fsub <9 x double> [[A_INS_8]], [[B_INS_8]]510; CHECK-NEXT: store <9 x double> [[RES]], ptr [[PTR_1]], align 8511; CHECK-NEXT: ret void512;513bb:514 %v.1 = load <9 x double>, ptr %ptr.1, align 8515 %v1.lane.0 = extractelement <9 x double> %v.1, i32 0516 %v1.lane.1 = extractelement <9 x double> %v.1, i32 1517 %v1.lane.2 = extractelement <9 x double> %v.1, i32 2518 %v1.lane.3 = extractelement <9 x double> %v.1, i32 3519 %v1.lane.4 = extractelement <9 x double> %v.1, i32 4520 %v1.lane.5 = extractelement <9 x double> %v.1, i32 5521 %v1.lane.6 = extractelement <9 x double> %v.1, i32 6522 %v1.lane.7 = extractelement <9 x double> %v.1, i32 7523 %v1.lane.8 = extractelement <9 x double> %v.1, i32 8524 525 %v.2 = load <4 x double>, ptr %ptr.2, align 16526 %v2.lane.0 = extractelement <4 x double> %v.2, i32 0527 %v2.lane.1 = extractelement <4 x double> %v.2, i32 1528 %v2.lane.2 = extractelement <4 x double> %v.2, i32 2529 530 %a.lane.0 = fmul double %v1.lane.4, %v2.lane.0531 %a.lane.1 = fmul double %v1.lane.3, %v2.lane.2532 %a.lane.2 = fmul double %v1.lane.5, %v2.lane.1533 %a.lane.3 = fmul double %v1.lane.6, %v2.lane.2534 %a.lane.4 = fmul double %v1.lane.8, %v2.lane.1535 %a.lane.5 = fmul double %v1.lane.7, %v2.lane.0536 %a.lane.6 = fmul double %v1.lane.1, %v2.lane.2537 %a.lane.7 = fmul double %v1.lane.0, %v2.lane.1538 %a.lane.8 = fmul double %v1.lane.2, %v2.lane.0539 540 %a.ins.0 = insertelement <9 x double> zeroinitializer, double %a.lane.0, i32 0541 %a.ins.1 = insertelement <9 x double> %a.ins.0, double %a.lane.1, i32 1542 %a.ins.2 = insertelement <9 x double> %a.ins.1, double %a.lane.2, i32 2543 %a.ins.3 = insertelement <9 x double> %a.ins.2, double %a.lane.3, i32 3544 %a.ins.4 = insertelement <9 x double> %a.ins.3, double %a.lane.4, i32 4545 %a.ins.5 = insertelement <9 x double> %a.ins.4, double %a.lane.5, i32 5546 %a.ins.6 = insertelement <9 x double> %a.ins.5, double %a.lane.6, i32 6547 %a.ins.7 = insertelement <9 x double> %a.ins.6, double %a.lane.7, i32 7548 %a.ins.8 = insertelement <9 x double> %a.ins.7, double %a.lane.8, i32 8549 550 %b.lane.0 = fmul double %v1.lane.7, %v2.lane.2551 %b.lane.1 = fmul double %v1.lane.6, %v2.lane.1552 %b.lane.2 = fmul double %v1.lane.8, %v2.lane.0553 %b.lane.3 = fmul double %v1.lane.1, %v2.lane.2554 %b.lane.4 = fmul double %v1.lane.0, %v2.lane.0555 %b.lane.5 = fmul double %v1.lane.3, %v2.lane.2556 %b.lane.6 = fmul double %v1.lane.2, %v2.lane.1557 %b.lane.7 = fmul double %v1.lane.5, %v2.lane.0558 %b.lane.8 = fmul double %v1.lane.4, %v2.lane.2559 560 %b.ins.0 = insertelement <9 x double> zeroinitializer, double %b.lane.0, i32 0561 %b.ins.1 = insertelement <9 x double> %b.ins.0, double %b.lane.1, i32 1562 %b.ins.2 = insertelement <9 x double> %b.ins.1, double %b.lane.2, i32 2563 %b.ins.3 = insertelement <9 x double> %b.ins.2, double %b.lane.3, i32 3564 %b.ins.4 = insertelement <9 x double> %b.ins.3, double %b.lane.4, i32 4565 %b.ins.5 = insertelement <9 x double> %b.ins.4, double %b.lane.5, i32 5566 %b.ins.6 = insertelement <9 x double> %b.ins.5, double %b.lane.6, i32 6567 %b.ins.7 = insertelement <9 x double> %b.ins.6, double %b.lane.7, i32 7568 %b.ins.8 = insertelement <9 x double> %b.ins.7, double %b.lane.8, i32 8569 570 %res = fsub <9 x double> %a.ins.8, %b.ins.8571 store <9 x double> %res, ptr %ptr.1, align 8572 ret void573}574