1700 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=i686-unknown-unknown -mattr=+avx -fixup-byte-word-insts=1 < %s | FileCheck -check-prefixes=X86,X86-BWON %s3; RUN: llc -mtriple=i686-unknown-unknown -mattr=+avx -fixup-byte-word-insts=0 < %s | FileCheck -check-prefixes=X86,X86-BWOFF %s4; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx -fixup-byte-word-insts=1 < %s | FileCheck -check-prefixes=X64,X64-BWON %s5; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx -fixup-byte-word-insts=0 < %s | FileCheck -check-prefixes=X64,X64-BWOFF %s6 7%struct.A = type { i8, i8, i8, i8, i8, i8, i8, i8 }8%struct.B = type { i32, i32, i32, i32, i32, i32, i32, i32 }9%struct.C = type { i8, i8, i8, i8, i32, i32, i32, i64 }10 11; save 1,2,3 ... as one big integer.12define void @merge_const_store(i32 %count, ptr nocapture %p) nounwind uwtable noinline ssp {13; X86-LABEL: merge_const_store:14; X86: # %bb.0:15; X86-NEXT: movl {{[0-9]+}}(%esp), %eax16; X86-NEXT: testl %eax, %eax17; X86-NEXT: jle .LBB0_318; X86-NEXT: # %bb.1: # %.lr.ph.preheader19; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx20; X86-NEXT: .p2align 421; X86-NEXT: .LBB0_2: # %.lr.ph22; X86-NEXT: # =>This Inner Loop Header: Depth=123; X86-NEXT: movl $67305985, (%ecx) # imm = 0x403020124; X86-NEXT: movl $134678021, 4(%ecx) # imm = 0x807060525; X86-NEXT: addl $8, %ecx26; X86-NEXT: decl %eax27; X86-NEXT: jne .LBB0_228; X86-NEXT: .LBB0_3: # %._crit_edge29; X86-NEXT: retl30;31; X64-LABEL: merge_const_store:32; X64: # %bb.0:33; X64-NEXT: testl %edi, %edi34; X64-NEXT: jle .LBB0_335; X64-NEXT: # %bb.1: # %.lr.ph.preheader36; X64-NEXT: movabsq $578437695752307201, %rax # imm = 0x80706050403020137; X64-NEXT: .p2align 438; X64-NEXT: .LBB0_2: # %.lr.ph39; X64-NEXT: # =>This Inner Loop Header: Depth=140; X64-NEXT: movq %rax, (%rsi)41; X64-NEXT: addq $8, %rsi42; X64-NEXT: decl %edi43; X64-NEXT: jne .LBB0_244; X64-NEXT: .LBB0_3: # %._crit_edge45; X64-NEXT: retq46 %1 = icmp sgt i32 %count, 047 br i1 %1, label %.lr.ph, label %._crit_edge48.lr.ph:49 %i.02 = phi i32 [ %10, %.lr.ph ], [ 0, %0 ]50 %.01 = phi ptr [ %11, %.lr.ph ], [ %p, %0 ]51 %2 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 052 store i8 1, ptr %2, align 153 %3 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 154 store i8 2, ptr %3, align 155 %4 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 256 store i8 3, ptr %4, align 157 %5 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 358 store i8 4, ptr %5, align 159 %6 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 460 store i8 5, ptr %6, align 161 %7 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 562 store i8 6, ptr %7, align 163 %8 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 664 store i8 7, ptr %8, align 165 %9 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 766 store i8 8, ptr %9, align 167 %10 = add nsw i32 %i.02, 168 %11 = getelementptr inbounds %struct.A, ptr %.01, i64 169 %exitcond = icmp eq i32 %10, %count70 br i1 %exitcond, label %._crit_edge, label %.lr.ph71._crit_edge:72 ret void73}74 75; No vectors because we use noimplicitfloat76define void @merge_const_store_no_vec(i32 %count, ptr nocapture %p) noimplicitfloat{77; X86-LABEL: merge_const_store_no_vec:78; X86: # %bb.0:79; X86-NEXT: movl {{[0-9]+}}(%esp), %eax80; X86-NEXT: testl %eax, %eax81; X86-NEXT: jle .LBB1_382; X86-NEXT: # %bb.1: # %.lr.ph.preheader83; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx84; X86-NEXT: .p2align 485; X86-NEXT: .LBB1_2: # %.lr.ph86; X86-NEXT: # =>This Inner Loop Header: Depth=187; X86-NEXT: movl $0, (%ecx)88; X86-NEXT: movl $0, 4(%ecx)89; X86-NEXT: movl $0, 8(%ecx)90; X86-NEXT: movl $0, 12(%ecx)91; X86-NEXT: movl $0, 16(%ecx)92; X86-NEXT: movl $0, 20(%ecx)93; X86-NEXT: movl $0, 24(%ecx)94; X86-NEXT: movl $0, 28(%ecx)95; X86-NEXT: addl $32, %ecx96; X86-NEXT: decl %eax97; X86-NEXT: jne .LBB1_298; X86-NEXT: .LBB1_3: # %._crit_edge99; X86-NEXT: retl100;101; X64-LABEL: merge_const_store_no_vec:102; X64: # %bb.0:103; X64-NEXT: testl %edi, %edi104; X64-NEXT: jle .LBB1_2105; X64-NEXT: .p2align 4106; X64-NEXT: .LBB1_1: # %.lr.ph107; X64-NEXT: # =>This Inner Loop Header: Depth=1108; X64-NEXT: movq $0, (%rsi)109; X64-NEXT: movq $0, 8(%rsi)110; X64-NEXT: movq $0, 16(%rsi)111; X64-NEXT: movq $0, 24(%rsi)112; X64-NEXT: addq $32, %rsi113; X64-NEXT: decl %edi114; X64-NEXT: jne .LBB1_1115; X64-NEXT: .LBB1_2: # %._crit_edge116; X64-NEXT: retq117 %1 = icmp sgt i32 %count, 0118 br i1 %1, label %.lr.ph, label %._crit_edge119.lr.ph:120 %i.02 = phi i32 [ %10, %.lr.ph ], [ 0, %0 ]121 %.01 = phi ptr [ %11, %.lr.ph ], [ %p, %0 ]122 %2 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 0123 store i32 0, ptr %2, align 4124 %3 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 1125 store i32 0, ptr %3, align 4126 %4 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 2127 store i32 0, ptr %4, align 4128 %5 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 3129 store i32 0, ptr %5, align 4130 %6 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 4131 store i32 0, ptr %6, align 4132 %7 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 5133 store i32 0, ptr %7, align 4134 %8 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 6135 store i32 0, ptr %8, align 4136 %9 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 7137 store i32 0, ptr %9, align 4138 %10 = add nsw i32 %i.02, 1139 %11 = getelementptr inbounds %struct.B, ptr %.01, i64 1140 %exitcond = icmp eq i32 %10, %count141 br i1 %exitcond, label %._crit_edge, label %.lr.ph142._crit_edge:143 ret void144}145 146; Move the constants using a single vector store.147define void @merge_const_store_vec(i32 %count, ptr nocapture %p) nounwind uwtable noinline ssp {148; X86-LABEL: merge_const_store_vec:149; X86: # %bb.0:150; X86-NEXT: movl {{[0-9]+}}(%esp), %eax151; X86-NEXT: testl %eax, %eax152; X86-NEXT: jle .LBB2_3153; X86-NEXT: # %bb.1: # %.lr.ph.preheader154; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx155; X86-NEXT: vxorps %xmm0, %xmm0, %xmm0156; X86-NEXT: .p2align 4157; X86-NEXT: .LBB2_2: # %.lr.ph158; X86-NEXT: # =>This Inner Loop Header: Depth=1159; X86-NEXT: vmovups %ymm0, (%ecx)160; X86-NEXT: addl $32, %ecx161; X86-NEXT: decl %eax162; X86-NEXT: jne .LBB2_2163; X86-NEXT: .LBB2_3: # %._crit_edge164; X86-NEXT: vzeroupper165; X86-NEXT: retl166;167; X64-LABEL: merge_const_store_vec:168; X64: # %bb.0:169; X64-NEXT: testl %edi, %edi170; X64-NEXT: jle .LBB2_3171; X64-NEXT: # %bb.1: # %.lr.ph.preheader172; X64-NEXT: vxorps %xmm0, %xmm0, %xmm0173; X64-NEXT: .p2align 4174; X64-NEXT: .LBB2_2: # %.lr.ph175; X64-NEXT: # =>This Inner Loop Header: Depth=1176; X64-NEXT: vmovups %ymm0, (%rsi)177; X64-NEXT: addq $32, %rsi178; X64-NEXT: decl %edi179; X64-NEXT: jne .LBB2_2180; X64-NEXT: .LBB2_3: # %._crit_edge181; X64-NEXT: vzeroupper182; X64-NEXT: retq183 %1 = icmp sgt i32 %count, 0184 br i1 %1, label %.lr.ph, label %._crit_edge185.lr.ph:186 %i.02 = phi i32 [ %10, %.lr.ph ], [ 0, %0 ]187 %.01 = phi ptr [ %11, %.lr.ph ], [ %p, %0 ]188 %2 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 0189 store i32 0, ptr %2, align 4190 %3 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 1191 store i32 0, ptr %3, align 4192 %4 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 2193 store i32 0, ptr %4, align 4194 %5 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 3195 store i32 0, ptr %5, align 4196 %6 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 4197 store i32 0, ptr %6, align 4198 %7 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 5199 store i32 0, ptr %7, align 4200 %8 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 6201 store i32 0, ptr %8, align 4202 %9 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 7203 store i32 0, ptr %9, align 4204 %10 = add nsw i32 %i.02, 1205 %11 = getelementptr inbounds %struct.B, ptr %.01, i64 1206 %exitcond = icmp eq i32 %10, %count207 br i1 %exitcond, label %._crit_edge, label %.lr.ph208._crit_edge:209 ret void210}211 212; Move the first 4 constants as a single vector. Move the rest as scalars.213define void @merge_nonconst_store(i32 %count, i8 %zz, ptr nocapture %p) nounwind uwtable noinline ssp {214; X86-BWON-LABEL: merge_nonconst_store:215; X86-BWON: # %bb.0:216; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %eax217; X86-BWON-NEXT: testl %eax, %eax218; X86-BWON-NEXT: jle .LBB3_3219; X86-BWON-NEXT: # %bb.1: # %.lr.ph.preheader220; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %ecx221; X86-BWON-NEXT: movzbl {{[0-9]+}}(%esp), %edx222; X86-BWON-NEXT: .p2align 4223; X86-BWON-NEXT: .LBB3_2: # %.lr.ph224; X86-BWON-NEXT: # =>This Inner Loop Header: Depth=1225; X86-BWON-NEXT: movl $67305985, (%ecx) # imm = 0x4030201226; X86-BWON-NEXT: movb %dl, 4(%ecx)227; X86-BWON-NEXT: movw $1798, 5(%ecx) # imm = 0x706228; X86-BWON-NEXT: movb $8, 7(%ecx)229; X86-BWON-NEXT: addl $8, %ecx230; X86-BWON-NEXT: decl %eax231; X86-BWON-NEXT: jne .LBB3_2232; X86-BWON-NEXT: .LBB3_3: # %._crit_edge233; X86-BWON-NEXT: retl234;235; X86-BWOFF-LABEL: merge_nonconst_store:236; X86-BWOFF: # %bb.0:237; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %eax238; X86-BWOFF-NEXT: testl %eax, %eax239; X86-BWOFF-NEXT: jle .LBB3_3240; X86-BWOFF-NEXT: # %bb.1: # %.lr.ph.preheader241; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %ecx242; X86-BWOFF-NEXT: movb {{[0-9]+}}(%esp), %dl243; X86-BWOFF-NEXT: .p2align 4244; X86-BWOFF-NEXT: .LBB3_2: # %.lr.ph245; X86-BWOFF-NEXT: # =>This Inner Loop Header: Depth=1246; X86-BWOFF-NEXT: movl $67305985, (%ecx) # imm = 0x4030201247; X86-BWOFF-NEXT: movb %dl, 4(%ecx)248; X86-BWOFF-NEXT: movw $1798, 5(%ecx) # imm = 0x706249; X86-BWOFF-NEXT: movb $8, 7(%ecx)250; X86-BWOFF-NEXT: addl $8, %ecx251; X86-BWOFF-NEXT: decl %eax252; X86-BWOFF-NEXT: jne .LBB3_2253; X86-BWOFF-NEXT: .LBB3_3: # %._crit_edge254; X86-BWOFF-NEXT: retl255;256; X64-LABEL: merge_nonconst_store:257; X64: # %bb.0:258; X64-NEXT: testl %edi, %edi259; X64-NEXT: jle .LBB3_2260; X64-NEXT: .p2align 4261; X64-NEXT: .LBB3_1: # %.lr.ph262; X64-NEXT: # =>This Inner Loop Header: Depth=1263; X64-NEXT: movl $67305985, (%rdx) # imm = 0x4030201264; X64-NEXT: movb %sil, 4(%rdx)265; X64-NEXT: movw $1798, 5(%rdx) # imm = 0x706266; X64-NEXT: movb $8, 7(%rdx)267; X64-NEXT: addq $8, %rdx268; X64-NEXT: decl %edi269; X64-NEXT: jne .LBB3_1270; X64-NEXT: .LBB3_2: # %._crit_edge271; X64-NEXT: retq272 %1 = icmp sgt i32 %count, 0273 br i1 %1, label %.lr.ph, label %._crit_edge274.lr.ph:275 %i.02 = phi i32 [ %10, %.lr.ph ], [ 0, %0 ]276 %.01 = phi ptr [ %11, %.lr.ph ], [ %p, %0 ]277 %2 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 0278 store i8 1, ptr %2, align 1279 %3 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 1280 store i8 2, ptr %3, align 1281 %4 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 2282 store i8 3, ptr %4, align 1283 %5 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 3284 store i8 4, ptr %5, align 1285 %6 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 4286 store i8 %zz, ptr %6, align 1 ; <----------- Not a const;287 %7 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 5288 store i8 6, ptr %7, align 1289 %8 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 6290 store i8 7, ptr %8, align 1291 %9 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 7292 store i8 8, ptr %9, align 1293 %10 = add nsw i32 %i.02, 1294 %11 = getelementptr inbounds %struct.A, ptr %.01, i64 1295 %exitcond = icmp eq i32 %10, %count296 br i1 %exitcond, label %._crit_edge, label %.lr.ph297._crit_edge:298 ret void299}300 301define void @merge_loads_i16(i32 %count, ptr noalias nocapture %q, ptr noalias nocapture %p) nounwind uwtable noinline ssp {302; X86-BWON-LABEL: merge_loads_i16:303; X86-BWON: # %bb.0:304; X86-BWON-NEXT: pushl %esi305; X86-BWON-NEXT: .cfi_def_cfa_offset 8306; X86-BWON-NEXT: .cfi_offset %esi, -8307; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %eax308; X86-BWON-NEXT: testl %eax, %eax309; X86-BWON-NEXT: jle .LBB4_3310; X86-BWON-NEXT: # %bb.1: # %.lr.ph311; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %ecx312; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %edx313; X86-BWON-NEXT: .p2align 4314; X86-BWON-NEXT: .LBB4_2: # =>This Inner Loop Header: Depth=1315; X86-BWON-NEXT: movzwl (%edx), %esi316; X86-BWON-NEXT: movw %si, (%ecx)317; X86-BWON-NEXT: addl $8, %ecx318; X86-BWON-NEXT: decl %eax319; X86-BWON-NEXT: jne .LBB4_2320; X86-BWON-NEXT: .LBB4_3: # %._crit_edge321; X86-BWON-NEXT: popl %esi322; X86-BWON-NEXT: .cfi_def_cfa_offset 4323; X86-BWON-NEXT: retl324;325; X86-BWOFF-LABEL: merge_loads_i16:326; X86-BWOFF: # %bb.0:327; X86-BWOFF-NEXT: pushl %esi328; X86-BWOFF-NEXT: .cfi_def_cfa_offset 8329; X86-BWOFF-NEXT: .cfi_offset %esi, -8330; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %eax331; X86-BWOFF-NEXT: testl %eax, %eax332; X86-BWOFF-NEXT: jle .LBB4_3333; X86-BWOFF-NEXT: # %bb.1: # %.lr.ph334; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %ecx335; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %edx336; X86-BWOFF-NEXT: .p2align 4337; X86-BWOFF-NEXT: .LBB4_2: # =>This Inner Loop Header: Depth=1338; X86-BWOFF-NEXT: movw (%edx), %si339; X86-BWOFF-NEXT: movw %si, (%ecx)340; X86-BWOFF-NEXT: addl $8, %ecx341; X86-BWOFF-NEXT: decl %eax342; X86-BWOFF-NEXT: jne .LBB4_2343; X86-BWOFF-NEXT: .LBB4_3: # %._crit_edge344; X86-BWOFF-NEXT: popl %esi345; X86-BWOFF-NEXT: .cfi_def_cfa_offset 4346; X86-BWOFF-NEXT: retl347;348; X64-BWON-LABEL: merge_loads_i16:349; X64-BWON: # %bb.0:350; X64-BWON-NEXT: testl %edi, %edi351; X64-BWON-NEXT: jle .LBB4_2352; X64-BWON-NEXT: .p2align 4353; X64-BWON-NEXT: .LBB4_1: # =>This Inner Loop Header: Depth=1354; X64-BWON-NEXT: movzwl (%rsi), %eax355; X64-BWON-NEXT: movw %ax, (%rdx)356; X64-BWON-NEXT: addq $8, %rdx357; X64-BWON-NEXT: decl %edi358; X64-BWON-NEXT: jne .LBB4_1359; X64-BWON-NEXT: .LBB4_2: # %._crit_edge360; X64-BWON-NEXT: retq361;362; X64-BWOFF-LABEL: merge_loads_i16:363; X64-BWOFF: # %bb.0:364; X64-BWOFF-NEXT: testl %edi, %edi365; X64-BWOFF-NEXT: jle .LBB4_2366; X64-BWOFF-NEXT: .p2align 4367; X64-BWOFF-NEXT: .LBB4_1: # =>This Inner Loop Header: Depth=1368; X64-BWOFF-NEXT: movw (%rsi), %ax369; X64-BWOFF-NEXT: movw %ax, (%rdx)370; X64-BWOFF-NEXT: addq $8, %rdx371; X64-BWOFF-NEXT: decl %edi372; X64-BWOFF-NEXT: jne .LBB4_1373; X64-BWOFF-NEXT: .LBB4_2: # %._crit_edge374; X64-BWOFF-NEXT: retq375 %1 = icmp sgt i32 %count, 0376 br i1 %1, label %.lr.ph, label %._crit_edge377 378.lr.ph: ; preds = %0379 %2 = getelementptr inbounds %struct.A, ptr %q, i64 0, i32 0380 %3 = getelementptr inbounds %struct.A, ptr %q, i64 0, i32 1381 br label %4382 383; <label>:4 ; preds = %4, %.lr.ph384 %i.02 = phi i32 [ 0, %.lr.ph ], [ %9, %4 ]385 %.01 = phi ptr [ %p, %.lr.ph ], [ %10, %4 ]386 %5 = load i8, ptr %2, align 1387 %6 = load i8, ptr %3, align 1388 %7 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 0389 store i8 %5, ptr %7, align 1390 %8 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 1391 store i8 %6, ptr %8, align 1392 %9 = add nsw i32 %i.02, 1393 %10 = getelementptr inbounds %struct.A, ptr %.01, i64 1394 %exitcond = icmp eq i32 %9, %count395 br i1 %exitcond, label %._crit_edge, label %4396 397._crit_edge: ; preds = %4, %0398 ret void399}400 401; The loads and the stores are interleaved. Can't merge them.402define void @no_merge_loads(i32 %count, ptr noalias nocapture %q, ptr noalias nocapture %p) nounwind uwtable noinline ssp {403; X86-BWON-LABEL: no_merge_loads:404; X86-BWON: # %bb.0:405; X86-BWON-NEXT: pushl %ebx406; X86-BWON-NEXT: .cfi_def_cfa_offset 8407; X86-BWON-NEXT: .cfi_offset %ebx, -8408; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %eax409; X86-BWON-NEXT: testl %eax, %eax410; X86-BWON-NEXT: jle .LBB5_3411; X86-BWON-NEXT: # %bb.1: # %.lr.ph412; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %ecx413; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %edx414; X86-BWON-NEXT: .p2align 4415; X86-BWON-NEXT: .LBB5_2: # %a4416; X86-BWON-NEXT: # =>This Inner Loop Header: Depth=1417; X86-BWON-NEXT: movzbl (%edx), %ebx418; X86-BWON-NEXT: movb %bl, (%ecx)419; X86-BWON-NEXT: movzbl 1(%edx), %ebx420; X86-BWON-NEXT: movb %bl, 1(%ecx)421; X86-BWON-NEXT: addl $8, %ecx422; X86-BWON-NEXT: decl %eax423; X86-BWON-NEXT: jne .LBB5_2424; X86-BWON-NEXT: .LBB5_3: # %._crit_edge425; X86-BWON-NEXT: popl %ebx426; X86-BWON-NEXT: .cfi_def_cfa_offset 4427; X86-BWON-NEXT: retl428;429; X86-BWOFF-LABEL: no_merge_loads:430; X86-BWOFF: # %bb.0:431; X86-BWOFF-NEXT: pushl %ebx432; X86-BWOFF-NEXT: .cfi_def_cfa_offset 8433; X86-BWOFF-NEXT: .cfi_offset %ebx, -8434; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %eax435; X86-BWOFF-NEXT: testl %eax, %eax436; X86-BWOFF-NEXT: jle .LBB5_3437; X86-BWOFF-NEXT: # %bb.1: # %.lr.ph438; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %ecx439; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %edx440; X86-BWOFF-NEXT: .p2align 4441; X86-BWOFF-NEXT: .LBB5_2: # %a4442; X86-BWOFF-NEXT: # =>This Inner Loop Header: Depth=1443; X86-BWOFF-NEXT: movb (%edx), %bl444; X86-BWOFF-NEXT: movb %bl, (%ecx)445; X86-BWOFF-NEXT: movb 1(%edx), %bl446; X86-BWOFF-NEXT: movb %bl, 1(%ecx)447; X86-BWOFF-NEXT: addl $8, %ecx448; X86-BWOFF-NEXT: decl %eax449; X86-BWOFF-NEXT: jne .LBB5_2450; X86-BWOFF-NEXT: .LBB5_3: # %._crit_edge451; X86-BWOFF-NEXT: popl %ebx452; X86-BWOFF-NEXT: .cfi_def_cfa_offset 4453; X86-BWOFF-NEXT: retl454;455; X64-BWON-LABEL: no_merge_loads:456; X64-BWON: # %bb.0:457; X64-BWON-NEXT: testl %edi, %edi458; X64-BWON-NEXT: jle .LBB5_2459; X64-BWON-NEXT: .p2align 4460; X64-BWON-NEXT: .LBB5_1: # %a4461; X64-BWON-NEXT: # =>This Inner Loop Header: Depth=1462; X64-BWON-NEXT: movzbl (%rsi), %eax463; X64-BWON-NEXT: movb %al, (%rdx)464; X64-BWON-NEXT: movzbl 1(%rsi), %eax465; X64-BWON-NEXT: movb %al, 1(%rdx)466; X64-BWON-NEXT: addq $8, %rdx467; X64-BWON-NEXT: decl %edi468; X64-BWON-NEXT: jne .LBB5_1469; X64-BWON-NEXT: .LBB5_2: # %._crit_edge470; X64-BWON-NEXT: retq471;472; X64-BWOFF-LABEL: no_merge_loads:473; X64-BWOFF: # %bb.0:474; X64-BWOFF-NEXT: testl %edi, %edi475; X64-BWOFF-NEXT: jle .LBB5_2476; X64-BWOFF-NEXT: .p2align 4477; X64-BWOFF-NEXT: .LBB5_1: # %a4478; X64-BWOFF-NEXT: # =>This Inner Loop Header: Depth=1479; X64-BWOFF-NEXT: movb (%rsi), %al480; X64-BWOFF-NEXT: movb %al, (%rdx)481; X64-BWOFF-NEXT: movb 1(%rsi), %al482; X64-BWOFF-NEXT: movb %al, 1(%rdx)483; X64-BWOFF-NEXT: addq $8, %rdx484; X64-BWOFF-NEXT: decl %edi485; X64-BWOFF-NEXT: jne .LBB5_1486; X64-BWOFF-NEXT: .LBB5_2: # %._crit_edge487; X64-BWOFF-NEXT: retq488 %1 = icmp sgt i32 %count, 0489 br i1 %1, label %.lr.ph, label %._crit_edge490 491.lr.ph: ; preds = %0492 %2 = getelementptr inbounds %struct.A, ptr %q, i64 0, i32 0493 %3 = getelementptr inbounds %struct.A, ptr %q, i64 0, i32 1494 br label %a4495 496a4: ; preds = %4, %.lr.ph497 %i.02 = phi i32 [ 0, %.lr.ph ], [ %a9, %a4 ]498 %.01 = phi ptr [ %p, %.lr.ph ], [ %a10, %a4 ]499 %a5 = load i8, ptr %2, align 1500 %a7 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 0501 store i8 %a5, ptr %a7, align 1502 %a8 = getelementptr inbounds %struct.A, ptr %.01, i64 0, i32 1503 %a6 = load i8, ptr %3, align 1504 store i8 %a6, ptr %a8, align 1505 %a9 = add nsw i32 %i.02, 1506 %a10 = getelementptr inbounds %struct.A, ptr %.01, i64 1507 %exitcond = icmp eq i32 %a9, %count508 br i1 %exitcond, label %._crit_edge, label %a4509 510._crit_edge: ; preds = %4, %0511 ret void512}513 514define void @merge_loads_integer(i32 %count, ptr noalias nocapture %q, ptr noalias nocapture %p) nounwind uwtable noinline ssp {515; X86-LABEL: merge_loads_integer:516; X86: # %bb.0:517; X86-NEXT: pushl %edi518; X86-NEXT: .cfi_def_cfa_offset 8519; X86-NEXT: pushl %esi520; X86-NEXT: .cfi_def_cfa_offset 12521; X86-NEXT: .cfi_offset %esi, -12522; X86-NEXT: .cfi_offset %edi, -8523; X86-NEXT: movl {{[0-9]+}}(%esp), %eax524; X86-NEXT: testl %eax, %eax525; X86-NEXT: jle .LBB6_3526; X86-NEXT: # %bb.1: # %.lr.ph527; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx528; X86-NEXT: movl {{[0-9]+}}(%esp), %edx529; X86-NEXT: .p2align 4530; X86-NEXT: .LBB6_2: # =>This Inner Loop Header: Depth=1531; X86-NEXT: movl (%edx), %esi532; X86-NEXT: movl 4(%edx), %edi533; X86-NEXT: movl %esi, (%ecx)534; X86-NEXT: movl %edi, 4(%ecx)535; X86-NEXT: addl $32, %ecx536; X86-NEXT: decl %eax537; X86-NEXT: jne .LBB6_2538; X86-NEXT: .LBB6_3: # %._crit_edge539; X86-NEXT: popl %esi540; X86-NEXT: .cfi_def_cfa_offset 8541; X86-NEXT: popl %edi542; X86-NEXT: .cfi_def_cfa_offset 4543; X86-NEXT: retl544;545; X64-LABEL: merge_loads_integer:546; X64: # %bb.0:547; X64-NEXT: testl %edi, %edi548; X64-NEXT: jle .LBB6_2549; X64-NEXT: .p2align 4550; X64-NEXT: .LBB6_1: # =>This Inner Loop Header: Depth=1551; X64-NEXT: movq (%rsi), %rax552; X64-NEXT: movq %rax, (%rdx)553; X64-NEXT: addq $32, %rdx554; X64-NEXT: decl %edi555; X64-NEXT: jne .LBB6_1556; X64-NEXT: .LBB6_2: # %._crit_edge557; X64-NEXT: retq558 %1 = icmp sgt i32 %count, 0559 br i1 %1, label %.lr.ph, label %._crit_edge560 561.lr.ph: ; preds = %0562 %2 = getelementptr inbounds %struct.B, ptr %q, i64 0, i32 0563 %3 = getelementptr inbounds %struct.B, ptr %q, i64 0, i32 1564 br label %4565 566; <label>:4 ; preds = %4, %.lr.ph567 %i.02 = phi i32 [ 0, %.lr.ph ], [ %9, %4 ]568 %.01 = phi ptr [ %p, %.lr.ph ], [ %10, %4 ]569 %5 = load i32, ptr %2570 %6 = load i32, ptr %3571 %7 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 0572 store i32 %5, ptr %7573 %8 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 1574 store i32 %6, ptr %8575 %9 = add nsw i32 %i.02, 1576 %10 = getelementptr inbounds %struct.B, ptr %.01, i64 1577 %exitcond = icmp eq i32 %9, %count578 br i1 %exitcond, label %._crit_edge, label %4579 580._crit_edge: ; preds = %4, %0581 ret void582}583 584define void @merge_loads_vector(i32 %count, ptr noalias nocapture %q, ptr noalias nocapture %p) nounwind uwtable noinline ssp {585; X86-LABEL: merge_loads_vector:586; X86: # %bb.0:587; X86-NEXT: movl {{[0-9]+}}(%esp), %eax588; X86-NEXT: testl %eax, %eax589; X86-NEXT: jle .LBB7_3590; X86-NEXT: # %bb.1: # %.lr.ph591; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx592; X86-NEXT: movl {{[0-9]+}}(%esp), %edx593; X86-NEXT: .p2align 4594; X86-NEXT: .LBB7_2: # %block4595; X86-NEXT: # =>This Inner Loop Header: Depth=1596; X86-NEXT: vmovups (%edx), %xmm0597; X86-NEXT: vmovups %xmm0, (%ecx)598; X86-NEXT: addl $32, %ecx599; X86-NEXT: decl %eax600; X86-NEXT: jne .LBB7_2601; X86-NEXT: .LBB7_3: # %._crit_edge602; X86-NEXT: retl603;604; X64-LABEL: merge_loads_vector:605; X64: # %bb.0:606; X64-NEXT: testl %edi, %edi607; X64-NEXT: jle .LBB7_2608; X64-NEXT: .p2align 4609; X64-NEXT: .LBB7_1: # %block4610; X64-NEXT: # =>This Inner Loop Header: Depth=1611; X64-NEXT: vmovups (%rsi), %xmm0612; X64-NEXT: vmovups %xmm0, (%rdx)613; X64-NEXT: addq $32, %rdx614; X64-NEXT: decl %edi615; X64-NEXT: jne .LBB7_1616; X64-NEXT: .LBB7_2: # %._crit_edge617; X64-NEXT: retq618 %a1 = icmp sgt i32 %count, 0619 br i1 %a1, label %.lr.ph, label %._crit_edge620 621.lr.ph: ; preds = %0622 %a2 = getelementptr inbounds %struct.B, ptr %q, i64 0, i32 0623 %a3 = getelementptr inbounds %struct.B, ptr %q, i64 0, i32 1624 %a4 = getelementptr inbounds %struct.B, ptr %q, i64 0, i32 2625 %a5 = getelementptr inbounds %struct.B, ptr %q, i64 0, i32 3626 br label %block4627 628block4: ; preds = %4, %.lr.ph629 %i.02 = phi i32 [ 0, %.lr.ph ], [ %c9, %block4 ]630 %.01 = phi ptr [ %p, %.lr.ph ], [ %c10, %block4 ]631 %a7 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 0632 %a8 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 1633 %a9 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 2634 %a10 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 3635 %b1 = load i32, ptr %a2636 %b2 = load i32, ptr %a3637 %b3 = load i32, ptr %a4638 %b4 = load i32, ptr %a5639 store i32 %b1, ptr %a7640 store i32 %b2, ptr %a8641 store i32 %b3, ptr %a9642 store i32 %b4, ptr %a10643 %c9 = add nsw i32 %i.02, 1644 %c10 = getelementptr inbounds %struct.B, ptr %.01, i64 1645 %exitcond = icmp eq i32 %c9, %count646 br i1 %exitcond, label %._crit_edge, label %block4647 648._crit_edge: ; preds = %4, %0649 ret void650}651 652; On x86, even unaligned copies can be merged to vector ops.653define void @merge_loads_no_align(i32 %count, ptr noalias nocapture %q, ptr noalias nocapture %p) nounwind uwtable noinline ssp {654; X86-LABEL: merge_loads_no_align:655; X86: # %bb.0:656; X86-NEXT: movl {{[0-9]+}}(%esp), %eax657; X86-NEXT: testl %eax, %eax658; X86-NEXT: jle .LBB8_3659; X86-NEXT: # %bb.1: # %.lr.ph660; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx661; X86-NEXT: movl {{[0-9]+}}(%esp), %edx662; X86-NEXT: .p2align 4663; X86-NEXT: .LBB8_2: # %block4664; X86-NEXT: # =>This Inner Loop Header: Depth=1665; X86-NEXT: vmovups (%edx), %xmm0666; X86-NEXT: vmovups %xmm0, (%ecx)667; X86-NEXT: addl $32, %ecx668; X86-NEXT: decl %eax669; X86-NEXT: jne .LBB8_2670; X86-NEXT: .LBB8_3: # %._crit_edge671; X86-NEXT: retl672;673; X64-LABEL: merge_loads_no_align:674; X64: # %bb.0:675; X64-NEXT: testl %edi, %edi676; X64-NEXT: jle .LBB8_2677; X64-NEXT: .p2align 4678; X64-NEXT: .LBB8_1: # %block4679; X64-NEXT: # =>This Inner Loop Header: Depth=1680; X64-NEXT: vmovups (%rsi), %xmm0681; X64-NEXT: vmovups %xmm0, (%rdx)682; X64-NEXT: addq $32, %rdx683; X64-NEXT: decl %edi684; X64-NEXT: jne .LBB8_1685; X64-NEXT: .LBB8_2: # %._crit_edge686; X64-NEXT: retq687 %a1 = icmp sgt i32 %count, 0688 br i1 %a1, label %.lr.ph, label %._crit_edge689 690.lr.ph: ; preds = %0691 %a2 = getelementptr inbounds %struct.B, ptr %q, i64 0, i32 0692 %a3 = getelementptr inbounds %struct.B, ptr %q, i64 0, i32 1693 %a4 = getelementptr inbounds %struct.B, ptr %q, i64 0, i32 2694 %a5 = getelementptr inbounds %struct.B, ptr %q, i64 0, i32 3695 br label %block4696 697block4: ; preds = %4, %.lr.ph698 %i.02 = phi i32 [ 0, %.lr.ph ], [ %c9, %block4 ]699 %.01 = phi ptr [ %p, %.lr.ph ], [ %c10, %block4 ]700 %a7 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 0701 %a8 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 1702 %a9 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 2703 %a10 = getelementptr inbounds %struct.B, ptr %.01, i64 0, i32 3704 %b1 = load i32, ptr %a2, align 1705 %b2 = load i32, ptr %a3, align 1706 %b3 = load i32, ptr %a4, align 1707 %b4 = load i32, ptr %a5, align 1708 store i32 %b1, ptr %a7, align 1709 store i32 %b2, ptr %a8, align 1710 store i32 %b3, ptr %a9, align 1711 store i32 %b4, ptr %a10, align 1712 %c9 = add nsw i32 %i.02, 1713 %c10 = getelementptr inbounds %struct.B, ptr %.01, i64 1714 %exitcond = icmp eq i32 %c9, %count715 br i1 %exitcond, label %._crit_edge, label %block4716 717._crit_edge: ; preds = %4, %0718 ret void719}720 721; Make sure that we merge the consecutive load/store sequence below and use a722; word (16 bit) instead of a byte copy.723define void @MergeLoadStoreBaseIndexOffset(ptr %a, ptr %b, ptr %c, i32 %n) {724; X86-BWON-LABEL: MergeLoadStoreBaseIndexOffset:725; X86-BWON: # %bb.0:726; X86-BWON-NEXT: pushl %ebx727; X86-BWON-NEXT: .cfi_def_cfa_offset 8728; X86-BWON-NEXT: pushl %edi729; X86-BWON-NEXT: .cfi_def_cfa_offset 12730; X86-BWON-NEXT: pushl %esi731; X86-BWON-NEXT: .cfi_def_cfa_offset 16732; X86-BWON-NEXT: .cfi_offset %esi, -16733; X86-BWON-NEXT: .cfi_offset %edi, -12734; X86-BWON-NEXT: .cfi_offset %ebx, -8735; X86-BWON-NEXT: xorl %eax, %eax736; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %ecx737; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %edx738; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %esi739; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %edi740; X86-BWON-NEXT: .p2align 4741; X86-BWON-NEXT: .LBB9_1: # =>This Inner Loop Header: Depth=1742; X86-BWON-NEXT: movl (%edi,%eax,8), %ebx743; X86-BWON-NEXT: movzwl (%edx,%ebx), %ebx744; X86-BWON-NEXT: movw %bx, (%esi,%eax,2)745; X86-BWON-NEXT: incl %eax746; X86-BWON-NEXT: cmpl %eax, %ecx747; X86-BWON-NEXT: jne .LBB9_1748; X86-BWON-NEXT: # %bb.2:749; X86-BWON-NEXT: popl %esi750; X86-BWON-NEXT: .cfi_def_cfa_offset 12751; X86-BWON-NEXT: popl %edi752; X86-BWON-NEXT: .cfi_def_cfa_offset 8753; X86-BWON-NEXT: popl %ebx754; X86-BWON-NEXT: .cfi_def_cfa_offset 4755; X86-BWON-NEXT: retl756;757; X86-BWOFF-LABEL: MergeLoadStoreBaseIndexOffset:758; X86-BWOFF: # %bb.0:759; X86-BWOFF-NEXT: pushl %ebx760; X86-BWOFF-NEXT: .cfi_def_cfa_offset 8761; X86-BWOFF-NEXT: pushl %edi762; X86-BWOFF-NEXT: .cfi_def_cfa_offset 12763; X86-BWOFF-NEXT: pushl %esi764; X86-BWOFF-NEXT: .cfi_def_cfa_offset 16765; X86-BWOFF-NEXT: .cfi_offset %esi, -16766; X86-BWOFF-NEXT: .cfi_offset %edi, -12767; X86-BWOFF-NEXT: .cfi_offset %ebx, -8768; X86-BWOFF-NEXT: xorl %eax, %eax769; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %ecx770; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %edx771; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %esi772; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %edi773; X86-BWOFF-NEXT: .p2align 4774; X86-BWOFF-NEXT: .LBB9_1: # =>This Inner Loop Header: Depth=1775; X86-BWOFF-NEXT: movl (%edi,%eax,8), %ebx776; X86-BWOFF-NEXT: movw (%edx,%ebx), %bx777; X86-BWOFF-NEXT: movw %bx, (%esi,%eax,2)778; X86-BWOFF-NEXT: incl %eax779; X86-BWOFF-NEXT: cmpl %eax, %ecx780; X86-BWOFF-NEXT: jne .LBB9_1781; X86-BWOFF-NEXT: # %bb.2:782; X86-BWOFF-NEXT: popl %esi783; X86-BWOFF-NEXT: .cfi_def_cfa_offset 12784; X86-BWOFF-NEXT: popl %edi785; X86-BWOFF-NEXT: .cfi_def_cfa_offset 8786; X86-BWOFF-NEXT: popl %ebx787; X86-BWOFF-NEXT: .cfi_def_cfa_offset 4788; X86-BWOFF-NEXT: retl789;790; X64-BWON-LABEL: MergeLoadStoreBaseIndexOffset:791; X64-BWON: # %bb.0:792; X64-BWON-NEXT: movl %ecx, %eax793; X64-BWON-NEXT: xorl %ecx, %ecx794; X64-BWON-NEXT: .p2align 4795; X64-BWON-NEXT: .LBB9_1: # =>This Inner Loop Header: Depth=1796; X64-BWON-NEXT: movq (%rdi,%rcx,8), %r8797; X64-BWON-NEXT: movzwl (%rdx,%r8), %r8d798; X64-BWON-NEXT: movw %r8w, (%rsi,%rcx,2)799; X64-BWON-NEXT: incq %rcx800; X64-BWON-NEXT: cmpl %ecx, %eax801; X64-BWON-NEXT: jne .LBB9_1802; X64-BWON-NEXT: # %bb.2:803; X64-BWON-NEXT: retq804;805; X64-BWOFF-LABEL: MergeLoadStoreBaseIndexOffset:806; X64-BWOFF: # %bb.0:807; X64-BWOFF-NEXT: movl %ecx, %eax808; X64-BWOFF-NEXT: xorl %ecx, %ecx809; X64-BWOFF-NEXT: .p2align 4810; X64-BWOFF-NEXT: .LBB9_1: # =>This Inner Loop Header: Depth=1811; X64-BWOFF-NEXT: movq (%rdi,%rcx,8), %r8812; X64-BWOFF-NEXT: movw (%rdx,%r8), %r8w813; X64-BWOFF-NEXT: movw %r8w, (%rsi,%rcx,2)814; X64-BWOFF-NEXT: incq %rcx815; X64-BWOFF-NEXT: cmpl %ecx, %eax816; X64-BWOFF-NEXT: jne .LBB9_1817; X64-BWOFF-NEXT: # %bb.2:818; X64-BWOFF-NEXT: retq819 br label %1820 821; <label>:1822 %.09 = phi i32 [ %n, %0 ], [ %11, %1 ]823 %.08 = phi ptr [ %b, %0 ], [ %10, %1 ]824 %.0 = phi ptr [ %a, %0 ], [ %2, %1 ]825 %2 = getelementptr inbounds i64, ptr %.0, i64 1826 %3 = load i64, ptr %.0, align 1827 %4 = getelementptr inbounds i8, ptr %c, i64 %3828 %5 = load i8, ptr %4, align 1829 %6 = add i64 %3, 1830 %7 = getelementptr inbounds i8, ptr %c, i64 %6831 %8 = load i8, ptr %7, align 1832 store i8 %5, ptr %.08, align 1833 %9 = getelementptr inbounds i8, ptr %.08, i64 1834 store i8 %8, ptr %9, align 1835 %10 = getelementptr inbounds i8, ptr %.08, i64 2836 %11 = add nsw i32 %.09, -1837 %12 = icmp eq i32 %11, 0838 br i1 %12, label %13, label %1839 840; <label>:13841 ret void842}843 844; Make sure that we merge the consecutive load/store sequence below and use a845; word (16 bit) instead of a byte copy for complicated address calculation.846define void @MergeLoadStoreBaseIndexOffsetComplicated(ptr %a, ptr %b, ptr %c, i64 %n) {847; X86-BWON-LABEL: MergeLoadStoreBaseIndexOffsetComplicated:848; X86-BWON: # %bb.0:849; X86-BWON-NEXT: pushl %ebp850; X86-BWON-NEXT: .cfi_def_cfa_offset 8851; X86-BWON-NEXT: pushl %ebx852; X86-BWON-NEXT: .cfi_def_cfa_offset 12853; X86-BWON-NEXT: pushl %edi854; X86-BWON-NEXT: .cfi_def_cfa_offset 16855; X86-BWON-NEXT: pushl %esi856; X86-BWON-NEXT: .cfi_def_cfa_offset 20857; X86-BWON-NEXT: .cfi_offset %esi, -20858; X86-BWON-NEXT: .cfi_offset %edi, -16859; X86-BWON-NEXT: .cfi_offset %ebx, -12860; X86-BWON-NEXT: .cfi_offset %ebp, -8861; X86-BWON-NEXT: xorl %eax, %eax862; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %esi863; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %edi864; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %ebx865; X86-BWON-NEXT: xorl %ebp, %ebp866; X86-BWON-NEXT: .p2align 4867; X86-BWON-NEXT: .LBB10_1: # =>This Inner Loop Header: Depth=1868; X86-BWON-NEXT: movsbl (%edi), %ecx869; X86-BWON-NEXT: movzbl (%esi,%ecx), %edx870; X86-BWON-NEXT: movzbl 1(%esi,%ecx), %ecx871; X86-BWON-NEXT: movb %dl, (%ebx,%eax)872; X86-BWON-NEXT: movl %eax, %edx873; X86-BWON-NEXT: orl $1, %edx874; X86-BWON-NEXT: movb %cl, (%ebx,%edx)875; X86-BWON-NEXT: incl %edi876; X86-BWON-NEXT: addl $2, %eax877; X86-BWON-NEXT: adcl $0, %ebp878; X86-BWON-NEXT: cmpl {{[0-9]+}}(%esp), %eax879; X86-BWON-NEXT: movl %ebp, %ecx880; X86-BWON-NEXT: sbbl {{[0-9]+}}(%esp), %ecx881; X86-BWON-NEXT: jl .LBB10_1882; X86-BWON-NEXT: # %bb.2:883; X86-BWON-NEXT: popl %esi884; X86-BWON-NEXT: .cfi_def_cfa_offset 16885; X86-BWON-NEXT: popl %edi886; X86-BWON-NEXT: .cfi_def_cfa_offset 12887; X86-BWON-NEXT: popl %ebx888; X86-BWON-NEXT: .cfi_def_cfa_offset 8889; X86-BWON-NEXT: popl %ebp890; X86-BWON-NEXT: .cfi_def_cfa_offset 4891; X86-BWON-NEXT: retl892;893; X86-BWOFF-LABEL: MergeLoadStoreBaseIndexOffsetComplicated:894; X86-BWOFF: # %bb.0:895; X86-BWOFF-NEXT: pushl %ebp896; X86-BWOFF-NEXT: .cfi_def_cfa_offset 8897; X86-BWOFF-NEXT: pushl %ebx898; X86-BWOFF-NEXT: .cfi_def_cfa_offset 12899; X86-BWOFF-NEXT: pushl %edi900; X86-BWOFF-NEXT: .cfi_def_cfa_offset 16901; X86-BWOFF-NEXT: pushl %esi902; X86-BWOFF-NEXT: .cfi_def_cfa_offset 20903; X86-BWOFF-NEXT: .cfi_offset %esi, -20904; X86-BWOFF-NEXT: .cfi_offset %edi, -16905; X86-BWOFF-NEXT: .cfi_offset %ebx, -12906; X86-BWOFF-NEXT: .cfi_offset %ebp, -8907; X86-BWOFF-NEXT: xorl %eax, %eax908; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %esi909; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %edi910; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %ebx911; X86-BWOFF-NEXT: xorl %ebp, %ebp912; X86-BWOFF-NEXT: .p2align 4913; X86-BWOFF-NEXT: .LBB10_1: # =>This Inner Loop Header: Depth=1914; X86-BWOFF-NEXT: movsbl (%edi), %ecx915; X86-BWOFF-NEXT: movb (%esi,%ecx), %dl916; X86-BWOFF-NEXT: movb 1(%esi,%ecx), %cl917; X86-BWOFF-NEXT: movb %dl, (%ebx,%eax)918; X86-BWOFF-NEXT: movl %eax, %edx919; X86-BWOFF-NEXT: orl $1, %edx920; X86-BWOFF-NEXT: movb %cl, (%ebx,%edx)921; X86-BWOFF-NEXT: incl %edi922; X86-BWOFF-NEXT: addl $2, %eax923; X86-BWOFF-NEXT: adcl $0, %ebp924; X86-BWOFF-NEXT: cmpl {{[0-9]+}}(%esp), %eax925; X86-BWOFF-NEXT: movl %ebp, %ecx926; X86-BWOFF-NEXT: sbbl {{[0-9]+}}(%esp), %ecx927; X86-BWOFF-NEXT: jl .LBB10_1928; X86-BWOFF-NEXT: # %bb.2:929; X86-BWOFF-NEXT: popl %esi930; X86-BWOFF-NEXT: .cfi_def_cfa_offset 16931; X86-BWOFF-NEXT: popl %edi932; X86-BWOFF-NEXT: .cfi_def_cfa_offset 12933; X86-BWOFF-NEXT: popl %ebx934; X86-BWOFF-NEXT: .cfi_def_cfa_offset 8935; X86-BWOFF-NEXT: popl %ebp936; X86-BWOFF-NEXT: .cfi_def_cfa_offset 4937; X86-BWOFF-NEXT: retl938;939; X64-BWON-LABEL: MergeLoadStoreBaseIndexOffsetComplicated:940; X64-BWON: # %bb.0:941; X64-BWON-NEXT: xorl %eax, %eax942; X64-BWON-NEXT: .p2align 4943; X64-BWON-NEXT: .LBB10_1: # =>This Inner Loop Header: Depth=1944; X64-BWON-NEXT: movsbq (%rsi), %r8945; X64-BWON-NEXT: movzwl (%rdx,%r8), %r8d946; X64-BWON-NEXT: movw %r8w, (%rdi,%rax)947; X64-BWON-NEXT: incq %rsi948; X64-BWON-NEXT: addq $2, %rax949; X64-BWON-NEXT: cmpq %rcx, %rax950; X64-BWON-NEXT: jl .LBB10_1951; X64-BWON-NEXT: # %bb.2:952; X64-BWON-NEXT: retq953;954; X64-BWOFF-LABEL: MergeLoadStoreBaseIndexOffsetComplicated:955; X64-BWOFF: # %bb.0:956; X64-BWOFF-NEXT: xorl %eax, %eax957; X64-BWOFF-NEXT: .p2align 4958; X64-BWOFF-NEXT: .LBB10_1: # =>This Inner Loop Header: Depth=1959; X64-BWOFF-NEXT: movsbq (%rsi), %r8960; X64-BWOFF-NEXT: movw (%rdx,%r8), %r8w961; X64-BWOFF-NEXT: movw %r8w, (%rdi,%rax)962; X64-BWOFF-NEXT: incq %rsi963; X64-BWOFF-NEXT: addq $2, %rax964; X64-BWOFF-NEXT: cmpq %rcx, %rax965; X64-BWOFF-NEXT: jl .LBB10_1966; X64-BWOFF-NEXT: # %bb.2:967; X64-BWOFF-NEXT: retq968 br label %1969 970; <label>:1971 %.09 = phi i64 [ 0, %0 ], [ %13, %1 ]972 %.08 = phi ptr [ %b, %0 ], [ %12, %1 ]973 %2 = load i8, ptr %.08, align 1974 %3 = sext i8 %2 to i64975 %4 = getelementptr inbounds i8, ptr %c, i64 %3976 %5 = load i8, ptr %4, align 1977 %6 = add nsw i64 %3, 1978 %7 = getelementptr inbounds i8, ptr %c, i64 %6979 %8 = load i8, ptr %7, align 1980 %9 = getelementptr inbounds i8, ptr %a, i64 %.09981 store i8 %5, ptr %9, align 1982 %10 = or disjoint i64 %.09, 1983 %11 = getelementptr inbounds i8, ptr %a, i64 %10984 store i8 %8, ptr %11, align 1985 %12 = getelementptr inbounds i8, ptr %.08, i64 1986 %13 = add nuw nsw i64 %.09, 2987 %14 = icmp slt i64 %13, %n988 br i1 %14, label %1, label %15989 990; <label>:15991 ret void992}993 994; Make sure that we merge the consecutive load/store sequence below and use a995; word (16 bit) instead of a byte copy even if there are intermediate sign996; extensions.997define void @MergeLoadStoreBaseIndexOffsetSext(ptr %a, ptr %b, ptr %c, i32 %n) {998; X86-BWON-LABEL: MergeLoadStoreBaseIndexOffsetSext:999; X86-BWON: # %bb.0:1000; X86-BWON-NEXT: pushl %ebx1001; X86-BWON-NEXT: .cfi_def_cfa_offset 81002; X86-BWON-NEXT: pushl %edi1003; X86-BWON-NEXT: .cfi_def_cfa_offset 121004; X86-BWON-NEXT: pushl %esi1005; X86-BWON-NEXT: .cfi_def_cfa_offset 161006; X86-BWON-NEXT: .cfi_offset %esi, -161007; X86-BWON-NEXT: .cfi_offset %edi, -121008; X86-BWON-NEXT: .cfi_offset %ebx, -81009; X86-BWON-NEXT: xorl %eax, %eax1010; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %ecx1011; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %edx1012; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %esi1013; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %edi1014; X86-BWON-NEXT: .p2align 41015; X86-BWON-NEXT: .LBB11_1: # =>This Inner Loop Header: Depth=11016; X86-BWON-NEXT: movsbl (%edi,%eax), %ebx1017; X86-BWON-NEXT: movzwl (%edx,%ebx), %ebx1018; X86-BWON-NEXT: movw %bx, (%esi,%eax,2)1019; X86-BWON-NEXT: incl %eax1020; X86-BWON-NEXT: cmpl %eax, %ecx1021; X86-BWON-NEXT: jne .LBB11_11022; X86-BWON-NEXT: # %bb.2:1023; X86-BWON-NEXT: popl %esi1024; X86-BWON-NEXT: .cfi_def_cfa_offset 121025; X86-BWON-NEXT: popl %edi1026; X86-BWON-NEXT: .cfi_def_cfa_offset 81027; X86-BWON-NEXT: popl %ebx1028; X86-BWON-NEXT: .cfi_def_cfa_offset 41029; X86-BWON-NEXT: retl1030;1031; X86-BWOFF-LABEL: MergeLoadStoreBaseIndexOffsetSext:1032; X86-BWOFF: # %bb.0:1033; X86-BWOFF-NEXT: pushl %ebx1034; X86-BWOFF-NEXT: .cfi_def_cfa_offset 81035; X86-BWOFF-NEXT: pushl %edi1036; X86-BWOFF-NEXT: .cfi_def_cfa_offset 121037; X86-BWOFF-NEXT: pushl %esi1038; X86-BWOFF-NEXT: .cfi_def_cfa_offset 161039; X86-BWOFF-NEXT: .cfi_offset %esi, -161040; X86-BWOFF-NEXT: .cfi_offset %edi, -121041; X86-BWOFF-NEXT: .cfi_offset %ebx, -81042; X86-BWOFF-NEXT: xorl %eax, %eax1043; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %ecx1044; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %edx1045; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %esi1046; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %edi1047; X86-BWOFF-NEXT: .p2align 41048; X86-BWOFF-NEXT: .LBB11_1: # =>This Inner Loop Header: Depth=11049; X86-BWOFF-NEXT: movsbl (%edi,%eax), %ebx1050; X86-BWOFF-NEXT: movw (%edx,%ebx), %bx1051; X86-BWOFF-NEXT: movw %bx, (%esi,%eax,2)1052; X86-BWOFF-NEXT: incl %eax1053; X86-BWOFF-NEXT: cmpl %eax, %ecx1054; X86-BWOFF-NEXT: jne .LBB11_11055; X86-BWOFF-NEXT: # %bb.2:1056; X86-BWOFF-NEXT: popl %esi1057; X86-BWOFF-NEXT: .cfi_def_cfa_offset 121058; X86-BWOFF-NEXT: popl %edi1059; X86-BWOFF-NEXT: .cfi_def_cfa_offset 81060; X86-BWOFF-NEXT: popl %ebx1061; X86-BWOFF-NEXT: .cfi_def_cfa_offset 41062; X86-BWOFF-NEXT: retl1063;1064; X64-BWON-LABEL: MergeLoadStoreBaseIndexOffsetSext:1065; X64-BWON: # %bb.0:1066; X64-BWON-NEXT: movl %ecx, %eax1067; X64-BWON-NEXT: xorl %ecx, %ecx1068; X64-BWON-NEXT: .p2align 41069; X64-BWON-NEXT: .LBB11_1: # =>This Inner Loop Header: Depth=11070; X64-BWON-NEXT: movsbq (%rdi,%rcx), %r81071; X64-BWON-NEXT: movzwl (%rdx,%r8), %r8d1072; X64-BWON-NEXT: movw %r8w, (%rsi,%rcx,2)1073; X64-BWON-NEXT: incq %rcx1074; X64-BWON-NEXT: cmpl %ecx, %eax1075; X64-BWON-NEXT: jne .LBB11_11076; X64-BWON-NEXT: # %bb.2:1077; X64-BWON-NEXT: retq1078;1079; X64-BWOFF-LABEL: MergeLoadStoreBaseIndexOffsetSext:1080; X64-BWOFF: # %bb.0:1081; X64-BWOFF-NEXT: movl %ecx, %eax1082; X64-BWOFF-NEXT: xorl %ecx, %ecx1083; X64-BWOFF-NEXT: .p2align 41084; X64-BWOFF-NEXT: .LBB11_1: # =>This Inner Loop Header: Depth=11085; X64-BWOFF-NEXT: movsbq (%rdi,%rcx), %r81086; X64-BWOFF-NEXT: movw (%rdx,%r8), %r8w1087; X64-BWOFF-NEXT: movw %r8w, (%rsi,%rcx,2)1088; X64-BWOFF-NEXT: incq %rcx1089; X64-BWOFF-NEXT: cmpl %ecx, %eax1090; X64-BWOFF-NEXT: jne .LBB11_11091; X64-BWOFF-NEXT: # %bb.2:1092; X64-BWOFF-NEXT: retq1093 br label %11094 1095; <label>:11096 %.09 = phi i32 [ %n, %0 ], [ %12, %1 ]1097 %.08 = phi ptr [ %b, %0 ], [ %11, %1 ]1098 %.0 = phi ptr [ %a, %0 ], [ %2, %1 ]1099 %2 = getelementptr inbounds i8, ptr %.0, i64 11100 %3 = load i8, ptr %.0, align 11101 %4 = sext i8 %3 to i641102 %5 = getelementptr inbounds i8, ptr %c, i64 %41103 %6 = load i8, ptr %5, align 11104 %7 = add i64 %4, 11105 %8 = getelementptr inbounds i8, ptr %c, i64 %71106 %9 = load i8, ptr %8, align 11107 store i8 %6, ptr %.08, align 11108 %10 = getelementptr inbounds i8, ptr %.08, i64 11109 store i8 %9, ptr %10, align 11110 %11 = getelementptr inbounds i8, ptr %.08, i64 21111 %12 = add nsw i32 %.09, -11112 %13 = icmp eq i32 %12, 01113 br i1 %13, label %14, label %11114 1115; <label>:141116 ret void1117}1118 1119; However, we can only merge ignore sign extensions when they are on all memory1120; computations;1121define void @loadStoreBaseIndexOffsetSextNoSex(ptr %a, ptr %b, ptr %c, i32 %n) {1122; X86-BWON-LABEL: loadStoreBaseIndexOffsetSextNoSex:1123; X86-BWON: # %bb.0:1124; X86-BWON-NEXT: pushl %ebp1125; X86-BWON-NEXT: .cfi_def_cfa_offset 81126; X86-BWON-NEXT: pushl %ebx1127; X86-BWON-NEXT: .cfi_def_cfa_offset 121128; X86-BWON-NEXT: pushl %edi1129; X86-BWON-NEXT: .cfi_def_cfa_offset 161130; X86-BWON-NEXT: pushl %esi1131; X86-BWON-NEXT: .cfi_def_cfa_offset 201132; X86-BWON-NEXT: .cfi_offset %esi, -201133; X86-BWON-NEXT: .cfi_offset %edi, -161134; X86-BWON-NEXT: .cfi_offset %ebx, -121135; X86-BWON-NEXT: .cfi_offset %ebp, -81136; X86-BWON-NEXT: xorl %eax, %eax1137; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %ebp1138; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %edx1139; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %esi1140; X86-BWON-NEXT: movl {{[0-9]+}}(%esp), %edi1141; X86-BWON-NEXT: .p2align 41142; X86-BWON-NEXT: .LBB12_1: # =>This Inner Loop Header: Depth=11143; X86-BWON-NEXT: movsbl (%edi,%eax), %ebx1144; X86-BWON-NEXT: movzbl (%edx,%ebx), %ecx1145; X86-BWON-NEXT: incb %bl1146; X86-BWON-NEXT: movsbl %bl, %ebx1147; X86-BWON-NEXT: movb (%edx,%ebx), %ch1148; X86-BWON-NEXT: movb %cl, (%esi,%eax,2)1149; X86-BWON-NEXT: movb %ch, 1(%esi,%eax,2)1150; X86-BWON-NEXT: incl %eax1151; X86-BWON-NEXT: cmpl %eax, %ebp1152; X86-BWON-NEXT: jne .LBB12_11153; X86-BWON-NEXT: # %bb.2:1154; X86-BWON-NEXT: popl %esi1155; X86-BWON-NEXT: .cfi_def_cfa_offset 161156; X86-BWON-NEXT: popl %edi1157; X86-BWON-NEXT: .cfi_def_cfa_offset 121158; X86-BWON-NEXT: popl %ebx1159; X86-BWON-NEXT: .cfi_def_cfa_offset 81160; X86-BWON-NEXT: popl %ebp1161; X86-BWON-NEXT: .cfi_def_cfa_offset 41162; X86-BWON-NEXT: retl1163;1164; X86-BWOFF-LABEL: loadStoreBaseIndexOffsetSextNoSex:1165; X86-BWOFF: # %bb.0:1166; X86-BWOFF-NEXT: pushl %ebp1167; X86-BWOFF-NEXT: .cfi_def_cfa_offset 81168; X86-BWOFF-NEXT: pushl %ebx1169; X86-BWOFF-NEXT: .cfi_def_cfa_offset 121170; X86-BWOFF-NEXT: pushl %edi1171; X86-BWOFF-NEXT: .cfi_def_cfa_offset 161172; X86-BWOFF-NEXT: pushl %esi1173; X86-BWOFF-NEXT: .cfi_def_cfa_offset 201174; X86-BWOFF-NEXT: .cfi_offset %esi, -201175; X86-BWOFF-NEXT: .cfi_offset %edi, -161176; X86-BWOFF-NEXT: .cfi_offset %ebx, -121177; X86-BWOFF-NEXT: .cfi_offset %ebp, -81178; X86-BWOFF-NEXT: xorl %eax, %eax1179; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %ebp1180; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %edx1181; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %esi1182; X86-BWOFF-NEXT: movl {{[0-9]+}}(%esp), %edi1183; X86-BWOFF-NEXT: .p2align 41184; X86-BWOFF-NEXT: .LBB12_1: # =>This Inner Loop Header: Depth=11185; X86-BWOFF-NEXT: movsbl (%edi,%eax), %ebx1186; X86-BWOFF-NEXT: movb (%edx,%ebx), %cl1187; X86-BWOFF-NEXT: incb %bl1188; X86-BWOFF-NEXT: movsbl %bl, %ebx1189; X86-BWOFF-NEXT: movb (%edx,%ebx), %ch1190; X86-BWOFF-NEXT: movb %cl, (%esi,%eax,2)1191; X86-BWOFF-NEXT: movb %ch, 1(%esi,%eax,2)1192; X86-BWOFF-NEXT: incl %eax1193; X86-BWOFF-NEXT: cmpl %eax, %ebp1194; X86-BWOFF-NEXT: jne .LBB12_11195; X86-BWOFF-NEXT: # %bb.2:1196; X86-BWOFF-NEXT: popl %esi1197; X86-BWOFF-NEXT: .cfi_def_cfa_offset 161198; X86-BWOFF-NEXT: popl %edi1199; X86-BWOFF-NEXT: .cfi_def_cfa_offset 121200; X86-BWOFF-NEXT: popl %ebx1201; X86-BWOFF-NEXT: .cfi_def_cfa_offset 81202; X86-BWOFF-NEXT: popl %ebp1203; X86-BWOFF-NEXT: .cfi_def_cfa_offset 41204; X86-BWOFF-NEXT: retl1205;1206; X64-BWON-LABEL: loadStoreBaseIndexOffsetSextNoSex:1207; X64-BWON: # %bb.0:1208; X64-BWON-NEXT: movl %ecx, %eax1209; X64-BWON-NEXT: xorl %ecx, %ecx1210; X64-BWON-NEXT: .p2align 41211; X64-BWON-NEXT: .LBB12_1: # =>This Inner Loop Header: Depth=11212; X64-BWON-NEXT: movsbq (%rdi,%rcx), %r81213; X64-BWON-NEXT: movzbl (%rdx,%r8), %r9d1214; X64-BWON-NEXT: incl %r8d1215; X64-BWON-NEXT: movsbq %r8b, %r81216; X64-BWON-NEXT: movzbl (%rdx,%r8), %r8d1217; X64-BWON-NEXT: movb %r9b, (%rsi,%rcx,2)1218; X64-BWON-NEXT: movb %r8b, 1(%rsi,%rcx,2)1219; X64-BWON-NEXT: incq %rcx1220; X64-BWON-NEXT: cmpl %ecx, %eax1221; X64-BWON-NEXT: jne .LBB12_11222; X64-BWON-NEXT: # %bb.2:1223; X64-BWON-NEXT: retq1224;1225; X64-BWOFF-LABEL: loadStoreBaseIndexOffsetSextNoSex:1226; X64-BWOFF: # %bb.0:1227; X64-BWOFF-NEXT: movl %ecx, %eax1228; X64-BWOFF-NEXT: xorl %ecx, %ecx1229; X64-BWOFF-NEXT: .p2align 41230; X64-BWOFF-NEXT: .LBB12_1: # =>This Inner Loop Header: Depth=11231; X64-BWOFF-NEXT: movsbq (%rdi,%rcx), %r81232; X64-BWOFF-NEXT: movb (%rdx,%r8), %r9b1233; X64-BWOFF-NEXT: incl %r8d1234; X64-BWOFF-NEXT: movsbq %r8b, %r81235; X64-BWOFF-NEXT: movb (%rdx,%r8), %r8b1236; X64-BWOFF-NEXT: movb %r9b, (%rsi,%rcx,2)1237; X64-BWOFF-NEXT: movb %r8b, 1(%rsi,%rcx,2)1238; X64-BWOFF-NEXT: incq %rcx1239; X64-BWOFF-NEXT: cmpl %ecx, %eax1240; X64-BWOFF-NEXT: jne .LBB12_11241; X64-BWOFF-NEXT: # %bb.2:1242; X64-BWOFF-NEXT: retq1243 br label %11244 1245; <label>:11246 %.09 = phi i32 [ %n, %0 ], [ %12, %1 ]1247 %.08 = phi ptr [ %b, %0 ], [ %11, %1 ]1248 %.0 = phi ptr [ %a, %0 ], [ %2, %1 ]1249 %2 = getelementptr inbounds i8, ptr %.0, i64 11250 %3 = load i8, ptr %.0, align 11251 %4 = sext i8 %3 to i641252 %5 = getelementptr inbounds i8, ptr %c, i64 %41253 %6 = load i8, ptr %5, align 11254 %7 = add i8 %3, 11255 %wrap.4 = sext i8 %7 to i641256 %8 = getelementptr inbounds i8, ptr %c, i64 %wrap.41257 %9 = load i8, ptr %8, align 11258 store i8 %6, ptr %.08, align 11259 %10 = getelementptr inbounds i8, ptr %.08, i64 11260 store i8 %9, ptr %10, align 11261 %11 = getelementptr inbounds i8, ptr %.08, i64 21262 %12 = add nsw i32 %.09, -11263 %13 = icmp eq i32 %12, 01264 br i1 %13, label %14, label %11265 1266; <label>:141267 ret void1268}1269 1270; PR21711 ( http://llvm.org/bugs/show_bug.cgi?id=21711 )1271define void @merge_vec_element_store(<8 x float> %v, ptr %ptr) {1272; X86-LABEL: merge_vec_element_store:1273; X86: # %bb.0:1274; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1275; X86-NEXT: vmovups %ymm0, (%eax)1276; X86-NEXT: vzeroupper1277; X86-NEXT: retl1278;1279; X64-LABEL: merge_vec_element_store:1280; X64: # %bb.0:1281; X64-NEXT: vmovups %ymm0, (%rdi)1282; X64-NEXT: vzeroupper1283; X64-NEXT: retq1284 %vecext0 = extractelement <8 x float> %v, i32 01285 %vecext1 = extractelement <8 x float> %v, i32 11286 %vecext2 = extractelement <8 x float> %v, i32 21287 %vecext3 = extractelement <8 x float> %v, i32 31288 %vecext4 = extractelement <8 x float> %v, i32 41289 %vecext5 = extractelement <8 x float> %v, i32 51290 %vecext6 = extractelement <8 x float> %v, i32 61291 %vecext7 = extractelement <8 x float> %v, i32 71292 %arrayidx1 = getelementptr inbounds float, ptr %ptr, i64 11293 %arrayidx2 = getelementptr inbounds float, ptr %ptr, i64 21294 %arrayidx3 = getelementptr inbounds float, ptr %ptr, i64 31295 %arrayidx4 = getelementptr inbounds float, ptr %ptr, i64 41296 %arrayidx5 = getelementptr inbounds float, ptr %ptr, i64 51297 %arrayidx6 = getelementptr inbounds float, ptr %ptr, i64 61298 %arrayidx7 = getelementptr inbounds float, ptr %ptr, i64 71299 store float %vecext0, ptr %ptr, align 41300 store float %vecext1, ptr %arrayidx1, align 41301 store float %vecext2, ptr %arrayidx2, align 41302 store float %vecext3, ptr %arrayidx3, align 41303 store float %vecext4, ptr %arrayidx4, align 41304 store float %vecext5, ptr %arrayidx5, align 41305 store float %vecext6, ptr %arrayidx6, align 41306 store float %vecext7, ptr %arrayidx7, align 41307 ret void1308 1309}1310 1311; PR21711 - Merge vector stores into wider vector stores.1312; These should be merged into 32-byte stores.1313define void @merge_vec_extract_stores(<8 x float> %v1, <8 x float> %v2, ptr %ptr) {1314; X86-LABEL: merge_vec_extract_stores:1315; X86: # %bb.0:1316; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1317; X86-NEXT: vmovups %ymm0, 48(%eax)1318; X86-NEXT: vmovups %ymm1, 80(%eax)1319; X86-NEXT: vzeroupper1320; X86-NEXT: retl1321;1322; X64-LABEL: merge_vec_extract_stores:1323; X64: # %bb.0:1324; X64-NEXT: vmovups %ymm0, 48(%rdi)1325; X64-NEXT: vmovups %ymm1, 80(%rdi)1326; X64-NEXT: vzeroupper1327; X64-NEXT: retq1328 %idx0 = getelementptr inbounds <4 x float>, ptr %ptr, i64 31329 %idx1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 41330 %idx2 = getelementptr inbounds <4 x float>, ptr %ptr, i64 51331 %idx3 = getelementptr inbounds <4 x float>, ptr %ptr, i64 61332 %shuffle0 = shufflevector <8 x float> %v1, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>1333 %shuffle1 = shufflevector <8 x float> %v1, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1334 %shuffle2 = shufflevector <8 x float> %v2, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>1335 %shuffle3 = shufflevector <8 x float> %v2, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1336 store <4 x float> %shuffle0, ptr %idx0, align 161337 store <4 x float> %shuffle1, ptr %idx1, align 161338 store <4 x float> %shuffle2, ptr %idx2, align 161339 store <4 x float> %shuffle3, ptr %idx3, align 161340 ret void1341 1342}1343 1344; Merging vector stores when sourced from vector loads.1345define void @merge_vec_stores_from_loads(ptr %v, ptr %ptr) {1346; X86-LABEL: merge_vec_stores_from_loads:1347; X86: # %bb.0:1348; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1349; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx1350; X86-NEXT: vmovups (%ecx), %ymm01351; X86-NEXT: vmovups %ymm0, (%eax)1352; X86-NEXT: vzeroupper1353; X86-NEXT: retl1354;1355; X64-LABEL: merge_vec_stores_from_loads:1356; X64: # %bb.0:1357; X64-NEXT: vmovups (%rdi), %ymm01358; X64-NEXT: vmovups %ymm0, (%rsi)1359; X64-NEXT: vzeroupper1360; X64-NEXT: retq1361 %load_idx0 = getelementptr inbounds <4 x float>, ptr %v, i64 01362 %load_idx1 = getelementptr inbounds <4 x float>, ptr %v, i64 11363 %v0 = load <4 x float>, ptr %load_idx01364 %v1 = load <4 x float>, ptr %load_idx11365 %store_idx0 = getelementptr inbounds <4 x float>, ptr %ptr, i64 01366 %store_idx1 = getelementptr inbounds <4 x float>, ptr %ptr, i64 11367 store <4 x float> %v0, ptr %store_idx0, align 161368 store <4 x float> %v1, ptr %store_idx1, align 161369 ret void1370 1371}1372 1373define void @merge_vec_stores_of_zero(ptr %ptr) {1374; X86-LABEL: merge_vec_stores_of_zero:1375; X86: # %bb.0:1376; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1377; X86-NEXT: vxorps %xmm0, %xmm0, %xmm01378; X86-NEXT: vmovups %ymm0, 48(%eax)1379; X86-NEXT: vzeroupper1380; X86-NEXT: retl1381;1382; X64-LABEL: merge_vec_stores_of_zero:1383; X64: # %bb.0:1384; X64-NEXT: vxorps %xmm0, %xmm0, %xmm01385; X64-NEXT: vmovups %ymm0, 48(%rdi)1386; X64-NEXT: vzeroupper1387; X64-NEXT: retq1388 %idx0 = getelementptr inbounds <4 x i32>, ptr %ptr, i64 31389 %idx1 = getelementptr inbounds <4 x i32>, ptr %ptr, i64 41390 store <4 x i32> zeroinitializer, ptr %idx0, align 161391 store <4 x i32> zeroinitializer, ptr %idx1, align 161392 ret void1393}1394 1395define void @merge_vec_stores_of_constant_splat(ptr %ptr) {1396; X86-LABEL: merge_vec_stores_of_constant_splat:1397; X86: # %bb.0:1398; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1399; X86-NEXT: vbroadcastss {{.*#+}} xmm0 = [42,42,42,42]1400; X86-NEXT: vmovaps %xmm0, 48(%eax)1401; X86-NEXT: vmovaps %xmm0, 64(%eax)1402; X86-NEXT: retl1403;1404; X64-LABEL: merge_vec_stores_of_constant_splat:1405; X64: # %bb.0:1406; X64-NEXT: vbroadcastss {{.*#+}} xmm0 = [42,42,42,42]1407; X64-NEXT: vmovaps %xmm0, 48(%rdi)1408; X64-NEXT: vmovaps %xmm0, 64(%rdi)1409; X64-NEXT: retq1410 %idx0 = getelementptr inbounds <4 x i32>, ptr %ptr, i64 31411 %idx1 = getelementptr inbounds <4 x i32>, ptr %ptr, i64 41412 store <4 x i32> <i32 42, i32 42, i32 42, i32 42>, ptr %idx0, align 161413 store <4 x i32> <i32 42, i32 42, i32 42, i32 42>, ptr %idx1, align 161414 ret void1415}1416 1417define void @merge_vec_stores_of_constants(ptr %ptr) {1418; X86-LABEL: merge_vec_stores_of_constants:1419; X86: # %bb.0:1420; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1421; X86-NEXT: vmovaps {{.*#+}} xmm0 = [25,51,45,0]1422; X86-NEXT: vmovaps %xmm0, 48(%eax)1423; X86-NEXT: vmovaps {{.*#+}} xmm0 = [0,265,26,0]1424; X86-NEXT: vmovaps %xmm0, 64(%eax)1425; X86-NEXT: retl1426;1427; X64-LABEL: merge_vec_stores_of_constants:1428; X64: # %bb.0:1429; X64-NEXT: vmovaps {{.*#+}} xmm0 = [25,51,45,0]1430; X64-NEXT: vmovaps %xmm0, 48(%rdi)1431; X64-NEXT: vmovaps {{.*#+}} xmm0 = [0,265,26,0]1432; X64-NEXT: vmovaps %xmm0, 64(%rdi)1433; X64-NEXT: retq1434 %idx0 = getelementptr inbounds <4 x i32>, ptr %ptr, i64 31435 %idx1 = getelementptr inbounds <4 x i32>, ptr %ptr, i64 41436 store <4 x i32> <i32 25, i32 51, i32 45, i32 0>, ptr %idx0, align 161437 store <4 x i32> <i32 0, i32 265, i32 26, i32 0>, ptr %idx1, align 161438 ret void1439}1440 1441define void @merge_vec_stores_of_constants_with_undefs(ptr %ptr) {1442; X86-LABEL: merge_vec_stores_of_constants_with_undefs:1443; X86: # %bb.0:1444; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1445; X86-NEXT: vxorps %xmm0, %xmm0, %xmm01446; X86-NEXT: vmovups %ymm0, 48(%eax)1447; X86-NEXT: vzeroupper1448; X86-NEXT: retl1449;1450; X64-LABEL: merge_vec_stores_of_constants_with_undefs:1451; X64: # %bb.0:1452; X64-NEXT: vxorps %xmm0, %xmm0, %xmm01453; X64-NEXT: vmovups %ymm0, 48(%rdi)1454; X64-NEXT: vzeroupper1455; X64-NEXT: retq1456 %idx0 = getelementptr inbounds <4 x i32>, ptr %ptr, i64 31457 %idx1 = getelementptr inbounds <4 x i32>, ptr %ptr, i64 41458 store <4 x i32> <i32 0, i32 0, i32 0, i32 undef>, ptr %idx0, align 161459 store <4 x i32> <i32 0, i32 undef, i32 0, i32 0>, ptr %idx1, align 161460 ret void1461}1462 1463; This is a minimized test based on real code that was failing.1464; This should now be merged.1465define void @merge_vec_element_and_scalar_load(ptr %array) {1466; X86-LABEL: merge_vec_element_and_scalar_load:1467; X86: # %bb.0:1468; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1469; X86-NEXT: movl (%eax), %ecx1470; X86-NEXT: movl 4(%eax), %edx1471; X86-NEXT: movl %edx, 36(%eax)1472; X86-NEXT: movl %ecx, 32(%eax)1473; X86-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero1474; X86-NEXT: vmovsd %xmm0, 40(%eax)1475; X86-NEXT: retl1476;1477; X64-LABEL: merge_vec_element_and_scalar_load:1478; X64: # %bb.0:1479; X64-NEXT: vmovups (%rdi), %xmm01480; X64-NEXT: vmovups %xmm0, 32(%rdi)1481; X64-NEXT: retq1482 %idx0 = getelementptr inbounds [6 x i64], ptr %array, i64 0, i64 01483 %idx1 = getelementptr inbounds [6 x i64], ptr %array, i64 0, i64 11484 %idx4 = getelementptr inbounds [6 x i64], ptr %array, i64 0, i64 41485 %idx5 = getelementptr inbounds [6 x i64], ptr %array, i64 0, i64 51486 1487 %a0 = load i64, ptr %idx0, align 81488 store i64 %a0, ptr %idx4, align 81489 1490 %b = bitcast ptr %idx1 to ptr1491 %v = load <2 x i64>, ptr %b, align 81492 %a1 = extractelement <2 x i64> %v, i32 01493 store i64 %a1, ptr %idx5, align 81494 ret void1495 1496}1497 1498; Don't let a non-consecutive store thwart merging of the last two.1499define void @almost_consecutive_stores(ptr %p) {1500; X86-LABEL: almost_consecutive_stores:1501; X86: # %bb.0:1502; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1503; X86-NEXT: movb $0, (%eax)1504; X86-NEXT: movb $1, 42(%eax)1505; X86-NEXT: movw $770, 2(%eax) # imm = 0x3021506; X86-NEXT: retl1507;1508; X64-LABEL: almost_consecutive_stores:1509; X64: # %bb.0:1510; X64-NEXT: movb $0, (%rdi)1511; X64-NEXT: movb $1, 42(%rdi)1512; X64-NEXT: movw $770, 2(%rdi) # imm = 0x3021513; X64-NEXT: retq1514 store i8 0, ptr %p1515 %p1 = getelementptr i8, ptr %p, i64 421516 store i8 1, ptr %p11517 %p2 = getelementptr i8, ptr %p, i64 21518 store i8 2, ptr %p21519 %p3 = getelementptr i8, ptr %p, i64 31520 store i8 3, ptr %p31521 ret void1522}1523 1524; We should be able to merge these.1525define void @merge_bitcast(<4 x i32> %v, ptr %ptr) {1526; X86-LABEL: merge_bitcast:1527; X86: # %bb.0:1528; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1529; X86-NEXT: vmovups %xmm0, (%eax)1530; X86-NEXT: retl1531;1532; X64-LABEL: merge_bitcast:1533; X64: # %bb.0:1534; X64-NEXT: vmovups %xmm0, (%rdi)1535; X64-NEXT: retq1536 %fv = bitcast <4 x i32> %v to <4 x float>1537 %vecext1 = extractelement <4 x i32> %v, i32 11538 %vecext2 = extractelement <4 x i32> %v, i32 21539 %vecext3 = extractelement <4 x i32> %v, i32 31540 %f0 = extractelement <4 x float> %fv, i32 01541 %f1 = bitcast i32 %vecext1 to float1542 %f2 = bitcast i32 %vecext2 to float1543 %f3 = bitcast i32 %vecext3 to float1544 %idx0 = getelementptr inbounds float, ptr %ptr, i64 01545 %idx1 = getelementptr inbounds float, ptr %ptr, i64 11546 %idx2 = getelementptr inbounds float, ptr %ptr, i64 21547 %idx3 = getelementptr inbounds float, ptr %ptr, i64 31548 store float %f0, ptr %idx0, align 41549 store float %f1, ptr %idx1, align 41550 store float %f2, ptr %idx2, align 41551 store float %f3, ptr %idx3, align 41552 ret void1553}1554 1555; same as @merge_const_store with heterogeneous types.1556define void @merge_const_store_heterogeneous(i32 %count, ptr nocapture %p) nounwind uwtable noinline ssp {1557; X86-LABEL: merge_const_store_heterogeneous:1558; X86: # %bb.0:1559; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1560; X86-NEXT: testl %eax, %eax1561; X86-NEXT: jle .LBB23_31562; X86-NEXT: # %bb.1: # %.lr.ph.preheader1563; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx1564; X86-NEXT: .p2align 41565; X86-NEXT: .LBB23_2: # %.lr.ph1566; X86-NEXT: # =>This Inner Loop Header: Depth=11567; X86-NEXT: movl $67305985, (%ecx) # imm = 0x40302011568; X86-NEXT: movl $134678021, 4(%ecx) # imm = 0x80706051569; X86-NEXT: addl $24, %ecx1570; X86-NEXT: decl %eax1571; X86-NEXT: jne .LBB23_21572; X86-NEXT: .LBB23_3: # %._crit_edge1573; X86-NEXT: retl1574;1575; X64-LABEL: merge_const_store_heterogeneous:1576; X64: # %bb.0:1577; X64-NEXT: testl %edi, %edi1578; X64-NEXT: jle .LBB23_31579; X64-NEXT: # %bb.1: # %.lr.ph.preheader1580; X64-NEXT: movabsq $578437695752307201, %rax # imm = 0x8070605040302011581; X64-NEXT: .p2align 41582; X64-NEXT: .LBB23_2: # %.lr.ph1583; X64-NEXT: # =>This Inner Loop Header: Depth=11584; X64-NEXT: movq %rax, (%rsi)1585; X64-NEXT: addq $24, %rsi1586; X64-NEXT: decl %edi1587; X64-NEXT: jne .LBB23_21588; X64-NEXT: .LBB23_3: # %._crit_edge1589; X64-NEXT: retq1590 %1 = icmp sgt i32 %count, 01591 br i1 %1, label %.lr.ph, label %._crit_edge1592.lr.ph:1593 %i.02 = phi i32 [ %7, %.lr.ph ], [ 0, %0 ]1594 %.01 = phi ptr [ %8, %.lr.ph ], [ %p, %0 ]1595 %2 = getelementptr inbounds %struct.C, ptr %.01, i64 0, i32 01596 store i8 1, ptr %2, align 11597 %3 = getelementptr inbounds %struct.C, ptr %.01, i64 0, i32 11598 store i8 2, ptr %3, align 11599 %4 = getelementptr inbounds %struct.C, ptr %.01, i64 0, i32 21600 store i8 3, ptr %4, align 11601 %5 = getelementptr inbounds %struct.C, ptr %.01, i64 0, i32 31602 store i8 4, ptr %5, align 11603 %6 = getelementptr inbounds %struct.C, ptr %.01, i64 0, i32 41604 store i32 134678021, ptr %6, align 11605 %7 = add nsw i32 %i.02, 11606 %8 = getelementptr inbounds %struct.C, ptr %.01, i64 11607 %exitcond = icmp eq i32 %7, %count1608 br i1 %exitcond, label %._crit_edge, label %.lr.ph1609._crit_edge:1610 ret void1611}1612 1613; Merging heterogeneous integer types.1614define void @merge_heterogeneous(ptr nocapture %p, ptr nocapture %q) {1615; X86-LABEL: merge_heterogeneous:1616; X86: # %bb.0:1617; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1618; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx1619; X86-NEXT: movl (%ecx), %edx1620; X86-NEXT: movl 4(%ecx), %ecx1621; X86-NEXT: movl %edx, (%eax)1622; X86-NEXT: movl %ecx, 4(%eax)1623; X86-NEXT: retl1624;1625; X64-LABEL: merge_heterogeneous:1626; X64: # %bb.0:1627; X64-NEXT: movq (%rdi), %rax1628; X64-NEXT: movq %rax, (%rsi)1629; X64-NEXT: retq1630 %s0 = getelementptr inbounds %struct.C, ptr %p, i64 0, i32 01631 %s1 = getelementptr inbounds %struct.C, ptr %p, i64 0, i32 11632 %s2 = getelementptr inbounds %struct.C, ptr %p, i64 0, i32 21633 %s3 = getelementptr inbounds %struct.C, ptr %p, i64 0, i32 31634 %s4 = getelementptr inbounds %struct.C, ptr %p, i64 0, i32 41635 %d0 = getelementptr inbounds %struct.C, ptr %q, i64 0, i32 01636 %d1 = getelementptr inbounds %struct.C, ptr %q, i64 0, i32 11637 %d2 = getelementptr inbounds %struct.C, ptr %q, i64 0, i32 21638 %d3 = getelementptr inbounds %struct.C, ptr %q, i64 0, i32 31639 %d4 = getelementptr inbounds %struct.C, ptr %q, i64 0, i32 41640 %v0 = load i8, ptr %s0, align 11641 %v1 = load i8, ptr %s1, align 11642 %v2 = load i8, ptr %s2, align 11643 %v3 = load i8, ptr %s3, align 11644 %v4 = load i32, ptr %s4, align 11645 store i8 %v0, ptr %d0, align 11646 store i8 %v1, ptr %d1, align 11647 store i8 %v2, ptr %d2, align 11648 store i8 %v3, ptr %d3, align 11649 store i32 %v4, ptr %d4, align 41650 ret void1651}1652 1653define i32 @merge_store_load_store_seq(ptr %buff) {1654; X86-LABEL: merge_store_load_store_seq:1655; X86: # %bb.0: # %entry1656; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx1657; X86-NEXT: movl $0, (%ecx)1658; X86-NEXT: movl 4(%ecx), %eax1659; X86-NEXT: movl $0, 4(%ecx)1660; X86-NEXT: retl1661;1662; X64-LABEL: merge_store_load_store_seq:1663; X64: # %bb.0: # %entry1664; X64-NEXT: movl 4(%rdi), %eax1665; X64-NEXT: movq $0, (%rdi)1666; X64-NEXT: retq1667entry:1668 1669 store i32 0, ptr %buff, align 41670 %arrayidx1 = getelementptr inbounds i32, ptr %buff, i64 11671 %0 = load i32, ptr %arrayidx1, align 41672 store i32 0, ptr %arrayidx1, align 41673 ret i32 %01674}1675 1676define i32 @merge_store_alias(ptr %buff, ptr %other) {1677; X86-LABEL: merge_store_alias:1678; X86: # %bb.0: # %entry1679; X86-NEXT: movl {{[0-9]+}}(%esp), %eax1680; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx1681; X86-NEXT: movl $0, (%ecx)1682; X86-NEXT: movl (%eax), %eax1683; X86-NEXT: movl $0, 4(%ecx)1684; X86-NEXT: retl1685;1686; X64-LABEL: merge_store_alias:1687; X64: # %bb.0: # %entry1688; X64-NEXT: movl $0, (%rdi)1689; X64-NEXT: movl (%rsi), %eax1690; X64-NEXT: movl $0, 4(%rdi)1691; X64-NEXT: retq1692entry:1693 1694 store i32 0, ptr %buff, align 41695 %arrayidx1 = getelementptr inbounds i32, ptr %buff, i64 11696 %0 = load i32, ptr %other, align 41697 store i32 0, ptr %arrayidx1, align 41698 ret i32 %01699}1700