928 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s3 4define ptr @test_memcpy1_generic(ptr %P, ptr %Q) {5; CHECK-LABEL: test_memcpy1_generic:6; CHECK: # %bb.0:7; CHECK-NEXT: pushq %rbx8; CHECK-NEXT: .cfi_def_cfa_offset 169; CHECK-NEXT: .cfi_offset %rbx, -1610; CHECK-NEXT: movq %rdi, %rbx11; CHECK-NEXT: movl $1024, %edx # imm = 0x40012; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_1@PLT13; CHECK-NEXT: movq %rbx, %rax14; CHECK-NEXT: popq %rbx15; CHECK-NEXT: .cfi_def_cfa_offset 816; CHECK-NEXT: retq17 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 4 %P, ptr align 4 %Q, i32 1024, i32 1)18 ret ptr %P19 ; 3rd arg (%edx) -- length20}21 22define ptr @test_memcpy2_generic(ptr %P, ptr %Q) {23; CHECK-LABEL: test_memcpy2_generic:24; CHECK: # %bb.0:25; CHECK-NEXT: pushq %rbx26; CHECK-NEXT: .cfi_def_cfa_offset 1627; CHECK-NEXT: .cfi_offset %rbx, -1628; CHECK-NEXT: movq %rdi, %rbx29; CHECK-NEXT: movl $1024, %edx # imm = 0x40030; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_2@PLT31; CHECK-NEXT: movq %rbx, %rax32; CHECK-NEXT: popq %rbx33; CHECK-NEXT: .cfi_def_cfa_offset 834; CHECK-NEXT: retq35 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 4 %P, ptr align 4 %Q, i32 1024, i32 2)36 ret ptr %P37 ; 3rd arg (%edx) -- length38}39 40define ptr @test_memcpy4_generic(ptr %P, ptr %Q) {41; CHECK-LABEL: test_memcpy4_generic:42; CHECK: # %bb.0:43; CHECK-NEXT: pushq %rbx44; CHECK-NEXT: .cfi_def_cfa_offset 1645; CHECK-NEXT: .cfi_offset %rbx, -1646; CHECK-NEXT: movq %rdi, %rbx47; CHECK-NEXT: movl $1024, %edx # imm = 0x40048; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_4@PLT49; CHECK-NEXT: movq %rbx, %rax50; CHECK-NEXT: popq %rbx51; CHECK-NEXT: .cfi_def_cfa_offset 852; CHECK-NEXT: retq53 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 4 %P, ptr align 4 %Q, i32 1024, i32 4)54 ret ptr %P55 ; 3rd arg (%edx) -- length56}57 58define ptr @test_memcpy8(ptr %P, ptr %Q) {59; CHECK-LABEL: test_memcpy8:60; CHECK: # %bb.0:61; CHECK-NEXT: pushq %rbx62; CHECK-NEXT: .cfi_def_cfa_offset 1663; CHECK-NEXT: .cfi_offset %rbx, -1664; CHECK-NEXT: movq %rdi, %rbx65; CHECK-NEXT: movl $1024, %edx # imm = 0x40066; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_8@PLT67; CHECK-NEXT: movq %rbx, %rax68; CHECK-NEXT: popq %rbx69; CHECK-NEXT: .cfi_def_cfa_offset 870; CHECK-NEXT: retq71 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 8 %P, ptr align 8 %Q, i32 1024, i32 8)72 ret ptr %P73 ; 3rd arg (%edx) -- length74}75 76define ptr @test_memcpy16_generic(ptr %P, ptr %Q) {77; CHECK-LABEL: test_memcpy16_generic:78; CHECK: # %bb.0:79; CHECK-NEXT: pushq %rbx80; CHECK-NEXT: .cfi_def_cfa_offset 1681; CHECK-NEXT: .cfi_offset %rbx, -1682; CHECK-NEXT: movq %rdi, %rbx83; CHECK-NEXT: movl $1024, %edx # imm = 0x40084; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_16@PLT85; CHECK-NEXT: movq %rbx, %rax86; CHECK-NEXT: popq %rbx87; CHECK-NEXT: .cfi_def_cfa_offset 888; CHECK-NEXT: retq89 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 16 %P, ptr align 16 %Q, i32 1024, i32 16)90 ret ptr %P91 ; 3rd arg (%edx) -- length92}93 94define void @test_memcpy_args(ptr %Storage) {95; CHECK-LABEL: test_memcpy_args:96; CHECK: # %bb.0:97; CHECK-NEXT: pushq %rax98; CHECK-NEXT: .cfi_def_cfa_offset 1699; CHECK-NEXT: movq (%rdi), %rax100; CHECK-NEXT: movq 8(%rdi), %rsi101; CHECK-NEXT: movq %rax, %rdi102; CHECK-NEXT: movl $1024, %edx # imm = 0x400103; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_4@PLT104; CHECK-NEXT: popq %rax105; CHECK-NEXT: .cfi_def_cfa_offset 8106; CHECK-NEXT: retq107 %Dst = load ptr, ptr %Storage108 %Src.addr = getelementptr ptr, ptr %Storage, i64 1109 %Src = load ptr, ptr %Src.addr110 111 ; 1st arg (%rdi)112 ; 2nd arg (%rsi)113 ; 3rd arg (%edx) -- length114 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 4 %Dst, ptr align 4 %Src, i32 1024, i32 4)115 ret void116}117 118define ptr @test_memmove1_generic(ptr %P, ptr %Q) {119; CHECK-LABEL: test_memmove1_generic:120; CHECK: # %bb.0:121; CHECK-NEXT: pushq %rbx122; CHECK-NEXT: .cfi_def_cfa_offset 16123; CHECK-NEXT: .cfi_offset %rbx, -16124; CHECK-NEXT: movq %rdi, %rbx125; CHECK-NEXT: movl $1024, %edx # imm = 0x400126; CHECK-NEXT: callq __llvm_memmove_element_unordered_atomic_1@PLT127; CHECK-NEXT: movq %rbx, %rax128; CHECK-NEXT: popq %rbx129; CHECK-NEXT: .cfi_def_cfa_offset 8130; CHECK-NEXT: retq131 call void @llvm.memmove.element.unordered.atomic.p0.p0.i32(ptr align 4 %P, ptr align 4 %Q, i32 1024, i32 1)132 ret ptr %P133 ; 3rd arg (%edx) -- length134}135 136define ptr @test_memmove2_generic(ptr %P, ptr %Q) {137; CHECK-LABEL: test_memmove2_generic:138; CHECK: # %bb.0:139; CHECK-NEXT: pushq %rbx140; CHECK-NEXT: .cfi_def_cfa_offset 16141; CHECK-NEXT: .cfi_offset %rbx, -16142; CHECK-NEXT: movq %rdi, %rbx143; CHECK-NEXT: movl $1024, %edx # imm = 0x400144; CHECK-NEXT: callq __llvm_memmove_element_unordered_atomic_2@PLT145; CHECK-NEXT: movq %rbx, %rax146; CHECK-NEXT: popq %rbx147; CHECK-NEXT: .cfi_def_cfa_offset 8148; CHECK-NEXT: retq149 call void @llvm.memmove.element.unordered.atomic.p0.p0.i32(ptr align 4 %P, ptr align 4 %Q, i32 1024, i32 2)150 ret ptr %P151 ; 3rd arg (%edx) -- length152}153 154define ptr @test_memmove4_generic(ptr %P, ptr %Q) {155; CHECK-LABEL: test_memmove4_generic:156; CHECK: # %bb.0:157; CHECK-NEXT: pushq %rbx158; CHECK-NEXT: .cfi_def_cfa_offset 16159; CHECK-NEXT: .cfi_offset %rbx, -16160; CHECK-NEXT: movq %rdi, %rbx161; CHECK-NEXT: movl $1024, %edx # imm = 0x400162; CHECK-NEXT: callq __llvm_memmove_element_unordered_atomic_4@PLT163; CHECK-NEXT: movq %rbx, %rax164; CHECK-NEXT: popq %rbx165; CHECK-NEXT: .cfi_def_cfa_offset 8166; CHECK-NEXT: retq167 call void @llvm.memmove.element.unordered.atomic.p0.p0.i32(ptr align 4 %P, ptr align 4 %Q, i32 1024, i32 4)168 ret ptr %P169 ; 3rd arg (%edx) -- length170}171 172define ptr @test_memmove8_generic(ptr %P, ptr %Q) {173; CHECK-LABEL: test_memmove8_generic:174; CHECK: # %bb.0:175; CHECK-NEXT: pushq %rbx176; CHECK-NEXT: .cfi_def_cfa_offset 16177; CHECK-NEXT: .cfi_offset %rbx, -16178; CHECK-NEXT: movq %rdi, %rbx179; CHECK-NEXT: movl $1024, %edx # imm = 0x400180; CHECK-NEXT: callq __llvm_memmove_element_unordered_atomic_8@PLT181; CHECK-NEXT: movq %rbx, %rax182; CHECK-NEXT: popq %rbx183; CHECK-NEXT: .cfi_def_cfa_offset 8184; CHECK-NEXT: retq185 call void @llvm.memmove.element.unordered.atomic.p0.p0.i32(ptr align 8 %P, ptr align 8 %Q, i32 1024, i32 8)186 ret ptr %P187 ; 3rd arg (%edx) -- length188}189 190define ptr @test_memmove16_generic(ptr %P, ptr %Q) {191; CHECK-LABEL: test_memmove16_generic:192; CHECK: # %bb.0:193; CHECK-NEXT: pushq %rbx194; CHECK-NEXT: .cfi_def_cfa_offset 16195; CHECK-NEXT: .cfi_offset %rbx, -16196; CHECK-NEXT: movq %rdi, %rbx197; CHECK-NEXT: movl $1024, %edx # imm = 0x400198; CHECK-NEXT: callq __llvm_memmove_element_unordered_atomic_16@PLT199; CHECK-NEXT: movq %rbx, %rax200; CHECK-NEXT: popq %rbx201; CHECK-NEXT: .cfi_def_cfa_offset 8202; CHECK-NEXT: retq203 call void @llvm.memmove.element.unordered.atomic.p0.p0.i32(ptr align 16 %P, ptr align 16 %Q, i32 1024, i32 16)204 ret ptr %P205 ; 3rd arg (%edx) -- length206}207 208define void @test_memmove_args(ptr %Storage) {209; CHECK-LABEL: test_memmove_args:210; CHECK: # %bb.0:211; CHECK-NEXT: pushq %rax212; CHECK-NEXT: .cfi_def_cfa_offset 16213; CHECK-NEXT: movq (%rdi), %rax214; CHECK-NEXT: movq 8(%rdi), %rsi215; CHECK-NEXT: movq %rax, %rdi216; CHECK-NEXT: movl $1024, %edx # imm = 0x400217; CHECK-NEXT: callq __llvm_memmove_element_unordered_atomic_4@PLT218; CHECK-NEXT: popq %rax219; CHECK-NEXT: .cfi_def_cfa_offset 8220; CHECK-NEXT: retq221 %Dst = load ptr, ptr %Storage222 %Src.addr = getelementptr ptr, ptr %Storage, i64 1223 %Src = load ptr, ptr %Src.addr224 225 ; 1st arg (%rdi)226 ; 2nd arg (%rsi)227 ; 3rd arg (%edx) -- length228 call void @llvm.memmove.element.unordered.atomic.p0.p0.i32(ptr align 4 %Dst, ptr align 4 %Src, i32 1024, i32 4)229 ret void230}231 232define ptr @test_memset1_generic(ptr %P, i8 %V) {233; CHECK-LABEL: test_memset1_generic:234; CHECK: # %bb.0:235; CHECK-NEXT: pushq %rbx236; CHECK-NEXT: .cfi_def_cfa_offset 16237; CHECK-NEXT: .cfi_offset %rbx, -16238; CHECK-NEXT: movq %rdi, %rbx239; CHECK-NEXT: movl $1024, %edx # imm = 0x400240; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_1@PLT241; CHECK-NEXT: movq %rbx, %rax242; CHECK-NEXT: popq %rbx243; CHECK-NEXT: .cfi_def_cfa_offset 8244; CHECK-NEXT: retq245 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 1 %P, i8 %V, i32 1024, i32 1)246 ret ptr %P247 ; 3rd arg (%edx) -- length248}249 250define ptr @test_memset2_generic(ptr %P, i8 %V) {251; CHECK-LABEL: test_memset2_generic:252; CHECK: # %bb.0:253; CHECK-NEXT: pushq %rbx254; CHECK-NEXT: .cfi_def_cfa_offset 16255; CHECK-NEXT: .cfi_offset %rbx, -16256; CHECK-NEXT: movq %rdi, %rbx257; CHECK-NEXT: movl $1024, %edx # imm = 0x400258; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_2@PLT259; CHECK-NEXT: movq %rbx, %rax260; CHECK-NEXT: popq %rbx261; CHECK-NEXT: .cfi_def_cfa_offset 8262; CHECK-NEXT: retq263 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 2 %P, i8 %V, i32 1024, i32 2)264 ret ptr %P265 ; 3rd arg (%edx) -- length266}267 268define ptr @test_memset4_generic(ptr %P, i8 %V) {269; CHECK-LABEL: test_memset4_generic:270; CHECK: # %bb.0:271; CHECK-NEXT: pushq %rbx272; CHECK-NEXT: .cfi_def_cfa_offset 16273; CHECK-NEXT: .cfi_offset %rbx, -16274; CHECK-NEXT: movq %rdi, %rbx275; CHECK-NEXT: movl $1024, %edx # imm = 0x400276; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_4@PLT277; CHECK-NEXT: movq %rbx, %rax278; CHECK-NEXT: popq %rbx279; CHECK-NEXT: .cfi_def_cfa_offset 8280; CHECK-NEXT: retq281 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 4 %P, i8 %V, i32 1024, i32 4)282 ret ptr %P283 ; 3rd arg (%edx) -- length284}285 286define ptr @test_memset8_generic(ptr %P, i8 %V) {287; CHECK-LABEL: test_memset8_generic:288; CHECK: # %bb.0:289; CHECK-NEXT: pushq %rbx290; CHECK-NEXT: .cfi_def_cfa_offset 16291; CHECK-NEXT: .cfi_offset %rbx, -16292; CHECK-NEXT: movq %rdi, %rbx293; CHECK-NEXT: movl $1024, %edx # imm = 0x400294; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_8@PLT295; CHECK-NEXT: movq %rbx, %rax296; CHECK-NEXT: popq %rbx297; CHECK-NEXT: .cfi_def_cfa_offset 8298; CHECK-NEXT: retq299 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 8 %P, i8 %V, i32 1024, i32 8)300 ret ptr %P301 ; 3rd arg (%edx) -- length302}303 304define ptr @test_memset16_generic(ptr %P, i8 %V) {305; CHECK-LABEL: test_memset16_generic:306; CHECK: # %bb.0:307; CHECK-NEXT: pushq %rbx308; CHECK-NEXT: .cfi_def_cfa_offset 16309; CHECK-NEXT: .cfi_offset %rbx, -16310; CHECK-NEXT: movq %rdi, %rbx311; CHECK-NEXT: movl $1024, %edx # imm = 0x400312; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_16@PLT313; CHECK-NEXT: movq %rbx, %rax314; CHECK-NEXT: popq %rbx315; CHECK-NEXT: .cfi_def_cfa_offset 8316; CHECK-NEXT: retq317 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 16 %P, i8 %V, i32 1024, i32 16)318 ret ptr %P319 ; 3rd arg (%edx) -- length320}321 322define void @test_memset_args(ptr %Storage, ptr %V) {323; CHECK-LABEL: test_memset_args:324; CHECK: # %bb.0:325; CHECK-NEXT: pushq %rax326; CHECK-NEXT: .cfi_def_cfa_offset 16327; CHECK-NEXT: movq (%rdi), %rdi328; CHECK-NEXT: movzbl (%rsi), %esi329; CHECK-NEXT: movl $1024, %edx # imm = 0x400330; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_4@PLT331; CHECK-NEXT: popq %rax332; CHECK-NEXT: .cfi_def_cfa_offset 8333; CHECK-NEXT: retq334 %Dst = load ptr, ptr %Storage335 %Val = load i8, ptr %V336 337 ; 1st arg (%rdi)338 ; 2nd arg (%rsi)339 ; 3rd arg (%edx) -- length340 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 4 %Dst, i8 %Val, i32 1024, i32 4)341 ret void342}343 344;; Next batch of tests are cases where we could profitably lower to345;; atomic loads and stores directly, just as we do for non-atomic ones for346;; non element.unorderered.atomic variants.347 348define ptr @test_memcpy1_64(ptr %P, ptr %Q) {349; CHECK-LABEL: test_memcpy1_64:350; CHECK: # %bb.0:351; CHECK-NEXT: pushq %rbx352; CHECK-NEXT: .cfi_def_cfa_offset 16353; CHECK-NEXT: .cfi_offset %rbx, -16354; CHECK-NEXT: movq %rdi, %rbx355; CHECK-NEXT: movl $64, %edx356; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_1@PLT357; CHECK-NEXT: movq %rbx, %rax358; CHECK-NEXT: popq %rbx359; CHECK-NEXT: .cfi_def_cfa_offset 8360; CHECK-NEXT: retq361 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 1 %P, ptr align 1 %Q, i32 64, i32 1)362 ret ptr %P363}364 365; Ensure align 16 generates vector load/stores even with small element size366define ptr @test_memcpy1_64_align4(ptr %P, ptr %Q) {367; CHECK-LABEL: test_memcpy1_64_align4:368; CHECK: # %bb.0:369; CHECK-NEXT: pushq %rbx370; CHECK-NEXT: .cfi_def_cfa_offset 16371; CHECK-NEXT: .cfi_offset %rbx, -16372; CHECK-NEXT: movq %rdi, %rbx373; CHECK-NEXT: movl $64, %edx374; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_1@PLT375; CHECK-NEXT: movq %rbx, %rax376; CHECK-NEXT: popq %rbx377; CHECK-NEXT: .cfi_def_cfa_offset 8378; CHECK-NEXT: retq379 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 4 %P, ptr align 4 %Q, i32 64, i32 1)380 ret ptr %P381}382 383define ptr @test_memcpy1_64_align8(ptr %P, ptr %Q) {384; CHECK-LABEL: test_memcpy1_64_align8:385; CHECK: # %bb.0:386; CHECK-NEXT: pushq %rbx387; CHECK-NEXT: .cfi_def_cfa_offset 16388; CHECK-NEXT: .cfi_offset %rbx, -16389; CHECK-NEXT: movq %rdi, %rbx390; CHECK-NEXT: movl $64, %edx391; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_1@PLT392; CHECK-NEXT: movq %rbx, %rax393; CHECK-NEXT: popq %rbx394; CHECK-NEXT: .cfi_def_cfa_offset 8395; CHECK-NEXT: retq396 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 8 %P, ptr align 8 %Q, i32 64, i32 1)397 ret ptr %P398}399 400 401define ptr @test_memcpy1_64_align16(ptr %P, ptr %Q) {402; CHECK-LABEL: test_memcpy1_64_align16:403; CHECK: # %bb.0:404; CHECK-NEXT: pushq %rbx405; CHECK-NEXT: .cfi_def_cfa_offset 16406; CHECK-NEXT: .cfi_offset %rbx, -16407; CHECK-NEXT: movq %rdi, %rbx408; CHECK-NEXT: movl $64, %edx409; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_1@PLT410; CHECK-NEXT: movq %rbx, %rax411; CHECK-NEXT: popq %rbx412; CHECK-NEXT: .cfi_def_cfa_offset 8413; CHECK-NEXT: retq414 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 16 %P, ptr align 16 %Q, i32 64, i32 1)415 ret ptr %P416}417 418 419; Make sure that different source & dest alignments are handled correctly.420define ptr @test_memcpy1_64_diff_aligns(ptr %P, ptr %Q) {421; CHECK-LABEL: test_memcpy1_64_diff_aligns:422; CHECK: # %bb.0:423; CHECK-NEXT: pushq %rbx424; CHECK-NEXT: .cfi_def_cfa_offset 16425; CHECK-NEXT: .cfi_offset %rbx, -16426; CHECK-NEXT: movq %rdi, %rbx427; CHECK-NEXT: movl $64, %edx428; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_1@PLT429; CHECK-NEXT: movq %rbx, %rax430; CHECK-NEXT: popq %rbx431; CHECK-NEXT: .cfi_def_cfa_offset 8432; CHECK-NEXT: retq433 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 1 %P, ptr align 16 %Q, i32 64, i32 1)434 ret ptr %P435}436 437define ptr @test_memcpy2_64(ptr %P, ptr %Q) {438; CHECK-LABEL: test_memcpy2_64:439; CHECK: # %bb.0:440; CHECK-NEXT: pushq %rbx441; CHECK-NEXT: .cfi_def_cfa_offset 16442; CHECK-NEXT: .cfi_offset %rbx, -16443; CHECK-NEXT: movq %rdi, %rbx444; CHECK-NEXT: movl $64, %edx445; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_2@PLT446; CHECK-NEXT: movq %rbx, %rax447; CHECK-NEXT: popq %rbx448; CHECK-NEXT: .cfi_def_cfa_offset 8449; CHECK-NEXT: retq450 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 2 %P, ptr align 2 %Q, i32 64, i32 2)451 ret ptr %P452}453 454define ptr @test_memcpy4_64(ptr %P, ptr %Q) {455; CHECK-LABEL: test_memcpy4_64:456; CHECK: # %bb.0:457; CHECK-NEXT: pushq %rbx458; CHECK-NEXT: .cfi_def_cfa_offset 16459; CHECK-NEXT: .cfi_offset %rbx, -16460; CHECK-NEXT: movq %rdi, %rbx461; CHECK-NEXT: movl $64, %edx462; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_4@PLT463; CHECK-NEXT: movq %rbx, %rax464; CHECK-NEXT: popq %rbx465; CHECK-NEXT: .cfi_def_cfa_offset 8466; CHECK-NEXT: retq467 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 4 %P, ptr align 4 %Q, i32 64, i32 4)468 ret ptr %P469}470 471define ptr @test_memcpy8_64(ptr %P, ptr %Q) {472; CHECK-LABEL: test_memcpy8_64:473; CHECK: # %bb.0:474; CHECK-NEXT: pushq %rbx475; CHECK-NEXT: .cfi_def_cfa_offset 16476; CHECK-NEXT: .cfi_offset %rbx, -16477; CHECK-NEXT: movq %rdi, %rbx478; CHECK-NEXT: movl $64, %edx479; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_8@PLT480; CHECK-NEXT: movq %rbx, %rax481; CHECK-NEXT: popq %rbx482; CHECK-NEXT: .cfi_def_cfa_offset 8483; CHECK-NEXT: retq484 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 8 %P, ptr align 8 %Q, i32 64, i32 8)485 ret ptr %P486}487 488define ptr @test_memcpy16_64(ptr %P, ptr %Q) {489; CHECK-LABEL: test_memcpy16_64:490; CHECK: # %bb.0:491; CHECK-NEXT: pushq %rbx492; CHECK-NEXT: .cfi_def_cfa_offset 16493; CHECK-NEXT: .cfi_offset %rbx, -16494; CHECK-NEXT: movq %rdi, %rbx495; CHECK-NEXT: movl $64, %edx496; CHECK-NEXT: callq __llvm_memcpy_element_unordered_atomic_16@PLT497; CHECK-NEXT: movq %rbx, %rax498; CHECK-NEXT: popq %rbx499; CHECK-NEXT: .cfi_def_cfa_offset 8500; CHECK-NEXT: retq501 call void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr align 16 %P, ptr align 16 %Q, i32 64, i32 16)502 ret ptr %P503}504 505; ==================================506 507define ptr @test_memmove1_64(ptr %P, ptr %Q) {508; CHECK-LABEL: test_memmove1_64:509; CHECK: # %bb.0:510; CHECK-NEXT: pushq %rbx511; CHECK-NEXT: .cfi_def_cfa_offset 16512; CHECK-NEXT: .cfi_offset %rbx, -16513; CHECK-NEXT: movq %rdi, %rbx514; CHECK-NEXT: movl $64, %edx515; CHECK-NEXT: callq __llvm_memmove_element_unordered_atomic_1@PLT516; CHECK-NEXT: movq %rbx, %rax517; CHECK-NEXT: popq %rbx518; CHECK-NEXT: .cfi_def_cfa_offset 8519; CHECK-NEXT: retq520 call void @llvm.memmove.element.unordered.atomic.p0.p0.i32(ptr align 1 %P, ptr align 1 %Q, i32 64, i32 1)521 ret ptr %P522}523 524; Ensure align 16 generates vector load/stores even with small element size525define ptr @test_memmove1_64_align16(ptr %P, ptr %Q) {526; CHECK-LABEL: test_memmove1_64_align16:527; CHECK: # %bb.0:528; CHECK-NEXT: pushq %rbx529; CHECK-NEXT: .cfi_def_cfa_offset 16530; CHECK-NEXT: .cfi_offset %rbx, -16531; CHECK-NEXT: movq %rdi, %rbx532; CHECK-NEXT: movl $64, %edx533; CHECK-NEXT: callq __llvm_memmove_element_unordered_atomic_1@PLT534; CHECK-NEXT: movq %rbx, %rax535; CHECK-NEXT: popq %rbx536; CHECK-NEXT: .cfi_def_cfa_offset 8537; CHECK-NEXT: retq538 call void @llvm.memmove.element.unordered.atomic.p0.p0.i32(ptr align 16 %P, ptr align 16 %Q, i32 64, i32 1)539 ret ptr %P540}541 542; Make sure that different source & dest alignments are handled correctly.543define ptr @test_memmove1_64_diff_aligns(ptr %P, ptr %Q) {544; CHECK-LABEL: test_memmove1_64_diff_aligns:545; CHECK: # %bb.0:546; CHECK-NEXT: pushq %rbx547; CHECK-NEXT: .cfi_def_cfa_offset 16548; CHECK-NEXT: .cfi_offset %rbx, -16549; CHECK-NEXT: movq %rdi, %rbx550; CHECK-NEXT: movl $64, %edx551; CHECK-NEXT: callq __llvm_memmove_element_unordered_atomic_1@PLT552; CHECK-NEXT: movq %rbx, %rax553; CHECK-NEXT: popq %rbx554; CHECK-NEXT: .cfi_def_cfa_offset 8555; CHECK-NEXT: retq556 call void @llvm.memmove.element.unordered.atomic.p0.p0.i32(ptr align 1 %P, ptr align 16 %Q, i32 64, i32 1)557 ret ptr %P558}559 560define ptr @test_memmove2_64(ptr %P, ptr %Q) {561; CHECK-LABEL: test_memmove2_64:562; CHECK: # %bb.0:563; CHECK-NEXT: pushq %rbx564; CHECK-NEXT: .cfi_def_cfa_offset 16565; CHECK-NEXT: .cfi_offset %rbx, -16566; CHECK-NEXT: movq %rdi, %rbx567; CHECK-NEXT: movl $64, %edx568; CHECK-NEXT: callq __llvm_memmove_element_unordered_atomic_2@PLT569; CHECK-NEXT: movq %rbx, %rax570; CHECK-NEXT: popq %rbx571; CHECK-NEXT: .cfi_def_cfa_offset 8572; CHECK-NEXT: retq573 call void @llvm.memmove.element.unordered.atomic.p0.p0.i32(ptr align 2 %P, ptr align 2 %Q, i32 64, i32 2)574 ret ptr %P575}576 577define ptr @test_memmove4_64(ptr %P, ptr %Q) {578; CHECK-LABEL: test_memmove4_64:579; CHECK: # %bb.0:580; CHECK-NEXT: pushq %rbx581; CHECK-NEXT: .cfi_def_cfa_offset 16582; CHECK-NEXT: .cfi_offset %rbx, -16583; CHECK-NEXT: movq %rdi, %rbx584; CHECK-NEXT: movl $64, %edx585; CHECK-NEXT: callq __llvm_memmove_element_unordered_atomic_4@PLT586; CHECK-NEXT: movq %rbx, %rax587; CHECK-NEXT: popq %rbx588; CHECK-NEXT: .cfi_def_cfa_offset 8589; CHECK-NEXT: retq590 call void @llvm.memmove.element.unordered.atomic.p0.p0.i32(ptr align 4 %P, ptr align 4 %Q, i32 64, i32 4)591 ret ptr %P592}593 594define ptr @test_memmove8_64(ptr %P, ptr %Q) {595; CHECK-LABEL: test_memmove8_64:596; CHECK: # %bb.0:597; CHECK-NEXT: pushq %rbx598; CHECK-NEXT: .cfi_def_cfa_offset 16599; CHECK-NEXT: .cfi_offset %rbx, -16600; CHECK-NEXT: movq %rdi, %rbx601; CHECK-NEXT: movl $64, %edx602; CHECK-NEXT: callq __llvm_memmove_element_unordered_atomic_8@PLT603; CHECK-NEXT: movq %rbx, %rax604; CHECK-NEXT: popq %rbx605; CHECK-NEXT: .cfi_def_cfa_offset 8606; CHECK-NEXT: retq607 call void @llvm.memmove.element.unordered.atomic.p0.p0.i32(ptr align 8 %P, ptr align 8 %Q, i32 64, i32 8)608 ret ptr %P609}610 611define ptr @test_memmove16_64(ptr %P, ptr %Q) {612; CHECK-LABEL: test_memmove16_64:613; CHECK: # %bb.0:614; CHECK-NEXT: pushq %rbx615; CHECK-NEXT: .cfi_def_cfa_offset 16616; CHECK-NEXT: .cfi_offset %rbx, -16617; CHECK-NEXT: movq %rdi, %rbx618; CHECK-NEXT: movl $64, %edx619; CHECK-NEXT: callq __llvm_memmove_element_unordered_atomic_16@PLT620; CHECK-NEXT: movq %rbx, %rax621; CHECK-NEXT: popq %rbx622; CHECK-NEXT: .cfi_def_cfa_offset 8623; CHECK-NEXT: retq624 call void @llvm.memmove.element.unordered.atomic.p0.p0.i32(ptr align 16 %P, ptr align 16 %Q, i32 64, i32 16)625 ret ptr %P626}627 628; ==============================629 630 631define ptr @test_memset1_64(ptr %P, i8 %V) {632; CHECK-LABEL: test_memset1_64:633; CHECK: # %bb.0:634; CHECK-NEXT: pushq %rbx635; CHECK-NEXT: .cfi_def_cfa_offset 16636; CHECK-NEXT: .cfi_offset %rbx, -16637; CHECK-NEXT: movq %rdi, %rbx638; CHECK-NEXT: movl $64, %edx639; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_1@PLT640; CHECK-NEXT: movq %rbx, %rax641; CHECK-NEXT: popq %rbx642; CHECK-NEXT: .cfi_def_cfa_offset 8643; CHECK-NEXT: retq644 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 1 %P, i8 %V, i32 64, i32 1)645 ret ptr %P646}647 648define ptr @test_memset1_64_align16(ptr %P, i8 %V) {649; CHECK-LABEL: test_memset1_64_align16:650; CHECK: # %bb.0:651; CHECK-NEXT: pushq %rbx652; CHECK-NEXT: .cfi_def_cfa_offset 16653; CHECK-NEXT: .cfi_offset %rbx, -16654; CHECK-NEXT: movq %rdi, %rbx655; CHECK-NEXT: movl $64, %edx656; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_1@PLT657; CHECK-NEXT: movq %rbx, %rax658; CHECK-NEXT: popq %rbx659; CHECK-NEXT: .cfi_def_cfa_offset 8660; CHECK-NEXT: retq661 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 16 %P, i8 %V, i32 64, i32 1)662 ret ptr %P663}664 665define ptr @test_memset2_64(ptr %P, i8 %V) {666; CHECK-LABEL: test_memset2_64:667; CHECK: # %bb.0:668; CHECK-NEXT: pushq %rbx669; CHECK-NEXT: .cfi_def_cfa_offset 16670; CHECK-NEXT: .cfi_offset %rbx, -16671; CHECK-NEXT: movq %rdi, %rbx672; CHECK-NEXT: movl $64, %edx673; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_2@PLT674; CHECK-NEXT: movq %rbx, %rax675; CHECK-NEXT: popq %rbx676; CHECK-NEXT: .cfi_def_cfa_offset 8677; CHECK-NEXT: retq678 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 2 %P, i8 %V, i32 64, i32 2)679 ret ptr %P680}681 682;; Use the memset4 case to explore alignment and sizing requirements in the683;; lowering684 685define ptr @test_memset4_64(ptr %P, i8 %V) {686; CHECK-LABEL: test_memset4_64:687; CHECK: # %bb.0:688; CHECK-NEXT: pushq %rbx689; CHECK-NEXT: .cfi_def_cfa_offset 16690; CHECK-NEXT: .cfi_offset %rbx, -16691; CHECK-NEXT: movq %rdi, %rbx692; CHECK-NEXT: movl $64, %edx693; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_4@PLT694; CHECK-NEXT: movq %rbx, %rax695; CHECK-NEXT: popq %rbx696; CHECK-NEXT: .cfi_def_cfa_offset 8697; CHECK-NEXT: retq698 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 4 %P, i8 %V, i32 64, i32 4)699 ret ptr %P700}701 702define ptr @test_memset4_64_align8(ptr %P, i8 %V) {703; CHECK-LABEL: test_memset4_64_align8:704; CHECK: # %bb.0:705; CHECK-NEXT: pushq %rbx706; CHECK-NEXT: .cfi_def_cfa_offset 16707; CHECK-NEXT: .cfi_offset %rbx, -16708; CHECK-NEXT: movq %rdi, %rbx709; CHECK-NEXT: movl $64, %edx710; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_4@PLT711; CHECK-NEXT: movq %rbx, %rax712; CHECK-NEXT: popq %rbx713; CHECK-NEXT: .cfi_def_cfa_offset 8714; CHECK-NEXT: retq715 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 8 %P, i8 %V, i32 64, i32 4)716 ret ptr %P717}718 719define ptr @test_memset4_64_align16(ptr %P, i8 %V) {720; CHECK-LABEL: test_memset4_64_align16:721; CHECK: # %bb.0:722; CHECK-NEXT: pushq %rbx723; CHECK-NEXT: .cfi_def_cfa_offset 16724; CHECK-NEXT: .cfi_offset %rbx, -16725; CHECK-NEXT: movq %rdi, %rbx726; CHECK-NEXT: movl $64, %edx727; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_4@PLT728; CHECK-NEXT: movq %rbx, %rax729; CHECK-NEXT: popq %rbx730; CHECK-NEXT: .cfi_def_cfa_offset 8731; CHECK-NEXT: retq732 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 16 %P, i8 %V, i32 64, i32 4)733 ret ptr %P734}735 736define ptr @test_memset4_64_align64(ptr %P, i8 %V) {737; CHECK-LABEL: test_memset4_64_align64:738; CHECK: # %bb.0:739; CHECK-NEXT: pushq %rbx740; CHECK-NEXT: .cfi_def_cfa_offset 16741; CHECK-NEXT: .cfi_offset %rbx, -16742; CHECK-NEXT: movq %rdi, %rbx743; CHECK-NEXT: movl $64, %edx744; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_4@PLT745; CHECK-NEXT: movq %rbx, %rax746; CHECK-NEXT: popq %rbx747; CHECK-NEXT: .cfi_def_cfa_offset 8748; CHECK-NEXT: retq749 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 64 %P, i8 %V, i32 64, i32 4)750 ret ptr %P751}752 753define ptr @test_memset4_4(ptr %P, i8 %V) {754; CHECK-LABEL: test_memset4_4:755; CHECK: # %bb.0:756; CHECK-NEXT: pushq %rbx757; CHECK-NEXT: .cfi_def_cfa_offset 16758; CHECK-NEXT: .cfi_offset %rbx, -16759; CHECK-NEXT: movq %rdi, %rbx760; CHECK-NEXT: movl $4, %edx761; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_4@PLT762; CHECK-NEXT: movq %rbx, %rax763; CHECK-NEXT: popq %rbx764; CHECK-NEXT: .cfi_def_cfa_offset 8765; CHECK-NEXT: retq766 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 4 %P, i8 %V, i32 4, i32 4)767 ret ptr %P768}769 770define ptr @test_memset4_8(ptr %P, i8 %V) {771; CHECK-LABEL: test_memset4_8:772; CHECK: # %bb.0:773; CHECK-NEXT: pushq %rbx774; CHECK-NEXT: .cfi_def_cfa_offset 16775; CHECK-NEXT: .cfi_offset %rbx, -16776; CHECK-NEXT: movq %rdi, %rbx777; CHECK-NEXT: movl $8, %edx778; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_4@PLT779; CHECK-NEXT: movq %rbx, %rax780; CHECK-NEXT: popq %rbx781; CHECK-NEXT: .cfi_def_cfa_offset 8782; CHECK-NEXT: retq783 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 4 %P, i8 %V, i32 8, i32 4)784 ret ptr %P785}786 787define ptr @test_memset4_8_align8(ptr %P, i8 %V) {788; CHECK-LABEL: test_memset4_8_align8:789; CHECK: # %bb.0:790; CHECK-NEXT: pushq %rbx791; CHECK-NEXT: .cfi_def_cfa_offset 16792; CHECK-NEXT: .cfi_offset %rbx, -16793; CHECK-NEXT: movq %rdi, %rbx794; CHECK-NEXT: movl $8, %edx795; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_4@PLT796; CHECK-NEXT: movq %rbx, %rax797; CHECK-NEXT: popq %rbx798; CHECK-NEXT: .cfi_def_cfa_offset 8799; CHECK-NEXT: retq800 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 8 %P, i8 %V, i32 8, i32 4)801 ret ptr %P802}803 804define ptr @test_memset4_12(ptr %P, i8 %V) {805; CHECK-LABEL: test_memset4_12:806; CHECK: # %bb.0:807; CHECK-NEXT: pushq %rbx808; CHECK-NEXT: .cfi_def_cfa_offset 16809; CHECK-NEXT: .cfi_offset %rbx, -16810; CHECK-NEXT: movq %rdi, %rbx811; CHECK-NEXT: movl $12, %edx812; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_4@PLT813; CHECK-NEXT: movq %rbx, %rax814; CHECK-NEXT: popq %rbx815; CHECK-NEXT: .cfi_def_cfa_offset 8816; CHECK-NEXT: retq817 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 4 %P, i8 %V, i32 12, i32 4)818 ret ptr %P819}820 821define ptr @test_memset4_16(ptr %P, i8 %V) {822; CHECK-LABEL: test_memset4_16:823; CHECK: # %bb.0:824; CHECK-NEXT: pushq %rbx825; CHECK-NEXT: .cfi_def_cfa_offset 16826; CHECK-NEXT: .cfi_offset %rbx, -16827; CHECK-NEXT: movq %rdi, %rbx828; CHECK-NEXT: movl $16, %edx829; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_4@PLT830; CHECK-NEXT: movq %rbx, %rax831; CHECK-NEXT: popq %rbx832; CHECK-NEXT: .cfi_def_cfa_offset 8833; CHECK-NEXT: retq834 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 4 %P, i8 %V, i32 16, i32 4)835 ret ptr %P836}837 838define ptr @test_memset4_16_align16(ptr %P, i8 %V) {839; CHECK-LABEL: test_memset4_16_align16:840; CHECK: # %bb.0:841; CHECK-NEXT: pushq %rbx842; CHECK-NEXT: .cfi_def_cfa_offset 16843; CHECK-NEXT: .cfi_offset %rbx, -16844; CHECK-NEXT: movq %rdi, %rbx845; CHECK-NEXT: movl $16, %edx846; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_4@PLT847; CHECK-NEXT: movq %rbx, %rax848; CHECK-NEXT: popq %rbx849; CHECK-NEXT: .cfi_def_cfa_offset 8850; CHECK-NEXT: retq851 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 16 %P, i8 %V, i32 16, i32 4)852 ret ptr %P853}854 855define ptr @test_memset4_60(ptr %P, i8 %V) {856; CHECK-LABEL: test_memset4_60:857; CHECK: # %bb.0:858; CHECK-NEXT: pushq %rbx859; CHECK-NEXT: .cfi_def_cfa_offset 16860; CHECK-NEXT: .cfi_offset %rbx, -16861; CHECK-NEXT: movq %rdi, %rbx862; CHECK-NEXT: movl $60, %edx863; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_4@PLT864; CHECK-NEXT: movq %rbx, %rax865; CHECK-NEXT: popq %rbx866; CHECK-NEXT: .cfi_def_cfa_offset 8867; CHECK-NEXT: retq868 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 4 %P, i8 %V, i32 60, i32 4)869 ret ptr %P870}871 872define ptr @test_memset8_64(ptr %P, i8 %V) {873; CHECK-LABEL: test_memset8_64:874; CHECK: # %bb.0:875; CHECK-NEXT: pushq %rbx876; CHECK-NEXT: .cfi_def_cfa_offset 16877; CHECK-NEXT: .cfi_offset %rbx, -16878; CHECK-NEXT: movq %rdi, %rbx879; CHECK-NEXT: movl $64, %edx880; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_8@PLT881; CHECK-NEXT: movq %rbx, %rax882; CHECK-NEXT: popq %rbx883; CHECK-NEXT: .cfi_def_cfa_offset 8884; CHECK-NEXT: retq885 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 8 %P, i8 %V, i32 64, i32 8)886 ret ptr %P887}888 889define ptr @test_memset16_64(ptr %P, i8 %V) {890; CHECK-LABEL: test_memset16_64:891; CHECK: # %bb.0:892; CHECK-NEXT: pushq %rbx893; CHECK-NEXT: .cfi_def_cfa_offset 16894; CHECK-NEXT: .cfi_offset %rbx, -16895; CHECK-NEXT: movq %rdi, %rbx896; CHECK-NEXT: movl $64, %edx897; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_16@PLT898; CHECK-NEXT: movq %rbx, %rax899; CHECK-NEXT: popq %rbx900; CHECK-NEXT: .cfi_def_cfa_offset 8901; CHECK-NEXT: retq902 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 16 %P, i8 %V, i32 64, i32 16)903 ret ptr %P904}905 906define ptr @test_memset16_64_zero(ptr %P) {907; CHECK-LABEL: test_memset16_64_zero:908; CHECK: # %bb.0:909; CHECK-NEXT: pushq %rbx910; CHECK-NEXT: .cfi_def_cfa_offset 16911; CHECK-NEXT: .cfi_offset %rbx, -16912; CHECK-NEXT: movq %rdi, %rbx913; CHECK-NEXT: xorl %esi, %esi914; CHECK-NEXT: movl $64, %edx915; CHECK-NEXT: callq __llvm_memset_element_unordered_atomic_16@PLT916; CHECK-NEXT: movq %rbx, %rax917; CHECK-NEXT: popq %rbx918; CHECK-NEXT: .cfi_def_cfa_offset 8919; CHECK-NEXT: retq920 call void @llvm.memset.element.unordered.atomic.p0.i32(ptr align 16 %P, i8 0, i32 64, i32 16)921 ret ptr %P922}923 924 925declare void @llvm.memcpy.element.unordered.atomic.p0.p0.i32(ptr nocapture, ptr nocapture, i32, i32) nounwind926declare void @llvm.memmove.element.unordered.atomic.p0.p0.i32(ptr nocapture, ptr nocapture, i32, i32) nounwind927declare void @llvm.memset.element.unordered.atomic.p0.i32(ptr nocapture, i8, i32, i32) nounwind928