4976 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 62; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX24; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 -mattr=+avx512vpopcntdq | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512POPCNT6 7;8; CTPOP9;10 11define i32 @test_ctpop_i128(i128 %a0) nounwind {12; CHECK-LABEL: test_ctpop_i128:13; CHECK: # %bb.0:14; CHECK-NEXT: popcntq %rsi, %rcx15; CHECK-NEXT: popcntq %rdi, %rax16; CHECK-NEXT: addl %ecx, %eax17; CHECK-NEXT: # kill: def $eax killed $eax killed $rax18; CHECK-NEXT: retq19 %cnt = call i128 @llvm.ctpop.i128(i128 %a0)20 %res = trunc i128 %cnt to i3221 ret i32 %res22}23 24define i32 @load_ctpop_i128(ptr %p0) nounwind {25; CHECK-LABEL: load_ctpop_i128:26; CHECK: # %bb.0:27; CHECK-NEXT: popcntq 8(%rdi), %rcx28; CHECK-NEXT: popcntq (%rdi), %rax29; CHECK-NEXT: addl %ecx, %eax30; CHECK-NEXT: # kill: def $eax killed $eax killed $rax31; CHECK-NEXT: retq32 %a0 = load i128, ptr %p033 %cnt = call i128 @llvm.ctpop.i128(i128 %a0)34 %res = trunc i128 %cnt to i3235 ret i32 %res36}37 38define i32 @test_ctpop_i256(i256 %a0) nounwind {39; CHECK-LABEL: test_ctpop_i256:40; CHECK: # %bb.0:41; CHECK-NEXT: popcntq %rcx, %rax42; CHECK-NEXT: xorl %ecx, %ecx43; CHECK-NEXT: popcntq %rdx, %rcx44; CHECK-NEXT: addl %eax, %ecx45; CHECK-NEXT: xorl %edx, %edx46; CHECK-NEXT: popcntq %rsi, %rdx47; CHECK-NEXT: xorl %eax, %eax48; CHECK-NEXT: popcntq %rdi, %rax49; CHECK-NEXT: addl %edx, %eax50; CHECK-NEXT: addl %ecx, %eax51; CHECK-NEXT: # kill: def $eax killed $eax killed $rax52; CHECK-NEXT: retq53 %cnt = call i256 @llvm.ctpop.i256(i256 %a0)54 %res = trunc i256 %cnt to i3255 ret i32 %res56}57 58define i32 @load_ctpop_i256(ptr %p0) nounwind {59; SSE-LABEL: load_ctpop_i256:60; SSE: # %bb.0:61; SSE-NEXT: popcntq 24(%rdi), %rcx62; SSE-NEXT: popcntq 16(%rdi), %rdx63; SSE-NEXT: popcntq 8(%rdi), %rsi64; SSE-NEXT: popcntq (%rdi), %rax65; SSE-NEXT: addl %ecx, %edx66; SSE-NEXT: addl %esi, %eax67; SSE-NEXT: addl %edx, %eax68; SSE-NEXT: # kill: def $eax killed $eax killed $rax69; SSE-NEXT: retq70;71; AVX2-LABEL: load_ctpop_i256:72; AVX2: # %bb.0:73; AVX2-NEXT: popcntq 24(%rdi), %rax74; AVX2-NEXT: popcntq 16(%rdi), %rcx75; AVX2-NEXT: addl %eax, %ecx76; AVX2-NEXT: popcntq 8(%rdi), %rdx77; AVX2-NEXT: xorl %eax, %eax78; AVX2-NEXT: popcntq (%rdi), %rax79; AVX2-NEXT: addl %edx, %eax80; AVX2-NEXT: addl %ecx, %eax81; AVX2-NEXT: # kill: def $eax killed $eax killed $rax82; AVX2-NEXT: retq83;84; AVX512-LABEL: load_ctpop_i256:85; AVX512: # %bb.0:86; AVX512-NEXT: popcntq 24(%rdi), %rax87; AVX512-NEXT: popcntq 16(%rdi), %rcx88; AVX512-NEXT: addl %eax, %ecx89; AVX512-NEXT: popcntq 8(%rdi), %rdx90; AVX512-NEXT: xorl %eax, %eax91; AVX512-NEXT: popcntq (%rdi), %rax92; AVX512-NEXT: addl %edx, %eax93; AVX512-NEXT: addl %ecx, %eax94; AVX512-NEXT: # kill: def $eax killed $eax killed $rax95; AVX512-NEXT: retq96 %a0 = load i256, ptr %p097 %cnt = call i256 @llvm.ctpop.i256(i256 %a0)98 %res = trunc i256 %cnt to i3299 ret i32 %res100}101 102define i32 @test_ctpop_i512(i512 %a0) nounwind {103; CHECK-LABEL: test_ctpop_i512:104; CHECK: # %bb.0:105; CHECK-NEXT: popcntq {{[0-9]+}}(%rsp), %rax106; CHECK-NEXT: popcntq {{[0-9]+}}(%rsp), %r10107; CHECK-NEXT: addl %eax, %r10d108; CHECK-NEXT: xorl %eax, %eax109; CHECK-NEXT: popcntq %r9, %rax110; CHECK-NEXT: popcntq %r8, %r8111; CHECK-NEXT: addl %eax, %r8d112; CHECK-NEXT: addl %r10d, %r8d113; CHECK-NEXT: xorl %eax, %eax114; CHECK-NEXT: popcntq %rcx, %rax115; CHECK-NEXT: xorl %ecx, %ecx116; CHECK-NEXT: popcntq %rdx, %rcx117; CHECK-NEXT: addl %eax, %ecx118; CHECK-NEXT: xorl %edx, %edx119; CHECK-NEXT: popcntq %rsi, %rdx120; CHECK-NEXT: xorl %eax, %eax121; CHECK-NEXT: popcntq %rdi, %rax122; CHECK-NEXT: addl %edx, %eax123; CHECK-NEXT: addl %ecx, %eax124; CHECK-NEXT: addl %r8d, %eax125; CHECK-NEXT: # kill: def $eax killed $eax killed $rax126; CHECK-NEXT: retq127 %cnt = call i512 @llvm.ctpop.i512(i512 %a0)128 %res = trunc i512 %cnt to i32129 ret i32 %res130}131 132define i32 @load_ctpop_i512(ptr %p0) nounwind {133; SSE-LABEL: load_ctpop_i512:134; SSE: # %bb.0:135; SSE-NEXT: popcntq 56(%rdi), %rax136; SSE-NEXT: popcntq 48(%rdi), %rcx137; SSE-NEXT: popcntq 40(%rdi), %rdx138; SSE-NEXT: popcntq 32(%rdi), %rsi139; SSE-NEXT: addl %eax, %ecx140; SSE-NEXT: addl %edx, %esi141; SSE-NEXT: xorl %eax, %eax142; SSE-NEXT: popcntq 24(%rdi), %rax143; SSE-NEXT: addl %ecx, %esi144; SSE-NEXT: xorl %ecx, %ecx145; SSE-NEXT: popcntq 16(%rdi), %rcx146; SSE-NEXT: addl %eax, %ecx147; SSE-NEXT: xorl %edx, %edx148; SSE-NEXT: popcntq 8(%rdi), %rdx149; SSE-NEXT: xorl %eax, %eax150; SSE-NEXT: popcntq (%rdi), %rax151; SSE-NEXT: addl %edx, %eax152; SSE-NEXT: addl %ecx, %eax153; SSE-NEXT: addl %esi, %eax154; SSE-NEXT: # kill: def $eax killed $eax killed $rax155; SSE-NEXT: retq156;157; AVX2-LABEL: load_ctpop_i512:158; AVX2: # %bb.0:159; AVX2-NEXT: popcntq 56(%rdi), %rax160; AVX2-NEXT: popcntq 48(%rdi), %rcx161; AVX2-NEXT: addl %eax, %ecx162; AVX2-NEXT: xorl %eax, %eax163; AVX2-NEXT: popcntq 40(%rdi), %rax164; AVX2-NEXT: popcntq 32(%rdi), %rdx165; AVX2-NEXT: addl %eax, %edx166; AVX2-NEXT: addl %ecx, %edx167; AVX2-NEXT: xorl %ecx, %ecx168; AVX2-NEXT: popcntq 24(%rdi), %rcx169; AVX2-NEXT: popcntq 16(%rdi), %rsi170; AVX2-NEXT: popcntq 8(%rdi), %r8171; AVX2-NEXT: xorl %eax, %eax172; AVX2-NEXT: popcntq (%rdi), %rax173; AVX2-NEXT: addl %ecx, %esi174; AVX2-NEXT: addl %r8d, %eax175; AVX2-NEXT: addl %esi, %eax176; AVX2-NEXT: addl %edx, %eax177; AVX2-NEXT: # kill: def $eax killed $eax killed $rax178; AVX2-NEXT: retq179;180; AVX512-LABEL: load_ctpop_i512:181; AVX512: # %bb.0:182; AVX512-NEXT: popcntq 56(%rdi), %rax183; AVX512-NEXT: popcntq 48(%rdi), %rcx184; AVX512-NEXT: addl %eax, %ecx185; AVX512-NEXT: xorl %eax, %eax186; AVX512-NEXT: popcntq 40(%rdi), %rax187; AVX512-NEXT: popcntq 32(%rdi), %rdx188; AVX512-NEXT: addl %eax, %edx189; AVX512-NEXT: addl %ecx, %edx190; AVX512-NEXT: xorl %eax, %eax191; AVX512-NEXT: popcntq 24(%rdi), %rax192; AVX512-NEXT: xorl %ecx, %ecx193; AVX512-NEXT: popcntq 16(%rdi), %rcx194; AVX512-NEXT: popcntq 8(%rdi), %rsi195; AVX512-NEXT: addl %eax, %ecx196; AVX512-NEXT: xorl %eax, %eax197; AVX512-NEXT: popcntq (%rdi), %rax198; AVX512-NEXT: addl %esi, %eax199; AVX512-NEXT: addl %ecx, %eax200; AVX512-NEXT: addl %edx, %eax201; AVX512-NEXT: # kill: def $eax killed $eax killed $rax202; AVX512-NEXT: retq203 %a0 = load i512, ptr %p0204 %cnt = call i512 @llvm.ctpop.i512(i512 %a0)205 %res = trunc i512 %cnt to i32206 ret i32 %res207}208 209define i32 @test_ctpop_i1024(i1024 %a0) nounwind {210; SSE-LABEL: test_ctpop_i1024:211; SSE: # %bb.0:212; SSE-NEXT: pushq %rbx213; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %rax214; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %r10215; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %r11216; SSE-NEXT: addl %eax, %r10d217; SSE-NEXT: xorl %eax, %eax218; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %rax219; SSE-NEXT: addl %r11d, %eax220; SSE-NEXT: xorl %r11d, %r11d221; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %r11222; SSE-NEXT: xorl %ebx, %ebx223; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx224; SSE-NEXT: addl %r10d, %eax225; SSE-NEXT: addl %r11d, %ebx226; SSE-NEXT: xorl %r11d, %r11d227; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %r11228; SSE-NEXT: xorl %r10d, %r10d229; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %r10230; SSE-NEXT: addl %r11d, %r10d231; SSE-NEXT: addl %ebx, %r10d232; SSE-NEXT: xorl %r11d, %r11d233; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %r11234; SSE-NEXT: xorl %ebx, %ebx235; SSE-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx236; SSE-NEXT: addl %eax, %r10d237; SSE-NEXT: addl %r11d, %ebx238; SSE-NEXT: xorl %eax, %eax239; SSE-NEXT: popcntq %r9, %rax240; SSE-NEXT: popcntq %r8, %r8241; SSE-NEXT: addl %eax, %r8d242; SSE-NEXT: addl %ebx, %r8d243; SSE-NEXT: xorl %eax, %eax244; SSE-NEXT: popcntq %rcx, %rax245; SSE-NEXT: xorl %ecx, %ecx246; SSE-NEXT: popcntq %rdx, %rcx247; SSE-NEXT: addl %eax, %ecx248; SSE-NEXT: xorl %edx, %edx249; SSE-NEXT: popcntq %rsi, %rdx250; SSE-NEXT: xorl %eax, %eax251; SSE-NEXT: popcntq %rdi, %rax252; SSE-NEXT: addl %edx, %eax253; SSE-NEXT: addl %ecx, %eax254; SSE-NEXT: addl %r8d, %eax255; SSE-NEXT: addl %r10d, %eax256; SSE-NEXT: # kill: def $eax killed $eax killed $rax257; SSE-NEXT: popq %rbx258; SSE-NEXT: retq259;260; AVX2-LABEL: test_ctpop_i1024:261; AVX2: # %bb.0:262; AVX2-NEXT: pushq %r14263; AVX2-NEXT: pushq %rbx264; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %rax265; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %r10266; AVX2-NEXT: addl %eax, %r10d267; AVX2-NEXT: xorl %eax, %eax268; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %rax269; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %r11270; AVX2-NEXT: addl %eax, %r11d271; AVX2-NEXT: addl %r10d, %r11d272; AVX2-NEXT: xorl %eax, %eax273; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %rax274; AVX2-NEXT: xorl %ebx, %ebx275; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx276; AVX2-NEXT: xorl %r14d, %r14d277; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %r14278; AVX2-NEXT: addl %eax, %ebx279; AVX2-NEXT: xorl %r10d, %r10d280; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %r10281; AVX2-NEXT: addl %r14d, %r10d282; AVX2-NEXT: addl %ebx, %r10d283; AVX2-NEXT: addl %r11d, %r10d284; AVX2-NEXT: xorl %eax, %eax285; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %rax286; AVX2-NEXT: xorl %r11d, %r11d287; AVX2-NEXT: popcntq {{[0-9]+}}(%rsp), %r11288; AVX2-NEXT: addl %eax, %r11d289; AVX2-NEXT: xorl %eax, %eax290; AVX2-NEXT: popcntq %r9, %rax291; AVX2-NEXT: popcntq %r8, %r8292; AVX2-NEXT: addl %eax, %r8d293; AVX2-NEXT: addl %r11d, %r8d294; AVX2-NEXT: xorl %eax, %eax295; AVX2-NEXT: popcntq %rcx, %rax296; AVX2-NEXT: xorl %ecx, %ecx297; AVX2-NEXT: popcntq %rdx, %rcx298; AVX2-NEXT: addl %eax, %ecx299; AVX2-NEXT: xorl %edx, %edx300; AVX2-NEXT: popcntq %rsi, %rdx301; AVX2-NEXT: xorl %eax, %eax302; AVX2-NEXT: popcntq %rdi, %rax303; AVX2-NEXT: addl %edx, %eax304; AVX2-NEXT: addl %ecx, %eax305; AVX2-NEXT: addl %r8d, %eax306; AVX2-NEXT: addl %r10d, %eax307; AVX2-NEXT: # kill: def $eax killed $eax killed $rax308; AVX2-NEXT: popq %rbx309; AVX2-NEXT: popq %r14310; AVX2-NEXT: retq311;312; AVX512-LABEL: test_ctpop_i1024:313; AVX512: # %bb.0:314; AVX512-NEXT: pushq %r14315; AVX512-NEXT: pushq %rbx316; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %rax317; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %r10318; AVX512-NEXT: addl %eax, %r10d319; AVX512-NEXT: xorl %eax, %eax320; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %rax321; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %r11322; AVX512-NEXT: addl %eax, %r11d323; AVX512-NEXT: addl %r10d, %r11d324; AVX512-NEXT: xorl %eax, %eax325; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %rax326; AVX512-NEXT: xorl %ebx, %ebx327; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx328; AVX512-NEXT: xorl %r14d, %r14d329; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %r14330; AVX512-NEXT: addl %eax, %ebx331; AVX512-NEXT: xorl %r10d, %r10d332; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %r10333; AVX512-NEXT: addl %r14d, %r10d334; AVX512-NEXT: addl %ebx, %r10d335; AVX512-NEXT: addl %r11d, %r10d336; AVX512-NEXT: xorl %eax, %eax337; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %rax338; AVX512-NEXT: xorl %r11d, %r11d339; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %r11340; AVX512-NEXT: addl %eax, %r11d341; AVX512-NEXT: xorl %eax, %eax342; AVX512-NEXT: popcntq %r9, %rax343; AVX512-NEXT: popcntq %r8, %r8344; AVX512-NEXT: addl %eax, %r8d345; AVX512-NEXT: addl %r11d, %r8d346; AVX512-NEXT: xorl %eax, %eax347; AVX512-NEXT: popcntq %rcx, %rax348; AVX512-NEXT: xorl %ecx, %ecx349; AVX512-NEXT: popcntq %rdx, %rcx350; AVX512-NEXT: addl %eax, %ecx351; AVX512-NEXT: xorl %edx, %edx352; AVX512-NEXT: popcntq %rsi, %rdx353; AVX512-NEXT: xorl %eax, %eax354; AVX512-NEXT: popcntq %rdi, %rax355; AVX512-NEXT: addl %edx, %eax356; AVX512-NEXT: addl %ecx, %eax357; AVX512-NEXT: addl %r8d, %eax358; AVX512-NEXT: addl %r10d, %eax359; AVX512-NEXT: # kill: def $eax killed $eax killed $rax360; AVX512-NEXT: popq %rbx361; AVX512-NEXT: popq %r14362; AVX512-NEXT: retq363 %cnt = call i1024 @llvm.ctpop.i1024(i1024 %a0)364 %res = trunc i1024 %cnt to i32365 ret i32 %res366}367 368define i32 @load_ctpop_i1024(ptr %p0) nounwind {369; SSE-LABEL: load_ctpop_i1024:370; SSE: # %bb.0:371; SSE-NEXT: popcntq 120(%rdi), %rax372; SSE-NEXT: popcntq 112(%rdi), %rcx373; SSE-NEXT: popcntq 104(%rdi), %rdx374; SSE-NEXT: popcntq 96(%rdi), %rsi375; SSE-NEXT: addl %eax, %ecx376; SSE-NEXT: addl %edx, %esi377; SSE-NEXT: xorl %eax, %eax378; SSE-NEXT: popcntq 88(%rdi), %rax379; SSE-NEXT: addl %ecx, %esi380; SSE-NEXT: xorl %edx, %edx381; SSE-NEXT: popcntq 80(%rdi), %rdx382; SSE-NEXT: addl %eax, %edx383; SSE-NEXT: xorl %eax, %eax384; SSE-NEXT: popcntq 72(%rdi), %rax385; SSE-NEXT: xorl %ecx, %ecx386; SSE-NEXT: popcntq 64(%rdi), %rcx387; SSE-NEXT: addl %eax, %ecx388; SSE-NEXT: addl %edx, %ecx389; SSE-NEXT: xorl %eax, %eax390; SSE-NEXT: popcntq 56(%rdi), %rax391; SSE-NEXT: addl %esi, %ecx392; SSE-NEXT: xorl %edx, %edx393; SSE-NEXT: popcntq 48(%rdi), %rdx394; SSE-NEXT: addl %eax, %edx395; SSE-NEXT: xorl %eax, %eax396; SSE-NEXT: popcntq 40(%rdi), %rax397; SSE-NEXT: xorl %esi, %esi398; SSE-NEXT: popcntq 32(%rdi), %rsi399; SSE-NEXT: addl %eax, %esi400; SSE-NEXT: addl %edx, %esi401; SSE-NEXT: xorl %eax, %eax402; SSE-NEXT: popcntq 24(%rdi), %rax403; SSE-NEXT: xorl %edx, %edx404; SSE-NEXT: popcntq 16(%rdi), %rdx405; SSE-NEXT: popcntq 8(%rdi), %r8406; SSE-NEXT: addl %eax, %edx407; SSE-NEXT: xorl %eax, %eax408; SSE-NEXT: popcntq (%rdi), %rax409; SSE-NEXT: addl %r8d, %eax410; SSE-NEXT: addl %edx, %eax411; SSE-NEXT: addl %esi, %eax412; SSE-NEXT: addl %ecx, %eax413; SSE-NEXT: # kill: def $eax killed $eax killed $rax414; SSE-NEXT: retq415;416; AVX2-LABEL: load_ctpop_i1024:417; AVX2: # %bb.0:418; AVX2-NEXT: popcntq 120(%rdi), %rax419; AVX2-NEXT: popcntq 112(%rdi), %rcx420; AVX2-NEXT: addl %eax, %ecx421; AVX2-NEXT: xorl %eax, %eax422; AVX2-NEXT: popcntq 104(%rdi), %rax423; AVX2-NEXT: popcntq 96(%rdi), %rdx424; AVX2-NEXT: addl %eax, %edx425; AVX2-NEXT: addl %ecx, %edx426; AVX2-NEXT: xorl %eax, %eax427; AVX2-NEXT: popcntq 88(%rdi), %rax428; AVX2-NEXT: popcntq 80(%rdi), %rsi429; AVX2-NEXT: popcntq 72(%rdi), %r8430; AVX2-NEXT: xorl %ecx, %ecx431; AVX2-NEXT: popcntq 64(%rdi), %rcx432; AVX2-NEXT: addl %eax, %esi433; AVX2-NEXT: addl %r8d, %ecx434; AVX2-NEXT: addl %esi, %ecx435; AVX2-NEXT: addl %edx, %ecx436; AVX2-NEXT: xorl %eax, %eax437; AVX2-NEXT: popcntq 56(%rdi), %rax438; AVX2-NEXT: xorl %edx, %edx439; AVX2-NEXT: popcntq 48(%rdi), %rdx440; AVX2-NEXT: xorl %esi, %esi441; AVX2-NEXT: popcntq 40(%rdi), %rsi442; AVX2-NEXT: xorl %r8d, %r8d443; AVX2-NEXT: popcntq 32(%rdi), %r8444; AVX2-NEXT: addl %eax, %edx445; AVX2-NEXT: addl %esi, %r8d446; AVX2-NEXT: xorl %eax, %eax447; AVX2-NEXT: popcntq 24(%rdi), %rax448; AVX2-NEXT: addl %edx, %r8d449; AVX2-NEXT: xorl %edx, %edx450; AVX2-NEXT: popcntq 16(%rdi), %rdx451; AVX2-NEXT: addl %eax, %edx452; AVX2-NEXT: xorl %esi, %esi453; AVX2-NEXT: popcntq 8(%rdi), %rsi454; AVX2-NEXT: xorl %eax, %eax455; AVX2-NEXT: popcntq (%rdi), %rax456; AVX2-NEXT: addl %esi, %eax457; AVX2-NEXT: addl %edx, %eax458; AVX2-NEXT: addl %r8d, %eax459; AVX2-NEXT: addl %ecx, %eax460; AVX2-NEXT: # kill: def $eax killed $eax killed $rax461; AVX2-NEXT: retq462;463; AVX512-LABEL: load_ctpop_i1024:464; AVX512: # %bb.0:465; AVX512-NEXT: popcntq 120(%rdi), %rax466; AVX512-NEXT: popcntq 112(%rdi), %rcx467; AVX512-NEXT: addl %eax, %ecx468; AVX512-NEXT: xorl %eax, %eax469; AVX512-NEXT: popcntq 104(%rdi), %rax470; AVX512-NEXT: popcntq 96(%rdi), %rdx471; AVX512-NEXT: addl %eax, %edx472; AVX512-NEXT: addl %ecx, %edx473; AVX512-NEXT: xorl %eax, %eax474; AVX512-NEXT: popcntq 88(%rdi), %rax475; AVX512-NEXT: popcntq 80(%rdi), %rsi476; AVX512-NEXT: popcntq 72(%rdi), %r8477; AVX512-NEXT: addl %eax, %esi478; AVX512-NEXT: xorl %ecx, %ecx479; AVX512-NEXT: popcntq 64(%rdi), %rcx480; AVX512-NEXT: addl %r8d, %ecx481; AVX512-NEXT: addl %esi, %ecx482; AVX512-NEXT: addl %edx, %ecx483; AVX512-NEXT: xorl %eax, %eax484; AVX512-NEXT: popcntq 56(%rdi), %rax485; AVX512-NEXT: xorl %edx, %edx486; AVX512-NEXT: popcntq 48(%rdi), %rdx487; AVX512-NEXT: xorl %esi, %esi488; AVX512-NEXT: popcntq 40(%rdi), %rsi489; AVX512-NEXT: addl %eax, %edx490; AVX512-NEXT: xorl %r8d, %r8d491; AVX512-NEXT: popcntq 32(%rdi), %r8492; AVX512-NEXT: addl %esi, %r8d493; AVX512-NEXT: xorl %eax, %eax494; AVX512-NEXT: popcntq 24(%rdi), %rax495; AVX512-NEXT: addl %edx, %r8d496; AVX512-NEXT: xorl %edx, %edx497; AVX512-NEXT: popcntq 16(%rdi), %rdx498; AVX512-NEXT: addl %eax, %edx499; AVX512-NEXT: xorl %esi, %esi500; AVX512-NEXT: popcntq 8(%rdi), %rsi501; AVX512-NEXT: xorl %eax, %eax502; AVX512-NEXT: popcntq (%rdi), %rax503; AVX512-NEXT: addl %esi, %eax504; AVX512-NEXT: addl %edx, %eax505; AVX512-NEXT: addl %r8d, %eax506; AVX512-NEXT: addl %ecx, %eax507; AVX512-NEXT: # kill: def $eax killed $eax killed $rax508; AVX512-NEXT: retq509 %a0 = load i1024, ptr %p0510 %cnt = call i1024 @llvm.ctpop.i1024(i1024 %a0)511 %res = trunc i1024 %cnt to i32512 ret i32 %res513}514 515;516; CTLZ517;518 519define i32 @test_ctlz_i128(i128 %a0) nounwind {520; SSE-LABEL: test_ctlz_i128:521; SSE: # %bb.0:522; SSE-NEXT: bsrq %rsi, %rcx523; SSE-NEXT: xorl $63, %ecx524; SSE-NEXT: movl $127, %eax525; SSE-NEXT: bsrq %rdi, %rax526; SSE-NEXT: xorl $63, %eax527; SSE-NEXT: addl $64, %eax528; SSE-NEXT: testq %rsi, %rsi529; SSE-NEXT: cmovnel %ecx, %eax530; SSE-NEXT: # kill: def $eax killed $eax killed $rax531; SSE-NEXT: retq532;533; AVX2-LABEL: test_ctlz_i128:534; AVX2: # %bb.0:535; AVX2-NEXT: lzcntq %rsi, %rcx536; AVX2-NEXT: lzcntq %rdi, %rax537; AVX2-NEXT: addl $64, %eax538; AVX2-NEXT: testq %rsi, %rsi539; AVX2-NEXT: cmovnel %ecx, %eax540; AVX2-NEXT: # kill: def $eax killed $eax killed $rax541; AVX2-NEXT: retq542;543; AVX512-LABEL: test_ctlz_i128:544; AVX512: # %bb.0:545; AVX512-NEXT: lzcntq %rsi, %rcx546; AVX512-NEXT: lzcntq %rdi, %rax547; AVX512-NEXT: addl $64, %eax548; AVX512-NEXT: testq %rsi, %rsi549; AVX512-NEXT: cmovnel %ecx, %eax550; AVX512-NEXT: # kill: def $eax killed $eax killed $rax551; AVX512-NEXT: retq552 %cnt = call i128 @llvm.ctlz.i128(i128 %a0, i1 0)553 %res = trunc i128 %cnt to i32554 ret i32 %res555}556 557define i32 @load_ctlz_i128(ptr %p0) nounwind {558; SSE-LABEL: load_ctlz_i128:559; SSE: # %bb.0:560; SSE-NEXT: movq 8(%rdi), %rcx561; SSE-NEXT: bsrq %rcx, %rdx562; SSE-NEXT: xorl $63, %edx563; SSE-NEXT: movl $127, %eax564; SSE-NEXT: bsrq (%rdi), %rax565; SSE-NEXT: xorl $63, %eax566; SSE-NEXT: addl $64, %eax567; SSE-NEXT: testq %rcx, %rcx568; SSE-NEXT: cmovnel %edx, %eax569; SSE-NEXT: # kill: def $eax killed $eax killed $rax570; SSE-NEXT: retq571;572; AVX2-LABEL: load_ctlz_i128:573; AVX2: # %bb.0:574; AVX2-NEXT: movq 8(%rdi), %rcx575; AVX2-NEXT: lzcntq %rcx, %rdx576; AVX2-NEXT: lzcntq (%rdi), %rax577; AVX2-NEXT: addl $64, %eax578; AVX2-NEXT: testq %rcx, %rcx579; AVX2-NEXT: cmovnel %edx, %eax580; AVX2-NEXT: # kill: def $eax killed $eax killed $rax581; AVX2-NEXT: retq582;583; AVX512-LABEL: load_ctlz_i128:584; AVX512: # %bb.0:585; AVX512-NEXT: movq 8(%rdi), %rcx586; AVX512-NEXT: lzcntq %rcx, %rdx587; AVX512-NEXT: lzcntq (%rdi), %rax588; AVX512-NEXT: addl $64, %eax589; AVX512-NEXT: testq %rcx, %rcx590; AVX512-NEXT: cmovnel %edx, %eax591; AVX512-NEXT: # kill: def $eax killed $eax killed $rax592; AVX512-NEXT: retq593 %a0 = load i128, ptr %p0594 %cnt = call i128 @llvm.ctlz.i128(i128 %a0, i1 0)595 %res = trunc i128 %cnt to i32596 ret i32 %res597}598 599define i32 @test_ctlz_i256(i256 %a0) nounwind {600; SSE-LABEL: test_ctlz_i256:601; SSE: # %bb.0:602; SSE-NEXT: bsrq %rcx, %rax603; SSE-NEXT: xorl $63, %eax604; SSE-NEXT: bsrq %rdx, %r8605; SSE-NEXT: xorl $63, %r8d606; SSE-NEXT: orl $64, %r8d607; SSE-NEXT: testq %rcx, %rcx608; SSE-NEXT: cmovnel %eax, %r8d609; SSE-NEXT: bsrq %rsi, %r9610; SSE-NEXT: xorl $63, %r9d611; SSE-NEXT: movl $127, %eax612; SSE-NEXT: bsrq %rdi, %rax613; SSE-NEXT: xorl $63, %eax614; SSE-NEXT: addl $64, %eax615; SSE-NEXT: testq %rsi, %rsi616; SSE-NEXT: cmovnel %r9d, %eax617; SSE-NEXT: subl $-128, %eax618; SSE-NEXT: orq %rcx, %rdx619; SSE-NEXT: cmovnel %r8d, %eax620; SSE-NEXT: # kill: def $eax killed $eax killed $rax621; SSE-NEXT: retq622;623; AVX2-LABEL: test_ctlz_i256:624; AVX2: # %bb.0:625; AVX2-NEXT: lzcntq %rcx, %rax626; AVX2-NEXT: lzcntq %rdx, %r8627; AVX2-NEXT: addl $64, %r8d628; AVX2-NEXT: testq %rcx, %rcx629; AVX2-NEXT: cmovnel %eax, %r8d630; AVX2-NEXT: lzcntq %rsi, %r9631; AVX2-NEXT: xorl %eax, %eax632; AVX2-NEXT: lzcntq %rdi, %rax633; AVX2-NEXT: addl $64, %eax634; AVX2-NEXT: testq %rsi, %rsi635; AVX2-NEXT: cmovnel %r9d, %eax636; AVX2-NEXT: subl $-128, %eax637; AVX2-NEXT: orq %rcx, %rdx638; AVX2-NEXT: cmovnel %r8d, %eax639; AVX2-NEXT: # kill: def $eax killed $eax killed $rax640; AVX2-NEXT: retq641;642; AVX512-LABEL: test_ctlz_i256:643; AVX512: # %bb.0:644; AVX512-NEXT: lzcntq %rcx, %rax645; AVX512-NEXT: lzcntq %rdx, %r8646; AVX512-NEXT: addl $64, %r8d647; AVX512-NEXT: testq %rcx, %rcx648; AVX512-NEXT: cmovnel %eax, %r8d649; AVX512-NEXT: lzcntq %rsi, %r9650; AVX512-NEXT: lzcntq %rdi, %rax651; AVX512-NEXT: addl $64, %eax652; AVX512-NEXT: testq %rsi, %rsi653; AVX512-NEXT: cmovnel %r9d, %eax654; AVX512-NEXT: subl $-128, %eax655; AVX512-NEXT: orq %rcx, %rdx656; AVX512-NEXT: cmovnel %r8d, %eax657; AVX512-NEXT: # kill: def $eax killed $eax killed $rax658; AVX512-NEXT: retq659 %cnt = call i256 @llvm.ctlz.i256(i256 %a0, i1 0)660 %res = trunc i256 %cnt to i32661 ret i32 %res662}663 664define i32 @load_ctlz_i256(ptr %p0) nounwind {665; SSE-LABEL: load_ctlz_i256:666; SSE: # %bb.0:667; SSE-NEXT: movq 16(%rdi), %rcx668; SSE-NEXT: movq 24(%rdi), %rdx669; SSE-NEXT: bsrq %rdx, %rax670; SSE-NEXT: xorl $63, %eax671; SSE-NEXT: bsrq %rcx, %rsi672; SSE-NEXT: xorl $63, %esi673; SSE-NEXT: orl $64, %esi674; SSE-NEXT: testq %rdx, %rdx675; SSE-NEXT: cmovnel %eax, %esi676; SSE-NEXT: movq 8(%rdi), %r8677; SSE-NEXT: bsrq %r8, %r9678; SSE-NEXT: xorl $63, %r9d679; SSE-NEXT: movl $127, %eax680; SSE-NEXT: bsrq (%rdi), %rax681; SSE-NEXT: xorl $63, %eax682; SSE-NEXT: addl $64, %eax683; SSE-NEXT: testq %r8, %r8684; SSE-NEXT: cmovnel %r9d, %eax685; SSE-NEXT: subl $-128, %eax686; SSE-NEXT: orq %rdx, %rcx687; SSE-NEXT: cmovnel %esi, %eax688; SSE-NEXT: # kill: def $eax killed $eax killed $rax689; SSE-NEXT: retq690;691; AVX2-LABEL: load_ctlz_i256:692; AVX2: # %bb.0:693; AVX2-NEXT: movq 16(%rdi), %rcx694; AVX2-NEXT: movq 24(%rdi), %rdx695; AVX2-NEXT: lzcntq %rdx, %rax696; AVX2-NEXT: lzcntq %rcx, %rsi697; AVX2-NEXT: addl $64, %esi698; AVX2-NEXT: testq %rdx, %rdx699; AVX2-NEXT: cmovnel %eax, %esi700; AVX2-NEXT: movq 8(%rdi), %r8701; AVX2-NEXT: lzcntq %r8, %r9702; AVX2-NEXT: xorl %eax, %eax703; AVX2-NEXT: lzcntq (%rdi), %rax704; AVX2-NEXT: addl $64, %eax705; AVX2-NEXT: testq %r8, %r8706; AVX2-NEXT: cmovnel %r9d, %eax707; AVX2-NEXT: subl $-128, %eax708; AVX2-NEXT: orq %rdx, %rcx709; AVX2-NEXT: cmovnel %esi, %eax710; AVX2-NEXT: # kill: def $eax killed $eax killed $rax711; AVX2-NEXT: retq712;713; AVX512-LABEL: load_ctlz_i256:714; AVX512: # %bb.0:715; AVX512-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0]716; AVX512-NEXT: vplzcntq %ymm0, %ymm1717; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1718; AVX512-NEXT: vptestmq %ymm0, %ymm0, %k1719; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]720; AVX512-NEXT: vpcompressq %ymm1, %ymm0 {%k1}721; AVX512-NEXT: vmovd %xmm0, %eax722; AVX512-NEXT: vzeroupper723; AVX512-NEXT: retq724 %a0 = load i256, ptr %p0725 %cnt = call i256 @llvm.ctlz.i256(i256 %a0, i1 0)726 %res = trunc i256 %cnt to i32727 ret i32 %res728}729 730define i32 @test_ctlz_i512(i512 %a0) nounwind {731; SSE-LABEL: test_ctlz_i512:732; SSE: # %bb.0:733; SSE-NEXT: pushq %r15734; SSE-NEXT: pushq %r14735; SSE-NEXT: pushq %rbx736; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10737; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11738; SSE-NEXT: bsrq %r11, %rax739; SSE-NEXT: xorl $63, %eax740; SSE-NEXT: bsrq %r10, %r14741; SSE-NEXT: xorl $63, %r14d742; SSE-NEXT: orl $64, %r14d743; SSE-NEXT: testq %r11, %r11744; SSE-NEXT: cmovnel %eax, %r14d745; SSE-NEXT: bsrq %r9, %rax746; SSE-NEXT: xorl $63, %eax747; SSE-NEXT: bsrq %r8, %rbx748; SSE-NEXT: xorl $63, %ebx749; SSE-NEXT: orl $64, %ebx750; SSE-NEXT: testq %r9, %r9751; SSE-NEXT: cmovnel %eax, %ebx752; SSE-NEXT: subl $-128, %ebx753; SSE-NEXT: movq %r10, %rax754; SSE-NEXT: orq %r11, %rax755; SSE-NEXT: cmovnel %r14d, %ebx756; SSE-NEXT: bsrq %rcx, %rax757; SSE-NEXT: xorl $63, %eax758; SSE-NEXT: bsrq %rdx, %r14759; SSE-NEXT: xorl $63, %r14d760; SSE-NEXT: orl $64, %r14d761; SSE-NEXT: testq %rcx, %rcx762; SSE-NEXT: cmovnel %eax, %r14d763; SSE-NEXT: bsrq %rsi, %r15764; SSE-NEXT: xorl $63, %r15d765; SSE-NEXT: movl $127, %eax766; SSE-NEXT: bsrq %rdi, %rax767; SSE-NEXT: xorl $63, %eax768; SSE-NEXT: addl $64, %eax769; SSE-NEXT: testq %rsi, %rsi770; SSE-NEXT: cmovnel %r15d, %eax771; SSE-NEXT: subl $-128, %eax772; SSE-NEXT: orq %rcx, %rdx773; SSE-NEXT: cmovnel %r14d, %eax774; SSE-NEXT: addl $256, %eax # imm = 0x100775; SSE-NEXT: orq %r11, %r9776; SSE-NEXT: orq %r10, %r8777; SSE-NEXT: orq %r9, %r8778; SSE-NEXT: cmovnel %ebx, %eax779; SSE-NEXT: # kill: def $eax killed $eax killed $rax780; SSE-NEXT: popq %rbx781; SSE-NEXT: popq %r14782; SSE-NEXT: popq %r15783; SSE-NEXT: retq784;785; AVX2-LABEL: test_ctlz_i512:786; AVX2: # %bb.0:787; AVX2-NEXT: pushq %r15788; AVX2-NEXT: pushq %r14789; AVX2-NEXT: pushq %rbx790; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10791; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11792; AVX2-NEXT: lzcntq %r11, %rax793; AVX2-NEXT: xorl %r14d, %r14d794; AVX2-NEXT: lzcntq %r10, %r14795; AVX2-NEXT: addl $64, %r14d796; AVX2-NEXT: testq %r11, %r11797; AVX2-NEXT: cmovnel %eax, %r14d798; AVX2-NEXT: xorl %eax, %eax799; AVX2-NEXT: lzcntq %r9, %rax800; AVX2-NEXT: xorl %ebx, %ebx801; AVX2-NEXT: lzcntq %r8, %rbx802; AVX2-NEXT: addl $64, %ebx803; AVX2-NEXT: testq %r9, %r9804; AVX2-NEXT: cmovnel %eax, %ebx805; AVX2-NEXT: subl $-128, %ebx806; AVX2-NEXT: movq %r10, %rax807; AVX2-NEXT: orq %r11, %rax808; AVX2-NEXT: cmovnel %r14d, %ebx809; AVX2-NEXT: xorl %eax, %eax810; AVX2-NEXT: lzcntq %rcx, %rax811; AVX2-NEXT: xorl %r14d, %r14d812; AVX2-NEXT: lzcntq %rdx, %r14813; AVX2-NEXT: addl $64, %r14d814; AVX2-NEXT: testq %rcx, %rcx815; AVX2-NEXT: cmovnel %eax, %r14d816; AVX2-NEXT: xorl %r15d, %r15d817; AVX2-NEXT: lzcntq %rsi, %r15818; AVX2-NEXT: xorl %eax, %eax819; AVX2-NEXT: lzcntq %rdi, %rax820; AVX2-NEXT: addl $64, %eax821; AVX2-NEXT: testq %rsi, %rsi822; AVX2-NEXT: cmovnel %r15d, %eax823; AVX2-NEXT: subl $-128, %eax824; AVX2-NEXT: orq %rcx, %rdx825; AVX2-NEXT: cmovnel %r14d, %eax826; AVX2-NEXT: addl $256, %eax # imm = 0x100827; AVX2-NEXT: orq %r11, %r9828; AVX2-NEXT: orq %r10, %r8829; AVX2-NEXT: orq %r9, %r8830; AVX2-NEXT: cmovnel %ebx, %eax831; AVX2-NEXT: # kill: def $eax killed $eax killed $rax832; AVX2-NEXT: popq %rbx833; AVX2-NEXT: popq %r14834; AVX2-NEXT: popq %r15835; AVX2-NEXT: retq836;837; AVX512-LABEL: test_ctlz_i512:838; AVX512: # %bb.0:839; AVX512-NEXT: vmovq %rdi, %xmm0840; AVX512-NEXT: vmovq %rsi, %xmm1841; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]842; AVX512-NEXT: vmovq %rdx, %xmm1843; AVX512-NEXT: vmovq %rcx, %xmm2844; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]845; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0846; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]847; AVX512-NEXT: vmovq %r8, %xmm2848; AVX512-NEXT: vmovq %r9, %xmm3849; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]850; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1851; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0852; AVX512-NEXT: vplzcntq %zmm0, %zmm1853; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1854; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k1855; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]856; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1}857; AVX512-NEXT: vmovd %xmm0, %eax858; AVX512-NEXT: vzeroupper859; AVX512-NEXT: retq860 %cnt = call i512 @llvm.ctlz.i512(i512 %a0, i1 0)861 %res = trunc i512 %cnt to i32862 ret i32 %res863}864 865define i32 @load_ctlz_i512(ptr %p0) nounwind {866; SSE-LABEL: load_ctlz_i512:867; SSE: # %bb.0:868; SSE-NEXT: pushq %r15869; SSE-NEXT: pushq %r14870; SSE-NEXT: pushq %rbx871; SSE-NEXT: movq 8(%rdi), %r10872; SSE-NEXT: movq 16(%rdi), %r9873; SSE-NEXT: movq 32(%rdi), %rcx874; SSE-NEXT: movq 40(%rdi), %rdx875; SSE-NEXT: movq 48(%rdi), %rsi876; SSE-NEXT: movq 56(%rdi), %r8877; SSE-NEXT: bsrq %r8, %rax878; SSE-NEXT: xorl $63, %eax879; SSE-NEXT: bsrq %rsi, %r14880; SSE-NEXT: xorl $63, %r14d881; SSE-NEXT: orl $64, %r14d882; SSE-NEXT: testq %r8, %r8883; SSE-NEXT: cmovnel %eax, %r14d884; SSE-NEXT: bsrq %rdx, %rax885; SSE-NEXT: xorl $63, %eax886; SSE-NEXT: bsrq %rcx, %r11887; SSE-NEXT: xorl $63, %r11d888; SSE-NEXT: orl $64, %r11d889; SSE-NEXT: testq %rdx, %rdx890; SSE-NEXT: cmovnel %eax, %r11d891; SSE-NEXT: movq 24(%rdi), %rbx892; SSE-NEXT: subl $-128, %r11d893; SSE-NEXT: movq %rsi, %rax894; SSE-NEXT: orq %r8, %rax895; SSE-NEXT: cmovnel %r14d, %r11d896; SSE-NEXT: bsrq %rbx, %rax897; SSE-NEXT: xorl $63, %eax898; SSE-NEXT: bsrq %r9, %r14899; SSE-NEXT: xorl $63, %r14d900; SSE-NEXT: orl $64, %r14d901; SSE-NEXT: testq %rbx, %rbx902; SSE-NEXT: cmovnel %eax, %r14d903; SSE-NEXT: bsrq %r10, %r15904; SSE-NEXT: xorl $63, %r15d905; SSE-NEXT: movl $127, %eax906; SSE-NEXT: bsrq (%rdi), %rax907; SSE-NEXT: xorl $63, %eax908; SSE-NEXT: addl $64, %eax909; SSE-NEXT: testq %r10, %r10910; SSE-NEXT: cmovnel %r15d, %eax911; SSE-NEXT: subl $-128, %eax912; SSE-NEXT: orq %rbx, %r9913; SSE-NEXT: cmovnel %r14d, %eax914; SSE-NEXT: addl $256, %eax # imm = 0x100915; SSE-NEXT: orq %r8, %rdx916; SSE-NEXT: orq %rsi, %rcx917; SSE-NEXT: orq %rdx, %rcx918; SSE-NEXT: cmovnel %r11d, %eax919; SSE-NEXT: # kill: def $eax killed $eax killed $rax920; SSE-NEXT: popq %rbx921; SSE-NEXT: popq %r14922; SSE-NEXT: popq %r15923; SSE-NEXT: retq924;925; AVX2-LABEL: load_ctlz_i512:926; AVX2: # %bb.0:927; AVX2-NEXT: pushq %r15928; AVX2-NEXT: pushq %r14929; AVX2-NEXT: pushq %rbx930; AVX2-NEXT: movq 8(%rdi), %r10931; AVX2-NEXT: movq 16(%rdi), %r9932; AVX2-NEXT: movq 32(%rdi), %rcx933; AVX2-NEXT: movq 40(%rdi), %rdx934; AVX2-NEXT: movq 48(%rdi), %rsi935; AVX2-NEXT: movq 56(%rdi), %r8936; AVX2-NEXT: lzcntq %r8, %rax937; AVX2-NEXT: xorl %ebx, %ebx938; AVX2-NEXT: lzcntq %rsi, %rbx939; AVX2-NEXT: addl $64, %ebx940; AVX2-NEXT: testq %r8, %r8941; AVX2-NEXT: cmovnel %eax, %ebx942; AVX2-NEXT: xorl %eax, %eax943; AVX2-NEXT: lzcntq %rdx, %rax944; AVX2-NEXT: lzcntq %rcx, %r11945; AVX2-NEXT: addl $64, %r11d946; AVX2-NEXT: testq %rdx, %rdx947; AVX2-NEXT: cmovnel %eax, %r11d948; AVX2-NEXT: subl $-128, %r11d949; AVX2-NEXT: movq %rsi, %rax950; AVX2-NEXT: orq %r8, %rax951; AVX2-NEXT: cmovnel %ebx, %r11d952; AVX2-NEXT: movq 24(%rdi), %rbx953; AVX2-NEXT: xorl %eax, %eax954; AVX2-NEXT: lzcntq %rbx, %rax955; AVX2-NEXT: xorl %r14d, %r14d956; AVX2-NEXT: lzcntq %r9, %r14957; AVX2-NEXT: addl $64, %r14d958; AVX2-NEXT: testq %rbx, %rbx959; AVX2-NEXT: cmovnel %eax, %r14d960; AVX2-NEXT: xorl %r15d, %r15d961; AVX2-NEXT: lzcntq %r10, %r15962; AVX2-NEXT: xorl %eax, %eax963; AVX2-NEXT: lzcntq (%rdi), %rax964; AVX2-NEXT: addl $64, %eax965; AVX2-NEXT: testq %r10, %r10966; AVX2-NEXT: cmovnel %r15d, %eax967; AVX2-NEXT: subl $-128, %eax968; AVX2-NEXT: orq %rbx, %r9969; AVX2-NEXT: cmovnel %r14d, %eax970; AVX2-NEXT: addl $256, %eax # imm = 0x100971; AVX2-NEXT: orq %r8, %rdx972; AVX2-NEXT: orq %rsi, %rcx973; AVX2-NEXT: orq %rdx, %rcx974; AVX2-NEXT: cmovnel %r11d, %eax975; AVX2-NEXT: # kill: def $eax killed $eax killed $rax976; AVX2-NEXT: popq %rbx977; AVX2-NEXT: popq %r14978; AVX2-NEXT: popq %r15979; AVX2-NEXT: retq980;981; AVX512-LABEL: load_ctlz_i512:982; AVX512: # %bb.0:983; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]984; AVX512-NEXT: vpermq (%rdi), %zmm0, %zmm0985; AVX512-NEXT: vplzcntq %zmm0, %zmm1986; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1987; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k1988; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]989; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1}990; AVX512-NEXT: vmovd %xmm0, %eax991; AVX512-NEXT: vzeroupper992; AVX512-NEXT: retq993 %a0 = load i512, ptr %p0994 %cnt = call i512 @llvm.ctlz.i512(i512 %a0, i1 0)995 %res = trunc i512 %cnt to i32996 ret i32 %res997}998 999define i32 @test_ctlz_i1024(i1024 %a0) nounwind {1000; SSE-LABEL: test_ctlz_i1024:1001; SSE: # %bb.0:1002; SSE-NEXT: pushq %rbp1003; SSE-NEXT: pushq %r151004; SSE-NEXT: pushq %r141005; SSE-NEXT: pushq %r131006; SSE-NEXT: pushq %r121007; SSE-NEXT: pushq %rbx1008; SSE-NEXT: movq %r9, %r111009; SSE-NEXT: movq %r8, %r91010; SSE-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1011; SSE-NEXT: movq %rdx, %r121012; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r101013; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx1014; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r141015; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r151016; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r81017; SSE-NEXT: bsrq %r8, %rax1018; SSE-NEXT: xorl $63, %eax1019; SSE-NEXT: bsrq %r15, %rcx1020; SSE-NEXT: xorl $63, %ecx1021; SSE-NEXT: orl $64, %ecx1022; SSE-NEXT: testq %r8, %r81023; SSE-NEXT: cmovnel %eax, %ecx1024; SSE-NEXT: bsrq %r14, %rdx1025; SSE-NEXT: xorl $63, %edx1026; SSE-NEXT: bsrq {{[0-9]+}}(%rsp), %rax1027; SSE-NEXT: xorl $63, %eax1028; SSE-NEXT: orl $64, %eax1029; SSE-NEXT: testq %r14, %r141030; SSE-NEXT: cmovnel %edx, %eax1031; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r131032; SSE-NEXT: subl $-128, %eax1033; SSE-NEXT: movq %r15, %rdx1034; SSE-NEXT: orq %r8, %rdx1035; SSE-NEXT: movq %r8, %r141036; SSE-NEXT: cmovnel %ecx, %eax1037; SSE-NEXT: bsrq %r13, %rcx1038; SSE-NEXT: xorl $63, %ecx1039; SSE-NEXT: bsrq %rbx, %rdx1040; SSE-NEXT: xorl $63, %edx1041; SSE-NEXT: orl $64, %edx1042; SSE-NEXT: testq %r13, %r131043; SSE-NEXT: cmovnel %ecx, %edx1044; SSE-NEXT: bsrq %r10, %rcx1045; SSE-NEXT: xorl $63, %ecx1046; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r81047; SSE-NEXT: bsrq %r8, %rbp1048; SSE-NEXT: xorl $63, %ebp1049; SSE-NEXT: orl $64, %ebp1050; SSE-NEXT: testq %r10, %r101051; SSE-NEXT: cmovnel %ecx, %ebp1052; SSE-NEXT: subl $-128, %ebp1053; SSE-NEXT: movq %rbx, %rcx1054; SSE-NEXT: orq %r13, %rcx1055; SSE-NEXT: cmovnel %edx, %ebp1056; SSE-NEXT: addl $256, %ebp # imm = 0x1001057; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx1058; SSE-NEXT: orq %r14, %rcx1059; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx1060; SSE-NEXT: orq %r15, %rdx1061; SSE-NEXT: orq %rcx, %rdx1062; SSE-NEXT: cmovnel %eax, %ebp1063; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r141064; SSE-NEXT: bsrq %r14, %rax1065; SSE-NEXT: xorl $63, %eax1066; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r151067; SSE-NEXT: bsrq %r15, %rcx1068; SSE-NEXT: xorl $63, %ecx1069; SSE-NEXT: orl $64, %ecx1070; SSE-NEXT: testq %r14, %r141071; SSE-NEXT: cmovnel %eax, %ecx1072; SSE-NEXT: bsrq %r11, %rax1073; SSE-NEXT: xorl $63, %eax1074; SSE-NEXT: bsrq %r9, %rdx1075; SSE-NEXT: xorl $63, %edx1076; SSE-NEXT: orl $64, %edx1077; SSE-NEXT: testq %r11, %r111078; SSE-NEXT: cmovnel %eax, %edx1079; SSE-NEXT: subl $-128, %edx1080; SSE-NEXT: movq %r15, %rax1081; SSE-NEXT: orq %r14, %rax1082; SSE-NEXT: cmovnel %ecx, %edx1083; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload1084; SSE-NEXT: bsrq %r15, %rax1085; SSE-NEXT: xorl $63, %eax1086; SSE-NEXT: bsrq %r12, %rcx1087; SSE-NEXT: xorl $63, %ecx1088; SSE-NEXT: orl $64, %ecx1089; SSE-NEXT: testq %r15, %r151090; SSE-NEXT: cmovnel %eax, %ecx1091; SSE-NEXT: movl $127, %eax1092; SSE-NEXT: bsrq %rdi, %rax1093; SSE-NEXT: bsrq %rsi, %rdi1094; SSE-NEXT: xorl $63, %edi1095; SSE-NEXT: xorl $63, %eax1096; SSE-NEXT: addl $64, %eax1097; SSE-NEXT: testq %rsi, %rsi1098; SSE-NEXT: cmovnel %edi, %eax1099; SSE-NEXT: subl $-128, %eax1100; SSE-NEXT: orq %r15, %r121101; SSE-NEXT: cmovnel %ecx, %eax1102; SSE-NEXT: orq %r14, %r111103; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r91104; SSE-NEXT: addl $256, %eax # imm = 0x1001105; SSE-NEXT: orq %r11, %r91106; SSE-NEXT: cmovnel %edx, %eax1107; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r131108; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r101109; SSE-NEXT: orq %r13, %r101110; SSE-NEXT: orq {{[0-9]+}}(%rsp), %rbx1111; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r81112; SSE-NEXT: orq %rbx, %r81113; SSE-NEXT: addl $512, %eax # imm = 0x2001114; SSE-NEXT: orq %r10, %r81115; SSE-NEXT: cmovnel %ebp, %eax1116; SSE-NEXT: # kill: def $eax killed $eax killed $rax1117; SSE-NEXT: popq %rbx1118; SSE-NEXT: popq %r121119; SSE-NEXT: popq %r131120; SSE-NEXT: popq %r141121; SSE-NEXT: popq %r151122; SSE-NEXT: popq %rbp1123; SSE-NEXT: retq1124;1125; AVX2-LABEL: test_ctlz_i1024:1126; AVX2: # %bb.0:1127; AVX2-NEXT: pushq %rbp1128; AVX2-NEXT: pushq %r151129; AVX2-NEXT: pushq %r141130; AVX2-NEXT: pushq %r131131; AVX2-NEXT: pushq %r121132; AVX2-NEXT: pushq %rbx1133; AVX2-NEXT: movq %r9, %r141134; AVX2-NEXT: movq %r8, %r111135; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1136; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1137; AVX2-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1138; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r151139; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx1140; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax1141; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r101142; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r81143; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r121144; AVX2-NEXT: xorl %ecx, %ecx1145; AVX2-NEXT: lzcntq %r12, %rcx1146; AVX2-NEXT: xorl %r9d, %r9d1147; AVX2-NEXT: lzcntq %r8, %r91148; AVX2-NEXT: addl $64, %r9d1149; AVX2-NEXT: testq %r12, %r121150; AVX2-NEXT: cmovnel %ecx, %r9d1151; AVX2-NEXT: xorl %esi, %esi1152; AVX2-NEXT: lzcntq %r10, %rsi1153; AVX2-NEXT: xorl %ecx, %ecx1154; AVX2-NEXT: lzcntq %rax, %rcx1155; AVX2-NEXT: addl $64, %ecx1156; AVX2-NEXT: testq %r10, %r101157; AVX2-NEXT: cmovnel %esi, %ecx1158; AVX2-NEXT: subl $-128, %ecx1159; AVX2-NEXT: movq %r8, %rsi1160; AVX2-NEXT: orq %r12, %rsi1161; AVX2-NEXT: cmovnel %r9d, %ecx1162; AVX2-NEXT: xorl %edi, %edi1163; AVX2-NEXT: lzcntq %rbx, %rdi1164; AVX2-NEXT: xorl %esi, %esi1165; AVX2-NEXT: lzcntq %r15, %rsi1166; AVX2-NEXT: addl $64, %esi1167; AVX2-NEXT: testq %rbx, %rbx1168; AVX2-NEXT: cmovnel %edi, %esi1169; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r131170; AVX2-NEXT: xorl %ebp, %ebp1171; AVX2-NEXT: lzcntq %r13, %rbp1172; AVX2-NEXT: addl $64, %ebp1173; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r91174; AVX2-NEXT: xorl %edi, %edi1175; AVX2-NEXT: lzcntq %r9, %rdi1176; AVX2-NEXT: testq %r9, %r91177; AVX2-NEXT: cmovnel %edi, %ebp1178; AVX2-NEXT: subl $-128, %ebp1179; AVX2-NEXT: movq %r15, %rdi1180; AVX2-NEXT: orq %rbx, %rdi1181; AVX2-NEXT: cmovnel %esi, %ebp1182; AVX2-NEXT: addl $256, %ebp # imm = 0x1001183; AVX2-NEXT: movq %r10, %rdi1184; AVX2-NEXT: orq %r12, %rdi1185; AVX2-NEXT: movq %rax, %rsi1186; AVX2-NEXT: orq %r8, %rsi1187; AVX2-NEXT: orq %rdi, %rsi1188; AVX2-NEXT: cmovnel %ecx, %ebp1189; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdi1190; AVX2-NEXT: xorl %eax, %eax1191; AVX2-NEXT: lzcntq %rdi, %rax1192; AVX2-NEXT: addl $64, %eax1193; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r121194; AVX2-NEXT: xorl %ecx, %ecx1195; AVX2-NEXT: lzcntq %r12, %rcx1196; AVX2-NEXT: testq %r12, %r121197; AVX2-NEXT: cmovnel %ecx, %eax1198; AVX2-NEXT: xorl %ecx, %ecx1199; AVX2-NEXT: lzcntq %r11, %rcx1200; AVX2-NEXT: addl $64, %ecx1201; AVX2-NEXT: xorl %esi, %esi1202; AVX2-NEXT: lzcntq %r14, %rsi1203; AVX2-NEXT: testq %r14, %r141204; AVX2-NEXT: cmovnel %esi, %ecx1205; AVX2-NEXT: subl $-128, %ecx1206; AVX2-NEXT: movq %rdi, %rsi1207; AVX2-NEXT: orq %r12, %rsi1208; AVX2-NEXT: cmovnel %eax, %ecx1209; AVX2-NEXT: movq %rdx, %rdi1210; AVX2-NEXT: lzcntq %rdx, %rdx1211; AVX2-NEXT: addl $64, %edx1212; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload1213; AVX2-NEXT: xorl %eax, %eax1214; AVX2-NEXT: lzcntq %r10, %rax1215; AVX2-NEXT: testq %r10, %r101216; AVX2-NEXT: cmovnel %eax, %edx1217; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload1218; AVX2-NEXT: lzcntq %rax, %rax1219; AVX2-NEXT: addl $64, %eax1220; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload1221; AVX2-NEXT: lzcntq %rsi, %r81222; AVX2-NEXT: testq %rsi, %rsi1223; AVX2-NEXT: cmovnel %r8d, %eax1224; AVX2-NEXT: subl $-128, %eax1225; AVX2-NEXT: orq %r10, %rdi1226; AVX2-NEXT: cmovnel %edx, %eax1227; AVX2-NEXT: orq %r12, %r141228; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r111229; AVX2-NEXT: addl $256, %eax # imm = 0x1001230; AVX2-NEXT: orq %r14, %r111231; AVX2-NEXT: cmovnel %ecx, %eax1232; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %rbx1233; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r91234; AVX2-NEXT: orq %rbx, %r91235; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r151236; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r131237; AVX2-NEXT: orq %r15, %r131238; AVX2-NEXT: addl $512, %eax # imm = 0x2001239; AVX2-NEXT: orq %r9, %r131240; AVX2-NEXT: cmovnel %ebp, %eax1241; AVX2-NEXT: # kill: def $eax killed $eax killed $rax1242; AVX2-NEXT: popq %rbx1243; AVX2-NEXT: popq %r121244; AVX2-NEXT: popq %r131245; AVX2-NEXT: popq %r141246; AVX2-NEXT: popq %r151247; AVX2-NEXT: popq %rbp1248; AVX2-NEXT: retq1249;1250; AVX512-LABEL: test_ctlz_i1024:1251; AVX512: # %bb.0:1252; AVX512-NEXT: pushq %r141253; AVX512-NEXT: pushq %rbx1254; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx1255; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r101256; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r111257; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r141258; AVX512-NEXT: vmovq %rdi, %xmm01259; AVX512-NEXT: vmovq %rsi, %xmm11260; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]1261; AVX512-NEXT: vmovq %rdx, %xmm11262; AVX512-NEXT: vmovq %rcx, %xmm21263; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]1264; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm01265; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]1266; AVX512-NEXT: vmovq %r8, %xmm21267; AVX512-NEXT: vmovq %r9, %xmm31268; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]1269; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm11270; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm01271; AVX512-NEXT: vplzcntq %zmm0, %zmm11272; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]1273; AVX512-NEXT: vpaddq %zmm2, %zmm1, %zmm11274; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k11275; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]1276; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1}1277; AVX512-NEXT: vmovd %xmm0, %ecx1278; AVX512-NEXT: addl $512, %ecx # imm = 0x2001279; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]1280; AVX512-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm01281; AVX512-NEXT: vplzcntq %zmm0, %zmm11282; AVX512-NEXT: vpaddq %zmm2, %zmm1, %zmm11283; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k11284; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}1285; AVX512-NEXT: vmovd %xmm0, %eax1286; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %r141287; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %r111288; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %rbx1289; AVX512-NEXT: orq %r14, %r111290; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %r101291; AVX512-NEXT: orq %rbx, %r101292; AVX512-NEXT: orq %r11, %r101293; AVX512-NEXT: cmovel %ecx, %eax1294; AVX512-NEXT: popq %rbx1295; AVX512-NEXT: popq %r141296; AVX512-NEXT: vzeroupper1297; AVX512-NEXT: retq1298 %cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 0)1299 %res = trunc i1024 %cnt to i321300 ret i32 %res1301}1302 1303define i32 @load_ctlz_i1024(ptr %p0) nounwind {1304; SSE-LABEL: load_ctlz_i1024:1305; SSE: # %bb.0:1306; SSE-NEXT: pushq %rbp1307; SSE-NEXT: pushq %r151308; SSE-NEXT: pushq %r141309; SSE-NEXT: pushq %r131310; SSE-NEXT: pushq %r121311; SSE-NEXT: pushq %rbx1312; SSE-NEXT: movq 40(%rdi), %rbp1313; SSE-NEXT: movq 64(%rdi), %rbx1314; SSE-NEXT: movq 72(%rdi), %r111315; SSE-NEXT: movq 80(%rdi), %r121316; SSE-NEXT: movq 88(%rdi), %r141317; SSE-NEXT: movq 96(%rdi), %rsi1318; SSE-NEXT: movq 104(%rdi), %r91319; SSE-NEXT: movq 112(%rdi), %r101320; SSE-NEXT: movq 120(%rdi), %r81321; SSE-NEXT: bsrq %r8, %rax1322; SSE-NEXT: xorl $63, %eax1323; SSE-NEXT: bsrq %r10, %rcx1324; SSE-NEXT: xorl $63, %ecx1325; SSE-NEXT: orl $64, %ecx1326; SSE-NEXT: testq %r8, %r81327; SSE-NEXT: cmovnel %eax, %ecx1328; SSE-NEXT: bsrq %r9, %rdx1329; SSE-NEXT: xorl $63, %edx1330; SSE-NEXT: bsrq %rsi, %rax1331; SSE-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1332; SSE-NEXT: xorl $63, %eax1333; SSE-NEXT: orl $64, %eax1334; SSE-NEXT: testq %r9, %r91335; SSE-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1336; SSE-NEXT: cmovnel %edx, %eax1337; SSE-NEXT: subl $-128, %eax1338; SSE-NEXT: movq %r10, %rdx1339; SSE-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1340; SSE-NEXT: orq %r8, %rdx1341; SSE-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1342; SSE-NEXT: cmovnel %ecx, %eax1343; SSE-NEXT: bsrq %r14, %rcx1344; SSE-NEXT: xorl $63, %ecx1345; SSE-NEXT: movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1346; SSE-NEXT: bsrq %r12, %rdx1347; SSE-NEXT: xorl $63, %edx1348; SSE-NEXT: orl $64, %edx1349; SSE-NEXT: testq %r14, %r141350; SSE-NEXT: cmovnel %ecx, %edx1351; SSE-NEXT: bsrq %r11, %rcx1352; SSE-NEXT: xorl $63, %ecx1353; SSE-NEXT: bsrq %rbx, %r151354; SSE-NEXT: xorl $63, %r15d1355; SSE-NEXT: orl $64, %r15d1356; SSE-NEXT: testq %r11, %r111357; SSE-NEXT: cmovnel %ecx, %r15d1358; SSE-NEXT: subl $-128, %r15d1359; SSE-NEXT: movq %r12, %rcx1360; SSE-NEXT: orq %r14, %rcx1361; SSE-NEXT: cmovnel %edx, %r15d1362; SSE-NEXT: movq 48(%rdi), %r121363; SSE-NEXT: addl $256, %r15d # imm = 0x1001364; SSE-NEXT: movq %r9, %rcx1365; SSE-NEXT: orq %r8, %rcx1366; SSE-NEXT: movq %rsi, %rdx1367; SSE-NEXT: orq %r10, %rdx1368; SSE-NEXT: orq %rcx, %rdx1369; SSE-NEXT: movq 56(%rdi), %r131370; SSE-NEXT: cmovnel %eax, %r15d1371; SSE-NEXT: bsrq %r13, %rax1372; SSE-NEXT: xorl $63, %eax1373; SSE-NEXT: bsrq %r12, %rcx1374; SSE-NEXT: xorl $63, %ecx1375; SSE-NEXT: orl $64, %ecx1376; SSE-NEXT: testq %r13, %r131377; SSE-NEXT: cmovnel %eax, %ecx1378; SSE-NEXT: movq %rbp, %r101379; SSE-NEXT: bsrq %rbp, %rax1380; SSE-NEXT: xorl $63, %eax1381; SSE-NEXT: movq 32(%rdi), %r81382; SSE-NEXT: bsrq %r8, %rbp1383; SSE-NEXT: xorl $63, %ebp1384; SSE-NEXT: orl $64, %ebp1385; SSE-NEXT: testq %r10, %r101386; SSE-NEXT: cmovnel %eax, %ebp1387; SSE-NEXT: subl $-128, %ebp1388; SSE-NEXT: movq %r12, %rax1389; SSE-NEXT: orq %r13, %rax1390; SSE-NEXT: cmovnel %ecx, %ebp1391; SSE-NEXT: movq 24(%rdi), %r91392; SSE-NEXT: bsrq %r9, %rax1393; SSE-NEXT: xorl $63, %eax1394; SSE-NEXT: movq 16(%rdi), %rsi1395; SSE-NEXT: bsrq %rsi, %rcx1396; SSE-NEXT: xorl $63, %ecx1397; SSE-NEXT: orl $64, %ecx1398; SSE-NEXT: testq %r9, %r91399; SSE-NEXT: cmovnel %eax, %ecx1400; SSE-NEXT: movl $127, %eax1401; SSE-NEXT: bsrq (%rdi), %rax1402; SSE-NEXT: movq 8(%rdi), %rdi1403; SSE-NEXT: bsrq %rdi, %rdx1404; SSE-NEXT: xorl $63, %edx1405; SSE-NEXT: xorl $63, %eax1406; SSE-NEXT: addl $64, %eax1407; SSE-NEXT: testq %rdi, %rdi1408; SSE-NEXT: cmovnel %edx, %eax1409; SSE-NEXT: subl $-128, %eax1410; SSE-NEXT: orq %r9, %rsi1411; SSE-NEXT: cmovnel %ecx, %eax1412; SSE-NEXT: orq %r13, %r101413; SSE-NEXT: orq %r12, %r81414; SSE-NEXT: addl $256, %eax # imm = 0x1001415; SSE-NEXT: orq %r10, %r81416; SSE-NEXT: cmovnel %ebp, %eax1417; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload1418; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload1419; SSE-NEXT: orq %r14, %r111420; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload1421; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload1422; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload1423; SSE-NEXT: orq %rcx, %rbx1424; SSE-NEXT: addl $512, %eax # imm = 0x2001425; SSE-NEXT: orq %r11, %rbx1426; SSE-NEXT: cmovnel %r15d, %eax1427; SSE-NEXT: # kill: def $eax killed $eax killed $rax1428; SSE-NEXT: popq %rbx1429; SSE-NEXT: popq %r121430; SSE-NEXT: popq %r131431; SSE-NEXT: popq %r141432; SSE-NEXT: popq %r151433; SSE-NEXT: popq %rbp1434; SSE-NEXT: retq1435;1436; AVX2-LABEL: load_ctlz_i1024:1437; AVX2: # %bb.0:1438; AVX2-NEXT: pushq %rbp1439; AVX2-NEXT: pushq %r151440; AVX2-NEXT: pushq %r141441; AVX2-NEXT: pushq %r131442; AVX2-NEXT: pushq %r121443; AVX2-NEXT: pushq %rbx1444; AVX2-NEXT: movq 48(%rdi), %r91445; AVX2-NEXT: movq 56(%rdi), %rbp1446; AVX2-NEXT: movq 64(%rdi), %r111447; AVX2-NEXT: movq 72(%rdi), %r101448; AVX2-NEXT: movq 80(%rdi), %r141449; AVX2-NEXT: movq 88(%rdi), %rbx1450; AVX2-NEXT: movq 96(%rdi), %rdx1451; AVX2-NEXT: movq 104(%rdi), %r81452; AVX2-NEXT: movq 112(%rdi), %rsi1453; AVX2-NEXT: movq 120(%rdi), %r151454; AVX2-NEXT: lzcntq %r15, %rax1455; AVX2-NEXT: lzcntq %rsi, %rcx1456; AVX2-NEXT: addl $64, %ecx1457; AVX2-NEXT: testq %r15, %r151458; AVX2-NEXT: cmovnel %eax, %ecx1459; AVX2-NEXT: xorl %r12d, %r12d1460; AVX2-NEXT: lzcntq %r8, %r121461; AVX2-NEXT: xorl %eax, %eax1462; AVX2-NEXT: lzcntq %rdx, %rax1463; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1464; AVX2-NEXT: addl $64, %eax1465; AVX2-NEXT: testq %r8, %r81466; AVX2-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1467; AVX2-NEXT: cmovnel %r12d, %eax1468; AVX2-NEXT: subl $-128, %eax1469; AVX2-NEXT: movq %rsi, %r121470; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1471; AVX2-NEXT: orq %r15, %r121472; AVX2-NEXT: cmovnel %ecx, %eax1473; AVX2-NEXT: xorl %ecx, %ecx1474; AVX2-NEXT: lzcntq %rbx, %rcx1475; AVX2-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1476; AVX2-NEXT: xorl %r13d, %r13d1477; AVX2-NEXT: lzcntq %r14, %r131478; AVX2-NEXT: addl $64, %r13d1479; AVX2-NEXT: testq %rbx, %rbx1480; AVX2-NEXT: cmovnel %ecx, %r13d1481; AVX2-NEXT: xorl %ecx, %ecx1482; AVX2-NEXT: lzcntq %r10, %rcx1483; AVX2-NEXT: xorl %r12d, %r12d1484; AVX2-NEXT: lzcntq %r11, %r121485; AVX2-NEXT: addl $64, %r12d1486; AVX2-NEXT: testq %r10, %r101487; AVX2-NEXT: cmovnel %ecx, %r12d1488; AVX2-NEXT: subl $-128, %r12d1489; AVX2-NEXT: movq %r14, %rcx1490; AVX2-NEXT: orq %rbx, %rcx1491; AVX2-NEXT: cmovnel %r13d, %r12d1492; AVX2-NEXT: addl $256, %r12d # imm = 0x1001493; AVX2-NEXT: movq %r8, %rcx1494; AVX2-NEXT: orq %r15, %rcx1495; AVX2-NEXT: orq %rsi, %rdx1496; AVX2-NEXT: orq %rcx, %rdx1497; AVX2-NEXT: cmovnel %eax, %r12d1498; AVX2-NEXT: movq %rbp, %r141499; AVX2-NEXT: xorl %ecx, %ecx1500; AVX2-NEXT: lzcntq %rbp, %rcx1501; AVX2-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill1502; AVX2-NEXT: xorl %eax, %eax1503; AVX2-NEXT: lzcntq %r9, %rax1504; AVX2-NEXT: addl $64, %eax1505; AVX2-NEXT: testq %rbp, %rbp1506; AVX2-NEXT: cmovnel %ecx, %eax1507; AVX2-NEXT: movq 32(%rdi), %r131508; AVX2-NEXT: xorl %ebp, %ebp1509; AVX2-NEXT: lzcntq %r13, %rbp1510; AVX2-NEXT: addl $64, %ebp1511; AVX2-NEXT: movq 40(%rdi), %r81512; AVX2-NEXT: xorl %edx, %edx1513; AVX2-NEXT: lzcntq %r8, %rdx1514; AVX2-NEXT: testq %r8, %r81515; AVX2-NEXT: cmovnel %edx, %ebp1516; AVX2-NEXT: subl $-128, %ebp1517; AVX2-NEXT: movq %r9, %rdx1518; AVX2-NEXT: orq %r14, %rdx1519; AVX2-NEXT: cmovnel %eax, %ebp1520; AVX2-NEXT: movq 16(%rdi), %r91521; AVX2-NEXT: xorl %ecx, %ecx1522; AVX2-NEXT: lzcntq %r9, %rcx1523; AVX2-NEXT: addl $64, %ecx1524; AVX2-NEXT: movq 24(%rdi), %rdx1525; AVX2-NEXT: xorl %eax, %eax1526; AVX2-NEXT: lzcntq %rdx, %rax1527; AVX2-NEXT: testq %rdx, %rdx1528; AVX2-NEXT: cmovnel %eax, %ecx1529; AVX2-NEXT: movq 8(%rdi), %rsi1530; AVX2-NEXT: xorl %eax, %eax1531; AVX2-NEXT: lzcntq (%rdi), %rax1532; AVX2-NEXT: addl $64, %eax1533; AVX2-NEXT: lzcntq %rsi, %rdi1534; AVX2-NEXT: testq %rsi, %rsi1535; AVX2-NEXT: cmovnel %edi, %eax1536; AVX2-NEXT: subl $-128, %eax1537; AVX2-NEXT: orq %rdx, %r91538; AVX2-NEXT: cmovnel %ecx, %eax1539; AVX2-NEXT: orq %r14, %r81540; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload1541; AVX2-NEXT: addl $256, %eax # imm = 0x1001542; AVX2-NEXT: orq %r8, %r131543; AVX2-NEXT: cmovnel %ebp, %eax1544; AVX2-NEXT: orq %r15, %rbx1545; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Folded Reload1546; AVX2-NEXT: orq %rbx, %r101547; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload1548; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload1549; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload1550; AVX2-NEXT: orq %rcx, %r111551; AVX2-NEXT: addl $512, %eax # imm = 0x2001552; AVX2-NEXT: orq %r10, %r111553; AVX2-NEXT: cmovnel %r12d, %eax1554; AVX2-NEXT: # kill: def $eax killed $eax killed $rax1555; AVX2-NEXT: popq %rbx1556; AVX2-NEXT: popq %r121557; AVX2-NEXT: popq %r131558; AVX2-NEXT: popq %r141559; AVX2-NEXT: popq %r151560; AVX2-NEXT: popq %rbp1561; AVX2-NEXT: retq1562;1563; AVX512-LABEL: load_ctlz_i1024:1564; AVX512: # %bb.0:1565; AVX512-NEXT: movq 80(%rdi), %rsi1566; AVX512-NEXT: movq 64(%rdi), %rcx1567; AVX512-NEXT: movq 72(%rdi), %rdx1568; AVX512-NEXT: movq 88(%rdi), %r81569; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]1570; AVX512-NEXT: vpermq 64(%rdi), %zmm0, %zmm11571; AVX512-NEXT: vplzcntq %zmm1, %zmm21572; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]1573; AVX512-NEXT: vpaddq %zmm3, %zmm2, %zmm21574; AVX512-NEXT: vptestmq %zmm1, %zmm1, %k11575; AVX512-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}1576; AVX512-NEXT: vmovd %xmm1, %r9d1577; AVX512-NEXT: vpermq (%rdi), %zmm0, %zmm01578; AVX512-NEXT: vplzcntq %zmm0, %zmm11579; AVX512-NEXT: vpaddq %zmm3, %zmm1, %zmm11580; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k11581; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]1582; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1}1583; AVX512-NEXT: vmovd %xmm0, %eax1584; AVX512-NEXT: addl $512, %eax # imm = 0x2001585; AVX512-NEXT: orq 120(%rdi), %r81586; AVX512-NEXT: orq 104(%rdi), %rdx1587; AVX512-NEXT: orq 112(%rdi), %rsi1588; AVX512-NEXT: orq %r8, %rdx1589; AVX512-NEXT: orq 96(%rdi), %rcx1590; AVX512-NEXT: orq %rsi, %rcx1591; AVX512-NEXT: orq %rdx, %rcx1592; AVX512-NEXT: cmovnel %r9d, %eax1593; AVX512-NEXT: vzeroupper1594; AVX512-NEXT: retq1595 %a0 = load i1024, ptr %p01596 %cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 0)1597 %res = trunc i1024 %cnt to i321598 ret i32 %res1599}1600 1601;1602; CTLZ_ZERO_UNDEF1603;1604 1605define i32 @test_ctlz_undef_i128(i128 %a0) nounwind {1606; SSE-LABEL: test_ctlz_undef_i128:1607; SSE: # %bb.0:1608; SSE-NEXT: bsrq %rsi, %rcx1609; SSE-NEXT: xorl $63, %ecx1610; SSE-NEXT: bsrq %rdi, %rax1611; SSE-NEXT: xorl $63, %eax1612; SSE-NEXT: orl $64, %eax1613; SSE-NEXT: testq %rsi, %rsi1614; SSE-NEXT: cmovnel %ecx, %eax1615; SSE-NEXT: # kill: def $eax killed $eax killed $rax1616; SSE-NEXT: retq1617;1618; AVX2-LABEL: test_ctlz_undef_i128:1619; AVX2: # %bb.0:1620; AVX2-NEXT: lzcntq %rsi, %rcx1621; AVX2-NEXT: lzcntq %rdi, %rax1622; AVX2-NEXT: addl $64, %eax1623; AVX2-NEXT: testq %rsi, %rsi1624; AVX2-NEXT: cmovnel %ecx, %eax1625; AVX2-NEXT: # kill: def $eax killed $eax killed $rax1626; AVX2-NEXT: retq1627;1628; AVX512-LABEL: test_ctlz_undef_i128:1629; AVX512: # %bb.0:1630; AVX512-NEXT: lzcntq %rsi, %rcx1631; AVX512-NEXT: lzcntq %rdi, %rax1632; AVX512-NEXT: addl $64, %eax1633; AVX512-NEXT: testq %rsi, %rsi1634; AVX512-NEXT: cmovnel %ecx, %eax1635; AVX512-NEXT: # kill: def $eax killed $eax killed $rax1636; AVX512-NEXT: retq1637 %cnt = call i128 @llvm.ctlz.i128(i128 %a0, i1 -1)1638 %res = trunc i128 %cnt to i321639 ret i32 %res1640}1641 1642define i32 @load_ctlz_undef_i128(ptr %p0) nounwind {1643; SSE-LABEL: load_ctlz_undef_i128:1644; SSE: # %bb.0:1645; SSE-NEXT: movq 8(%rdi), %rcx1646; SSE-NEXT: bsrq %rcx, %rdx1647; SSE-NEXT: xorl $63, %edx1648; SSE-NEXT: bsrq (%rdi), %rax1649; SSE-NEXT: xorl $63, %eax1650; SSE-NEXT: orl $64, %eax1651; SSE-NEXT: testq %rcx, %rcx1652; SSE-NEXT: cmovnel %edx, %eax1653; SSE-NEXT: # kill: def $eax killed $eax killed $rax1654; SSE-NEXT: retq1655;1656; AVX2-LABEL: load_ctlz_undef_i128:1657; AVX2: # %bb.0:1658; AVX2-NEXT: movq 8(%rdi), %rcx1659; AVX2-NEXT: lzcntq %rcx, %rdx1660; AVX2-NEXT: lzcntq (%rdi), %rax1661; AVX2-NEXT: addl $64, %eax1662; AVX2-NEXT: testq %rcx, %rcx1663; AVX2-NEXT: cmovnel %edx, %eax1664; AVX2-NEXT: # kill: def $eax killed $eax killed $rax1665; AVX2-NEXT: retq1666;1667; AVX512-LABEL: load_ctlz_undef_i128:1668; AVX512: # %bb.0:1669; AVX512-NEXT: movq 8(%rdi), %rcx1670; AVX512-NEXT: lzcntq %rcx, %rdx1671; AVX512-NEXT: lzcntq (%rdi), %rax1672; AVX512-NEXT: addl $64, %eax1673; AVX512-NEXT: testq %rcx, %rcx1674; AVX512-NEXT: cmovnel %edx, %eax1675; AVX512-NEXT: # kill: def $eax killed $eax killed $rax1676; AVX512-NEXT: retq1677 %a0 = load i128, ptr %p01678 %cnt = call i128 @llvm.ctlz.i128(i128 %a0, i1 -1)1679 %res = trunc i128 %cnt to i321680 ret i32 %res1681}1682 1683define i32 @test_ctlz_undef_i256(i256 %a0) nounwind {1684; SSE-LABEL: test_ctlz_undef_i256:1685; SSE: # %bb.0:1686; SSE-NEXT: bsrq %rcx, %rax1687; SSE-NEXT: xorl $63, %eax1688; SSE-NEXT: bsrq %rdx, %r81689; SSE-NEXT: xorl $63, %r8d1690; SSE-NEXT: orl $64, %r8d1691; SSE-NEXT: testq %rcx, %rcx1692; SSE-NEXT: cmovnel %eax, %r8d1693; SSE-NEXT: bsrq %rsi, %r91694; SSE-NEXT: xorl $63, %r9d1695; SSE-NEXT: bsrq %rdi, %rax1696; SSE-NEXT: xorl $63, %eax1697; SSE-NEXT: orl $64, %eax1698; SSE-NEXT: testq %rsi, %rsi1699; SSE-NEXT: cmovnel %r9d, %eax1700; SSE-NEXT: subl $-128, %eax1701; SSE-NEXT: orq %rcx, %rdx1702; SSE-NEXT: cmovnel %r8d, %eax1703; SSE-NEXT: # kill: def $eax killed $eax killed $rax1704; SSE-NEXT: retq1705;1706; AVX2-LABEL: test_ctlz_undef_i256:1707; AVX2: # %bb.0:1708; AVX2-NEXT: lzcntq %rcx, %rax1709; AVX2-NEXT: lzcntq %rdx, %r81710; AVX2-NEXT: addl $64, %r8d1711; AVX2-NEXT: testq %rcx, %rcx1712; AVX2-NEXT: cmovnel %eax, %r8d1713; AVX2-NEXT: lzcntq %rsi, %r91714; AVX2-NEXT: xorl %eax, %eax1715; AVX2-NEXT: lzcntq %rdi, %rax1716; AVX2-NEXT: addl $64, %eax1717; AVX2-NEXT: testq %rsi, %rsi1718; AVX2-NEXT: cmovnel %r9d, %eax1719; AVX2-NEXT: subl $-128, %eax1720; AVX2-NEXT: orq %rcx, %rdx1721; AVX2-NEXT: cmovnel %r8d, %eax1722; AVX2-NEXT: # kill: def $eax killed $eax killed $rax1723; AVX2-NEXT: retq1724;1725; AVX512-LABEL: test_ctlz_undef_i256:1726; AVX512: # %bb.0:1727; AVX512-NEXT: lzcntq %rcx, %rax1728; AVX512-NEXT: lzcntq %rdx, %r81729; AVX512-NEXT: addl $64, %r8d1730; AVX512-NEXT: testq %rcx, %rcx1731; AVX512-NEXT: cmovnel %eax, %r8d1732; AVX512-NEXT: lzcntq %rsi, %r91733; AVX512-NEXT: lzcntq %rdi, %rax1734; AVX512-NEXT: addl $64, %eax1735; AVX512-NEXT: testq %rsi, %rsi1736; AVX512-NEXT: cmovnel %r9d, %eax1737; AVX512-NEXT: subl $-128, %eax1738; AVX512-NEXT: orq %rcx, %rdx1739; AVX512-NEXT: cmovnel %r8d, %eax1740; AVX512-NEXT: # kill: def $eax killed $eax killed $rax1741; AVX512-NEXT: retq1742 %cnt = call i256 @llvm.ctlz.i256(i256 %a0, i1 -1)1743 %res = trunc i256 %cnt to i321744 ret i32 %res1745}1746 1747define i32 @load_ctlz_undef_i256(ptr %p0) nounwind {1748; SSE-LABEL: load_ctlz_undef_i256:1749; SSE: # %bb.0:1750; SSE-NEXT: movq 8(%rdi), %rdx1751; SSE-NEXT: movq 16(%rdi), %rcx1752; SSE-NEXT: movq 24(%rdi), %rsi1753; SSE-NEXT: bsrq %rsi, %rax1754; SSE-NEXT: xorl $63, %eax1755; SSE-NEXT: bsrq %rcx, %r81756; SSE-NEXT: xorl $63, %r8d1757; SSE-NEXT: orl $64, %r8d1758; SSE-NEXT: testq %rsi, %rsi1759; SSE-NEXT: cmovnel %eax, %r8d1760; SSE-NEXT: bsrq %rdx, %r91761; SSE-NEXT: xorl $63, %r9d1762; SSE-NEXT: bsrq (%rdi), %rax1763; SSE-NEXT: xorl $63, %eax1764; SSE-NEXT: orl $64, %eax1765; SSE-NEXT: testq %rdx, %rdx1766; SSE-NEXT: cmovnel %r9d, %eax1767; SSE-NEXT: subl $-128, %eax1768; SSE-NEXT: orq %rsi, %rcx1769; SSE-NEXT: cmovnel %r8d, %eax1770; SSE-NEXT: # kill: def $eax killed $eax killed $rax1771; SSE-NEXT: retq1772;1773; AVX2-LABEL: load_ctlz_undef_i256:1774; AVX2: # %bb.0:1775; AVX2-NEXT: movq 16(%rdi), %rcx1776; AVX2-NEXT: movq 24(%rdi), %rdx1777; AVX2-NEXT: lzcntq %rdx, %rax1778; AVX2-NEXT: lzcntq %rcx, %rsi1779; AVX2-NEXT: addl $64, %esi1780; AVX2-NEXT: testq %rdx, %rdx1781; AVX2-NEXT: cmovnel %eax, %esi1782; AVX2-NEXT: movq 8(%rdi), %r81783; AVX2-NEXT: lzcntq %r8, %r91784; AVX2-NEXT: xorl %eax, %eax1785; AVX2-NEXT: lzcntq (%rdi), %rax1786; AVX2-NEXT: addl $64, %eax1787; AVX2-NEXT: testq %r8, %r81788; AVX2-NEXT: cmovnel %r9d, %eax1789; AVX2-NEXT: subl $-128, %eax1790; AVX2-NEXT: orq %rdx, %rcx1791; AVX2-NEXT: cmovnel %esi, %eax1792; AVX2-NEXT: # kill: def $eax killed $eax killed $rax1793; AVX2-NEXT: retq1794;1795; AVX512-LABEL: load_ctlz_undef_i256:1796; AVX512: # %bb.0:1797; AVX512-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0]1798; AVX512-NEXT: vptestmq %ymm0, %ymm0, %k11799; AVX512-NEXT: vplzcntq %ymm0, %ymm01800; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm01801; AVX512-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}1802; AVX512-NEXT: vmovd %xmm0, %eax1803; AVX512-NEXT: vzeroupper1804; AVX512-NEXT: retq1805 %a0 = load i256, ptr %p01806 %cnt = call i256 @llvm.ctlz.i256(i256 %a0, i1 -1)1807 %res = trunc i256 %cnt to i321808 ret i32 %res1809}1810 1811define i32 @test_ctlz_undef_i512(i512 %a0) nounwind {1812; SSE-LABEL: test_ctlz_undef_i512:1813; SSE: # %bb.0:1814; SSE-NEXT: pushq %r151815; SSE-NEXT: pushq %r141816; SSE-NEXT: pushq %rbx1817; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r101818; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r111819; SSE-NEXT: bsrq %r11, %rax1820; SSE-NEXT: xorl $63, %eax1821; SSE-NEXT: bsrq %r10, %r141822; SSE-NEXT: xorl $63, %r14d1823; SSE-NEXT: orl $64, %r14d1824; SSE-NEXT: testq %r11, %r111825; SSE-NEXT: cmovnel %eax, %r14d1826; SSE-NEXT: bsrq %r9, %rax1827; SSE-NEXT: xorl $63, %eax1828; SSE-NEXT: bsrq %r8, %rbx1829; SSE-NEXT: xorl $63, %ebx1830; SSE-NEXT: orl $64, %ebx1831; SSE-NEXT: testq %r9, %r91832; SSE-NEXT: cmovnel %eax, %ebx1833; SSE-NEXT: subl $-128, %ebx1834; SSE-NEXT: movq %r10, %rax1835; SSE-NEXT: orq %r11, %rax1836; SSE-NEXT: cmovnel %r14d, %ebx1837; SSE-NEXT: bsrq %rcx, %rax1838; SSE-NEXT: xorl $63, %eax1839; SSE-NEXT: bsrq %rdx, %r141840; SSE-NEXT: xorl $63, %r14d1841; SSE-NEXT: orl $64, %r14d1842; SSE-NEXT: testq %rcx, %rcx1843; SSE-NEXT: cmovnel %eax, %r14d1844; SSE-NEXT: bsrq %rsi, %r151845; SSE-NEXT: xorl $63, %r15d1846; SSE-NEXT: bsrq %rdi, %rax1847; SSE-NEXT: xorl $63, %eax1848; SSE-NEXT: orl $64, %eax1849; SSE-NEXT: testq %rsi, %rsi1850; SSE-NEXT: cmovnel %r15d, %eax1851; SSE-NEXT: subl $-128, %eax1852; SSE-NEXT: orq %rcx, %rdx1853; SSE-NEXT: cmovnel %r14d, %eax1854; SSE-NEXT: addl $256, %eax # imm = 0x1001855; SSE-NEXT: orq %r11, %r91856; SSE-NEXT: orq %r10, %r81857; SSE-NEXT: orq %r9, %r81858; SSE-NEXT: cmovnel %ebx, %eax1859; SSE-NEXT: # kill: def $eax killed $eax killed $rax1860; SSE-NEXT: popq %rbx1861; SSE-NEXT: popq %r141862; SSE-NEXT: popq %r151863; SSE-NEXT: retq1864;1865; AVX2-LABEL: test_ctlz_undef_i512:1866; AVX2: # %bb.0:1867; AVX2-NEXT: pushq %r151868; AVX2-NEXT: pushq %r141869; AVX2-NEXT: pushq %rbx1870; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r101871; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r111872; AVX2-NEXT: lzcntq %r11, %rax1873; AVX2-NEXT: xorl %r14d, %r14d1874; AVX2-NEXT: lzcntq %r10, %r141875; AVX2-NEXT: addl $64, %r14d1876; AVX2-NEXT: testq %r11, %r111877; AVX2-NEXT: cmovnel %eax, %r14d1878; AVX2-NEXT: xorl %eax, %eax1879; AVX2-NEXT: lzcntq %r9, %rax1880; AVX2-NEXT: xorl %ebx, %ebx1881; AVX2-NEXT: lzcntq %r8, %rbx1882; AVX2-NEXT: addl $64, %ebx1883; AVX2-NEXT: testq %r9, %r91884; AVX2-NEXT: cmovnel %eax, %ebx1885; AVX2-NEXT: subl $-128, %ebx1886; AVX2-NEXT: movq %r10, %rax1887; AVX2-NEXT: orq %r11, %rax1888; AVX2-NEXT: cmovnel %r14d, %ebx1889; AVX2-NEXT: xorl %eax, %eax1890; AVX2-NEXT: lzcntq %rcx, %rax1891; AVX2-NEXT: xorl %r14d, %r14d1892; AVX2-NEXT: lzcntq %rdx, %r141893; AVX2-NEXT: addl $64, %r14d1894; AVX2-NEXT: testq %rcx, %rcx1895; AVX2-NEXT: cmovnel %eax, %r14d1896; AVX2-NEXT: xorl %r15d, %r15d1897; AVX2-NEXT: lzcntq %rsi, %r151898; AVX2-NEXT: xorl %eax, %eax1899; AVX2-NEXT: lzcntq %rdi, %rax1900; AVX2-NEXT: addl $64, %eax1901; AVX2-NEXT: testq %rsi, %rsi1902; AVX2-NEXT: cmovnel %r15d, %eax1903; AVX2-NEXT: subl $-128, %eax1904; AVX2-NEXT: orq %rcx, %rdx1905; AVX2-NEXT: cmovnel %r14d, %eax1906; AVX2-NEXT: addl $256, %eax # imm = 0x1001907; AVX2-NEXT: orq %r11, %r91908; AVX2-NEXT: orq %r10, %r81909; AVX2-NEXT: orq %r9, %r81910; AVX2-NEXT: cmovnel %ebx, %eax1911; AVX2-NEXT: # kill: def $eax killed $eax killed $rax1912; AVX2-NEXT: popq %rbx1913; AVX2-NEXT: popq %r141914; AVX2-NEXT: popq %r151915; AVX2-NEXT: retq1916;1917; AVX512-LABEL: test_ctlz_undef_i512:1918; AVX512: # %bb.0:1919; AVX512-NEXT: vmovq %rdi, %xmm01920; AVX512-NEXT: vmovq %rsi, %xmm11921; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]1922; AVX512-NEXT: vmovq %rdx, %xmm11923; AVX512-NEXT: vmovq %rcx, %xmm21924; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]1925; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm01926; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]1927; AVX512-NEXT: vmovq %r8, %xmm21928; AVX512-NEXT: vmovq %r9, %xmm31929; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]1930; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm11931; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm01932; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k11933; AVX512-NEXT: vplzcntq %zmm0, %zmm01934; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm01935; AVX512-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}1936; AVX512-NEXT: vmovd %xmm0, %eax1937; AVX512-NEXT: vzeroupper1938; AVX512-NEXT: retq1939 %cnt = call i512 @llvm.ctlz.i512(i512 %a0, i1 -1)1940 %res = trunc i512 %cnt to i321941 ret i32 %res1942}1943 1944define i32 @load_ctlz_undef_i512(ptr %p0) nounwind {1945; SSE-LABEL: load_ctlz_undef_i512:1946; SSE: # %bb.0:1947; SSE-NEXT: pushq %r151948; SSE-NEXT: pushq %r141949; SSE-NEXT: pushq %rbx1950; SSE-NEXT: movq 8(%rdi), %r111951; SSE-NEXT: movq 16(%rdi), %r91952; SSE-NEXT: movq 24(%rdi), %r101953; SSE-NEXT: movq 32(%rdi), %rcx1954; SSE-NEXT: movq 40(%rdi), %rdx1955; SSE-NEXT: movq 48(%rdi), %rsi1956; SSE-NEXT: movq 56(%rdi), %r81957; SSE-NEXT: bsrq %r8, %rax1958; SSE-NEXT: xorl $63, %eax1959; SSE-NEXT: bsrq %rsi, %r141960; SSE-NEXT: xorl $63, %r14d1961; SSE-NEXT: orl $64, %r14d1962; SSE-NEXT: testq %r8, %r81963; SSE-NEXT: cmovnel %eax, %r14d1964; SSE-NEXT: bsrq %rdx, %rax1965; SSE-NEXT: xorl $63, %eax1966; SSE-NEXT: bsrq %rcx, %rbx1967; SSE-NEXT: xorl $63, %ebx1968; SSE-NEXT: orl $64, %ebx1969; SSE-NEXT: testq %rdx, %rdx1970; SSE-NEXT: cmovnel %eax, %ebx1971; SSE-NEXT: subl $-128, %ebx1972; SSE-NEXT: movq %rsi, %rax1973; SSE-NEXT: orq %r8, %rax1974; SSE-NEXT: cmovnel %r14d, %ebx1975; SSE-NEXT: bsrq %r10, %rax1976; SSE-NEXT: xorl $63, %eax1977; SSE-NEXT: bsrq %r9, %r141978; SSE-NEXT: xorl $63, %r14d1979; SSE-NEXT: orl $64, %r14d1980; SSE-NEXT: testq %r10, %r101981; SSE-NEXT: cmovnel %eax, %r14d1982; SSE-NEXT: bsrq %r11, %r151983; SSE-NEXT: xorl $63, %r15d1984; SSE-NEXT: bsrq (%rdi), %rax1985; SSE-NEXT: xorl $63, %eax1986; SSE-NEXT: orl $64, %eax1987; SSE-NEXT: testq %r11, %r111988; SSE-NEXT: cmovnel %r15d, %eax1989; SSE-NEXT: subl $-128, %eax1990; SSE-NEXT: orq %r10, %r91991; SSE-NEXT: cmovnel %r14d, %eax1992; SSE-NEXT: addl $256, %eax # imm = 0x1001993; SSE-NEXT: orq %r8, %rdx1994; SSE-NEXT: orq %rsi, %rcx1995; SSE-NEXT: orq %rdx, %rcx1996; SSE-NEXT: cmovnel %ebx, %eax1997; SSE-NEXT: # kill: def $eax killed $eax killed $rax1998; SSE-NEXT: popq %rbx1999; SSE-NEXT: popq %r142000; SSE-NEXT: popq %r152001; SSE-NEXT: retq2002;2003; AVX2-LABEL: load_ctlz_undef_i512:2004; AVX2: # %bb.0:2005; AVX2-NEXT: pushq %r152006; AVX2-NEXT: pushq %r142007; AVX2-NEXT: pushq %rbx2008; AVX2-NEXT: movq 8(%rdi), %r102009; AVX2-NEXT: movq 16(%rdi), %r92010; AVX2-NEXT: movq 32(%rdi), %rcx2011; AVX2-NEXT: movq 40(%rdi), %rdx2012; AVX2-NEXT: movq 48(%rdi), %rsi2013; AVX2-NEXT: movq 56(%rdi), %r82014; AVX2-NEXT: lzcntq %r8, %rax2015; AVX2-NEXT: xorl %ebx, %ebx2016; AVX2-NEXT: lzcntq %rsi, %rbx2017; AVX2-NEXT: addl $64, %ebx2018; AVX2-NEXT: testq %r8, %r82019; AVX2-NEXT: cmovnel %eax, %ebx2020; AVX2-NEXT: xorl %eax, %eax2021; AVX2-NEXT: lzcntq %rdx, %rax2022; AVX2-NEXT: lzcntq %rcx, %r112023; AVX2-NEXT: addl $64, %r11d2024; AVX2-NEXT: testq %rdx, %rdx2025; AVX2-NEXT: cmovnel %eax, %r11d2026; AVX2-NEXT: subl $-128, %r11d2027; AVX2-NEXT: movq %rsi, %rax2028; AVX2-NEXT: orq %r8, %rax2029; AVX2-NEXT: cmovnel %ebx, %r11d2030; AVX2-NEXT: movq 24(%rdi), %rbx2031; AVX2-NEXT: xorl %eax, %eax2032; AVX2-NEXT: lzcntq %rbx, %rax2033; AVX2-NEXT: xorl %r14d, %r14d2034; AVX2-NEXT: lzcntq %r9, %r142035; AVX2-NEXT: addl $64, %r14d2036; AVX2-NEXT: testq %rbx, %rbx2037; AVX2-NEXT: cmovnel %eax, %r14d2038; AVX2-NEXT: xorl %r15d, %r15d2039; AVX2-NEXT: lzcntq %r10, %r152040; AVX2-NEXT: xorl %eax, %eax2041; AVX2-NEXT: lzcntq (%rdi), %rax2042; AVX2-NEXT: addl $64, %eax2043; AVX2-NEXT: testq %r10, %r102044; AVX2-NEXT: cmovnel %r15d, %eax2045; AVX2-NEXT: subl $-128, %eax2046; AVX2-NEXT: orq %rbx, %r92047; AVX2-NEXT: cmovnel %r14d, %eax2048; AVX2-NEXT: addl $256, %eax # imm = 0x1002049; AVX2-NEXT: orq %r8, %rdx2050; AVX2-NEXT: orq %rsi, %rcx2051; AVX2-NEXT: orq %rdx, %rcx2052; AVX2-NEXT: cmovnel %r11d, %eax2053; AVX2-NEXT: # kill: def $eax killed $eax killed $rax2054; AVX2-NEXT: popq %rbx2055; AVX2-NEXT: popq %r142056; AVX2-NEXT: popq %r152057; AVX2-NEXT: retq2058;2059; AVX512-LABEL: load_ctlz_undef_i512:2060; AVX512: # %bb.0:2061; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]2062; AVX512-NEXT: vpermq (%rdi), %zmm0, %zmm02063; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k12064; AVX512-NEXT: vplzcntq %zmm0, %zmm02065; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm02066; AVX512-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}2067; AVX512-NEXT: vmovd %xmm0, %eax2068; AVX512-NEXT: vzeroupper2069; AVX512-NEXT: retq2070 %a0 = load i512, ptr %p02071 %cnt = call i512 @llvm.ctlz.i512(i512 %a0, i1 -1)2072 %res = trunc i512 %cnt to i322073 ret i32 %res2074}2075 2076define i32 @test_ctlz_undef_i1024(i1024 %a0) nounwind {2077; SSE-LABEL: test_ctlz_undef_i1024:2078; SSE: # %bb.0:2079; SSE-NEXT: pushq %rbp2080; SSE-NEXT: pushq %r152081; SSE-NEXT: pushq %r142082; SSE-NEXT: pushq %r132083; SSE-NEXT: pushq %r122084; SSE-NEXT: pushq %rbx2085; SSE-NEXT: movq %r9, %r122086; SSE-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2087; SSE-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2088; SSE-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2089; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r142090; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r132091; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r152092; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx2093; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rsi2094; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r112095; SSE-NEXT: bsrq %r11, %rax2096; SSE-NEXT: xorl $63, %eax2097; SSE-NEXT: bsrq %rsi, %rcx2098; SSE-NEXT: xorl $63, %ecx2099; SSE-NEXT: orl $64, %ecx2100; SSE-NEXT: testq %r11, %r112101; SSE-NEXT: cmovnel %eax, %ecx2102; SSE-NEXT: bsrq %rdx, %r102103; SSE-NEXT: xorl $63, %r10d2104; SSE-NEXT: bsrq {{[0-9]+}}(%rsp), %rax2105; SSE-NEXT: xorl $63, %eax2106; SSE-NEXT: orl $64, %eax2107; SSE-NEXT: testq %rdx, %rdx2108; SSE-NEXT: cmovnel %r10d, %eax2109; SSE-NEXT: subl $-128, %eax2110; SSE-NEXT: movq %rsi, %r92111; SSE-NEXT: movq %rsi, %rbx2112; SSE-NEXT: orq %r11, %r92113; SSE-NEXT: cmovnel %ecx, %eax2114; SSE-NEXT: bsrq %r15, %rcx2115; SSE-NEXT: xorl $63, %ecx2116; SSE-NEXT: bsrq %r13, %rsi2117; SSE-NEXT: xorl $63, %esi2118; SSE-NEXT: orl $64, %esi2119; SSE-NEXT: testq %r15, %r152120; SSE-NEXT: cmovnel %ecx, %esi2121; SSE-NEXT: bsrq %r14, %rcx2122; SSE-NEXT: xorl $63, %ecx2123; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r92124; SSE-NEXT: bsrq %r9, %rbp2125; SSE-NEXT: xorl $63, %ebp2126; SSE-NEXT: orl $64, %ebp2127; SSE-NEXT: testq %r14, %r142128; SSE-NEXT: cmovnel %ecx, %ebp2129; SSE-NEXT: movq %r8, %r102130; SSE-NEXT: subl $-128, %ebp2131; SSE-NEXT: movq %r13, %rcx2132; SSE-NEXT: orq %r15, %rcx2133; SSE-NEXT: cmovnel %esi, %ebp2134; SSE-NEXT: addl $256, %ebp # imm = 0x1002135; SSE-NEXT: orq %r11, %rdx2136; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rsi2137; SSE-NEXT: orq %rbx, %rsi2138; SSE-NEXT: orq %rdx, %rsi2139; SSE-NEXT: cmovnel %eax, %ebp2140; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx2141; SSE-NEXT: bsrq %rdx, %rcx2142; SSE-NEXT: xorl $63, %ecx2143; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r82144; SSE-NEXT: bsrq %r8, %rax2145; SSE-NEXT: xorl $63, %eax2146; SSE-NEXT: orl $64, %eax2147; SSE-NEXT: testq %rdx, %rdx2148; SSE-NEXT: cmovnel %ecx, %eax2149; SSE-NEXT: bsrq %r12, %rsi2150; SSE-NEXT: xorl $63, %esi2151; SSE-NEXT: bsrq %r10, %rcx2152; SSE-NEXT: xorl $63, %ecx2153; SSE-NEXT: orl $64, %ecx2154; SSE-NEXT: testq %r12, %r122155; SSE-NEXT: cmovnel %esi, %ecx2156; SSE-NEXT: movq %rdi, %rbx2157; SSE-NEXT: subl $-128, %ecx2158; SSE-NEXT: movq %r8, %rsi2159; SSE-NEXT: orq %rdx, %rsi2160; SSE-NEXT: cmovnel %eax, %ecx2161; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload2162; SSE-NEXT: bsrq %r11, %rax2163; SSE-NEXT: xorl $63, %eax2164; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload2165; SSE-NEXT: bsrq %r8, %rdx2166; SSE-NEXT: xorl $63, %edx2167; SSE-NEXT: orl $64, %edx2168; SSE-NEXT: testq %r11, %r112169; SSE-NEXT: cmovnel %eax, %edx2170; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload2171; SSE-NEXT: bsrq %rdi, %rsi2172; SSE-NEXT: xorl $63, %esi2173; SSE-NEXT: bsrq %rbx, %rax2174; SSE-NEXT: xorl $63, %eax2175; SSE-NEXT: orl $64, %eax2176; SSE-NEXT: testq %rdi, %rdi2177; SSE-NEXT: cmovnel %esi, %eax2178; SSE-NEXT: subl $-128, %eax2179; SSE-NEXT: orq %r11, %r82180; SSE-NEXT: cmovnel %edx, %eax2181; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r122182; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r102183; SSE-NEXT: addl $256, %eax # imm = 0x1002184; SSE-NEXT: orq %r12, %r102185; SSE-NEXT: cmovnel %ecx, %eax2186; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r152187; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r142188; SSE-NEXT: orq %r15, %r142189; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r132190; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r92191; SSE-NEXT: orq %r13, %r92192; SSE-NEXT: addl $512, %eax # imm = 0x2002193; SSE-NEXT: orq %r14, %r92194; SSE-NEXT: cmovnel %ebp, %eax2195; SSE-NEXT: # kill: def $eax killed $eax killed $rax2196; SSE-NEXT: popq %rbx2197; SSE-NEXT: popq %r122198; SSE-NEXT: popq %r132199; SSE-NEXT: popq %r142200; SSE-NEXT: popq %r152201; SSE-NEXT: popq %rbp2202; SSE-NEXT: retq2203;2204; AVX2-LABEL: test_ctlz_undef_i1024:2205; AVX2: # %bb.0:2206; AVX2-NEXT: pushq %rbp2207; AVX2-NEXT: pushq %r152208; AVX2-NEXT: pushq %r142209; AVX2-NEXT: pushq %r132210; AVX2-NEXT: pushq %r122211; AVX2-NEXT: pushq %rbx2212; AVX2-NEXT: movq %r9, %r142213; AVX2-NEXT: movq %r8, %r112214; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2215; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2216; AVX2-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2217; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r152218; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx2219; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax2220; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r102221; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r82222; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r122223; AVX2-NEXT: xorl %ecx, %ecx2224; AVX2-NEXT: lzcntq %r12, %rcx2225; AVX2-NEXT: xorl %r9d, %r9d2226; AVX2-NEXT: lzcntq %r8, %r92227; AVX2-NEXT: addl $64, %r9d2228; AVX2-NEXT: testq %r12, %r122229; AVX2-NEXT: cmovnel %ecx, %r9d2230; AVX2-NEXT: xorl %esi, %esi2231; AVX2-NEXT: lzcntq %r10, %rsi2232; AVX2-NEXT: xorl %ecx, %ecx2233; AVX2-NEXT: lzcntq %rax, %rcx2234; AVX2-NEXT: addl $64, %ecx2235; AVX2-NEXT: testq %r10, %r102236; AVX2-NEXT: cmovnel %esi, %ecx2237; AVX2-NEXT: subl $-128, %ecx2238; AVX2-NEXT: movq %r8, %rsi2239; AVX2-NEXT: orq %r12, %rsi2240; AVX2-NEXT: cmovnel %r9d, %ecx2241; AVX2-NEXT: xorl %edi, %edi2242; AVX2-NEXT: lzcntq %rbx, %rdi2243; AVX2-NEXT: xorl %esi, %esi2244; AVX2-NEXT: lzcntq %r15, %rsi2245; AVX2-NEXT: addl $64, %esi2246; AVX2-NEXT: testq %rbx, %rbx2247; AVX2-NEXT: cmovnel %edi, %esi2248; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r132249; AVX2-NEXT: xorl %ebp, %ebp2250; AVX2-NEXT: lzcntq %r13, %rbp2251; AVX2-NEXT: addl $64, %ebp2252; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r92253; AVX2-NEXT: xorl %edi, %edi2254; AVX2-NEXT: lzcntq %r9, %rdi2255; AVX2-NEXT: testq %r9, %r92256; AVX2-NEXT: cmovnel %edi, %ebp2257; AVX2-NEXT: subl $-128, %ebp2258; AVX2-NEXT: movq %r15, %rdi2259; AVX2-NEXT: orq %rbx, %rdi2260; AVX2-NEXT: cmovnel %esi, %ebp2261; AVX2-NEXT: addl $256, %ebp # imm = 0x1002262; AVX2-NEXT: movq %r10, %rdi2263; AVX2-NEXT: orq %r12, %rdi2264; AVX2-NEXT: movq %rax, %rsi2265; AVX2-NEXT: orq %r8, %rsi2266; AVX2-NEXT: orq %rdi, %rsi2267; AVX2-NEXT: cmovnel %ecx, %ebp2268; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdi2269; AVX2-NEXT: xorl %eax, %eax2270; AVX2-NEXT: lzcntq %rdi, %rax2271; AVX2-NEXT: addl $64, %eax2272; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r122273; AVX2-NEXT: xorl %ecx, %ecx2274; AVX2-NEXT: lzcntq %r12, %rcx2275; AVX2-NEXT: testq %r12, %r122276; AVX2-NEXT: cmovnel %ecx, %eax2277; AVX2-NEXT: xorl %ecx, %ecx2278; AVX2-NEXT: lzcntq %r11, %rcx2279; AVX2-NEXT: addl $64, %ecx2280; AVX2-NEXT: xorl %esi, %esi2281; AVX2-NEXT: lzcntq %r14, %rsi2282; AVX2-NEXT: testq %r14, %r142283; AVX2-NEXT: cmovnel %esi, %ecx2284; AVX2-NEXT: subl $-128, %ecx2285; AVX2-NEXT: movq %rdi, %rsi2286; AVX2-NEXT: orq %r12, %rsi2287; AVX2-NEXT: cmovnel %eax, %ecx2288; AVX2-NEXT: movq %rdx, %rdi2289; AVX2-NEXT: lzcntq %rdx, %rdx2290; AVX2-NEXT: addl $64, %edx2291; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload2292; AVX2-NEXT: xorl %eax, %eax2293; AVX2-NEXT: lzcntq %r10, %rax2294; AVX2-NEXT: testq %r10, %r102295; AVX2-NEXT: cmovnel %eax, %edx2296; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload2297; AVX2-NEXT: lzcntq %rax, %rax2298; AVX2-NEXT: addl $64, %eax2299; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload2300; AVX2-NEXT: lzcntq %rsi, %r82301; AVX2-NEXT: testq %rsi, %rsi2302; AVX2-NEXT: cmovnel %r8d, %eax2303; AVX2-NEXT: subl $-128, %eax2304; AVX2-NEXT: orq %r10, %rdi2305; AVX2-NEXT: cmovnel %edx, %eax2306; AVX2-NEXT: orq %r12, %r142307; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r112308; AVX2-NEXT: addl $256, %eax # imm = 0x1002309; AVX2-NEXT: orq %r14, %r112310; AVX2-NEXT: cmovnel %ecx, %eax2311; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %rbx2312; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r92313; AVX2-NEXT: orq %rbx, %r92314; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r152315; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r132316; AVX2-NEXT: orq %r15, %r132317; AVX2-NEXT: addl $512, %eax # imm = 0x2002318; AVX2-NEXT: orq %r9, %r132319; AVX2-NEXT: cmovnel %ebp, %eax2320; AVX2-NEXT: # kill: def $eax killed $eax killed $rax2321; AVX2-NEXT: popq %rbx2322; AVX2-NEXT: popq %r122323; AVX2-NEXT: popq %r132324; AVX2-NEXT: popq %r142325; AVX2-NEXT: popq %r152326; AVX2-NEXT: popq %rbp2327; AVX2-NEXT: retq2328;2329; AVX512-LABEL: test_ctlz_undef_i1024:2330; AVX512: # %bb.0:2331; AVX512-NEXT: pushq %r142332; AVX512-NEXT: pushq %rbx2333; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx2334; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r102335; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r112336; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r142337; AVX512-NEXT: vmovq %rdi, %xmm02338; AVX512-NEXT: vmovq %rsi, %xmm12339; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]2340; AVX512-NEXT: vmovq %rdx, %xmm12341; AVX512-NEXT: vmovq %rcx, %xmm22342; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]2343; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm02344; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]2345; AVX512-NEXT: vmovq %r8, %xmm22346; AVX512-NEXT: vmovq %r9, %xmm32347; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]2348; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm12349; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm02350; AVX512-NEXT: vplzcntq %zmm0, %zmm12351; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]2352; AVX512-NEXT: vpaddq %zmm2, %zmm1, %zmm12353; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k12354; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}2355; AVX512-NEXT: vmovd %xmm0, %ecx2356; AVX512-NEXT: addl $512, %ecx # imm = 0x2002357; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]2358; AVX512-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm02359; AVX512-NEXT: vplzcntq %zmm0, %zmm12360; AVX512-NEXT: vpaddq %zmm2, %zmm1, %zmm12361; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k12362; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}2363; AVX512-NEXT: vmovd %xmm0, %eax2364; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %r142365; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %r112366; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %rbx2367; AVX512-NEXT: orq %r14, %r112368; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %r102369; AVX512-NEXT: orq %rbx, %r102370; AVX512-NEXT: orq %r11, %r102371; AVX512-NEXT: cmovel %ecx, %eax2372; AVX512-NEXT: popq %rbx2373; AVX512-NEXT: popq %r142374; AVX512-NEXT: vzeroupper2375; AVX512-NEXT: retq2376 %cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 -1)2377 %res = trunc i1024 %cnt to i322378 ret i32 %res2379}2380 2381define i32 @load_ctlz_undef_i1024(ptr %p0) nounwind {2382; SSE-LABEL: load_ctlz_undef_i1024:2383; SSE: # %bb.0:2384; SSE-NEXT: pushq %rbp2385; SSE-NEXT: pushq %r152386; SSE-NEXT: pushq %r142387; SSE-NEXT: pushq %r132388; SSE-NEXT: pushq %r122389; SSE-NEXT: pushq %rbx2390; SSE-NEXT: movq 40(%rdi), %rbp2391; SSE-NEXT: movq 64(%rdi), %rbx2392; SSE-NEXT: movq 72(%rdi), %r112393; SSE-NEXT: movq 80(%rdi), %r122394; SSE-NEXT: movq 88(%rdi), %r142395; SSE-NEXT: movq 96(%rdi), %r132396; SSE-NEXT: movq 104(%rdi), %r92397; SSE-NEXT: movq 112(%rdi), %r102398; SSE-NEXT: movq 120(%rdi), %r82399; SSE-NEXT: bsrq %r8, %rax2400; SSE-NEXT: xorl $63, %eax2401; SSE-NEXT: bsrq %r10, %rcx2402; SSE-NEXT: xorl $63, %ecx2403; SSE-NEXT: orl $64, %ecx2404; SSE-NEXT: testq %r8, %r82405; SSE-NEXT: cmovnel %eax, %ecx2406; SSE-NEXT: bsrq %r9, %rdx2407; SSE-NEXT: xorl $63, %edx2408; SSE-NEXT: bsrq %r13, %rax2409; SSE-NEXT: movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2410; SSE-NEXT: xorl $63, %eax2411; SSE-NEXT: orl $64, %eax2412; SSE-NEXT: testq %r9, %r92413; SSE-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2414; SSE-NEXT: cmovnel %edx, %eax2415; SSE-NEXT: subl $-128, %eax2416; SSE-NEXT: movq %r10, %rdx2417; SSE-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2418; SSE-NEXT: orq %r8, %rdx2419; SSE-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2420; SSE-NEXT: cmovnel %ecx, %eax2421; SSE-NEXT: bsrq %r14, %rcx2422; SSE-NEXT: xorl $63, %ecx2423; SSE-NEXT: movq %r12, %rsi2424; SSE-NEXT: movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2425; SSE-NEXT: bsrq %r12, %rdx2426; SSE-NEXT: xorl $63, %edx2427; SSE-NEXT: orl $64, %edx2428; SSE-NEXT: testq %r14, %r142429; SSE-NEXT: cmovnel %ecx, %edx2430; SSE-NEXT: bsrq %r11, %rcx2431; SSE-NEXT: xorl $63, %ecx2432; SSE-NEXT: bsrq %rbx, %r152433; SSE-NEXT: xorl $63, %r15d2434; SSE-NEXT: orl $64, %r15d2435; SSE-NEXT: testq %r11, %r112436; SSE-NEXT: cmovnel %ecx, %r15d2437; SSE-NEXT: movq 48(%rdi), %r122438; SSE-NEXT: subl $-128, %r15d2439; SSE-NEXT: movq %rsi, %rcx2440; SSE-NEXT: orq %r14, %rcx2441; SSE-NEXT: cmovnel %edx, %r15d2442; SSE-NEXT: addl $256, %r15d # imm = 0x1002443; SSE-NEXT: movq %r9, %rcx2444; SSE-NEXT: orq %r8, %rcx2445; SSE-NEXT: movq %r13, %rdx2446; SSE-NEXT: orq %r10, %rdx2447; SSE-NEXT: orq %rcx, %rdx2448; SSE-NEXT: movq 56(%rdi), %r132449; SSE-NEXT: cmovnel %eax, %r15d2450; SSE-NEXT: bsrq %r13, %rax2451; SSE-NEXT: xorl $63, %eax2452; SSE-NEXT: bsrq %r12, %rdx2453; SSE-NEXT: xorl $63, %edx2454; SSE-NEXT: orl $64, %edx2455; SSE-NEXT: testq %r13, %r132456; SSE-NEXT: cmovnel %eax, %edx2457; SSE-NEXT: movq %rbp, %r102458; SSE-NEXT: bsrq %rbp, %rax2459; SSE-NEXT: xorl $63, %eax2460; SSE-NEXT: movq 32(%rdi), %r82461; SSE-NEXT: bsrq %r8, %rbp2462; SSE-NEXT: xorl $63, %ebp2463; SSE-NEXT: orl $64, %ebp2464; SSE-NEXT: testq %r10, %r102465; SSE-NEXT: cmovnel %eax, %ebp2466; SSE-NEXT: subl $-128, %ebp2467; SSE-NEXT: movq %r12, %rax2468; SSE-NEXT: orq %r13, %rax2469; SSE-NEXT: cmovnel %edx, %ebp2470; SSE-NEXT: movq 24(%rdi), %r92471; SSE-NEXT: bsrq %r9, %rax2472; SSE-NEXT: xorl $63, %eax2473; SSE-NEXT: movq 16(%rdi), %rsi2474; SSE-NEXT: bsrq %rsi, %rcx2475; SSE-NEXT: xorl $63, %ecx2476; SSE-NEXT: orl $64, %ecx2477; SSE-NEXT: testq %r9, %r92478; SSE-NEXT: cmovnel %eax, %ecx2479; SSE-NEXT: movq 8(%rdi), %rdx2480; SSE-NEXT: bsrq (%rdi), %rax2481; SSE-NEXT: bsrq %rdx, %rdi2482; SSE-NEXT: xorl $63, %edi2483; SSE-NEXT: xorl $63, %eax2484; SSE-NEXT: orl $64, %eax2485; SSE-NEXT: testq %rdx, %rdx2486; SSE-NEXT: cmovnel %edi, %eax2487; SSE-NEXT: subl $-128, %eax2488; SSE-NEXT: orq %r9, %rsi2489; SSE-NEXT: cmovnel %ecx, %eax2490; SSE-NEXT: orq %r13, %r102491; SSE-NEXT: orq %r12, %r82492; SSE-NEXT: addl $256, %eax # imm = 0x1002493; SSE-NEXT: orq %r10, %r82494; SSE-NEXT: cmovnel %ebp, %eax2495; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload2496; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload2497; SSE-NEXT: orq %r14, %r112498; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload2499; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload2500; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload2501; SSE-NEXT: orq %rcx, %rbx2502; SSE-NEXT: addl $512, %eax # imm = 0x2002503; SSE-NEXT: orq %r11, %rbx2504; SSE-NEXT: cmovnel %r15d, %eax2505; SSE-NEXT: # kill: def $eax killed $eax killed $rax2506; SSE-NEXT: popq %rbx2507; SSE-NEXT: popq %r122508; SSE-NEXT: popq %r132509; SSE-NEXT: popq %r142510; SSE-NEXT: popq %r152511; SSE-NEXT: popq %rbp2512; SSE-NEXT: retq2513;2514; AVX2-LABEL: load_ctlz_undef_i1024:2515; AVX2: # %bb.0:2516; AVX2-NEXT: pushq %rbp2517; AVX2-NEXT: pushq %r152518; AVX2-NEXT: pushq %r142519; AVX2-NEXT: pushq %r132520; AVX2-NEXT: pushq %r122521; AVX2-NEXT: pushq %rbx2522; AVX2-NEXT: movq 48(%rdi), %r92523; AVX2-NEXT: movq 56(%rdi), %rbp2524; AVX2-NEXT: movq 64(%rdi), %r112525; AVX2-NEXT: movq 72(%rdi), %r102526; AVX2-NEXT: movq 80(%rdi), %r142527; AVX2-NEXT: movq 88(%rdi), %rbx2528; AVX2-NEXT: movq 96(%rdi), %rdx2529; AVX2-NEXT: movq 104(%rdi), %r82530; AVX2-NEXT: movq 112(%rdi), %rsi2531; AVX2-NEXT: movq 120(%rdi), %r152532; AVX2-NEXT: lzcntq %r15, %rax2533; AVX2-NEXT: lzcntq %rsi, %rcx2534; AVX2-NEXT: addl $64, %ecx2535; AVX2-NEXT: testq %r15, %r152536; AVX2-NEXT: cmovnel %eax, %ecx2537; AVX2-NEXT: xorl %r12d, %r12d2538; AVX2-NEXT: lzcntq %r8, %r122539; AVX2-NEXT: xorl %eax, %eax2540; AVX2-NEXT: lzcntq %rdx, %rax2541; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2542; AVX2-NEXT: addl $64, %eax2543; AVX2-NEXT: testq %r8, %r82544; AVX2-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2545; AVX2-NEXT: cmovnel %r12d, %eax2546; AVX2-NEXT: subl $-128, %eax2547; AVX2-NEXT: movq %rsi, %r122548; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2549; AVX2-NEXT: orq %r15, %r122550; AVX2-NEXT: cmovnel %ecx, %eax2551; AVX2-NEXT: xorl %ecx, %ecx2552; AVX2-NEXT: lzcntq %rbx, %rcx2553; AVX2-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2554; AVX2-NEXT: xorl %r13d, %r13d2555; AVX2-NEXT: lzcntq %r14, %r132556; AVX2-NEXT: addl $64, %r13d2557; AVX2-NEXT: testq %rbx, %rbx2558; AVX2-NEXT: cmovnel %ecx, %r13d2559; AVX2-NEXT: xorl %ecx, %ecx2560; AVX2-NEXT: lzcntq %r10, %rcx2561; AVX2-NEXT: xorl %r12d, %r12d2562; AVX2-NEXT: lzcntq %r11, %r122563; AVX2-NEXT: addl $64, %r12d2564; AVX2-NEXT: testq %r10, %r102565; AVX2-NEXT: cmovnel %ecx, %r12d2566; AVX2-NEXT: subl $-128, %r12d2567; AVX2-NEXT: movq %r14, %rcx2568; AVX2-NEXT: orq %rbx, %rcx2569; AVX2-NEXT: cmovnel %r13d, %r12d2570; AVX2-NEXT: addl $256, %r12d # imm = 0x1002571; AVX2-NEXT: movq %r8, %rcx2572; AVX2-NEXT: orq %r15, %rcx2573; AVX2-NEXT: orq %rsi, %rdx2574; AVX2-NEXT: orq %rcx, %rdx2575; AVX2-NEXT: cmovnel %eax, %r12d2576; AVX2-NEXT: movq %rbp, %r142577; AVX2-NEXT: xorl %ecx, %ecx2578; AVX2-NEXT: lzcntq %rbp, %rcx2579; AVX2-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2580; AVX2-NEXT: xorl %eax, %eax2581; AVX2-NEXT: lzcntq %r9, %rax2582; AVX2-NEXT: addl $64, %eax2583; AVX2-NEXT: testq %rbp, %rbp2584; AVX2-NEXT: cmovnel %ecx, %eax2585; AVX2-NEXT: movq 32(%rdi), %r132586; AVX2-NEXT: xorl %ebp, %ebp2587; AVX2-NEXT: lzcntq %r13, %rbp2588; AVX2-NEXT: addl $64, %ebp2589; AVX2-NEXT: movq 40(%rdi), %r82590; AVX2-NEXT: xorl %edx, %edx2591; AVX2-NEXT: lzcntq %r8, %rdx2592; AVX2-NEXT: testq %r8, %r82593; AVX2-NEXT: cmovnel %edx, %ebp2594; AVX2-NEXT: subl $-128, %ebp2595; AVX2-NEXT: movq %r9, %rdx2596; AVX2-NEXT: orq %r14, %rdx2597; AVX2-NEXT: cmovnel %eax, %ebp2598; AVX2-NEXT: movq 16(%rdi), %r92599; AVX2-NEXT: xorl %ecx, %ecx2600; AVX2-NEXT: lzcntq %r9, %rcx2601; AVX2-NEXT: addl $64, %ecx2602; AVX2-NEXT: movq 24(%rdi), %rdx2603; AVX2-NEXT: xorl %eax, %eax2604; AVX2-NEXT: lzcntq %rdx, %rax2605; AVX2-NEXT: testq %rdx, %rdx2606; AVX2-NEXT: cmovnel %eax, %ecx2607; AVX2-NEXT: movq 8(%rdi), %rsi2608; AVX2-NEXT: xorl %eax, %eax2609; AVX2-NEXT: lzcntq (%rdi), %rax2610; AVX2-NEXT: addl $64, %eax2611; AVX2-NEXT: lzcntq %rsi, %rdi2612; AVX2-NEXT: testq %rsi, %rsi2613; AVX2-NEXT: cmovnel %edi, %eax2614; AVX2-NEXT: subl $-128, %eax2615; AVX2-NEXT: orq %rdx, %r92616; AVX2-NEXT: cmovnel %ecx, %eax2617; AVX2-NEXT: orq %r14, %r82618; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload2619; AVX2-NEXT: addl $256, %eax # imm = 0x1002620; AVX2-NEXT: orq %r8, %r132621; AVX2-NEXT: cmovnel %ebp, %eax2622; AVX2-NEXT: orq %r15, %rbx2623; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Folded Reload2624; AVX2-NEXT: orq %rbx, %r102625; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload2626; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload2627; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload2628; AVX2-NEXT: orq %rcx, %r112629; AVX2-NEXT: addl $512, %eax # imm = 0x2002630; AVX2-NEXT: orq %r10, %r112631; AVX2-NEXT: cmovnel %r12d, %eax2632; AVX2-NEXT: # kill: def $eax killed $eax killed $rax2633; AVX2-NEXT: popq %rbx2634; AVX2-NEXT: popq %r122635; AVX2-NEXT: popq %r132636; AVX2-NEXT: popq %r142637; AVX2-NEXT: popq %r152638; AVX2-NEXT: popq %rbp2639; AVX2-NEXT: retq2640;2641; AVX512-LABEL: load_ctlz_undef_i1024:2642; AVX512: # %bb.0:2643; AVX512-NEXT: movq 80(%rdi), %rsi2644; AVX512-NEXT: movq 64(%rdi), %rcx2645; AVX512-NEXT: movq 72(%rdi), %rdx2646; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]2647; AVX512-NEXT: vpermq 64(%rdi), %zmm0, %zmm12648; AVX512-NEXT: movq 88(%rdi), %r82649; AVX512-NEXT: vplzcntq %zmm1, %zmm22650; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]2651; AVX512-NEXT: vpaddq %zmm3, %zmm2, %zmm22652; AVX512-NEXT: vptestmq %zmm1, %zmm1, %k12653; AVX512-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}2654; AVX512-NEXT: vmovd %xmm1, %r9d2655; AVX512-NEXT: vpermq (%rdi), %zmm0, %zmm02656; AVX512-NEXT: vplzcntq %zmm0, %zmm12657; AVX512-NEXT: vpaddq %zmm3, %zmm1, %zmm12658; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k12659; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}2660; AVX512-NEXT: vmovd %xmm0, %eax2661; AVX512-NEXT: addl $512, %eax # imm = 0x2002662; AVX512-NEXT: orq 120(%rdi), %r82663; AVX512-NEXT: orq 104(%rdi), %rdx2664; AVX512-NEXT: orq 112(%rdi), %rsi2665; AVX512-NEXT: orq %r8, %rdx2666; AVX512-NEXT: orq 96(%rdi), %rcx2667; AVX512-NEXT: orq %rsi, %rcx2668; AVX512-NEXT: orq %rdx, %rcx2669; AVX512-NEXT: cmovnel %r9d, %eax2670; AVX512-NEXT: vzeroupper2671; AVX512-NEXT: retq2672 %a0 = load i1024, ptr %p02673 %cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 -1)2674 %res = trunc i1024 %cnt to i322675 ret i32 %res2676}2677 2678;2679; CTTZ2680;2681 2682define i32 @test_cttz_i128(i128 %a0) nounwind {2683; SSE-LABEL: test_cttz_i128:2684; SSE: # %bb.0:2685; SSE-NEXT: rep bsfq %rdi, %rcx2686; SSE-NEXT: movl $64, %eax2687; SSE-NEXT: rep bsfq %rsi, %rax2688; SSE-NEXT: addl $64, %eax2689; SSE-NEXT: testq %rdi, %rdi2690; SSE-NEXT: cmovnel %ecx, %eax2691; SSE-NEXT: # kill: def $eax killed $eax killed $rax2692; SSE-NEXT: retq2693;2694; AVX2-LABEL: test_cttz_i128:2695; AVX2: # %bb.0:2696; AVX2-NEXT: tzcntq %rdi, %rcx2697; AVX2-NEXT: tzcntq %rsi, %rax2698; AVX2-NEXT: addl $64, %eax2699; AVX2-NEXT: testq %rdi, %rdi2700; AVX2-NEXT: cmovnel %ecx, %eax2701; AVX2-NEXT: # kill: def $eax killed $eax killed $rax2702; AVX2-NEXT: retq2703;2704; AVX512-LABEL: test_cttz_i128:2705; AVX512: # %bb.0:2706; AVX512-NEXT: tzcntq %rdi, %rcx2707; AVX512-NEXT: tzcntq %rsi, %rax2708; AVX512-NEXT: addl $64, %eax2709; AVX512-NEXT: testq %rdi, %rdi2710; AVX512-NEXT: cmovnel %ecx, %eax2711; AVX512-NEXT: # kill: def $eax killed $eax killed $rax2712; AVX512-NEXT: retq2713 %cnt = call i128 @llvm.cttz.i128(i128 %a0, i1 0)2714 %res = trunc i128 %cnt to i322715 ret i32 %res2716}2717 2718define i32 @load_cttz_i128(ptr %p0) nounwind {2719; SSE-LABEL: load_cttz_i128:2720; SSE: # %bb.0:2721; SSE-NEXT: movq (%rdi), %rcx2722; SSE-NEXT: rep bsfq %rcx, %rdx2723; SSE-NEXT: movl $64, %eax2724; SSE-NEXT: rep bsfq 8(%rdi), %rax2725; SSE-NEXT: addl $64, %eax2726; SSE-NEXT: testq %rcx, %rcx2727; SSE-NEXT: cmovnel %edx, %eax2728; SSE-NEXT: # kill: def $eax killed $eax killed $rax2729; SSE-NEXT: retq2730;2731; AVX2-LABEL: load_cttz_i128:2732; AVX2: # %bb.0:2733; AVX2-NEXT: movq (%rdi), %rcx2734; AVX2-NEXT: tzcntq %rcx, %rdx2735; AVX2-NEXT: tzcntq 8(%rdi), %rax2736; AVX2-NEXT: addl $64, %eax2737; AVX2-NEXT: testq %rcx, %rcx2738; AVX2-NEXT: cmovnel %edx, %eax2739; AVX2-NEXT: # kill: def $eax killed $eax killed $rax2740; AVX2-NEXT: retq2741;2742; AVX512-LABEL: load_cttz_i128:2743; AVX512: # %bb.0:2744; AVX512-NEXT: movq (%rdi), %rcx2745; AVX512-NEXT: tzcntq %rcx, %rdx2746; AVX512-NEXT: tzcntq 8(%rdi), %rax2747; AVX512-NEXT: addl $64, %eax2748; AVX512-NEXT: testq %rcx, %rcx2749; AVX512-NEXT: cmovnel %edx, %eax2750; AVX512-NEXT: # kill: def $eax killed $eax killed $rax2751; AVX512-NEXT: retq2752 %a0 = load i128, ptr %p02753 %cnt = call i128 @llvm.cttz.i128(i128 %a0, i1 0)2754 %res = trunc i128 %cnt to i322755 ret i32 %res2756}2757 2758define i32 @test_cttz_i256(i256 %a0) nounwind {2759; SSE-LABEL: test_cttz_i256:2760; SSE: # %bb.0:2761; SSE-NEXT: rep bsfq %rdi, %rax2762; SSE-NEXT: rep bsfq %rsi, %r82763; SSE-NEXT: addl $64, %r8d2764; SSE-NEXT: testq %rdi, %rdi2765; SSE-NEXT: cmovnel %eax, %r8d2766; SSE-NEXT: rep bsfq %rdx, %r92767; SSE-NEXT: movl $64, %eax2768; SSE-NEXT: rep bsfq %rcx, %rax2769; SSE-NEXT: addl $64, %eax2770; SSE-NEXT: testq %rdx, %rdx2771; SSE-NEXT: cmovnel %r9d, %eax2772; SSE-NEXT: subl $-128, %eax2773; SSE-NEXT: orq %rsi, %rdi2774; SSE-NEXT: cmovnel %r8d, %eax2775; SSE-NEXT: # kill: def $eax killed $eax killed $rax2776; SSE-NEXT: retq2777;2778; AVX2-LABEL: test_cttz_i256:2779; AVX2: # %bb.0:2780; AVX2-NEXT: tzcntq %rdi, %rax2781; AVX2-NEXT: tzcntq %rsi, %r82782; AVX2-NEXT: addl $64, %r8d2783; AVX2-NEXT: testq %rdi, %rdi2784; AVX2-NEXT: cmovnel %eax, %r8d2785; AVX2-NEXT: tzcntq %rdx, %r92786; AVX2-NEXT: xorl %eax, %eax2787; AVX2-NEXT: tzcntq %rcx, %rax2788; AVX2-NEXT: addl $64, %eax2789; AVX2-NEXT: testq %rdx, %rdx2790; AVX2-NEXT: cmovnel %r9d, %eax2791; AVX2-NEXT: subl $-128, %eax2792; AVX2-NEXT: orq %rsi, %rdi2793; AVX2-NEXT: cmovnel %r8d, %eax2794; AVX2-NEXT: # kill: def $eax killed $eax killed $rax2795; AVX2-NEXT: retq2796;2797; AVX512-LABEL: test_cttz_i256:2798; AVX512: # %bb.0:2799; AVX512-NEXT: tzcntq %rdi, %rax2800; AVX512-NEXT: tzcntq %rsi, %r82801; AVX512-NEXT: addl $64, %r8d2802; AVX512-NEXT: testq %rdi, %rdi2803; AVX512-NEXT: cmovnel %eax, %r8d2804; AVX512-NEXT: tzcntq %rdx, %r92805; AVX512-NEXT: tzcntq %rcx, %rax2806; AVX512-NEXT: addl $64, %eax2807; AVX512-NEXT: testq %rdx, %rdx2808; AVX512-NEXT: cmovnel %r9d, %eax2809; AVX512-NEXT: subl $-128, %eax2810; AVX512-NEXT: orq %rsi, %rdi2811; AVX512-NEXT: cmovnel %r8d, %eax2812; AVX512-NEXT: # kill: def $eax killed $eax killed $rax2813; AVX512-NEXT: retq2814 %cnt = call i256 @llvm.cttz.i256(i256 %a0, i1 0)2815 %res = trunc i256 %cnt to i322816 ret i32 %res2817}2818 2819define i32 @load_cttz_i256(ptr %p0) nounwind {2820; SSE-LABEL: load_cttz_i256:2821; SSE: # %bb.0:2822; SSE-NEXT: movq 16(%rdi), %rcx2823; SSE-NEXT: movq (%rdi), %rdx2824; SSE-NEXT: movq 8(%rdi), %rsi2825; SSE-NEXT: rep bsfq %rdx, %rax2826; SSE-NEXT: rep bsfq %rsi, %r82827; SSE-NEXT: addl $64, %r8d2828; SSE-NEXT: testq %rdx, %rdx2829; SSE-NEXT: cmovnel %eax, %r8d2830; SSE-NEXT: rep bsfq %rcx, %r92831; SSE-NEXT: movl $64, %eax2832; SSE-NEXT: rep bsfq 24(%rdi), %rax2833; SSE-NEXT: addl $64, %eax2834; SSE-NEXT: testq %rcx, %rcx2835; SSE-NEXT: cmovnel %r9d, %eax2836; SSE-NEXT: subl $-128, %eax2837; SSE-NEXT: orq %rsi, %rdx2838; SSE-NEXT: cmovnel %r8d, %eax2839; SSE-NEXT: # kill: def $eax killed $eax killed $rax2840; SSE-NEXT: retq2841;2842; AVX2-LABEL: load_cttz_i256:2843; AVX2: # %bb.0:2844; AVX2-NEXT: movq (%rdi), %rcx2845; AVX2-NEXT: movq 8(%rdi), %rdx2846; AVX2-NEXT: tzcntq %rcx, %rax2847; AVX2-NEXT: tzcntq %rdx, %rsi2848; AVX2-NEXT: addl $64, %esi2849; AVX2-NEXT: testq %rcx, %rcx2850; AVX2-NEXT: cmovnel %eax, %esi2851; AVX2-NEXT: movq 16(%rdi), %r82852; AVX2-NEXT: tzcntq %r8, %r92853; AVX2-NEXT: xorl %eax, %eax2854; AVX2-NEXT: tzcntq 24(%rdi), %rax2855; AVX2-NEXT: addl $64, %eax2856; AVX2-NEXT: testq %r8, %r82857; AVX2-NEXT: cmovnel %r9d, %eax2858; AVX2-NEXT: subl $-128, %eax2859; AVX2-NEXT: orq %rdx, %rcx2860; AVX2-NEXT: cmovnel %esi, %eax2861; AVX2-NEXT: # kill: def $eax killed $eax killed $rax2862; AVX2-NEXT: retq2863;2864; AVX512F-LABEL: load_cttz_i256:2865; AVX512F: # %bb.0:2866; AVX512F-NEXT: vmovdqu (%rdi), %ymm02867; AVX512F-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm12868; AVX512F-NEXT: vpaddq %ymm1, %ymm0, %ymm12869; AVX512F-NEXT: vpandn %ymm1, %ymm0, %ymm12870; AVX512F-NEXT: vplzcntq %ymm1, %ymm12871; AVX512F-NEXT: vmovdqa {{.*#+}} ymm2 = [64,128,192,256]2872; AVX512F-NEXT: vpsubq %ymm1, %ymm2, %ymm12873; AVX512F-NEXT: vptestmq %ymm0, %ymm0, %k12874; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]2875; AVX512F-NEXT: vpcompressq %ymm1, %ymm0 {%k1}2876; AVX512F-NEXT: vmovd %xmm0, %eax2877; AVX512F-NEXT: vzeroupper2878; AVX512F-NEXT: retq2879;2880; AVX512POPCNT-LABEL: load_cttz_i256:2881; AVX512POPCNT: # %bb.0:2882; AVX512POPCNT-NEXT: vmovdqu (%rdi), %ymm02883; AVX512POPCNT-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm12884; AVX512POPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm12885; AVX512POPCNT-NEXT: vpandn %ymm1, %ymm0, %ymm12886; AVX512POPCNT-NEXT: vpopcntq %ymm1, %ymm12887; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm12888; AVX512POPCNT-NEXT: vptestmq %ymm0, %ymm0, %k12889; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]2890; AVX512POPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1}2891; AVX512POPCNT-NEXT: vmovd %xmm0, %eax2892; AVX512POPCNT-NEXT: vzeroupper2893; AVX512POPCNT-NEXT: retq2894 %a0 = load i256, ptr %p02895 %cnt = call i256 @llvm.cttz.i256(i256 %a0, i1 0)2896 %res = trunc i256 %cnt to i322897 ret i32 %res2898}2899 2900define i32 @test_cttz_i512(i512 %a0) nounwind {2901; SSE-LABEL: test_cttz_i512:2902; SSE: # %bb.0:2903; SSE-NEXT: pushq %r142904; SSE-NEXT: pushq %rbx2905; SSE-NEXT: rep bsfq %rdi, %rax2906; SSE-NEXT: rep bsfq %rsi, %r112907; SSE-NEXT: addl $64, %r11d2908; SSE-NEXT: testq %rdi, %rdi2909; SSE-NEXT: cmovnel %eax, %r11d2910; SSE-NEXT: rep bsfq %rdx, %rax2911; SSE-NEXT: rep bsfq %rcx, %r102912; SSE-NEXT: addl $64, %r10d2913; SSE-NEXT: testq %rdx, %rdx2914; SSE-NEXT: cmovnel %eax, %r10d2915; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx2916; SSE-NEXT: subl $-128, %r10d2917; SSE-NEXT: movq %rdi, %rax2918; SSE-NEXT: orq %rsi, %rax2919; SSE-NEXT: cmovnel %r11d, %r10d2920; SSE-NEXT: rep bsfq %r8, %rax2921; SSE-NEXT: rep bsfq %r9, %r112922; SSE-NEXT: addl $64, %r11d2923; SSE-NEXT: testq %r8, %r82924; SSE-NEXT: cmovnel %eax, %r11d2925; SSE-NEXT: rep bsfq %rbx, %r142926; SSE-NEXT: movl $64, %eax2927; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rax2928; SSE-NEXT: addl $64, %eax2929; SSE-NEXT: testq %rbx, %rbx2930; SSE-NEXT: cmovnel %r14d, %eax2931; SSE-NEXT: subl $-128, %eax2932; SSE-NEXT: orq %r9, %r82933; SSE-NEXT: cmovnel %r11d, %eax2934; SSE-NEXT: addl $256, %eax # imm = 0x1002935; SSE-NEXT: orq %rcx, %rsi2936; SSE-NEXT: orq %rdx, %rdi2937; SSE-NEXT: orq %rsi, %rdi2938; SSE-NEXT: cmovnel %r10d, %eax2939; SSE-NEXT: # kill: def $eax killed $eax killed $rax2940; SSE-NEXT: popq %rbx2941; SSE-NEXT: popq %r142942; SSE-NEXT: retq2943;2944; AVX2-LABEL: test_cttz_i512:2945; AVX2: # %bb.0:2946; AVX2-NEXT: pushq %r142947; AVX2-NEXT: pushq %rbx2948; AVX2-NEXT: tzcntq %rdi, %rax2949; AVX2-NEXT: tzcntq %rsi, %r112950; AVX2-NEXT: addl $64, %r11d2951; AVX2-NEXT: testq %rdi, %rdi2952; AVX2-NEXT: cmovnel %eax, %r11d2953; AVX2-NEXT: xorl %eax, %eax2954; AVX2-NEXT: tzcntq %rdx, %rax2955; AVX2-NEXT: tzcntq %rcx, %r102956; AVX2-NEXT: addl $64, %r10d2957; AVX2-NEXT: testq %rdx, %rdx2958; AVX2-NEXT: cmovnel %eax, %r10d2959; AVX2-NEXT: subl $-128, %r10d2960; AVX2-NEXT: movq %rdi, %rax2961; AVX2-NEXT: orq %rsi, %rax2962; AVX2-NEXT: cmovnel %r11d, %r10d2963; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r112964; AVX2-NEXT: xorl %eax, %eax2965; AVX2-NEXT: tzcntq %r8, %rax2966; AVX2-NEXT: xorl %ebx, %ebx2967; AVX2-NEXT: tzcntq %r9, %rbx2968; AVX2-NEXT: addl $64, %ebx2969; AVX2-NEXT: testq %r8, %r82970; AVX2-NEXT: cmovnel %eax, %ebx2971; AVX2-NEXT: xorl %r14d, %r14d2972; AVX2-NEXT: tzcntq %r11, %r142973; AVX2-NEXT: xorl %eax, %eax2974; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %rax2975; AVX2-NEXT: addl $64, %eax2976; AVX2-NEXT: testq %r11, %r112977; AVX2-NEXT: cmovnel %r14d, %eax2978; AVX2-NEXT: subl $-128, %eax2979; AVX2-NEXT: orq %r9, %r82980; AVX2-NEXT: cmovnel %ebx, %eax2981; AVX2-NEXT: addl $256, %eax # imm = 0x1002982; AVX2-NEXT: orq %rcx, %rsi2983; AVX2-NEXT: orq %rdx, %rdi2984; AVX2-NEXT: orq %rsi, %rdi2985; AVX2-NEXT: cmovnel %r10d, %eax2986; AVX2-NEXT: # kill: def $eax killed $eax killed $rax2987; AVX2-NEXT: popq %rbx2988; AVX2-NEXT: popq %r142989; AVX2-NEXT: retq2990;2991; AVX512F-LABEL: test_cttz_i512:2992; AVX512F: # %bb.0:2993; AVX512F-NEXT: vmovq %rcx, %xmm02994; AVX512F-NEXT: vmovq %rdx, %xmm12995; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]2996; AVX512F-NEXT: vmovq %rsi, %xmm12997; AVX512F-NEXT: vmovq %rdi, %xmm22998; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]2999; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm03000; AVX512F-NEXT: vmovq %r9, %xmm13001; AVX512F-NEXT: vmovq %r8, %xmm23002; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]3003; AVX512F-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm13004; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm03005; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -13006; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm13007; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm13008; AVX512F-NEXT: vplzcntq %zmm1, %zmm13009; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]3010; AVX512F-NEXT: vpsubq %zmm1, %zmm2, %zmm13011; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k13012; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]3013; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1}3014; AVX512F-NEXT: vmovd %xmm0, %eax3015; AVX512F-NEXT: vzeroupper3016; AVX512F-NEXT: retq3017;3018; AVX512POPCNT-LABEL: test_cttz_i512:3019; AVX512POPCNT: # %bb.0:3020; AVX512POPCNT-NEXT: vmovq %rcx, %xmm03021; AVX512POPCNT-NEXT: vmovq %rdx, %xmm13022; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]3023; AVX512POPCNT-NEXT: vmovq %rsi, %xmm13024; AVX512POPCNT-NEXT: vmovq %rdi, %xmm23025; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]3026; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm03027; AVX512POPCNT-NEXT: vmovq %r9, %xmm13028; AVX512POPCNT-NEXT: vmovq %r8, %xmm23029; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]3030; AVX512POPCNT-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm13031; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm03032; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -13033; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm13034; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm13035; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm13036; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm13037; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k13038; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]3039; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}3040; AVX512POPCNT-NEXT: vmovd %xmm0, %eax3041; AVX512POPCNT-NEXT: vzeroupper3042; AVX512POPCNT-NEXT: retq3043 %cnt = call i512 @llvm.cttz.i512(i512 %a0, i1 0)3044 %res = trunc i512 %cnt to i323045 ret i32 %res3046}3047 3048define i32 @load_cttz_i512(ptr %p0) nounwind {3049; SSE-LABEL: load_cttz_i512:3050; SSE: # %bb.0:3051; SSE-NEXT: pushq %r153052; SSE-NEXT: pushq %r143053; SSE-NEXT: pushq %rbx3054; SSE-NEXT: movq 48(%rdi), %r103055; SSE-NEXT: movq 40(%rdi), %r93056; SSE-NEXT: movq 24(%rdi), %r83057; SSE-NEXT: movq 16(%rdi), %rdx3058; SSE-NEXT: movq (%rdi), %rcx3059; SSE-NEXT: movq 8(%rdi), %rsi3060; SSE-NEXT: rep bsfq %rcx, %rax3061; SSE-NEXT: rep bsfq %rsi, %rbx3062; SSE-NEXT: addl $64, %ebx3063; SSE-NEXT: testq %rcx, %rcx3064; SSE-NEXT: cmovnel %eax, %ebx3065; SSE-NEXT: rep bsfq %rdx, %rax3066; SSE-NEXT: rep bsfq %r8, %r113067; SSE-NEXT: addl $64, %r11d3068; SSE-NEXT: testq %rdx, %rdx3069; SSE-NEXT: cmovnel %eax, %r11d3070; SSE-NEXT: movq 32(%rdi), %r143071; SSE-NEXT: subl $-128, %r11d3072; SSE-NEXT: movq %rcx, %rax3073; SSE-NEXT: orq %rsi, %rax3074; SSE-NEXT: cmovnel %ebx, %r11d3075; SSE-NEXT: rep bsfq %r14, %rax3076; SSE-NEXT: rep bsfq %r9, %rbx3077; SSE-NEXT: addl $64, %ebx3078; SSE-NEXT: testq %r14, %r143079; SSE-NEXT: cmovnel %eax, %ebx3080; SSE-NEXT: rep bsfq %r10, %r153081; SSE-NEXT: movl $64, %eax3082; SSE-NEXT: rep bsfq 56(%rdi), %rax3083; SSE-NEXT: addl $64, %eax3084; SSE-NEXT: testq %r10, %r103085; SSE-NEXT: cmovnel %r15d, %eax3086; SSE-NEXT: subl $-128, %eax3087; SSE-NEXT: orq %r9, %r143088; SSE-NEXT: cmovnel %ebx, %eax3089; SSE-NEXT: addl $256, %eax # imm = 0x1003090; SSE-NEXT: orq %r8, %rsi3091; SSE-NEXT: orq %rdx, %rcx3092; SSE-NEXT: orq %rsi, %rcx3093; SSE-NEXT: cmovnel %r11d, %eax3094; SSE-NEXT: # kill: def $eax killed $eax killed $rax3095; SSE-NEXT: popq %rbx3096; SSE-NEXT: popq %r143097; SSE-NEXT: popq %r153098; SSE-NEXT: retq3099;3100; AVX2-LABEL: load_cttz_i512:3101; AVX2: # %bb.0:3102; AVX2-NEXT: pushq %r153103; AVX2-NEXT: pushq %r143104; AVX2-NEXT: pushq %rbx3105; AVX2-NEXT: movq 48(%rdi), %r103106; AVX2-NEXT: movq 40(%rdi), %r93107; AVX2-NEXT: movq 24(%rdi), %r83108; AVX2-NEXT: movq 16(%rdi), %rdx3109; AVX2-NEXT: movq (%rdi), %rcx3110; AVX2-NEXT: movq 8(%rdi), %rsi3111; AVX2-NEXT: tzcntq %rcx, %rax3112; AVX2-NEXT: xorl %ebx, %ebx3113; AVX2-NEXT: tzcntq %rsi, %rbx3114; AVX2-NEXT: addl $64, %ebx3115; AVX2-NEXT: testq %rcx, %rcx3116; AVX2-NEXT: cmovnel %eax, %ebx3117; AVX2-NEXT: xorl %eax, %eax3118; AVX2-NEXT: tzcntq %rdx, %rax3119; AVX2-NEXT: tzcntq %r8, %r113120; AVX2-NEXT: addl $64, %r11d3121; AVX2-NEXT: testq %rdx, %rdx3122; AVX2-NEXT: cmovnel %eax, %r11d3123; AVX2-NEXT: subl $-128, %r11d3124; AVX2-NEXT: movq %rcx, %rax3125; AVX2-NEXT: orq %rsi, %rax3126; AVX2-NEXT: cmovnel %ebx, %r11d3127; AVX2-NEXT: movq 32(%rdi), %rbx3128; AVX2-NEXT: xorl %eax, %eax3129; AVX2-NEXT: tzcntq %rbx, %rax3130; AVX2-NEXT: xorl %r14d, %r14d3131; AVX2-NEXT: tzcntq %r9, %r143132; AVX2-NEXT: addl $64, %r14d3133; AVX2-NEXT: testq %rbx, %rbx3134; AVX2-NEXT: cmovnel %eax, %r14d3135; AVX2-NEXT: xorl %r15d, %r15d3136; AVX2-NEXT: tzcntq %r10, %r153137; AVX2-NEXT: xorl %eax, %eax3138; AVX2-NEXT: tzcntq 56(%rdi), %rax3139; AVX2-NEXT: addl $64, %eax3140; AVX2-NEXT: testq %r10, %r103141; AVX2-NEXT: cmovnel %r15d, %eax3142; AVX2-NEXT: subl $-128, %eax3143; AVX2-NEXT: orq %r9, %rbx3144; AVX2-NEXT: cmovnel %r14d, %eax3145; AVX2-NEXT: addl $256, %eax # imm = 0x1003146; AVX2-NEXT: orq %r8, %rsi3147; AVX2-NEXT: orq %rdx, %rcx3148; AVX2-NEXT: orq %rsi, %rcx3149; AVX2-NEXT: cmovnel %r11d, %eax3150; AVX2-NEXT: # kill: def $eax killed $eax killed $rax3151; AVX2-NEXT: popq %rbx3152; AVX2-NEXT: popq %r143153; AVX2-NEXT: popq %r153154; AVX2-NEXT: retq3155;3156; AVX512F-LABEL: load_cttz_i512:3157; AVX512F: # %bb.0:3158; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm03159; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -13160; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm13161; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm13162; AVX512F-NEXT: vplzcntq %zmm1, %zmm13163; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]3164; AVX512F-NEXT: vpsubq %zmm1, %zmm2, %zmm13165; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k13166; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]3167; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1}3168; AVX512F-NEXT: vmovd %xmm0, %eax3169; AVX512F-NEXT: vzeroupper3170; AVX512F-NEXT: retq3171;3172; AVX512POPCNT-LABEL: load_cttz_i512:3173; AVX512POPCNT: # %bb.0:3174; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm03175; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -13176; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm13177; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm13178; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm13179; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm13180; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k13181; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]3182; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}3183; AVX512POPCNT-NEXT: vmovd %xmm0, %eax3184; AVX512POPCNT-NEXT: vzeroupper3185; AVX512POPCNT-NEXT: retq3186 %a0 = load i512, ptr %p03187 %cnt = call i512 @llvm.cttz.i512(i512 %a0, i1 0)3188 %res = trunc i512 %cnt to i323189 ret i32 %res3190}3191 3192define i32 @test_cttz_i1024(i1024 %a0) nounwind {3193; SSE-LABEL: test_cttz_i1024:3194; SSE: # %bb.0:3195; SSE-NEXT: pushq %rbp3196; SSE-NEXT: pushq %r153197; SSE-NEXT: pushq %r143198; SSE-NEXT: pushq %r133199; SSE-NEXT: pushq %r123200; SSE-NEXT: pushq %rbx3201; SSE-NEXT: movq %r9, %r133202; SSE-NEXT: movq %r8, %r143203; SSE-NEXT: movq %rcx, %rbx3204; SSE-NEXT: movq %rdx, %r103205; SSE-NEXT: movq %rsi, %r93206; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rsi3207; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r113208; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx3209; SSE-NEXT: rep bsfq %rdi, %rax3210; SSE-NEXT: rep bsfq %r9, %r153211; SSE-NEXT: addl $64, %r15d3212; SSE-NEXT: testq %rdi, %rdi3213; SSE-NEXT: cmovnel %eax, %r15d3214; SSE-NEXT: rep bsfq %r10, %r123215; SSE-NEXT: rep bsfq %rcx, %rax3216; SSE-NEXT: addl $64, %eax3217; SSE-NEXT: testq %r10, %r103218; SSE-NEXT: cmovnel %r12d, %eax3219; SSE-NEXT: subl $-128, %eax3220; SSE-NEXT: movq %rdi, %r123221; SSE-NEXT: orq %r9, %r123222; SSE-NEXT: cmovnel %r15d, %eax3223; SSE-NEXT: rep bsfq %r8, %r153224; SSE-NEXT: movq %r13, %rcx3225; SSE-NEXT: movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3226; SSE-NEXT: rep bsfq %r13, %r133227; SSE-NEXT: addl $64, %r13d3228; SSE-NEXT: testq %r8, %r83229; SSE-NEXT: cmovnel %r15d, %r13d3230; SSE-NEXT: rep bsfq %rdx, %r123231; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %r153232; SSE-NEXT: addl $64, %r15d3233; SSE-NEXT: testq %rdx, %rdx3234; SSE-NEXT: cmovnel %r12d, %r15d3235; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r123236; SSE-NEXT: subl $-128, %r15d3237; SSE-NEXT: movq %r8, %rbp3238; SSE-NEXT: orq %rcx, %rbp3239; SSE-NEXT: cmovnel %r13d, %r15d3240; SSE-NEXT: addl $256, %r15d # imm = 0x1003241; SSE-NEXT: movq %r9, %r133242; SSE-NEXT: orq %rbx, %r133243; SSE-NEXT: movq %rdi, %rbp3244; SSE-NEXT: orq %r10, %rbp3245; SSE-NEXT: orq %r13, %rbp3246; SSE-NEXT: cmovnel %eax, %r15d3247; SSE-NEXT: rep bsfq %r11, %r133248; SSE-NEXT: rep bsfq %r12, %rax3249; SSE-NEXT: addl $64, %eax3250; SSE-NEXT: testq %r11, %r113251; SSE-NEXT: cmovnel %r13d, %eax3252; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %r133253; SSE-NEXT: addl $64, %r13d3254; SSE-NEXT: rep bsfq %rsi, %rcx3255; SSE-NEXT: testq %rsi, %rsi3256; SSE-NEXT: cmovnel %ecx, %r13d3257; SSE-NEXT: subl $-128, %r13d3258; SSE-NEXT: movq %r11, %rcx3259; SSE-NEXT: orq %r12, %rcx3260; SSE-NEXT: cmovnel %eax, %r13d3261; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbp3262; SSE-NEXT: rep bsfq %rbp, %rcx3263; SSE-NEXT: addl $64, %ecx3264; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx3265; SSE-NEXT: rep bsfq %rdx, %rax3266; SSE-NEXT: testq %rdx, %rdx3267; SSE-NEXT: cmovnel %eax, %ecx3268; SSE-NEXT: movl $64, %eax3269; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rax3270; SSE-NEXT: addl $64, %eax3271; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r83272; SSE-NEXT: rep bsfq %r8, %rsi3273; SSE-NEXT: testq %r8, %r83274; SSE-NEXT: cmovnel %esi, %eax3275; SSE-NEXT: subl $-128, %eax3276; SSE-NEXT: orq %rbp, %rdx3277; SSE-NEXT: cmovnel %ecx, %eax3278; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r123279; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r113280; SSE-NEXT: addl $256, %eax # imm = 0x1003281; SSE-NEXT: orq %r12, %r113282; SSE-NEXT: cmovnel %r13d, %eax3283; SSE-NEXT: orq {{[0-9]+}}(%rsp), %rbx3284; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload3285; SSE-NEXT: orq %rbx, %r93286; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r103287; SSE-NEXT: orq %r14, %rdi3288; SSE-NEXT: orq %r10, %rdi3289; SSE-NEXT: addl $512, %eax # imm = 0x2003290; SSE-NEXT: orq %r9, %rdi3291; SSE-NEXT: cmovnel %r15d, %eax3292; SSE-NEXT: # kill: def $eax killed $eax killed $rax3293; SSE-NEXT: popq %rbx3294; SSE-NEXT: popq %r123295; SSE-NEXT: popq %r133296; SSE-NEXT: popq %r143297; SSE-NEXT: popq %r153298; SSE-NEXT: popq %rbp3299; SSE-NEXT: retq3300;3301; AVX2-LABEL: test_cttz_i1024:3302; AVX2: # %bb.0:3303; AVX2-NEXT: pushq %rbp3304; AVX2-NEXT: pushq %r153305; AVX2-NEXT: pushq %r143306; AVX2-NEXT: pushq %r133307; AVX2-NEXT: pushq %r123308; AVX2-NEXT: pushq %rbx3309; AVX2-NEXT: movq %r9, %rbx3310; AVX2-NEXT: movq %r8, %r143311; AVX2-NEXT: movq %rcx, %r113312; AVX2-NEXT: movq %rdx, %r103313; AVX2-NEXT: movq %rsi, %r93314; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r83315; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rsi3316; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx3317; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx3318; AVX2-NEXT: tzcntq %rdi, %rax3319; AVX2-NEXT: xorl %r15d, %r15d3320; AVX2-NEXT: tzcntq %r9, %r153321; AVX2-NEXT: addl $64, %r15d3322; AVX2-NEXT: testq %rdi, %rdi3323; AVX2-NEXT: cmovnel %eax, %r15d3324; AVX2-NEXT: xorl %r12d, %r12d3325; AVX2-NEXT: tzcntq %r10, %r123326; AVX2-NEXT: xorl %eax, %eax3327; AVX2-NEXT: tzcntq %r11, %rax3328; AVX2-NEXT: addl $64, %eax3329; AVX2-NEXT: testq %r10, %r103330; AVX2-NEXT: cmovnel %r12d, %eax3331; AVX2-NEXT: subl $-128, %eax3332; AVX2-NEXT: movq %rdi, %r123333; AVX2-NEXT: orq %r9, %r123334; AVX2-NEXT: cmovnel %r15d, %eax3335; AVX2-NEXT: xorl %r15d, %r15d3336; AVX2-NEXT: tzcntq %r14, %r153337; AVX2-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3338; AVX2-NEXT: xorl %r12d, %r12d3339; AVX2-NEXT: tzcntq %rbx, %r123340; AVX2-NEXT: addl $64, %r12d3341; AVX2-NEXT: testq %r14, %r143342; AVX2-NEXT: cmovnel %r15d, %r12d3343; AVX2-NEXT: xorl %r13d, %r13d3344; AVX2-NEXT: tzcntq %rcx, %r133345; AVX2-NEXT: xorl %r15d, %r15d3346; AVX2-NEXT: tzcntq %rdx, %r153347; AVX2-NEXT: addl $64, %r15d3348; AVX2-NEXT: testq %rcx, %rcx3349; AVX2-NEXT: cmovnel %r13d, %r15d3350; AVX2-NEXT: subl $-128, %r15d3351; AVX2-NEXT: movq %r14, %r133352; AVX2-NEXT: orq %rbx, %r133353; AVX2-NEXT: cmovnel %r12d, %r15d3354; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r123355; AVX2-NEXT: addl $256, %r15d # imm = 0x1003356; AVX2-NEXT: movq %r9, %r133357; AVX2-NEXT: orq %r11, %r133358; AVX2-NEXT: movq %rdi, %rbp3359; AVX2-NEXT: orq %r10, %rbp3360; AVX2-NEXT: orq %r13, %rbp3361; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r133362; AVX2-NEXT: cmovnel %eax, %r15d3363; AVX2-NEXT: xorl %ebp, %ebp3364; AVX2-NEXT: tzcntq %r12, %rbp3365; AVX2-NEXT: xorl %eax, %eax3366; AVX2-NEXT: tzcntq %r13, %rax3367; AVX2-NEXT: addl $64, %eax3368; AVX2-NEXT: testq %r12, %r123369; AVX2-NEXT: cmovnel %ebp, %eax3370; AVX2-NEXT: xorl %ebp, %ebp3371; AVX2-NEXT: tzcntq %r8, %rbp3372; AVX2-NEXT: addl $64, %ebp3373; AVX2-NEXT: xorl %ecx, %ecx3374; AVX2-NEXT: tzcntq %rsi, %rcx3375; AVX2-NEXT: testq %rsi, %rsi3376; AVX2-NEXT: cmovnel %ecx, %ebp3377; AVX2-NEXT: subl $-128, %ebp3378; AVX2-NEXT: movq %r12, %rcx3379; AVX2-NEXT: orq %r13, %rcx3380; AVX2-NEXT: cmovnel %eax, %ebp3381; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx3382; AVX2-NEXT: xorl %ecx, %ecx3383; AVX2-NEXT: tzcntq %rbx, %rcx3384; AVX2-NEXT: addl $64, %ecx3385; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx3386; AVX2-NEXT: xorl %eax, %eax3387; AVX2-NEXT: tzcntq %rdx, %rax3388; AVX2-NEXT: testq %rdx, %rdx3389; AVX2-NEXT: cmovnel %eax, %ecx3390; AVX2-NEXT: xorl %eax, %eax3391; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %rax3392; AVX2-NEXT: addl $64, %eax3393; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r83394; AVX2-NEXT: tzcntq %r8, %rsi3395; AVX2-NEXT: testq %r8, %r83396; AVX2-NEXT: cmovnel %esi, %eax3397; AVX2-NEXT: subl $-128, %eax3398; AVX2-NEXT: orq %rbx, %rdx3399; AVX2-NEXT: cmovnel %ecx, %eax3400; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r133401; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r123402; AVX2-NEXT: addl $256, %eax # imm = 0x1003403; AVX2-NEXT: orq %r13, %r123404; AVX2-NEXT: cmovnel %ebp, %eax3405; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r113406; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload3407; AVX2-NEXT: orq %r11, %r93408; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r103409; AVX2-NEXT: orq %r14, %rdi3410; AVX2-NEXT: orq %r10, %rdi3411; AVX2-NEXT: addl $512, %eax # imm = 0x2003412; AVX2-NEXT: orq %r9, %rdi3413; AVX2-NEXT: cmovnel %r15d, %eax3414; AVX2-NEXT: # kill: def $eax killed $eax killed $rax3415; AVX2-NEXT: popq %rbx3416; AVX2-NEXT: popq %r123417; AVX2-NEXT: popq %r133418; AVX2-NEXT: popq %r143419; AVX2-NEXT: popq %r153420; AVX2-NEXT: popq %rbp3421; AVX2-NEXT: retq3422;3423; AVX512F-LABEL: test_cttz_i1024:3424; AVX512F: # %bb.0:3425; AVX512F-NEXT: vmovq %rcx, %xmm03426; AVX512F-NEXT: vmovq %rdx, %xmm13427; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]3428; AVX512F-NEXT: vmovq %rsi, %xmm13429; AVX512F-NEXT: vmovq %rdi, %xmm23430; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]3431; AVX512F-NEXT: vmovq %r9, %xmm23432; AVX512F-NEXT: vmovq %r8, %xmm33433; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]3434; AVX512F-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm2, %ymm23435; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm03436; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm03437; AVX512F-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm13438; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -13439; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm33440; AVX512F-NEXT: vpandnq %zmm3, %zmm0, %zmm33441; AVX512F-NEXT: vplzcntq %zmm3, %zmm33442; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]3443; AVX512F-NEXT: vpsubq %zmm3, %zmm4, %zmm33444; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k13445; AVX512F-NEXT: vpcompressq %zmm3, %zmm0 {%k1} {z}3446; AVX512F-NEXT: vmovd %xmm0, %r10d3447; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm03448; AVX512F-NEXT: vpandnq %zmm0, %zmm1, %zmm03449; AVX512F-NEXT: vplzcntq %zmm0, %zmm03450; AVX512F-NEXT: vpsubq %zmm0, %zmm4, %zmm03451; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k13452; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]3453; AVX512F-NEXT: vpcompressq %zmm0, %zmm1 {%k1}3454; AVX512F-NEXT: vmovd %xmm1, %eax3455; AVX512F-NEXT: addl $512, %eax # imm = 0x2003456; AVX512F-NEXT: orq %r9, %rsi3457; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rcx3458; AVX512F-NEXT: orq %rsi, %rcx3459; AVX512F-NEXT: orq %r8, %rdi3460; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rdx3461; AVX512F-NEXT: orq %rdi, %rdx3462; AVX512F-NEXT: orq %rcx, %rdx3463; AVX512F-NEXT: cmovnel %r10d, %eax3464; AVX512F-NEXT: vzeroupper3465; AVX512F-NEXT: retq3466;3467; AVX512POPCNT-LABEL: test_cttz_i1024:3468; AVX512POPCNT: # %bb.0:3469; AVX512POPCNT-NEXT: vmovq %rcx, %xmm03470; AVX512POPCNT-NEXT: vmovq %rdx, %xmm13471; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]3472; AVX512POPCNT-NEXT: vmovq %rsi, %xmm13473; AVX512POPCNT-NEXT: vmovq %rdi, %xmm23474; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]3475; AVX512POPCNT-NEXT: vmovq %r9, %xmm23476; AVX512POPCNT-NEXT: vmovq %r8, %xmm33477; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]3478; AVX512POPCNT-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm2, %ymm23479; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm03480; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm03481; AVX512POPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm13482; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -13483; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm33484; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm33485; AVX512POPCNT-NEXT: vpopcntq %zmm3, %zmm33486; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]3487; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm33488; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k13489; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm0 {%k1} {z}3490; AVX512POPCNT-NEXT: vmovd %xmm0, %r10d3491; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm03492; AVX512POPCNT-NEXT: vpandnq %zmm0, %zmm1, %zmm03493; AVX512POPCNT-NEXT: vpopcntq %zmm0, %zmm03494; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm0, %zmm03495; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k13496; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]3497; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm1 {%k1}3498; AVX512POPCNT-NEXT: vmovd %xmm1, %eax3499; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x2003500; AVX512POPCNT-NEXT: orq %r9, %rsi3501; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rcx3502; AVX512POPCNT-NEXT: orq %rsi, %rcx3503; AVX512POPCNT-NEXT: orq %r8, %rdi3504; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rdx3505; AVX512POPCNT-NEXT: orq %rdi, %rdx3506; AVX512POPCNT-NEXT: orq %rcx, %rdx3507; AVX512POPCNT-NEXT: cmovnel %r10d, %eax3508; AVX512POPCNT-NEXT: vzeroupper3509; AVX512POPCNT-NEXT: retq3510 %cnt = call i1024 @llvm.cttz.i1024(i1024 %a0, i1 0)3511 %res = trunc i1024 %cnt to i323512 ret i32 %res3513}3514 3515define i32 @load_cttz_i1024(ptr %p0) nounwind {3516; SSE-LABEL: load_cttz_i1024:3517; SSE: # %bb.0:3518; SSE-NEXT: pushq %rbp3519; SSE-NEXT: pushq %r153520; SSE-NEXT: pushq %r143521; SSE-NEXT: pushq %r133522; SSE-NEXT: pushq %r123523; SSE-NEXT: pushq %rbx3524; SSE-NEXT: movq 88(%rdi), %r103525; SSE-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3526; SSE-NEXT: movq 56(%rdi), %rcx3527; SSE-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3528; SSE-NEXT: movq 40(%rdi), %rsi3529; SSE-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3530; SSE-NEXT: movq 24(%rdi), %r93531; SSE-NEXT: movq 16(%rdi), %r153532; SSE-NEXT: movq (%rdi), %r83533; SSE-NEXT: movq 8(%rdi), %r113534; SSE-NEXT: rep bsfq %r8, %rax3535; SSE-NEXT: rep bsfq %r11, %rdx3536; SSE-NEXT: addl $64, %edx3537; SSE-NEXT: testq %r8, %r83538; SSE-NEXT: cmovnel %eax, %edx3539; SSE-NEXT: rep bsfq %r15, %rbx3540; SSE-NEXT: rep bsfq %r9, %rax3541; SSE-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3542; SSE-NEXT: addl $64, %eax3543; SSE-NEXT: testq %r15, %r153544; SSE-NEXT: cmovnel %ebx, %eax3545; SSE-NEXT: movq 32(%rdi), %rbx3546; SSE-NEXT: subl $-128, %eax3547; SSE-NEXT: movq %r8, %r143548; SSE-NEXT: orq %r11, %r143549; SSE-NEXT: cmovnel %edx, %eax3550; SSE-NEXT: rep bsfq %rbx, %rdx3551; SSE-NEXT: rep bsfq %rsi, %r123552; SSE-NEXT: addl $64, %r12d3553; SSE-NEXT: testq %rbx, %rbx3554; SSE-NEXT: cmovnel %edx, %r12d3555; SSE-NEXT: movq 48(%rdi), %r133556; SSE-NEXT: movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3557; SSE-NEXT: rep bsfq %r13, %rdx3558; SSE-NEXT: rep bsfq %rcx, %r143559; SSE-NEXT: addl $64, %r14d3560; SSE-NEXT: testq %r13, %r133561; SSE-NEXT: cmovnel %edx, %r14d3562; SSE-NEXT: subl $-128, %r14d3563; SSE-NEXT: movq %rbx, %rdx3564; SSE-NEXT: orq %rsi, %rdx3565; SSE-NEXT: cmovnel %r12d, %r14d3566; SSE-NEXT: movq 72(%rdi), %r123567; SSE-NEXT: addl $256, %r14d # imm = 0x1003568; SSE-NEXT: movq %r11, %rdx3569; SSE-NEXT: orq %r9, %rdx3570; SSE-NEXT: movq %r8, %r133571; SSE-NEXT: orq %r15, %r133572; SSE-NEXT: orq %rdx, %r133573; SSE-NEXT: movq 64(%rdi), %r133574; SSE-NEXT: cmovnel %eax, %r14d3575; SSE-NEXT: rep bsfq %r13, %rdx3576; SSE-NEXT: rep bsfq %r12, %rax3577; SSE-NEXT: addl $64, %eax3578; SSE-NEXT: testq %r13, %r133579; SSE-NEXT: cmovnel %edx, %eax3580; SSE-NEXT: rep bsfq %r10, %rbp3581; SSE-NEXT: addl $64, %ebp3582; SSE-NEXT: movq 80(%rdi), %r103583; SSE-NEXT: rep bsfq %r10, %rcx3584; SSE-NEXT: testq %r10, %r103585; SSE-NEXT: cmovnel %ecx, %ebp3586; SSE-NEXT: subl $-128, %ebp3587; SSE-NEXT: movq %r13, %rcx3588; SSE-NEXT: orq %r12, %rcx3589; SSE-NEXT: cmovnel %eax, %ebp3590; SSE-NEXT: movq 104(%rdi), %r93591; SSE-NEXT: rep bsfq %r9, %rcx3592; SSE-NEXT: addl $64, %ecx3593; SSE-NEXT: movq 96(%rdi), %rdx3594; SSE-NEXT: rep bsfq %rdx, %rax3595; SSE-NEXT: testq %rdx, %rdx3596; SSE-NEXT: cmovnel %eax, %ecx3597; SSE-NEXT: movl $64, %eax3598; SSE-NEXT: rep bsfq 120(%rdi), %rax3599; SSE-NEXT: movq 112(%rdi), %rdi3600; SSE-NEXT: addl $64, %eax3601; SSE-NEXT: rep bsfq %rdi, %rsi3602; SSE-NEXT: testq %rdi, %rdi3603; SSE-NEXT: cmovnel %esi, %eax3604; SSE-NEXT: subl $-128, %eax3605; SSE-NEXT: orq %r9, %rdx3606; SSE-NEXT: cmovnel %ecx, %eax3607; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Folded Reload3608; SSE-NEXT: orq %r10, %r133609; SSE-NEXT: addl $256, %eax # imm = 0x1003610; SSE-NEXT: orq %r12, %r133611; SSE-NEXT: cmovnel %ebp, %eax3612; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload3613; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload3614; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload3615; SSE-NEXT: orq %rcx, %r113616; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Folded Reload3617; SSE-NEXT: orq %rbx, %r83618; SSE-NEXT: orq %r15, %r83619; SSE-NEXT: addl $512, %eax # imm = 0x2003620; SSE-NEXT: orq %r11, %r83621; SSE-NEXT: cmovnel %r14d, %eax3622; SSE-NEXT: # kill: def $eax killed $eax killed $rax3623; SSE-NEXT: popq %rbx3624; SSE-NEXT: popq %r123625; SSE-NEXT: popq %r133626; SSE-NEXT: popq %r143627; SSE-NEXT: popq %r153628; SSE-NEXT: popq %rbp3629; SSE-NEXT: retq3630;3631; AVX2-LABEL: load_cttz_i1024:3632; AVX2: # %bb.0:3633; AVX2-NEXT: pushq %rbp3634; AVX2-NEXT: pushq %r153635; AVX2-NEXT: pushq %r143636; AVX2-NEXT: pushq %r133637; AVX2-NEXT: pushq %r123638; AVX2-NEXT: pushq %rbx3639; AVX2-NEXT: movq 72(%rdi), %r143640; AVX2-NEXT: movq 64(%rdi), %r153641; AVX2-NEXT: movq 56(%rdi), %r93642; AVX2-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3643; AVX2-NEXT: movq 48(%rdi), %rcx3644; AVX2-NEXT: movq 40(%rdi), %r103645; AVX2-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3646; AVX2-NEXT: movq 32(%rdi), %rsi3647; AVX2-NEXT: movq 24(%rdi), %rbp3648; AVX2-NEXT: movq 16(%rdi), %rbx3649; AVX2-NEXT: movq (%rdi), %r83650; AVX2-NEXT: movq 8(%rdi), %r113651; AVX2-NEXT: tzcntq %r8, %rax3652; AVX2-NEXT: tzcntq %r11, %rdx3653; AVX2-NEXT: addl $64, %edx3654; AVX2-NEXT: testq %r8, %r83655; AVX2-NEXT: cmovnel %eax, %edx3656; AVX2-NEXT: xorl %r12d, %r12d3657; AVX2-NEXT: tzcntq %rbx, %r123658; AVX2-NEXT: xorl %eax, %eax3659; AVX2-NEXT: tzcntq %rbp, %rax3660; AVX2-NEXT: movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3661; AVX2-NEXT: addl $64, %eax3662; AVX2-NEXT: testq %rbx, %rbx3663; AVX2-NEXT: cmovnel %r12d, %eax3664; AVX2-NEXT: subl $-128, %eax3665; AVX2-NEXT: movq %r8, %r123666; AVX2-NEXT: orq %r11, %r123667; AVX2-NEXT: cmovnel %edx, %eax3668; AVX2-NEXT: xorl %edx, %edx3669; AVX2-NEXT: tzcntq %rsi, %rdx3670; AVX2-NEXT: xorl %r13d, %r13d3671; AVX2-NEXT: tzcntq %r10, %r133672; AVX2-NEXT: addl $64, %r13d3673; AVX2-NEXT: testq %rsi, %rsi3674; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3675; AVX2-NEXT: cmovnel %edx, %r13d3676; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill3677; AVX2-NEXT: xorl %edx, %edx3678; AVX2-NEXT: tzcntq %rcx, %rdx3679; AVX2-NEXT: xorl %r12d, %r12d3680; AVX2-NEXT: tzcntq %r9, %r123681; AVX2-NEXT: addl $64, %r12d3682; AVX2-NEXT: testq %rcx, %rcx3683; AVX2-NEXT: cmovnel %edx, %r12d3684; AVX2-NEXT: subl $-128, %r12d3685; AVX2-NEXT: movq %rsi, %rdx3686; AVX2-NEXT: orq %r10, %rdx3687; AVX2-NEXT: cmovnel %r13d, %r12d3688; AVX2-NEXT: addl $256, %r12d # imm = 0x1003689; AVX2-NEXT: movq %r11, %rdx3690; AVX2-NEXT: orq %rbp, %rdx3691; AVX2-NEXT: movq %r8, %r133692; AVX2-NEXT: orq %rbx, %r133693; AVX2-NEXT: orq %rdx, %r133694; AVX2-NEXT: cmovnel %eax, %r12d3695; AVX2-NEXT: xorl %edx, %edx3696; AVX2-NEXT: tzcntq %r15, %rdx3697; AVX2-NEXT: xorl %eax, %eax3698; AVX2-NEXT: tzcntq %r14, %rax3699; AVX2-NEXT: addl $64, %eax3700; AVX2-NEXT: testq %r15, %r153701; AVX2-NEXT: cmovnel %edx, %eax3702; AVX2-NEXT: movq 88(%rdi), %rbp3703; AVX2-NEXT: xorl %r13d, %r13d3704; AVX2-NEXT: tzcntq %rbp, %r133705; AVX2-NEXT: addl $64, %r13d3706; AVX2-NEXT: movq 80(%rdi), %r103707; AVX2-NEXT: xorl %ecx, %ecx3708; AVX2-NEXT: tzcntq %r10, %rcx3709; AVX2-NEXT: testq %r10, %r103710; AVX2-NEXT: cmovnel %ecx, %r13d3711; AVX2-NEXT: subl $-128, %r13d3712; AVX2-NEXT: movq %r15, %rcx3713; AVX2-NEXT: orq %r14, %rcx3714; AVX2-NEXT: cmovnel %eax, %r13d3715; AVX2-NEXT: movq 104(%rdi), %r93716; AVX2-NEXT: xorl %ecx, %ecx3717; AVX2-NEXT: tzcntq %r9, %rcx3718; AVX2-NEXT: addl $64, %ecx3719; AVX2-NEXT: movq 96(%rdi), %rdx3720; AVX2-NEXT: xorl %eax, %eax3721; AVX2-NEXT: tzcntq %rdx, %rax3722; AVX2-NEXT: testq %rdx, %rdx3723; AVX2-NEXT: cmovnel %eax, %ecx3724; AVX2-NEXT: movq 112(%rdi), %rsi3725; AVX2-NEXT: xorl %eax, %eax3726; AVX2-NEXT: tzcntq 120(%rdi), %rax3727; AVX2-NEXT: addl $64, %eax3728; AVX2-NEXT: tzcntq %rsi, %rdi3729; AVX2-NEXT: testq %rsi, %rsi3730; AVX2-NEXT: cmovnel %edi, %eax3731; AVX2-NEXT: subl $-128, %eax3732; AVX2-NEXT: orq %r9, %rdx3733; AVX2-NEXT: cmovnel %ecx, %eax3734; AVX2-NEXT: orq %rbp, %r143735; AVX2-NEXT: orq %r10, %r153736; AVX2-NEXT: addl $256, %eax # imm = 0x1003737; AVX2-NEXT: orq %r14, %r153738; AVX2-NEXT: cmovnel %r13d, %eax3739; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload3740; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload3741; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload3742; AVX2-NEXT: orq %rcx, %r113743; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload3744; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload3745; AVX2-NEXT: orq %rbx, %r83746; AVX2-NEXT: addl $512, %eax # imm = 0x2003747; AVX2-NEXT: orq %r11, %r83748; AVX2-NEXT: cmovnel %r12d, %eax3749; AVX2-NEXT: # kill: def $eax killed $eax killed $rax3750; AVX2-NEXT: popq %rbx3751; AVX2-NEXT: popq %r123752; AVX2-NEXT: popq %r133753; AVX2-NEXT: popq %r143754; AVX2-NEXT: popq %r153755; AVX2-NEXT: popq %rbp3756; AVX2-NEXT: retq3757;3758; AVX512F-LABEL: load_cttz_i1024:3759; AVX512F: # %bb.0:3760; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm03761; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm13762; AVX512F-NEXT: movq 16(%rdi), %rax3763; AVX512F-NEXT: movq (%rdi), %rcx3764; AVX512F-NEXT: movq 8(%rdi), %rdx3765; AVX512F-NEXT: movq 24(%rdi), %rsi3766; AVX512F-NEXT: orq 56(%rdi), %rsi3767; AVX512F-NEXT: orq 40(%rdi), %rdx3768; AVX512F-NEXT: orq %rsi, %rdx3769; AVX512F-NEXT: orq 48(%rdi), %rax3770; AVX512F-NEXT: orq 32(%rdi), %rcx3771; AVX512F-NEXT: orq %rax, %rcx3772; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -13773; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm33774; AVX512F-NEXT: vpandnq %zmm3, %zmm1, %zmm33775; AVX512F-NEXT: vplzcntq %zmm3, %zmm33776; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]3777; AVX512F-NEXT: vpsubq %zmm3, %zmm4, %zmm33778; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k13779; AVX512F-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}3780; AVX512F-NEXT: vmovd %xmm1, %esi3781; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm13782; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm13783; AVX512F-NEXT: vplzcntq %zmm1, %zmm13784; AVX512F-NEXT: vpsubq %zmm1, %zmm4, %zmm13785; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k13786; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]3787; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1}3788; AVX512F-NEXT: vmovd %xmm0, %eax3789; AVX512F-NEXT: addl $512, %eax # imm = 0x2003790; AVX512F-NEXT: orq %rdx, %rcx3791; AVX512F-NEXT: cmovnel %esi, %eax3792; AVX512F-NEXT: vzeroupper3793; AVX512F-NEXT: retq3794;3795; AVX512POPCNT-LABEL: load_cttz_i1024:3796; AVX512POPCNT: # %bb.0:3797; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm03798; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm13799; AVX512POPCNT-NEXT: movq 16(%rdi), %rax3800; AVX512POPCNT-NEXT: movq (%rdi), %rcx3801; AVX512POPCNT-NEXT: movq 8(%rdi), %rdx3802; AVX512POPCNT-NEXT: movq 24(%rdi), %rsi3803; AVX512POPCNT-NEXT: orq 56(%rdi), %rsi3804; AVX512POPCNT-NEXT: orq 40(%rdi), %rdx3805; AVX512POPCNT-NEXT: orq %rsi, %rdx3806; AVX512POPCNT-NEXT: orq 48(%rdi), %rax3807; AVX512POPCNT-NEXT: orq 32(%rdi), %rcx3808; AVX512POPCNT-NEXT: orq %rax, %rcx3809; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -13810; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm33811; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm1, %zmm33812; AVX512POPCNT-NEXT: vpopcntq %zmm3, %zmm33813; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]3814; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm33815; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k13816; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}3817; AVX512POPCNT-NEXT: vmovd %xmm1, %esi3818; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm13819; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm13820; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm13821; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm1, %zmm13822; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k13823; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]3824; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}3825; AVX512POPCNT-NEXT: vmovd %xmm0, %eax3826; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x2003827; AVX512POPCNT-NEXT: orq %rdx, %rcx3828; AVX512POPCNT-NEXT: cmovnel %esi, %eax3829; AVX512POPCNT-NEXT: vzeroupper3830; AVX512POPCNT-NEXT: retq3831 %a0 = load i1024, ptr %p03832 %cnt = call i1024 @llvm.cttz.i1024(i1024 %a0, i1 0)3833 %res = trunc i1024 %cnt to i323834 ret i32 %res3835}3836 3837;3838; CTTZ_ZERO_UNDEF3839;3840 3841define i32 @test_cttz_undef_i128(i128 %a0) nounwind {3842; SSE-LABEL: test_cttz_undef_i128:3843; SSE: # %bb.0:3844; SSE-NEXT: rep bsfq %rdi, %rcx3845; SSE-NEXT: rep bsfq %rsi, %rax3846; SSE-NEXT: addl $64, %eax3847; SSE-NEXT: testq %rdi, %rdi3848; SSE-NEXT: cmovnel %ecx, %eax3849; SSE-NEXT: # kill: def $eax killed $eax killed $rax3850; SSE-NEXT: retq3851;3852; AVX2-LABEL: test_cttz_undef_i128:3853; AVX2: # %bb.0:3854; AVX2-NEXT: tzcntq %rdi, %rcx3855; AVX2-NEXT: tzcntq %rsi, %rax3856; AVX2-NEXT: addl $64, %eax3857; AVX2-NEXT: testq %rdi, %rdi3858; AVX2-NEXT: cmovnel %ecx, %eax3859; AVX2-NEXT: # kill: def $eax killed $eax killed $rax3860; AVX2-NEXT: retq3861;3862; AVX512-LABEL: test_cttz_undef_i128:3863; AVX512: # %bb.0:3864; AVX512-NEXT: tzcntq %rdi, %rcx3865; AVX512-NEXT: tzcntq %rsi, %rax3866; AVX512-NEXT: addl $64, %eax3867; AVX512-NEXT: testq %rdi, %rdi3868; AVX512-NEXT: cmovnel %ecx, %eax3869; AVX512-NEXT: # kill: def $eax killed $eax killed $rax3870; AVX512-NEXT: retq3871 %cnt = call i128 @llvm.cttz.i128(i128 %a0, i1 -1)3872 %res = trunc i128 %cnt to i323873 ret i32 %res3874}3875 3876define i32 @load_cttz_undef_i128(ptr %p0) nounwind {3877; SSE-LABEL: load_cttz_undef_i128:3878; SSE: # %bb.0:3879; SSE-NEXT: movq (%rdi), %rcx3880; SSE-NEXT: rep bsfq %rcx, %rdx3881; SSE-NEXT: rep bsfq 8(%rdi), %rax3882; SSE-NEXT: addl $64, %eax3883; SSE-NEXT: testq %rcx, %rcx3884; SSE-NEXT: cmovnel %edx, %eax3885; SSE-NEXT: # kill: def $eax killed $eax killed $rax3886; SSE-NEXT: retq3887;3888; AVX2-LABEL: load_cttz_undef_i128:3889; AVX2: # %bb.0:3890; AVX2-NEXT: movq (%rdi), %rcx3891; AVX2-NEXT: tzcntq %rcx, %rdx3892; AVX2-NEXT: tzcntq 8(%rdi), %rax3893; AVX2-NEXT: addl $64, %eax3894; AVX2-NEXT: testq %rcx, %rcx3895; AVX2-NEXT: cmovnel %edx, %eax3896; AVX2-NEXT: # kill: def $eax killed $eax killed $rax3897; AVX2-NEXT: retq3898;3899; AVX512-LABEL: load_cttz_undef_i128:3900; AVX512: # %bb.0:3901; AVX512-NEXT: movq (%rdi), %rcx3902; AVX512-NEXT: tzcntq %rcx, %rdx3903; AVX512-NEXT: tzcntq 8(%rdi), %rax3904; AVX512-NEXT: addl $64, %eax3905; AVX512-NEXT: testq %rcx, %rcx3906; AVX512-NEXT: cmovnel %edx, %eax3907; AVX512-NEXT: # kill: def $eax killed $eax killed $rax3908; AVX512-NEXT: retq3909 %a0 = load i128, ptr %p03910 %cnt = call i128 @llvm.cttz.i128(i128 %a0, i1 -1)3911 %res = trunc i128 %cnt to i323912 ret i32 %res3913}3914 3915define i32 @test_cttz_undef_i256(i256 %a0) nounwind {3916; SSE-LABEL: test_cttz_undef_i256:3917; SSE: # %bb.0:3918; SSE-NEXT: rep bsfq %rdi, %rax3919; SSE-NEXT: rep bsfq %rsi, %r83920; SSE-NEXT: addl $64, %r8d3921; SSE-NEXT: testq %rdi, %rdi3922; SSE-NEXT: cmovnel %eax, %r8d3923; SSE-NEXT: rep bsfq %rdx, %r93924; SSE-NEXT: rep bsfq %rcx, %rax3925; SSE-NEXT: addl $64, %eax3926; SSE-NEXT: testq %rdx, %rdx3927; SSE-NEXT: cmovnel %r9d, %eax3928; SSE-NEXT: subl $-128, %eax3929; SSE-NEXT: orq %rsi, %rdi3930; SSE-NEXT: cmovnel %r8d, %eax3931; SSE-NEXT: # kill: def $eax killed $eax killed $rax3932; SSE-NEXT: retq3933;3934; AVX2-LABEL: test_cttz_undef_i256:3935; AVX2: # %bb.0:3936; AVX2-NEXT: tzcntq %rdi, %rax3937; AVX2-NEXT: tzcntq %rsi, %r83938; AVX2-NEXT: addl $64, %r8d3939; AVX2-NEXT: testq %rdi, %rdi3940; AVX2-NEXT: cmovnel %eax, %r8d3941; AVX2-NEXT: tzcntq %rdx, %r93942; AVX2-NEXT: xorl %eax, %eax3943; AVX2-NEXT: tzcntq %rcx, %rax3944; AVX2-NEXT: addl $64, %eax3945; AVX2-NEXT: testq %rdx, %rdx3946; AVX2-NEXT: cmovnel %r9d, %eax3947; AVX2-NEXT: subl $-128, %eax3948; AVX2-NEXT: orq %rsi, %rdi3949; AVX2-NEXT: cmovnel %r8d, %eax3950; AVX2-NEXT: # kill: def $eax killed $eax killed $rax3951; AVX2-NEXT: retq3952;3953; AVX512-LABEL: test_cttz_undef_i256:3954; AVX512: # %bb.0:3955; AVX512-NEXT: tzcntq %rdi, %rax3956; AVX512-NEXT: tzcntq %rsi, %r83957; AVX512-NEXT: addl $64, %r8d3958; AVX512-NEXT: testq %rdi, %rdi3959; AVX512-NEXT: cmovnel %eax, %r8d3960; AVX512-NEXT: tzcntq %rdx, %r93961; AVX512-NEXT: tzcntq %rcx, %rax3962; AVX512-NEXT: addl $64, %eax3963; AVX512-NEXT: testq %rdx, %rdx3964; AVX512-NEXT: cmovnel %r9d, %eax3965; AVX512-NEXT: subl $-128, %eax3966; AVX512-NEXT: orq %rsi, %rdi3967; AVX512-NEXT: cmovnel %r8d, %eax3968; AVX512-NEXT: # kill: def $eax killed $eax killed $rax3969; AVX512-NEXT: retq3970 %cnt = call i256 @llvm.cttz.i256(i256 %a0, i1 -1)3971 %res = trunc i256 %cnt to i323972 ret i32 %res3973}3974 3975define i32 @load_cttz_undef_i256(ptr %p0) nounwind {3976; SSE-LABEL: load_cttz_undef_i256:3977; SSE: # %bb.0:3978; SSE-NEXT: movq 16(%rdi), %rcx3979; SSE-NEXT: movq (%rdi), %rdx3980; SSE-NEXT: movq 8(%rdi), %rsi3981; SSE-NEXT: rep bsfq %rdx, %rax3982; SSE-NEXT: rep bsfq %rsi, %r83983; SSE-NEXT: addl $64, %r8d3984; SSE-NEXT: testq %rdx, %rdx3985; SSE-NEXT: cmovnel %eax, %r8d3986; SSE-NEXT: rep bsfq %rcx, %r93987; SSE-NEXT: rep bsfq 24(%rdi), %rax3988; SSE-NEXT: addl $64, %eax3989; SSE-NEXT: testq %rcx, %rcx3990; SSE-NEXT: cmovnel %r9d, %eax3991; SSE-NEXT: subl $-128, %eax3992; SSE-NEXT: orq %rsi, %rdx3993; SSE-NEXT: cmovnel %r8d, %eax3994; SSE-NEXT: # kill: def $eax killed $eax killed $rax3995; SSE-NEXT: retq3996;3997; AVX2-LABEL: load_cttz_undef_i256:3998; AVX2: # %bb.0:3999; AVX2-NEXT: movq (%rdi), %rcx4000; AVX2-NEXT: movq 8(%rdi), %rdx4001; AVX2-NEXT: tzcntq %rcx, %rax4002; AVX2-NEXT: tzcntq %rdx, %rsi4003; AVX2-NEXT: addl $64, %esi4004; AVX2-NEXT: testq %rcx, %rcx4005; AVX2-NEXT: cmovnel %eax, %esi4006; AVX2-NEXT: movq 16(%rdi), %r84007; AVX2-NEXT: tzcntq %r8, %r94008; AVX2-NEXT: xorl %eax, %eax4009; AVX2-NEXT: tzcntq 24(%rdi), %rax4010; AVX2-NEXT: addl $64, %eax4011; AVX2-NEXT: testq %r8, %r84012; AVX2-NEXT: cmovnel %r9d, %eax4013; AVX2-NEXT: subl $-128, %eax4014; AVX2-NEXT: orq %rdx, %rcx4015; AVX2-NEXT: cmovnel %esi, %eax4016; AVX2-NEXT: # kill: def $eax killed $eax killed $rax4017; AVX2-NEXT: retq4018;4019; AVX512F-LABEL: load_cttz_undef_i256:4020; AVX512F: # %bb.0:4021; AVX512F-NEXT: vmovdqu (%rdi), %ymm04022; AVX512F-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm14023; AVX512F-NEXT: vpaddq %ymm1, %ymm0, %ymm14024; AVX512F-NEXT: vpandn %ymm1, %ymm0, %ymm14025; AVX512F-NEXT: vmovdqa {{.*#+}} ymm2 = [64,128,192,256]4026; AVX512F-NEXT: vplzcntq %ymm1, %ymm14027; AVX512F-NEXT: vpsubq %ymm1, %ymm2, %ymm14028; AVX512F-NEXT: vptestmq %ymm0, %ymm0, %k14029; AVX512F-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}4030; AVX512F-NEXT: vmovd %xmm0, %eax4031; AVX512F-NEXT: vzeroupper4032; AVX512F-NEXT: retq4033;4034; AVX512POPCNT-LABEL: load_cttz_undef_i256:4035; AVX512POPCNT: # %bb.0:4036; AVX512POPCNT-NEXT: vmovdqu (%rdi), %ymm04037; AVX512POPCNT-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm14038; AVX512POPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm14039; AVX512POPCNT-NEXT: vpandn %ymm1, %ymm0, %ymm14040; AVX512POPCNT-NEXT: vpopcntq %ymm1, %ymm14041; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm14042; AVX512POPCNT-NEXT: vptestmq %ymm0, %ymm0, %k14043; AVX512POPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}4044; AVX512POPCNT-NEXT: vmovd %xmm0, %eax4045; AVX512POPCNT-NEXT: vzeroupper4046; AVX512POPCNT-NEXT: retq4047 %a0 = load i256, ptr %p04048 %cnt = call i256 @llvm.cttz.i256(i256 %a0, i1 -1)4049 %res = trunc i256 %cnt to i324050 ret i32 %res4051}4052 4053define i32 @test_cttz_undef_i512(i512 %a0) nounwind {4054; SSE-LABEL: test_cttz_undef_i512:4055; SSE: # %bb.0:4056; SSE-NEXT: pushq %r144057; SSE-NEXT: pushq %rbx4058; SSE-NEXT: rep bsfq %rdi, %rax4059; SSE-NEXT: rep bsfq %rsi, %r114060; SSE-NEXT: addl $64, %r11d4061; SSE-NEXT: testq %rdi, %rdi4062; SSE-NEXT: cmovnel %eax, %r11d4063; SSE-NEXT: rep bsfq %rdx, %rax4064; SSE-NEXT: rep bsfq %rcx, %r104065; SSE-NEXT: addl $64, %r10d4066; SSE-NEXT: testq %rdx, %rdx4067; SSE-NEXT: cmovnel %eax, %r10d4068; SSE-NEXT: subl $-128, %r10d4069; SSE-NEXT: movq %rdi, %rax4070; SSE-NEXT: orq %rsi, %rax4071; SSE-NEXT: cmovnel %r11d, %r10d4072; SSE-NEXT: rep bsfq %r8, %rax4073; SSE-NEXT: rep bsfq %r9, %r114074; SSE-NEXT: addl $64, %r11d4075; SSE-NEXT: testq %r8, %r84076; SSE-NEXT: cmovnel %eax, %r11d4077; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx4078; SSE-NEXT: rep bsfq %rbx, %r144079; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rax4080; SSE-NEXT: addl $64, %eax4081; SSE-NEXT: testq %rbx, %rbx4082; SSE-NEXT: cmovnel %r14d, %eax4083; SSE-NEXT: subl $-128, %eax4084; SSE-NEXT: orq %r9, %r84085; SSE-NEXT: cmovnel %r11d, %eax4086; SSE-NEXT: addl $256, %eax # imm = 0x1004087; SSE-NEXT: orq %rcx, %rsi4088; SSE-NEXT: orq %rdx, %rdi4089; SSE-NEXT: orq %rsi, %rdi4090; SSE-NEXT: cmovnel %r10d, %eax4091; SSE-NEXT: # kill: def $eax killed $eax killed $rax4092; SSE-NEXT: popq %rbx4093; SSE-NEXT: popq %r144094; SSE-NEXT: retq4095;4096; AVX2-LABEL: test_cttz_undef_i512:4097; AVX2: # %bb.0:4098; AVX2-NEXT: pushq %r144099; AVX2-NEXT: pushq %rbx4100; AVX2-NEXT: tzcntq %rdi, %rax4101; AVX2-NEXT: tzcntq %rsi, %r114102; AVX2-NEXT: addl $64, %r11d4103; AVX2-NEXT: testq %rdi, %rdi4104; AVX2-NEXT: cmovnel %eax, %r11d4105; AVX2-NEXT: xorl %eax, %eax4106; AVX2-NEXT: tzcntq %rdx, %rax4107; AVX2-NEXT: tzcntq %rcx, %r104108; AVX2-NEXT: addl $64, %r10d4109; AVX2-NEXT: testq %rdx, %rdx4110; AVX2-NEXT: cmovnel %eax, %r10d4111; AVX2-NEXT: subl $-128, %r10d4112; AVX2-NEXT: movq %rdi, %rax4113; AVX2-NEXT: orq %rsi, %rax4114; AVX2-NEXT: cmovnel %r11d, %r10d4115; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r114116; AVX2-NEXT: xorl %eax, %eax4117; AVX2-NEXT: tzcntq %r8, %rax4118; AVX2-NEXT: xorl %ebx, %ebx4119; AVX2-NEXT: tzcntq %r9, %rbx4120; AVX2-NEXT: addl $64, %ebx4121; AVX2-NEXT: testq %r8, %r84122; AVX2-NEXT: cmovnel %eax, %ebx4123; AVX2-NEXT: xorl %r14d, %r14d4124; AVX2-NEXT: tzcntq %r11, %r144125; AVX2-NEXT: xorl %eax, %eax4126; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %rax4127; AVX2-NEXT: addl $64, %eax4128; AVX2-NEXT: testq %r11, %r114129; AVX2-NEXT: cmovnel %r14d, %eax4130; AVX2-NEXT: subl $-128, %eax4131; AVX2-NEXT: orq %r9, %r84132; AVX2-NEXT: cmovnel %ebx, %eax4133; AVX2-NEXT: addl $256, %eax # imm = 0x1004134; AVX2-NEXT: orq %rcx, %rsi4135; AVX2-NEXT: orq %rdx, %rdi4136; AVX2-NEXT: orq %rsi, %rdi4137; AVX2-NEXT: cmovnel %r10d, %eax4138; AVX2-NEXT: # kill: def $eax killed $eax killed $rax4139; AVX2-NEXT: popq %rbx4140; AVX2-NEXT: popq %r144141; AVX2-NEXT: retq4142;4143; AVX512F-LABEL: test_cttz_undef_i512:4144; AVX512F: # %bb.0:4145; AVX512F-NEXT: vmovq %rcx, %xmm04146; AVX512F-NEXT: vmovq %rdx, %xmm14147; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]4148; AVX512F-NEXT: vmovq %rsi, %xmm14149; AVX512F-NEXT: vmovq %rdi, %xmm24150; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]4151; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm04152; AVX512F-NEXT: vmovq %r9, %xmm14153; AVX512F-NEXT: vmovq %r8, %xmm24154; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]4155; AVX512F-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm14156; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm04157; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -14158; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm14159; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm14160; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]4161; AVX512F-NEXT: vplzcntq %zmm1, %zmm14162; AVX512F-NEXT: vpsubq %zmm1, %zmm2, %zmm14163; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k14164; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}4165; AVX512F-NEXT: vmovd %xmm0, %eax4166; AVX512F-NEXT: vzeroupper4167; AVX512F-NEXT: retq4168;4169; AVX512POPCNT-LABEL: test_cttz_undef_i512:4170; AVX512POPCNT: # %bb.0:4171; AVX512POPCNT-NEXT: vmovq %rcx, %xmm04172; AVX512POPCNT-NEXT: vmovq %rdx, %xmm14173; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]4174; AVX512POPCNT-NEXT: vmovq %rsi, %xmm14175; AVX512POPCNT-NEXT: vmovq %rdi, %xmm24176; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]4177; AVX512POPCNT-NEXT: vmovq %r9, %xmm24178; AVX512POPCNT-NEXT: vmovq %r8, %xmm34179; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]4180; AVX512POPCNT-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm2, %ymm24181; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm04182; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm04183; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -14184; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm14185; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm14186; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm14187; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm14188; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k14189; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}4190; AVX512POPCNT-NEXT: vmovd %xmm0, %eax4191; AVX512POPCNT-NEXT: vzeroupper4192; AVX512POPCNT-NEXT: retq4193 %cnt = call i512 @llvm.cttz.i512(i512 %a0, i1 -1)4194 %res = trunc i512 %cnt to i324195 ret i32 %res4196}4197 4198define i32 @load_cttz_undef_i512(ptr %p0) nounwind {4199; SSE-LABEL: load_cttz_undef_i512:4200; SSE: # %bb.0:4201; SSE-NEXT: pushq %r154202; SSE-NEXT: pushq %r144203; SSE-NEXT: pushq %rbx4204; SSE-NEXT: movq 40(%rdi), %r94205; SSE-NEXT: movq 24(%rdi), %r84206; SSE-NEXT: movq 16(%rdi), %rdx4207; SSE-NEXT: movq (%rdi), %rcx4208; SSE-NEXT: movq 8(%rdi), %rsi4209; SSE-NEXT: rep bsfq %rcx, %rax4210; SSE-NEXT: rep bsfq %rsi, %r114211; SSE-NEXT: addl $64, %r11d4212; SSE-NEXT: testq %rcx, %rcx4213; SSE-NEXT: cmovnel %eax, %r11d4214; SSE-NEXT: rep bsfq %rdx, %rax4215; SSE-NEXT: rep bsfq %r8, %r104216; SSE-NEXT: addl $64, %r10d4217; SSE-NEXT: testq %rdx, %rdx4218; SSE-NEXT: cmovnel %eax, %r10d4219; SSE-NEXT: movq 32(%rdi), %rbx4220; SSE-NEXT: subl $-128, %r10d4221; SSE-NEXT: movq %rcx, %rax4222; SSE-NEXT: orq %rsi, %rax4223; SSE-NEXT: cmovnel %r11d, %r10d4224; SSE-NEXT: rep bsfq %rbx, %rax4225; SSE-NEXT: rep bsfq %r9, %r114226; SSE-NEXT: addl $64, %r11d4227; SSE-NEXT: testq %rbx, %rbx4228; SSE-NEXT: cmovnel %eax, %r11d4229; SSE-NEXT: movq 48(%rdi), %r144230; SSE-NEXT: rep bsfq %r14, %r154231; SSE-NEXT: rep bsfq 56(%rdi), %rax4232; SSE-NEXT: addl $64, %eax4233; SSE-NEXT: testq %r14, %r144234; SSE-NEXT: cmovnel %r15d, %eax4235; SSE-NEXT: subl $-128, %eax4236; SSE-NEXT: orq %r9, %rbx4237; SSE-NEXT: cmovnel %r11d, %eax4238; SSE-NEXT: addl $256, %eax # imm = 0x1004239; SSE-NEXT: orq %r8, %rsi4240; SSE-NEXT: orq %rdx, %rcx4241; SSE-NEXT: orq %rsi, %rcx4242; SSE-NEXT: cmovnel %r10d, %eax4243; SSE-NEXT: # kill: def $eax killed $eax killed $rax4244; SSE-NEXT: popq %rbx4245; SSE-NEXT: popq %r144246; SSE-NEXT: popq %r154247; SSE-NEXT: retq4248;4249; AVX2-LABEL: load_cttz_undef_i512:4250; AVX2: # %bb.0:4251; AVX2-NEXT: pushq %r154252; AVX2-NEXT: pushq %r144253; AVX2-NEXT: pushq %rbx4254; AVX2-NEXT: movq 48(%rdi), %r104255; AVX2-NEXT: movq 40(%rdi), %r94256; AVX2-NEXT: movq 24(%rdi), %r84257; AVX2-NEXT: movq 16(%rdi), %rdx4258; AVX2-NEXT: movq (%rdi), %rcx4259; AVX2-NEXT: movq 8(%rdi), %rsi4260; AVX2-NEXT: tzcntq %rcx, %rax4261; AVX2-NEXT: xorl %ebx, %ebx4262; AVX2-NEXT: tzcntq %rsi, %rbx4263; AVX2-NEXT: addl $64, %ebx4264; AVX2-NEXT: testq %rcx, %rcx4265; AVX2-NEXT: cmovnel %eax, %ebx4266; AVX2-NEXT: xorl %eax, %eax4267; AVX2-NEXT: tzcntq %rdx, %rax4268; AVX2-NEXT: tzcntq %r8, %r114269; AVX2-NEXT: addl $64, %r11d4270; AVX2-NEXT: testq %rdx, %rdx4271; AVX2-NEXT: cmovnel %eax, %r11d4272; AVX2-NEXT: subl $-128, %r11d4273; AVX2-NEXT: movq %rcx, %rax4274; AVX2-NEXT: orq %rsi, %rax4275; AVX2-NEXT: cmovnel %ebx, %r11d4276; AVX2-NEXT: movq 32(%rdi), %rbx4277; AVX2-NEXT: xorl %eax, %eax4278; AVX2-NEXT: tzcntq %rbx, %rax4279; AVX2-NEXT: xorl %r14d, %r14d4280; AVX2-NEXT: tzcntq %r9, %r144281; AVX2-NEXT: addl $64, %r14d4282; AVX2-NEXT: testq %rbx, %rbx4283; AVX2-NEXT: cmovnel %eax, %r14d4284; AVX2-NEXT: xorl %r15d, %r15d4285; AVX2-NEXT: tzcntq %r10, %r154286; AVX2-NEXT: xorl %eax, %eax4287; AVX2-NEXT: tzcntq 56(%rdi), %rax4288; AVX2-NEXT: addl $64, %eax4289; AVX2-NEXT: testq %r10, %r104290; AVX2-NEXT: cmovnel %r15d, %eax4291; AVX2-NEXT: subl $-128, %eax4292; AVX2-NEXT: orq %r9, %rbx4293; AVX2-NEXT: cmovnel %r14d, %eax4294; AVX2-NEXT: addl $256, %eax # imm = 0x1004295; AVX2-NEXT: orq %r8, %rsi4296; AVX2-NEXT: orq %rdx, %rcx4297; AVX2-NEXT: orq %rsi, %rcx4298; AVX2-NEXT: cmovnel %r11d, %eax4299; AVX2-NEXT: # kill: def $eax killed $eax killed $rax4300; AVX2-NEXT: popq %rbx4301; AVX2-NEXT: popq %r144302; AVX2-NEXT: popq %r154303; AVX2-NEXT: retq4304;4305; AVX512F-LABEL: load_cttz_undef_i512:4306; AVX512F: # %bb.0:4307; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm04308; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -14309; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm14310; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm14311; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512]4312; AVX512F-NEXT: vplzcntq %zmm1, %zmm14313; AVX512F-NEXT: vpsubq %zmm1, %zmm2, %zmm14314; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k14315; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}4316; AVX512F-NEXT: vmovd %xmm0, %eax4317; AVX512F-NEXT: vzeroupper4318; AVX512F-NEXT: retq4319;4320; AVX512POPCNT-LABEL: load_cttz_undef_i512:4321; AVX512POPCNT: # %bb.0:4322; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm04323; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm1 = -14324; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm14325; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm14326; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm14327; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm14328; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k14329; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}4330; AVX512POPCNT-NEXT: vmovd %xmm0, %eax4331; AVX512POPCNT-NEXT: vzeroupper4332; AVX512POPCNT-NEXT: retq4333 %a0 = load i512, ptr %p04334 %cnt = call i512 @llvm.cttz.i512(i512 %a0, i1 -1)4335 %res = trunc i512 %cnt to i324336 ret i32 %res4337}4338 4339define i32 @test_cttz_undef_i1024(i1024 %a0) nounwind {4340; SSE-LABEL: test_cttz_undef_i1024:4341; SSE: # %bb.0:4342; SSE-NEXT: pushq %rbp4343; SSE-NEXT: pushq %r154344; SSE-NEXT: pushq %r144345; SSE-NEXT: pushq %r134346; SSE-NEXT: pushq %r124347; SSE-NEXT: pushq %rbx4348; SSE-NEXT: movq %r9, %r144349; SSE-NEXT: movq %rcx, %rbx4350; SSE-NEXT: movq %rdx, %r104351; SSE-NEXT: movq %rsi, %r94352; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx4353; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r114354; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx4355; SSE-NEXT: rep bsfq %rdi, %rax4356; SSE-NEXT: rep bsfq %rsi, %r124357; SSE-NEXT: addl $64, %r12d4358; SSE-NEXT: testq %rdi, %rdi4359; SSE-NEXT: cmovnel %eax, %r12d4360; SSE-NEXT: rep bsfq %r10, %r154361; SSE-NEXT: rep bsfq %rbx, %rax4362; SSE-NEXT: addl $64, %eax4363; SSE-NEXT: testq %r10, %r104364; SSE-NEXT: cmovnel %r15d, %eax4365; SSE-NEXT: subl $-128, %eax4366; SSE-NEXT: movq %rdi, %r134367; SSE-NEXT: orq %rsi, %r134368; SSE-NEXT: cmovnel %r12d, %eax4369; SSE-NEXT: movq %r8, %r154370; SSE-NEXT: rep bsfq %r8, %r124371; SSE-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill4372; SSE-NEXT: rep bsfq %r14, %r134373; SSE-NEXT: addl $64, %r13d4374; SSE-NEXT: testq %r8, %r84375; SSE-NEXT: cmovnel %r12d, %r13d4376; SSE-NEXT: rep bsfq %rcx, %rbp4377; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %r124378; SSE-NEXT: addl $64, %r12d4379; SSE-NEXT: testq %rcx, %rcx4380; SSE-NEXT: cmovnel %ebp, %r12d4381; SSE-NEXT: subl $-128, %r12d4382; SSE-NEXT: movq %r8, %rbp4383; SSE-NEXT: orq %r14, %rbp4384; SSE-NEXT: cmovnel %r13d, %r12d4385; SSE-NEXT: addl $256, %r12d # imm = 0x1004386; SSE-NEXT: movq %rsi, %r134387; SSE-NEXT: orq %rbx, %r134388; SSE-NEXT: movq %rdi, %rbp4389; SSE-NEXT: orq %r10, %rbp4390; SSE-NEXT: orq %r13, %rbp4391; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r134392; SSE-NEXT: cmovnel %eax, %r12d4393; SSE-NEXT: rep bsfq %r11, %rbp4394; SSE-NEXT: rep bsfq %r13, %rax4395; SSE-NEXT: addl $64, %eax4396; SSE-NEXT: testq %r11, %r114397; SSE-NEXT: cmovnel %ebp, %eax4398; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rbp4399; SSE-NEXT: addl $64, %ebp4400; SSE-NEXT: rep bsfq %rdx, %rcx4401; SSE-NEXT: testq %rdx, %rdx4402; SSE-NEXT: cmovnel %ecx, %ebp4403; SSE-NEXT: subl $-128, %ebp4404; SSE-NEXT: movq %r11, %rcx4405; SSE-NEXT: orq %r13, %rcx4406; SSE-NEXT: cmovnel %eax, %ebp4407; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r144408; SSE-NEXT: rep bsfq %r14, %rcx4409; SSE-NEXT: addl $64, %ecx4410; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx4411; SSE-NEXT: rep bsfq %rdx, %rax4412; SSE-NEXT: testq %rdx, %rdx4413; SSE-NEXT: cmovnel %eax, %ecx4414; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rax4415; SSE-NEXT: addl $64, %eax4416; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r84417; SSE-NEXT: rep bsfq %r8, %rsi4418; SSE-NEXT: testq %r8, %r84419; SSE-NEXT: cmovnel %esi, %eax4420; SSE-NEXT: subl $-128, %eax4421; SSE-NEXT: orq %r14, %rdx4422; SSE-NEXT: cmovnel %ecx, %eax4423; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r134424; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r114425; SSE-NEXT: addl $256, %eax # imm = 0x1004426; SSE-NEXT: orq %r13, %r114427; SSE-NEXT: cmovnel %ebp, %eax4428; SSE-NEXT: orq {{[0-9]+}}(%rsp), %rbx4429; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload4430; SSE-NEXT: orq %rbx, %r94431; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r104432; SSE-NEXT: orq %r15, %rdi4433; SSE-NEXT: orq %r10, %rdi4434; SSE-NEXT: addl $512, %eax # imm = 0x2004435; SSE-NEXT: orq %r9, %rdi4436; SSE-NEXT: cmovnel %r12d, %eax4437; SSE-NEXT: # kill: def $eax killed $eax killed $rax4438; SSE-NEXT: popq %rbx4439; SSE-NEXT: popq %r124440; SSE-NEXT: popq %r134441; SSE-NEXT: popq %r144442; SSE-NEXT: popq %r154443; SSE-NEXT: popq %rbp4444; SSE-NEXT: retq4445;4446; AVX2-LABEL: test_cttz_undef_i1024:4447; AVX2: # %bb.0:4448; AVX2-NEXT: pushq %rbp4449; AVX2-NEXT: pushq %r154450; AVX2-NEXT: pushq %r144451; AVX2-NEXT: pushq %r134452; AVX2-NEXT: pushq %r124453; AVX2-NEXT: pushq %rbx4454; AVX2-NEXT: movq %r9, %rbx4455; AVX2-NEXT: movq %r8, %r144456; AVX2-NEXT: movq %rcx, %r114457; AVX2-NEXT: movq %rdx, %r104458; AVX2-NEXT: movq %rsi, %r94459; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r84460; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rsi4461; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx4462; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx4463; AVX2-NEXT: tzcntq %rdi, %rax4464; AVX2-NEXT: xorl %r15d, %r15d4465; AVX2-NEXT: tzcntq %r9, %r154466; AVX2-NEXT: addl $64, %r15d4467; AVX2-NEXT: testq %rdi, %rdi4468; AVX2-NEXT: cmovnel %eax, %r15d4469; AVX2-NEXT: xorl %r12d, %r12d4470; AVX2-NEXT: tzcntq %r10, %r124471; AVX2-NEXT: xorl %eax, %eax4472; AVX2-NEXT: tzcntq %r11, %rax4473; AVX2-NEXT: addl $64, %eax4474; AVX2-NEXT: testq %r10, %r104475; AVX2-NEXT: cmovnel %r12d, %eax4476; AVX2-NEXT: subl $-128, %eax4477; AVX2-NEXT: movq %rdi, %r124478; AVX2-NEXT: orq %r9, %r124479; AVX2-NEXT: cmovnel %r15d, %eax4480; AVX2-NEXT: xorl %r15d, %r15d4481; AVX2-NEXT: tzcntq %r14, %r154482; AVX2-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill4483; AVX2-NEXT: xorl %r12d, %r12d4484; AVX2-NEXT: tzcntq %rbx, %r124485; AVX2-NEXT: addl $64, %r12d4486; AVX2-NEXT: testq %r14, %r144487; AVX2-NEXT: cmovnel %r15d, %r12d4488; AVX2-NEXT: xorl %r13d, %r13d4489; AVX2-NEXT: tzcntq %rcx, %r134490; AVX2-NEXT: xorl %r15d, %r15d4491; AVX2-NEXT: tzcntq %rdx, %r154492; AVX2-NEXT: addl $64, %r15d4493; AVX2-NEXT: testq %rcx, %rcx4494; AVX2-NEXT: cmovnel %r13d, %r15d4495; AVX2-NEXT: subl $-128, %r15d4496; AVX2-NEXT: movq %r14, %r134497; AVX2-NEXT: orq %rbx, %r134498; AVX2-NEXT: cmovnel %r12d, %r15d4499; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r124500; AVX2-NEXT: addl $256, %r15d # imm = 0x1004501; AVX2-NEXT: movq %r9, %r134502; AVX2-NEXT: orq %r11, %r134503; AVX2-NEXT: movq %rdi, %rbp4504; AVX2-NEXT: orq %r10, %rbp4505; AVX2-NEXT: orq %r13, %rbp4506; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r134507; AVX2-NEXT: cmovnel %eax, %r15d4508; AVX2-NEXT: xorl %ebp, %ebp4509; AVX2-NEXT: tzcntq %r12, %rbp4510; AVX2-NEXT: xorl %eax, %eax4511; AVX2-NEXT: tzcntq %r13, %rax4512; AVX2-NEXT: addl $64, %eax4513; AVX2-NEXT: testq %r12, %r124514; AVX2-NEXT: cmovnel %ebp, %eax4515; AVX2-NEXT: xorl %ebp, %ebp4516; AVX2-NEXT: tzcntq %r8, %rbp4517; AVX2-NEXT: addl $64, %ebp4518; AVX2-NEXT: xorl %ecx, %ecx4519; AVX2-NEXT: tzcntq %rsi, %rcx4520; AVX2-NEXT: testq %rsi, %rsi4521; AVX2-NEXT: cmovnel %ecx, %ebp4522; AVX2-NEXT: subl $-128, %ebp4523; AVX2-NEXT: movq %r12, %rcx4524; AVX2-NEXT: orq %r13, %rcx4525; AVX2-NEXT: cmovnel %eax, %ebp4526; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx4527; AVX2-NEXT: xorl %ecx, %ecx4528; AVX2-NEXT: tzcntq %rbx, %rcx4529; AVX2-NEXT: addl $64, %ecx4530; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx4531; AVX2-NEXT: xorl %eax, %eax4532; AVX2-NEXT: tzcntq %rdx, %rax4533; AVX2-NEXT: testq %rdx, %rdx4534; AVX2-NEXT: cmovnel %eax, %ecx4535; AVX2-NEXT: xorl %eax, %eax4536; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %rax4537; AVX2-NEXT: addl $64, %eax4538; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r84539; AVX2-NEXT: tzcntq %r8, %rsi4540; AVX2-NEXT: testq %r8, %r84541; AVX2-NEXT: cmovnel %esi, %eax4542; AVX2-NEXT: subl $-128, %eax4543; AVX2-NEXT: orq %rbx, %rdx4544; AVX2-NEXT: cmovnel %ecx, %eax4545; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r134546; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r124547; AVX2-NEXT: addl $256, %eax # imm = 0x1004548; AVX2-NEXT: orq %r13, %r124549; AVX2-NEXT: cmovnel %ebp, %eax4550; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r114551; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload4552; AVX2-NEXT: orq %r11, %r94553; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r104554; AVX2-NEXT: orq %r14, %rdi4555; AVX2-NEXT: orq %r10, %rdi4556; AVX2-NEXT: addl $512, %eax # imm = 0x2004557; AVX2-NEXT: orq %r9, %rdi4558; AVX2-NEXT: cmovnel %r15d, %eax4559; AVX2-NEXT: # kill: def $eax killed $eax killed $rax4560; AVX2-NEXT: popq %rbx4561; AVX2-NEXT: popq %r124562; AVX2-NEXT: popq %r134563; AVX2-NEXT: popq %r144564; AVX2-NEXT: popq %r154565; AVX2-NEXT: popq %rbp4566; AVX2-NEXT: retq4567;4568; AVX512F-LABEL: test_cttz_undef_i1024:4569; AVX512F: # %bb.0:4570; AVX512F-NEXT: vmovq %rcx, %xmm04571; AVX512F-NEXT: vmovq %rdx, %xmm14572; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]4573; AVX512F-NEXT: vmovq %rsi, %xmm14574; AVX512F-NEXT: vmovq %rdi, %xmm24575; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]4576; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm04577; AVX512F-NEXT: vmovq %r9, %xmm14578; AVX512F-NEXT: vmovq %r8, %xmm24579; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]4580; AVX512F-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm14581; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm04582; AVX512F-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm14583; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -14584; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm34585; AVX512F-NEXT: vpandnq %zmm3, %zmm0, %zmm34586; AVX512F-NEXT: vplzcntq %zmm3, %zmm34587; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]4588; AVX512F-NEXT: vpsubq %zmm3, %zmm4, %zmm34589; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k14590; AVX512F-NEXT: vpcompressq %zmm3, %zmm0 {%k1} {z}4591; AVX512F-NEXT: vmovd %xmm0, %r10d4592; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm04593; AVX512F-NEXT: vpandnq %zmm0, %zmm1, %zmm04594; AVX512F-NEXT: vplzcntq %zmm0, %zmm04595; AVX512F-NEXT: vpsubq %zmm0, %zmm4, %zmm04596; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k14597; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}4598; AVX512F-NEXT: vmovd %xmm0, %eax4599; AVX512F-NEXT: addl $512, %eax # imm = 0x2004600; AVX512F-NEXT: orq %r9, %rsi4601; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rcx4602; AVX512F-NEXT: orq %rsi, %rcx4603; AVX512F-NEXT: orq %r8, %rdi4604; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rdx4605; AVX512F-NEXT: orq %rdi, %rdx4606; AVX512F-NEXT: orq %rcx, %rdx4607; AVX512F-NEXT: cmovnel %r10d, %eax4608; AVX512F-NEXT: vzeroupper4609; AVX512F-NEXT: retq4610;4611; AVX512POPCNT-LABEL: test_cttz_undef_i1024:4612; AVX512POPCNT: # %bb.0:4613; AVX512POPCNT-NEXT: vmovq %rcx, %xmm04614; AVX512POPCNT-NEXT: vmovq %rdx, %xmm14615; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]4616; AVX512POPCNT-NEXT: vmovq %rsi, %xmm14617; AVX512POPCNT-NEXT: vmovq %rdi, %xmm24618; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]4619; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm04620; AVX512POPCNT-NEXT: vmovq %r9, %xmm14621; AVX512POPCNT-NEXT: vmovq %r8, %xmm24622; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]4623; AVX512POPCNT-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm14624; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm04625; AVX512POPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm14626; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -14627; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm34628; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm34629; AVX512POPCNT-NEXT: vpopcntq %zmm3, %zmm34630; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]4631; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm34632; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k14633; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm0 {%k1} {z}4634; AVX512POPCNT-NEXT: vmovd %xmm0, %r10d4635; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm04636; AVX512POPCNT-NEXT: vpandnq %zmm0, %zmm1, %zmm04637; AVX512POPCNT-NEXT: vpopcntq %zmm0, %zmm04638; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm0, %zmm04639; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k14640; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}4641; AVX512POPCNT-NEXT: vmovd %xmm0, %eax4642; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x2004643; AVX512POPCNT-NEXT: orq %r9, %rsi4644; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rcx4645; AVX512POPCNT-NEXT: orq %rsi, %rcx4646; AVX512POPCNT-NEXT: orq %r8, %rdi4647; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rdx4648; AVX512POPCNT-NEXT: orq %rdi, %rdx4649; AVX512POPCNT-NEXT: orq %rcx, %rdx4650; AVX512POPCNT-NEXT: cmovnel %r10d, %eax4651; AVX512POPCNT-NEXT: vzeroupper4652; AVX512POPCNT-NEXT: retq4653 %cnt = call i1024 @llvm.cttz.i1024(i1024 %a0, i1 -1)4654 %res = trunc i1024 %cnt to i324655 ret i32 %res4656}4657 4658define i32 @load_cttz_undef_i1024(ptr %p0) nounwind {4659; SSE-LABEL: load_cttz_undef_i1024:4660; SSE: # %bb.0:4661; SSE-NEXT: pushq %rbp4662; SSE-NEXT: pushq %r154663; SSE-NEXT: pushq %r144664; SSE-NEXT: pushq %r134665; SSE-NEXT: pushq %r124666; SSE-NEXT: pushq %rbx4667; SSE-NEXT: movq 72(%rdi), %rbx4668; SSE-NEXT: movq 56(%rdi), %r94669; SSE-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill4670; SSE-NEXT: movq 48(%rdi), %rcx4671; SSE-NEXT: movq 40(%rdi), %r104672; SSE-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill4673; SSE-NEXT: movq 32(%rdi), %rsi4674; SSE-NEXT: movq 24(%rdi), %rbp4675; SSE-NEXT: movq (%rdi), %r84676; SSE-NEXT: movq 8(%rdi), %r114677; SSE-NEXT: rep bsfq %r8, %rax4678; SSE-NEXT: rep bsfq %r11, %rdx4679; SSE-NEXT: addl $64, %edx4680; SSE-NEXT: testq %r8, %r84681; SSE-NEXT: cmovnel %eax, %edx4682; SSE-NEXT: movq 16(%rdi), %r144683; SSE-NEXT: rep bsfq %r14, %r154684; SSE-NEXT: rep bsfq %rbp, %rax4685; SSE-NEXT: movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill4686; SSE-NEXT: addl $64, %eax4687; SSE-NEXT: testq %r14, %r144688; SSE-NEXT: cmovnel %r15d, %eax4689; SSE-NEXT: subl $-128, %eax4690; SSE-NEXT: movq %r8, %r154691; SSE-NEXT: orq %r11, %r154692; SSE-NEXT: cmovnel %edx, %eax4693; SSE-NEXT: rep bsfq %rsi, %rdx4694; SSE-NEXT: rep bsfq %r10, %r134695; SSE-NEXT: addl $64, %r13d4696; SSE-NEXT: testq %rsi, %rsi4697; SSE-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill4698; SSE-NEXT: cmovnel %edx, %r13d4699; SSE-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill4700; SSE-NEXT: rep bsfq %rcx, %rdx4701; SSE-NEXT: rep bsfq %r9, %r154702; SSE-NEXT: addl $64, %r15d4703; SSE-NEXT: testq %rcx, %rcx4704; SSE-NEXT: cmovnel %edx, %r15d4705; SSE-NEXT: movq 64(%rdi), %r124706; SSE-NEXT: subl $-128, %r15d4707; SSE-NEXT: movq %rsi, %rdx4708; SSE-NEXT: orq %r10, %rdx4709; SSE-NEXT: cmovnel %r13d, %r15d4710; SSE-NEXT: addl $256, %r15d # imm = 0x1004711; SSE-NEXT: movq %r11, %rdx4712; SSE-NEXT: orq %rbp, %rdx4713; SSE-NEXT: movq %r8, %r134714; SSE-NEXT: orq %r14, %r134715; SSE-NEXT: orq %rdx, %r134716; SSE-NEXT: cmovnel %eax, %r15d4717; SSE-NEXT: rep bsfq %r12, %rdx4718; SSE-NEXT: rep bsfq %rbx, %rax4719; SSE-NEXT: addl $64, %eax4720; SSE-NEXT: testq %r12, %r124721; SSE-NEXT: cmovnel %edx, %eax4722; SSE-NEXT: movq 88(%rdi), %rbp4723; SSE-NEXT: rep bsfq %rbp, %r134724; SSE-NEXT: addl $64, %r13d4725; SSE-NEXT: movq 80(%rdi), %r104726; SSE-NEXT: rep bsfq %r10, %rcx4727; SSE-NEXT: testq %r10, %r104728; SSE-NEXT: cmovnel %ecx, %r13d4729; SSE-NEXT: subl $-128, %r13d4730; SSE-NEXT: movq %r12, %rcx4731; SSE-NEXT: orq %rbx, %rcx4732; SSE-NEXT: cmovnel %eax, %r13d4733; SSE-NEXT: movq 104(%rdi), %r94734; SSE-NEXT: rep bsfq %r9, %rcx4735; SSE-NEXT: addl $64, %ecx4736; SSE-NEXT: movq 96(%rdi), %rdx4737; SSE-NEXT: rep bsfq %rdx, %rax4738; SSE-NEXT: testq %rdx, %rdx4739; SSE-NEXT: cmovnel %eax, %ecx4740; SSE-NEXT: rep bsfq 120(%rdi), %rax4741; SSE-NEXT: movq 112(%rdi), %rdi4742; SSE-NEXT: addl $64, %eax4743; SSE-NEXT: rep bsfq %rdi, %rsi4744; SSE-NEXT: testq %rdi, %rdi4745; SSE-NEXT: cmovnel %esi, %eax4746; SSE-NEXT: subl $-128, %eax4747; SSE-NEXT: orq %r9, %rdx4748; SSE-NEXT: cmovnel %ecx, %eax4749; SSE-NEXT: orq %rbp, %rbx4750; SSE-NEXT: orq %r10, %r124751; SSE-NEXT: addl $256, %eax # imm = 0x1004752; SSE-NEXT: orq %rbx, %r124753; SSE-NEXT: cmovnel %r13d, %eax4754; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload4755; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload4756; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload4757; SSE-NEXT: orq %rcx, %r114758; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload4759; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload4760; SSE-NEXT: orq %r14, %r84761; SSE-NEXT: addl $512, %eax # imm = 0x2004762; SSE-NEXT: orq %r11, %r84763; SSE-NEXT: cmovnel %r15d, %eax4764; SSE-NEXT: # kill: def $eax killed $eax killed $rax4765; SSE-NEXT: popq %rbx4766; SSE-NEXT: popq %r124767; SSE-NEXT: popq %r134768; SSE-NEXT: popq %r144769; SSE-NEXT: popq %r154770; SSE-NEXT: popq %rbp4771; SSE-NEXT: retq4772;4773; AVX2-LABEL: load_cttz_undef_i1024:4774; AVX2: # %bb.0:4775; AVX2-NEXT: pushq %rbp4776; AVX2-NEXT: pushq %r154777; AVX2-NEXT: pushq %r144778; AVX2-NEXT: pushq %r134779; AVX2-NEXT: pushq %r124780; AVX2-NEXT: pushq %rbx4781; AVX2-NEXT: movq 72(%rdi), %r144782; AVX2-NEXT: movq 64(%rdi), %r154783; AVX2-NEXT: movq 56(%rdi), %r94784; AVX2-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill4785; AVX2-NEXT: movq 48(%rdi), %rcx4786; AVX2-NEXT: movq 40(%rdi), %r104787; AVX2-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill4788; AVX2-NEXT: movq 32(%rdi), %rsi4789; AVX2-NEXT: movq 24(%rdi), %rbp4790; AVX2-NEXT: movq 16(%rdi), %rbx4791; AVX2-NEXT: movq (%rdi), %r84792; AVX2-NEXT: movq 8(%rdi), %r114793; AVX2-NEXT: tzcntq %r8, %rax4794; AVX2-NEXT: tzcntq %r11, %rdx4795; AVX2-NEXT: addl $64, %edx4796; AVX2-NEXT: testq %r8, %r84797; AVX2-NEXT: cmovnel %eax, %edx4798; AVX2-NEXT: xorl %r12d, %r12d4799; AVX2-NEXT: tzcntq %rbx, %r124800; AVX2-NEXT: xorl %eax, %eax4801; AVX2-NEXT: tzcntq %rbp, %rax4802; AVX2-NEXT: movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill4803; AVX2-NEXT: addl $64, %eax4804; AVX2-NEXT: testq %rbx, %rbx4805; AVX2-NEXT: cmovnel %r12d, %eax4806; AVX2-NEXT: subl $-128, %eax4807; AVX2-NEXT: movq %r8, %r124808; AVX2-NEXT: orq %r11, %r124809; AVX2-NEXT: cmovnel %edx, %eax4810; AVX2-NEXT: xorl %edx, %edx4811; AVX2-NEXT: tzcntq %rsi, %rdx4812; AVX2-NEXT: xorl %r13d, %r13d4813; AVX2-NEXT: tzcntq %r10, %r134814; AVX2-NEXT: addl $64, %r13d4815; AVX2-NEXT: testq %rsi, %rsi4816; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill4817; AVX2-NEXT: cmovnel %edx, %r13d4818; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill4819; AVX2-NEXT: xorl %edx, %edx4820; AVX2-NEXT: tzcntq %rcx, %rdx4821; AVX2-NEXT: xorl %r12d, %r12d4822; AVX2-NEXT: tzcntq %r9, %r124823; AVX2-NEXT: addl $64, %r12d4824; AVX2-NEXT: testq %rcx, %rcx4825; AVX2-NEXT: cmovnel %edx, %r12d4826; AVX2-NEXT: subl $-128, %r12d4827; AVX2-NEXT: movq %rsi, %rdx4828; AVX2-NEXT: orq %r10, %rdx4829; AVX2-NEXT: cmovnel %r13d, %r12d4830; AVX2-NEXT: addl $256, %r12d # imm = 0x1004831; AVX2-NEXT: movq %r11, %rdx4832; AVX2-NEXT: orq %rbp, %rdx4833; AVX2-NEXT: movq %r8, %r134834; AVX2-NEXT: orq %rbx, %r134835; AVX2-NEXT: orq %rdx, %r134836; AVX2-NEXT: cmovnel %eax, %r12d4837; AVX2-NEXT: xorl %edx, %edx4838; AVX2-NEXT: tzcntq %r15, %rdx4839; AVX2-NEXT: xorl %eax, %eax4840; AVX2-NEXT: tzcntq %r14, %rax4841; AVX2-NEXT: addl $64, %eax4842; AVX2-NEXT: testq %r15, %r154843; AVX2-NEXT: cmovnel %edx, %eax4844; AVX2-NEXT: movq 88(%rdi), %rbp4845; AVX2-NEXT: xorl %r13d, %r13d4846; AVX2-NEXT: tzcntq %rbp, %r134847; AVX2-NEXT: addl $64, %r13d4848; AVX2-NEXT: movq 80(%rdi), %r104849; AVX2-NEXT: xorl %ecx, %ecx4850; AVX2-NEXT: tzcntq %r10, %rcx4851; AVX2-NEXT: testq %r10, %r104852; AVX2-NEXT: cmovnel %ecx, %r13d4853; AVX2-NEXT: subl $-128, %r13d4854; AVX2-NEXT: movq %r15, %rcx4855; AVX2-NEXT: orq %r14, %rcx4856; AVX2-NEXT: cmovnel %eax, %r13d4857; AVX2-NEXT: movq 104(%rdi), %r94858; AVX2-NEXT: xorl %ecx, %ecx4859; AVX2-NEXT: tzcntq %r9, %rcx4860; AVX2-NEXT: addl $64, %ecx4861; AVX2-NEXT: movq 96(%rdi), %rdx4862; AVX2-NEXT: xorl %eax, %eax4863; AVX2-NEXT: tzcntq %rdx, %rax4864; AVX2-NEXT: testq %rdx, %rdx4865; AVX2-NEXT: cmovnel %eax, %ecx4866; AVX2-NEXT: movq 112(%rdi), %rsi4867; AVX2-NEXT: xorl %eax, %eax4868; AVX2-NEXT: tzcntq 120(%rdi), %rax4869; AVX2-NEXT: addl $64, %eax4870; AVX2-NEXT: tzcntq %rsi, %rdi4871; AVX2-NEXT: testq %rsi, %rsi4872; AVX2-NEXT: cmovnel %edi, %eax4873; AVX2-NEXT: subl $-128, %eax4874; AVX2-NEXT: orq %r9, %rdx4875; AVX2-NEXT: cmovnel %ecx, %eax4876; AVX2-NEXT: orq %rbp, %r144877; AVX2-NEXT: orq %r10, %r154878; AVX2-NEXT: addl $256, %eax # imm = 0x1004879; AVX2-NEXT: orq %r14, %r154880; AVX2-NEXT: cmovnel %r13d, %eax4881; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload4882; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload4883; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload4884; AVX2-NEXT: orq %rcx, %r114885; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload4886; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload4887; AVX2-NEXT: orq %rbx, %r84888; AVX2-NEXT: addl $512, %eax # imm = 0x2004889; AVX2-NEXT: orq %r11, %r84890; AVX2-NEXT: cmovnel %r12d, %eax4891; AVX2-NEXT: # kill: def $eax killed $eax killed $rax4892; AVX2-NEXT: popq %rbx4893; AVX2-NEXT: popq %r124894; AVX2-NEXT: popq %r134895; AVX2-NEXT: popq %r144896; AVX2-NEXT: popq %r154897; AVX2-NEXT: popq %rbp4898; AVX2-NEXT: retq4899;4900; AVX512F-LABEL: load_cttz_undef_i1024:4901; AVX512F: # %bb.0:4902; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm04903; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm14904; AVX512F-NEXT: movq 16(%rdi), %rax4905; AVX512F-NEXT: movq (%rdi), %rcx4906; AVX512F-NEXT: movq 8(%rdi), %rdx4907; AVX512F-NEXT: movq 24(%rdi), %rsi4908; AVX512F-NEXT: orq 56(%rdi), %rsi4909; AVX512F-NEXT: orq 40(%rdi), %rdx4910; AVX512F-NEXT: orq 48(%rdi), %rax4911; AVX512F-NEXT: orq 32(%rdi), %rcx4912; AVX512F-NEXT: orq %rsi, %rdx4913; AVX512F-NEXT: orq %rax, %rcx4914; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -14915; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm34916; AVX512F-NEXT: vpandnq %zmm3, %zmm1, %zmm34917; AVX512F-NEXT: vplzcntq %zmm3, %zmm34918; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]4919; AVX512F-NEXT: vpsubq %zmm3, %zmm4, %zmm34920; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k14921; AVX512F-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}4922; AVX512F-NEXT: vmovd %xmm1, %esi4923; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm14924; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm14925; AVX512F-NEXT: vplzcntq %zmm1, %zmm14926; AVX512F-NEXT: vpsubq %zmm1, %zmm4, %zmm14927; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k14928; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}4929; AVX512F-NEXT: vmovd %xmm0, %eax4930; AVX512F-NEXT: addl $512, %eax # imm = 0x2004931; AVX512F-NEXT: orq %rdx, %rcx4932; AVX512F-NEXT: cmovnel %esi, %eax4933; AVX512F-NEXT: vzeroupper4934; AVX512F-NEXT: retq4935;4936; AVX512POPCNT-LABEL: load_cttz_undef_i1024:4937; AVX512POPCNT: # %bb.0:4938; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm04939; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm14940; AVX512POPCNT-NEXT: movq 16(%rdi), %rax4941; AVX512POPCNT-NEXT: movq (%rdi), %rcx4942; AVX512POPCNT-NEXT: movq 8(%rdi), %rdx4943; AVX512POPCNT-NEXT: movq 24(%rdi), %rsi4944; AVX512POPCNT-NEXT: orq 56(%rdi), %rsi4945; AVX512POPCNT-NEXT: orq 40(%rdi), %rdx4946; AVX512POPCNT-NEXT: orq 48(%rdi), %rax4947; AVX512POPCNT-NEXT: orq 32(%rdi), %rcx4948; AVX512POPCNT-NEXT: orq %rsi, %rdx4949; AVX512POPCNT-NEXT: orq %rax, %rcx4950; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -14951; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm34952; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm1, %zmm34953; AVX512POPCNT-NEXT: vpopcntq %zmm3, %zmm34954; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]4955; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm34956; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k14957; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}4958; AVX512POPCNT-NEXT: vmovd %xmm1, %esi4959; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm14960; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm14961; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm14962; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm1, %zmm14963; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k14964; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}4965; AVX512POPCNT-NEXT: vmovd %xmm0, %eax4966; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x2004967; AVX512POPCNT-NEXT: orq %rdx, %rcx4968; AVX512POPCNT-NEXT: cmovnel %esi, %eax4969; AVX512POPCNT-NEXT: vzeroupper4970; AVX512POPCNT-NEXT: retq4971 %a0 = load i1024, ptr %p04972 %cnt = call i1024 @llvm.cttz.i1024(i1024 %a0, i1 -1)4973 %res = trunc i1024 %cnt to i324974 ret i32 %res4975}4976