brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mattr=+sve -force-streaming-compatible < %s | FileCheck %s3; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s4; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE5 6target triple = "aarch64-unknown-linux-gnu"7 8;9; CLZ10;11 12define <4 x i8> @ctlz_v4i8(<4 x i8> %op) {13; CHECK-LABEL: ctlz_v4i8:14; CHECK:       // %bb.0:15; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z016; CHECK-NEXT:    ptrue p0.h, vl417; CHECK-NEXT:    and z0.h, z0.h, #0xff18; CHECK-NEXT:    clz z0.h, p0/m, z0.h19; CHECK-NEXT:    sub z0.h, z0.h, #8 // =0x820; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z021; CHECK-NEXT:    ret22;23; NONEON-NOSVE-LABEL: ctlz_v4i8:24; NONEON-NOSVE:       // %bb.0:25; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!26; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 1627; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #6]28; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #4]29; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #2]30; NONEON-NOSVE-NEXT:    ldrb w11, [sp]31; NONEON-NOSVE-NEXT:    clz w8, w832; NONEON-NOSVE-NEXT:    clz w9, w933; NONEON-NOSVE-NEXT:    clz w10, w1034; NONEON-NOSVE-NEXT:    sub w8, w8, #2435; NONEON-NOSVE-NEXT:    sub w9, w9, #2436; NONEON-NOSVE-NEXT:    sub w10, w10, #2437; NONEON-NOSVE-NEXT:    strh w8, [sp, #14]38; NONEON-NOSVE-NEXT:    clz w8, w1139; NONEON-NOSVE-NEXT:    sub w8, w8, #2440; NONEON-NOSVE-NEXT:    strh w9, [sp, #12]41; NONEON-NOSVE-NEXT:    strh w10, [sp, #10]42; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]43; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]44; NONEON-NOSVE-NEXT:    add sp, sp, #1645; NONEON-NOSVE-NEXT:    ret46  %res = call <4 x i8> @llvm.ctlz.v4i8(<4 x i8> %op)47  ret <4 x i8> %res48}49 50define <8 x i8> @ctlz_v8i8(<8 x i8> %op) {51; CHECK-LABEL: ctlz_v8i8:52; CHECK:       // %bb.0:53; CHECK-NEXT:    ptrue p0.b, vl854; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z055; CHECK-NEXT:    clz z0.b, p0/m, z0.b56; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z057; CHECK-NEXT:    ret58;59; NONEON-NOSVE-LABEL: ctlz_v8i8:60; NONEON-NOSVE:       // %bb.0:61; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!62; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 1663; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #7]64; NONEON-NOSVE-NEXT:    clz w8, w865; NONEON-NOSVE-NEXT:    sub w8, w8, #2466; NONEON-NOSVE-NEXT:    strb w8, [sp, #15]67; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #6]68; NONEON-NOSVE-NEXT:    clz w8, w869; NONEON-NOSVE-NEXT:    sub w8, w8, #2470; NONEON-NOSVE-NEXT:    strb w8, [sp, #14]71; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #5]72; NONEON-NOSVE-NEXT:    clz w8, w873; NONEON-NOSVE-NEXT:    sub w8, w8, #2474; NONEON-NOSVE-NEXT:    strb w8, [sp, #13]75; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #4]76; NONEON-NOSVE-NEXT:    clz w8, w877; NONEON-NOSVE-NEXT:    sub w8, w8, #2478; NONEON-NOSVE-NEXT:    strb w8, [sp, #12]79; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #3]80; NONEON-NOSVE-NEXT:    clz w8, w881; NONEON-NOSVE-NEXT:    sub w8, w8, #2482; NONEON-NOSVE-NEXT:    strb w8, [sp, #11]83; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]84; NONEON-NOSVE-NEXT:    clz w8, w885; NONEON-NOSVE-NEXT:    sub w8, w8, #2486; NONEON-NOSVE-NEXT:    strb w8, [sp, #10]87; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #1]88; NONEON-NOSVE-NEXT:    clz w8, w889; NONEON-NOSVE-NEXT:    sub w8, w8, #2490; NONEON-NOSVE-NEXT:    strb w8, [sp, #9]91; NONEON-NOSVE-NEXT:    ldrb w8, [sp]92; NONEON-NOSVE-NEXT:    clz w8, w893; NONEON-NOSVE-NEXT:    sub w8, w8, #2494; NONEON-NOSVE-NEXT:    strb w8, [sp, #8]95; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]96; NONEON-NOSVE-NEXT:    add sp, sp, #1697; NONEON-NOSVE-NEXT:    ret98  %res = call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %op)99  ret <8 x i8> %res100}101 102define <16 x i8> @ctlz_v16i8(<16 x i8> %op) {103; CHECK-LABEL: ctlz_v16i8:104; CHECK:       // %bb.0:105; CHECK-NEXT:    ptrue p0.b, vl16106; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0107; CHECK-NEXT:    clz z0.b, p0/m, z0.b108; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0109; CHECK-NEXT:    ret110;111; NONEON-NOSVE-LABEL: ctlz_v16i8:112; NONEON-NOSVE:       // %bb.0:113; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!114; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32115; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #15]116; NONEON-NOSVE-NEXT:    clz w8, w8117; NONEON-NOSVE-NEXT:    sub w8, w8, #24118; NONEON-NOSVE-NEXT:    strb w8, [sp, #31]119; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]120; NONEON-NOSVE-NEXT:    clz w8, w8121; NONEON-NOSVE-NEXT:    sub w8, w8, #24122; NONEON-NOSVE-NEXT:    strb w8, [sp, #30]123; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #13]124; NONEON-NOSVE-NEXT:    clz w8, w8125; NONEON-NOSVE-NEXT:    sub w8, w8, #24126; NONEON-NOSVE-NEXT:    strb w8, [sp, #29]127; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]128; NONEON-NOSVE-NEXT:    clz w8, w8129; NONEON-NOSVE-NEXT:    sub w8, w8, #24130; NONEON-NOSVE-NEXT:    strb w8, [sp, #28]131; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #11]132; NONEON-NOSVE-NEXT:    clz w8, w8133; NONEON-NOSVE-NEXT:    sub w8, w8, #24134; NONEON-NOSVE-NEXT:    strb w8, [sp, #27]135; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #10]136; NONEON-NOSVE-NEXT:    clz w8, w8137; NONEON-NOSVE-NEXT:    sub w8, w8, #24138; NONEON-NOSVE-NEXT:    strb w8, [sp, #26]139; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #9]140; NONEON-NOSVE-NEXT:    clz w8, w8141; NONEON-NOSVE-NEXT:    sub w8, w8, #24142; NONEON-NOSVE-NEXT:    strb w8, [sp, #25]143; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]144; NONEON-NOSVE-NEXT:    clz w8, w8145; NONEON-NOSVE-NEXT:    sub w8, w8, #24146; NONEON-NOSVE-NEXT:    strb w8, [sp, #24]147; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #7]148; NONEON-NOSVE-NEXT:    clz w8, w8149; NONEON-NOSVE-NEXT:    sub w8, w8, #24150; NONEON-NOSVE-NEXT:    strb w8, [sp, #23]151; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #6]152; NONEON-NOSVE-NEXT:    clz w8, w8153; NONEON-NOSVE-NEXT:    sub w8, w8, #24154; NONEON-NOSVE-NEXT:    strb w8, [sp, #22]155; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #5]156; NONEON-NOSVE-NEXT:    clz w8, w8157; NONEON-NOSVE-NEXT:    sub w8, w8, #24158; NONEON-NOSVE-NEXT:    strb w8, [sp, #21]159; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #4]160; NONEON-NOSVE-NEXT:    clz w8, w8161; NONEON-NOSVE-NEXT:    sub w8, w8, #24162; NONEON-NOSVE-NEXT:    strb w8, [sp, #20]163; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #3]164; NONEON-NOSVE-NEXT:    clz w8, w8165; NONEON-NOSVE-NEXT:    sub w8, w8, #24166; NONEON-NOSVE-NEXT:    strb w8, [sp, #19]167; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]168; NONEON-NOSVE-NEXT:    clz w8, w8169; NONEON-NOSVE-NEXT:    sub w8, w8, #24170; NONEON-NOSVE-NEXT:    strb w8, [sp, #18]171; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #1]172; NONEON-NOSVE-NEXT:    clz w8, w8173; NONEON-NOSVE-NEXT:    sub w8, w8, #24174; NONEON-NOSVE-NEXT:    strb w8, [sp, #17]175; NONEON-NOSVE-NEXT:    ldrb w8, [sp]176; NONEON-NOSVE-NEXT:    clz w8, w8177; NONEON-NOSVE-NEXT:    sub w8, w8, #24178; NONEON-NOSVE-NEXT:    strb w8, [sp, #16]179; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]180; NONEON-NOSVE-NEXT:    add sp, sp, #32181; NONEON-NOSVE-NEXT:    ret182  %res = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %op)183  ret <16 x i8> %res184}185 186define void @ctlz_v32i8(ptr %a) {187; CHECK-LABEL: ctlz_v32i8:188; CHECK:       // %bb.0:189; CHECK-NEXT:    ldp q0, q1, [x0]190; CHECK-NEXT:    ptrue p0.b, vl16191; CHECK-NEXT:    clz z0.b, p0/m, z0.b192; CHECK-NEXT:    clz z1.b, p0/m, z1.b193; CHECK-NEXT:    stp q0, q1, [x0]194; CHECK-NEXT:    ret195;196; NONEON-NOSVE-LABEL: ctlz_v32i8:197; NONEON-NOSVE:       // %bb.0:198; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]199; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!200; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64201; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]202; NONEON-NOSVE-NEXT:    clz w8, w8203; NONEON-NOSVE-NEXT:    sub w8, w8, #24204; NONEON-NOSVE-NEXT:    strb w8, [sp, #63]205; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]206; NONEON-NOSVE-NEXT:    clz w8, w8207; NONEON-NOSVE-NEXT:    sub w8, w8, #24208; NONEON-NOSVE-NEXT:    strb w8, [sp, #62]209; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]210; NONEON-NOSVE-NEXT:    clz w8, w8211; NONEON-NOSVE-NEXT:    sub w8, w8, #24212; NONEON-NOSVE-NEXT:    strb w8, [sp, #61]213; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]214; NONEON-NOSVE-NEXT:    clz w8, w8215; NONEON-NOSVE-NEXT:    sub w8, w8, #24216; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]217; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]218; NONEON-NOSVE-NEXT:    clz w8, w8219; NONEON-NOSVE-NEXT:    sub w8, w8, #24220; NONEON-NOSVE-NEXT:    strb w8, [sp, #59]221; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]222; NONEON-NOSVE-NEXT:    clz w8, w8223; NONEON-NOSVE-NEXT:    sub w8, w8, #24224; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]225; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]226; NONEON-NOSVE-NEXT:    clz w8, w8227; NONEON-NOSVE-NEXT:    sub w8, w8, #24228; NONEON-NOSVE-NEXT:    strb w8, [sp, #57]229; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]230; NONEON-NOSVE-NEXT:    clz w8, w8231; NONEON-NOSVE-NEXT:    sub w8, w8, #24232; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]233; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]234; NONEON-NOSVE-NEXT:    clz w8, w8235; NONEON-NOSVE-NEXT:    sub w8, w8, #24236; NONEON-NOSVE-NEXT:    strb w8, [sp, #55]237; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]238; NONEON-NOSVE-NEXT:    clz w8, w8239; NONEON-NOSVE-NEXT:    sub w8, w8, #24240; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]241; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]242; NONEON-NOSVE-NEXT:    clz w8, w8243; NONEON-NOSVE-NEXT:    sub w8, w8, #24244; NONEON-NOSVE-NEXT:    strb w8, [sp, #53]245; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]246; NONEON-NOSVE-NEXT:    clz w8, w8247; NONEON-NOSVE-NEXT:    sub w8, w8, #24248; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]249; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]250; NONEON-NOSVE-NEXT:    clz w8, w8251; NONEON-NOSVE-NEXT:    sub w8, w8, #24252; NONEON-NOSVE-NEXT:    strb w8, [sp, #51]253; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]254; NONEON-NOSVE-NEXT:    clz w8, w8255; NONEON-NOSVE-NEXT:    sub w8, w8, #24256; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]257; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]258; NONEON-NOSVE-NEXT:    clz w8, w8259; NONEON-NOSVE-NEXT:    sub w8, w8, #24260; NONEON-NOSVE-NEXT:    strb w8, [sp, #49]261; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]262; NONEON-NOSVE-NEXT:    clz w8, w8263; NONEON-NOSVE-NEXT:    sub w8, w8, #24264; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]265; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #15]266; NONEON-NOSVE-NEXT:    clz w8, w8267; NONEON-NOSVE-NEXT:    sub w8, w8, #24268; NONEON-NOSVE-NEXT:    strb w8, [sp, #47]269; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]270; NONEON-NOSVE-NEXT:    clz w8, w8271; NONEON-NOSVE-NEXT:    sub w8, w8, #24272; NONEON-NOSVE-NEXT:    strb w8, [sp, #46]273; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #13]274; NONEON-NOSVE-NEXT:    clz w8, w8275; NONEON-NOSVE-NEXT:    sub w8, w8, #24276; NONEON-NOSVE-NEXT:    strb w8, [sp, #45]277; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]278; NONEON-NOSVE-NEXT:    clz w8, w8279; NONEON-NOSVE-NEXT:    sub w8, w8, #24280; NONEON-NOSVE-NEXT:    strb w8, [sp, #44]281; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #11]282; NONEON-NOSVE-NEXT:    clz w8, w8283; NONEON-NOSVE-NEXT:    sub w8, w8, #24284; NONEON-NOSVE-NEXT:    strb w8, [sp, #43]285; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #10]286; NONEON-NOSVE-NEXT:    clz w8, w8287; NONEON-NOSVE-NEXT:    sub w8, w8, #24288; NONEON-NOSVE-NEXT:    strb w8, [sp, #42]289; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #9]290; NONEON-NOSVE-NEXT:    clz w8, w8291; NONEON-NOSVE-NEXT:    sub w8, w8, #24292; NONEON-NOSVE-NEXT:    strb w8, [sp, #41]293; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]294; NONEON-NOSVE-NEXT:    clz w8, w8295; NONEON-NOSVE-NEXT:    sub w8, w8, #24296; NONEON-NOSVE-NEXT:    strb w8, [sp, #40]297; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #7]298; NONEON-NOSVE-NEXT:    clz w8, w8299; NONEON-NOSVE-NEXT:    sub w8, w8, #24300; NONEON-NOSVE-NEXT:    strb w8, [sp, #39]301; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #6]302; NONEON-NOSVE-NEXT:    clz w8, w8303; NONEON-NOSVE-NEXT:    sub w8, w8, #24304; NONEON-NOSVE-NEXT:    strb w8, [sp, #38]305; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #5]306; NONEON-NOSVE-NEXT:    clz w8, w8307; NONEON-NOSVE-NEXT:    sub w8, w8, #24308; NONEON-NOSVE-NEXT:    strb w8, [sp, #37]309; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #4]310; NONEON-NOSVE-NEXT:    clz w8, w8311; NONEON-NOSVE-NEXT:    sub w8, w8, #24312; NONEON-NOSVE-NEXT:    strb w8, [sp, #36]313; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #3]314; NONEON-NOSVE-NEXT:    clz w8, w8315; NONEON-NOSVE-NEXT:    sub w8, w8, #24316; NONEON-NOSVE-NEXT:    strb w8, [sp, #35]317; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]318; NONEON-NOSVE-NEXT:    clz w8, w8319; NONEON-NOSVE-NEXT:    sub w8, w8, #24320; NONEON-NOSVE-NEXT:    strb w8, [sp, #34]321; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #1]322; NONEON-NOSVE-NEXT:    clz w8, w8323; NONEON-NOSVE-NEXT:    sub w8, w8, #24324; NONEON-NOSVE-NEXT:    strb w8, [sp, #33]325; NONEON-NOSVE-NEXT:    ldrb w8, [sp]326; NONEON-NOSVE-NEXT:    clz w8, w8327; NONEON-NOSVE-NEXT:    sub w8, w8, #24328; NONEON-NOSVE-NEXT:    strb w8, [sp, #32]329; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]330; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]331; NONEON-NOSVE-NEXT:    add sp, sp, #64332; NONEON-NOSVE-NEXT:    ret333  %op = load <32 x i8>, ptr %a334  %res = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %op)335  store <32 x i8> %res, ptr %a336  ret void337}338 339define <2 x i16> @ctlz_v2i16(<2 x i16> %op) {340; CHECK-LABEL: ctlz_v2i16:341; CHECK:       // %bb.0:342; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0343; CHECK-NEXT:    ptrue p0.s, vl2344; CHECK-NEXT:    and z0.s, z0.s, #0xffff345; CHECK-NEXT:    clz z0.s, p0/m, z0.s346; CHECK-NEXT:    sub z0.s, z0.s, #16 // =0x10347; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0348; CHECK-NEXT:    ret349;350; NONEON-NOSVE-LABEL: ctlz_v2i16:351; NONEON-NOSVE:       // %bb.0:352; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!353; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16354; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]355; NONEON-NOSVE-NEXT:    ldrh w9, [sp]356; NONEON-NOSVE-NEXT:    clz w8, w8357; NONEON-NOSVE-NEXT:    clz w9, w9358; NONEON-NOSVE-NEXT:    sub w8, w8, #16359; NONEON-NOSVE-NEXT:    sub w9, w9, #16360; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #8]361; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]362; NONEON-NOSVE-NEXT:    add sp, sp, #16363; NONEON-NOSVE-NEXT:    ret364  %res = call <2 x i16> @llvm.ctlz.v2i16(<2 x i16> %op)365  ret <2 x i16> %res366}367 368define <4 x i16> @ctlz_v4i16(<4 x i16> %op) {369; CHECK-LABEL: ctlz_v4i16:370; CHECK:       // %bb.0:371; CHECK-NEXT:    ptrue p0.h, vl4372; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0373; CHECK-NEXT:    clz z0.h, p0/m, z0.h374; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0375; CHECK-NEXT:    ret376;377; NONEON-NOSVE-LABEL: ctlz_v4i16:378; NONEON-NOSVE:       // %bb.0:379; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!380; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16381; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #6]382; NONEON-NOSVE-NEXT:    clz w8, w8383; NONEON-NOSVE-NEXT:    sub w8, w8, #16384; NONEON-NOSVE-NEXT:    strh w8, [sp, #14]385; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]386; NONEON-NOSVE-NEXT:    clz w8, w8387; NONEON-NOSVE-NEXT:    sub w8, w8, #16388; NONEON-NOSVE-NEXT:    strh w8, [sp, #12]389; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]390; NONEON-NOSVE-NEXT:    clz w8, w8391; NONEON-NOSVE-NEXT:    sub w8, w8, #16392; NONEON-NOSVE-NEXT:    strh w8, [sp, #10]393; NONEON-NOSVE-NEXT:    ldrh w8, [sp]394; NONEON-NOSVE-NEXT:    clz w8, w8395; NONEON-NOSVE-NEXT:    sub w8, w8, #16396; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]397; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]398; NONEON-NOSVE-NEXT:    add sp, sp, #16399; NONEON-NOSVE-NEXT:    ret400  %res = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %op)401  ret <4 x i16> %res402}403 404define <8 x i16> @ctlz_v8i16(<8 x i16> %op) {405; CHECK-LABEL: ctlz_v8i16:406; CHECK:       // %bb.0:407; CHECK-NEXT:    ptrue p0.h, vl8408; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0409; CHECK-NEXT:    clz z0.h, p0/m, z0.h410; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0411; CHECK-NEXT:    ret412;413; NONEON-NOSVE-LABEL: ctlz_v8i16:414; NONEON-NOSVE:       // %bb.0:415; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!416; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32417; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #14]418; NONEON-NOSVE-NEXT:    clz w8, w8419; NONEON-NOSVE-NEXT:    sub w8, w8, #16420; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]421; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]422; NONEON-NOSVE-NEXT:    clz w8, w8423; NONEON-NOSVE-NEXT:    sub w8, w8, #16424; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]425; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #10]426; NONEON-NOSVE-NEXT:    clz w8, w8427; NONEON-NOSVE-NEXT:    sub w8, w8, #16428; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]429; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #8]430; NONEON-NOSVE-NEXT:    clz w8, w8431; NONEON-NOSVE-NEXT:    sub w8, w8, #16432; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]433; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #6]434; NONEON-NOSVE-NEXT:    clz w8, w8435; NONEON-NOSVE-NEXT:    sub w8, w8, #16436; NONEON-NOSVE-NEXT:    strh w8, [sp, #22]437; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]438; NONEON-NOSVE-NEXT:    clz w8, w8439; NONEON-NOSVE-NEXT:    sub w8, w8, #16440; NONEON-NOSVE-NEXT:    strh w8, [sp, #20]441; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]442; NONEON-NOSVE-NEXT:    clz w8, w8443; NONEON-NOSVE-NEXT:    sub w8, w8, #16444; NONEON-NOSVE-NEXT:    strh w8, [sp, #18]445; NONEON-NOSVE-NEXT:    ldrh w8, [sp]446; NONEON-NOSVE-NEXT:    clz w8, w8447; NONEON-NOSVE-NEXT:    sub w8, w8, #16448; NONEON-NOSVE-NEXT:    strh w8, [sp, #16]449; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]450; NONEON-NOSVE-NEXT:    add sp, sp, #32451; NONEON-NOSVE-NEXT:    ret452  %res = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %op)453  ret <8 x i16> %res454}455 456define void @ctlz_v16i16(ptr %a) {457; CHECK-LABEL: ctlz_v16i16:458; CHECK:       // %bb.0:459; CHECK-NEXT:    ldp q0, q1, [x0]460; CHECK-NEXT:    ptrue p0.h, vl8461; CHECK-NEXT:    clz z0.h, p0/m, z0.h462; CHECK-NEXT:    clz z1.h, p0/m, z1.h463; CHECK-NEXT:    stp q0, q1, [x0]464; CHECK-NEXT:    ret465;466; NONEON-NOSVE-LABEL: ctlz_v16i16:467; NONEON-NOSVE:       // %bb.0:468; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]469; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!470; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64471; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #30]472; NONEON-NOSVE-NEXT:    clz w8, w8473; NONEON-NOSVE-NEXT:    sub w8, w8, #16474; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]475; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]476; NONEON-NOSVE-NEXT:    clz w8, w8477; NONEON-NOSVE-NEXT:    sub w8, w8, #16478; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]479; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #26]480; NONEON-NOSVE-NEXT:    clz w8, w8481; NONEON-NOSVE-NEXT:    sub w8, w8, #16482; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]483; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]484; NONEON-NOSVE-NEXT:    clz w8, w8485; NONEON-NOSVE-NEXT:    sub w8, w8, #16486; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]487; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]488; NONEON-NOSVE-NEXT:    clz w8, w8489; NONEON-NOSVE-NEXT:    sub w8, w8, #16490; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]491; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]492; NONEON-NOSVE-NEXT:    clz w8, w8493; NONEON-NOSVE-NEXT:    sub w8, w8, #16494; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]495; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]496; NONEON-NOSVE-NEXT:    clz w8, w8497; NONEON-NOSVE-NEXT:    sub w8, w8, #16498; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]499; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]500; NONEON-NOSVE-NEXT:    clz w8, w8501; NONEON-NOSVE-NEXT:    sub w8, w8, #16502; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]503; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #14]504; NONEON-NOSVE-NEXT:    clz w8, w8505; NONEON-NOSVE-NEXT:    sub w8, w8, #16506; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]507; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]508; NONEON-NOSVE-NEXT:    clz w8, w8509; NONEON-NOSVE-NEXT:    sub w8, w8, #16510; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]511; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #10]512; NONEON-NOSVE-NEXT:    clz w8, w8513; NONEON-NOSVE-NEXT:    sub w8, w8, #16514; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]515; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #8]516; NONEON-NOSVE-NEXT:    clz w8, w8517; NONEON-NOSVE-NEXT:    sub w8, w8, #16518; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]519; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #6]520; NONEON-NOSVE-NEXT:    clz w8, w8521; NONEON-NOSVE-NEXT:    sub w8, w8, #16522; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]523; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]524; NONEON-NOSVE-NEXT:    clz w8, w8525; NONEON-NOSVE-NEXT:    sub w8, w8, #16526; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]527; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]528; NONEON-NOSVE-NEXT:    clz w8, w8529; NONEON-NOSVE-NEXT:    sub w8, w8, #16530; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]531; NONEON-NOSVE-NEXT:    ldrh w8, [sp]532; NONEON-NOSVE-NEXT:    clz w8, w8533; NONEON-NOSVE-NEXT:    sub w8, w8, #16534; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]535; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]536; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]537; NONEON-NOSVE-NEXT:    add sp, sp, #64538; NONEON-NOSVE-NEXT:    ret539  %op = load <16 x i16>, ptr %a540  %res = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %op)541  store <16 x i16> %res, ptr %a542  ret void543}544 545define <2 x i32> @ctlz_v2i32(<2 x i32> %op) {546; CHECK-LABEL: ctlz_v2i32:547; CHECK:       // %bb.0:548; CHECK-NEXT:    ptrue p0.s, vl2549; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0550; CHECK-NEXT:    clz z0.s, p0/m, z0.s551; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0552; CHECK-NEXT:    ret553;554; NONEON-NOSVE-LABEL: ctlz_v2i32:555; NONEON-NOSVE:       // %bb.0:556; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!557; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16558; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]559; NONEON-NOSVE-NEXT:    clz w9, w8560; NONEON-NOSVE-NEXT:    ldr w8, [sp]561; NONEON-NOSVE-NEXT:    clz w8, w8562; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8]563; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]564; NONEON-NOSVE-NEXT:    add sp, sp, #16565; NONEON-NOSVE-NEXT:    ret566  %res = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %op)567  ret <2 x i32> %res568}569 570define <4 x i32> @ctlz_v4i32(<4 x i32> %op) {571; CHECK-LABEL: ctlz_v4i32:572; CHECK:       // %bb.0:573; CHECK-NEXT:    ptrue p0.s, vl4574; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0575; CHECK-NEXT:    clz z0.s, p0/m, z0.s576; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0577; CHECK-NEXT:    ret578;579; NONEON-NOSVE-LABEL: ctlz_v4i32:580; NONEON-NOSVE:       // %bb.0:581; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!582; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32583; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12]584; NONEON-NOSVE-NEXT:    clz w9, w8585; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]586; NONEON-NOSVE-NEXT:    clz w8, w8587; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]588; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]589; NONEON-NOSVE-NEXT:    clz w9, w8590; NONEON-NOSVE-NEXT:    ldr w8, [sp]591; NONEON-NOSVE-NEXT:    clz w8, w8592; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]593; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]594; NONEON-NOSVE-NEXT:    add sp, sp, #32595; NONEON-NOSVE-NEXT:    ret596  %res = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %op)597  ret <4 x i32> %res598}599 600define void @ctlz_v8i32(ptr %a) {601; CHECK-LABEL: ctlz_v8i32:602; CHECK:       // %bb.0:603; CHECK-NEXT:    ldp q0, q1, [x0]604; CHECK-NEXT:    ptrue p0.s, vl4605; CHECK-NEXT:    clz z0.s, p0/m, z0.s606; CHECK-NEXT:    clz z1.s, p0/m, z1.s607; CHECK-NEXT:    stp q0, q1, [x0]608; CHECK-NEXT:    ret609;610; NONEON-NOSVE-LABEL: ctlz_v8i32:611; NONEON-NOSVE:       // %bb.0:612; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]613; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!614; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64615; NONEON-NOSVE-NEXT:    ldr w8, [sp, #28]616; NONEON-NOSVE-NEXT:    clz w9, w8617; NONEON-NOSVE-NEXT:    ldr w8, [sp, #24]618; NONEON-NOSVE-NEXT:    clz w8, w8619; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]620; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]621; NONEON-NOSVE-NEXT:    clz w9, w8622; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]623; NONEON-NOSVE-NEXT:    clz w8, w8624; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]625; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12]626; NONEON-NOSVE-NEXT:    clz w9, w8627; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]628; NONEON-NOSVE-NEXT:    clz w8, w8629; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]630; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]631; NONEON-NOSVE-NEXT:    clz w9, w8632; NONEON-NOSVE-NEXT:    ldr w8, [sp]633; NONEON-NOSVE-NEXT:    clz w8, w8634; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]635; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]636; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]637; NONEON-NOSVE-NEXT:    add sp, sp, #64638; NONEON-NOSVE-NEXT:    ret639  %op = load <8 x i32>, ptr %a640  %res = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %op)641  store <8 x i32> %res, ptr %a642  ret void643}644 645define <1 x i64> @ctlz_v1i64(<1 x i64> %op) {646; CHECK-LABEL: ctlz_v1i64:647; CHECK:       // %bb.0:648; CHECK-NEXT:    ptrue p0.d, vl1649; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0650; CHECK-NEXT:    clz z0.d, p0/m, z0.d651; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0652; CHECK-NEXT:    ret653;654; NONEON-NOSVE-LABEL: ctlz_v1i64:655; NONEON-NOSVE:       // %bb.0:656; NONEON-NOSVE-NEXT:    sub sp, sp, #16657; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16658; NONEON-NOSVE-NEXT:    fmov x8, d0659; NONEON-NOSVE-NEXT:    clz x8, x8660; NONEON-NOSVE-NEXT:    str x8, [sp, #8]661; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]662; NONEON-NOSVE-NEXT:    add sp, sp, #16663; NONEON-NOSVE-NEXT:    ret664  %res = call <1 x i64> @llvm.ctlz.v1i64(<1 x i64> %op)665  ret <1 x i64> %res666}667 668define <2 x i64> @ctlz_v2i64(<2 x i64> %op) {669; CHECK-LABEL: ctlz_v2i64:670; CHECK:       // %bb.0:671; CHECK-NEXT:    ptrue p0.d, vl2672; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0673; CHECK-NEXT:    clz z0.d, p0/m, z0.d674; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0675; CHECK-NEXT:    ret676;677; NONEON-NOSVE-LABEL: ctlz_v2i64:678; NONEON-NOSVE:       // %bb.0:679; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!680; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32681; NONEON-NOSVE-NEXT:    ldr x8, [sp, #8]682; NONEON-NOSVE-NEXT:    clz x9, x8683; NONEON-NOSVE-NEXT:    ldr x8, [sp]684; NONEON-NOSVE-NEXT:    clz x8, x8685; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #16]686; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]687; NONEON-NOSVE-NEXT:    add sp, sp, #32688; NONEON-NOSVE-NEXT:    ret689  %res = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %op)690  ret <2 x i64> %res691}692 693define void @ctlz_v4i64(ptr %a) {694; CHECK-LABEL: ctlz_v4i64:695; CHECK:       // %bb.0:696; CHECK-NEXT:    ldp q0, q1, [x0]697; CHECK-NEXT:    ptrue p0.d, vl2698; CHECK-NEXT:    clz z0.d, p0/m, z0.d699; CHECK-NEXT:    clz z1.d, p0/m, z1.d700; CHECK-NEXT:    stp q0, q1, [x0]701; CHECK-NEXT:    ret702;703; NONEON-NOSVE-LABEL: ctlz_v4i64:704; NONEON-NOSVE:       // %bb.0:705; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]706; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!707; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64708; NONEON-NOSVE-NEXT:    ldr x8, [sp, #24]709; NONEON-NOSVE-NEXT:    clz x9, x8710; NONEON-NOSVE-NEXT:    ldr x8, [sp, #16]711; NONEON-NOSVE-NEXT:    clz x8, x8712; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #48]713; NONEON-NOSVE-NEXT:    ldr x8, [sp, #8]714; NONEON-NOSVE-NEXT:    clz x9, x8715; NONEON-NOSVE-NEXT:    ldr x8, [sp]716; NONEON-NOSVE-NEXT:    clz x8, x8717; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #32]718; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]719; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]720; NONEON-NOSVE-NEXT:    add sp, sp, #64721; NONEON-NOSVE-NEXT:    ret722  %op = load <4 x i64>, ptr %a723  %res = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %op)724  store <4 x i64> %res, ptr %a725  ret void726}727 728;729; CNT730;731 732define <4 x i8> @ctpop_v4i8(<4 x i8> %op) {733; CHECK-LABEL: ctpop_v4i8:734; CHECK:       // %bb.0:735; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0736; CHECK-NEXT:    ptrue p0.h, vl4737; CHECK-NEXT:    and z0.h, z0.h, #0xff738; CHECK-NEXT:    cnt z0.h, p0/m, z0.h739; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0740; CHECK-NEXT:    ret741;742; NONEON-NOSVE-LABEL: ctpop_v4i8:743; NONEON-NOSVE:       // %bb.0:744; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!745; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16746; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #6]747; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #4]748; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x1010101749; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #2]750; NONEON-NOSVE-NEXT:    ldrb w12, [sp]751; NONEON-NOSVE-NEXT:    lsr w13, w9, #1752; NONEON-NOSVE-NEXT:    lsr w14, w11, #1753; NONEON-NOSVE-NEXT:    lsr w15, w10, #1754; NONEON-NOSVE-NEXT:    lsr w16, w12, #1755; NONEON-NOSVE-NEXT:    and w13, w13, #0x55555555756; NONEON-NOSVE-NEXT:    sub w9, w9, w13757; NONEON-NOSVE-NEXT:    and w13, w14, #0x55555555758; NONEON-NOSVE-NEXT:    and w14, w15, #0x55555555759; NONEON-NOSVE-NEXT:    sub w11, w11, w13760; NONEON-NOSVE-NEXT:    lsr w13, w9, #2761; NONEON-NOSVE-NEXT:    and w15, w16, #0x55555555762; NONEON-NOSVE-NEXT:    sub w10, w10, w14763; NONEON-NOSVE-NEXT:    sub w12, w12, w15764; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333765; NONEON-NOSVE-NEXT:    and w13, w13, #0x33333333766; NONEON-NOSVE-NEXT:    lsr w14, w11, #2767; NONEON-NOSVE-NEXT:    lsr w15, w10, #2768; NONEON-NOSVE-NEXT:    add w9, w9, w13769; NONEON-NOSVE-NEXT:    lsr w13, w12, #2770; NONEON-NOSVE-NEXT:    and w11, w11, #0x33333333771; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333772; NONEON-NOSVE-NEXT:    and w14, w14, #0x33333333773; NONEON-NOSVE-NEXT:    and w15, w15, #0x33333333774; NONEON-NOSVE-NEXT:    and w12, w12, #0x33333333775; NONEON-NOSVE-NEXT:    and w13, w13, #0x33333333776; NONEON-NOSVE-NEXT:    add w11, w11, w14777; NONEON-NOSVE-NEXT:    add w10, w10, w15778; NONEON-NOSVE-NEXT:    add w12, w12, w13779; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4780; NONEON-NOSVE-NEXT:    add w11, w11, w11, lsr #4781; NONEON-NOSVE-NEXT:    add w10, w10, w10, lsr #4782; NONEON-NOSVE-NEXT:    add w12, w12, w12, lsr #4783; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f784; NONEON-NOSVE-NEXT:    and w11, w11, #0xf0f0f0f785; NONEON-NOSVE-NEXT:    and w10, w10, #0xf0f0f0f786; NONEON-NOSVE-NEXT:    and w12, w12, #0xf0f0f0f787; NONEON-NOSVE-NEXT:    mul w9, w9, w8788; NONEON-NOSVE-NEXT:    mul w11, w11, w8789; NONEON-NOSVE-NEXT:    mul w10, w10, w8790; NONEON-NOSVE-NEXT:    mul w8, w12, w8791; NONEON-NOSVE-NEXT:    lsr w9, w9, #24792; NONEON-NOSVE-NEXT:    lsr w11, w11, #24793; NONEON-NOSVE-NEXT:    lsr w10, w10, #24794; NONEON-NOSVE-NEXT:    strh w9, [sp, #14]795; NONEON-NOSVE-NEXT:    lsr w8, w8, #24796; NONEON-NOSVE-NEXT:    strh w11, [sp, #12]797; NONEON-NOSVE-NEXT:    strh w10, [sp, #10]798; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]799; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]800; NONEON-NOSVE-NEXT:    add sp, sp, #16801; NONEON-NOSVE-NEXT:    ret802  %res = call <4 x i8> @llvm.ctpop.v4i8(<4 x i8> %op)803  ret <4 x i8> %res804}805 806define <8 x i8> @ctpop_v8i8(<8 x i8> %op) {807; CHECK-LABEL: ctpop_v8i8:808; CHECK:       // %bb.0:809; CHECK-NEXT:    ptrue p0.b, vl8810; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0811; CHECK-NEXT:    cnt z0.b, p0/m, z0.b812; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0813; CHECK-NEXT:    ret814;815; NONEON-NOSVE-LABEL: ctpop_v8i8:816; NONEON-NOSVE:       // %bb.0:817; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!818; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16819; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]820; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x1010101821; NONEON-NOSVE-NEXT:    lsr w10, w9, #1822; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555823; NONEON-NOSVE-NEXT:    sub w9, w9, w10824; NONEON-NOSVE-NEXT:    lsr w10, w9, #2825; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333826; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333827; NONEON-NOSVE-NEXT:    add w9, w9, w10828; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4829; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f830; NONEON-NOSVE-NEXT:    mul w9, w9, w8831; NONEON-NOSVE-NEXT:    lsr w9, w9, #24832; NONEON-NOSVE-NEXT:    strb w9, [sp, #15]833; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #6]834; NONEON-NOSVE-NEXT:    lsr w10, w9, #1835; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555836; NONEON-NOSVE-NEXT:    sub w9, w9, w10837; NONEON-NOSVE-NEXT:    lsr w10, w9, #2838; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333839; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333840; NONEON-NOSVE-NEXT:    add w9, w9, w10841; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4842; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f843; NONEON-NOSVE-NEXT:    mul w9, w9, w8844; NONEON-NOSVE-NEXT:    lsr w9, w9, #24845; NONEON-NOSVE-NEXT:    strb w9, [sp, #14]846; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #5]847; NONEON-NOSVE-NEXT:    lsr w10, w9, #1848; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555849; NONEON-NOSVE-NEXT:    sub w9, w9, w10850; NONEON-NOSVE-NEXT:    lsr w10, w9, #2851; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333852; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333853; NONEON-NOSVE-NEXT:    add w9, w9, w10854; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4855; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f856; NONEON-NOSVE-NEXT:    mul w9, w9, w8857; NONEON-NOSVE-NEXT:    lsr w9, w9, #24858; NONEON-NOSVE-NEXT:    strb w9, [sp, #13]859; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #4]860; NONEON-NOSVE-NEXT:    lsr w10, w9, #1861; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555862; NONEON-NOSVE-NEXT:    sub w9, w9, w10863; NONEON-NOSVE-NEXT:    lsr w10, w9, #2864; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333865; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333866; NONEON-NOSVE-NEXT:    add w9, w9, w10867; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4868; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f869; NONEON-NOSVE-NEXT:    mul w9, w9, w8870; NONEON-NOSVE-NEXT:    lsr w9, w9, #24871; NONEON-NOSVE-NEXT:    strb w9, [sp, #12]872; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]873; NONEON-NOSVE-NEXT:    lsr w10, w9, #1874; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555875; NONEON-NOSVE-NEXT:    sub w9, w9, w10876; NONEON-NOSVE-NEXT:    lsr w10, w9, #2877; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333878; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333879; NONEON-NOSVE-NEXT:    add w9, w9, w10880; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4881; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f882; NONEON-NOSVE-NEXT:    mul w9, w9, w8883; NONEON-NOSVE-NEXT:    lsr w9, w9, #24884; NONEON-NOSVE-NEXT:    strb w9, [sp, #11]885; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #2]886; NONEON-NOSVE-NEXT:    lsr w10, w9, #1887; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555888; NONEON-NOSVE-NEXT:    sub w9, w9, w10889; NONEON-NOSVE-NEXT:    lsr w10, w9, #2890; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333891; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333892; NONEON-NOSVE-NEXT:    add w9, w9, w10893; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4894; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f895; NONEON-NOSVE-NEXT:    mul w9, w9, w8896; NONEON-NOSVE-NEXT:    lsr w9, w9, #24897; NONEON-NOSVE-NEXT:    strb w9, [sp, #10]898; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1]899; NONEON-NOSVE-NEXT:    lsr w10, w9, #1900; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555901; NONEON-NOSVE-NEXT:    sub w9, w9, w10902; NONEON-NOSVE-NEXT:    lsr w10, w9, #2903; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333904; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333905; NONEON-NOSVE-NEXT:    add w9, w9, w10906; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4907; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f908; NONEON-NOSVE-NEXT:    mul w9, w9, w8909; NONEON-NOSVE-NEXT:    lsr w9, w9, #24910; NONEON-NOSVE-NEXT:    strb w9, [sp, #9]911; NONEON-NOSVE-NEXT:    ldrb w9, [sp]912; NONEON-NOSVE-NEXT:    lsr w10, w9, #1913; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555914; NONEON-NOSVE-NEXT:    sub w9, w9, w10915; NONEON-NOSVE-NEXT:    lsr w10, w9, #2916; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333917; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333918; NONEON-NOSVE-NEXT:    add w9, w9, w10919; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4920; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f921; NONEON-NOSVE-NEXT:    mul w8, w9, w8922; NONEON-NOSVE-NEXT:    lsr w8, w8, #24923; NONEON-NOSVE-NEXT:    strb w8, [sp, #8]924; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]925; NONEON-NOSVE-NEXT:    add sp, sp, #16926; NONEON-NOSVE-NEXT:    ret927  %res = call <8 x i8> @llvm.ctpop.v8i8(<8 x i8> %op)928  ret <8 x i8> %res929}930 931define <16 x i8> @ctpop_v16i8(<16 x i8> %op) {932; CHECK-LABEL: ctpop_v16i8:933; CHECK:       // %bb.0:934; CHECK-NEXT:    ptrue p0.b, vl16935; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0936; CHECK-NEXT:    cnt z0.b, p0/m, z0.b937; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0938; CHECK-NEXT:    ret939;940; NONEON-NOSVE-LABEL: ctpop_v16i8:941; NONEON-NOSVE:       // %bb.0:942; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!943; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32944; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]945; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x1010101946; NONEON-NOSVE-NEXT:    lsr w10, w9, #1947; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555948; NONEON-NOSVE-NEXT:    sub w9, w9, w10949; NONEON-NOSVE-NEXT:    lsr w10, w9, #2950; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333951; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333952; NONEON-NOSVE-NEXT:    add w9, w9, w10953; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4954; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f955; NONEON-NOSVE-NEXT:    mul w9, w9, w8956; NONEON-NOSVE-NEXT:    lsr w9, w9, #24957; NONEON-NOSVE-NEXT:    strb w9, [sp, #31]958; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]959; NONEON-NOSVE-NEXT:    lsr w10, w9, #1960; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555961; NONEON-NOSVE-NEXT:    sub w9, w9, w10962; NONEON-NOSVE-NEXT:    lsr w10, w9, #2963; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333964; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333965; NONEON-NOSVE-NEXT:    add w9, w9, w10966; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4967; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f968; NONEON-NOSVE-NEXT:    mul w9, w9, w8969; NONEON-NOSVE-NEXT:    lsr w9, w9, #24970; NONEON-NOSVE-NEXT:    strb w9, [sp, #30]971; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]972; NONEON-NOSVE-NEXT:    lsr w10, w9, #1973; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555974; NONEON-NOSVE-NEXT:    sub w9, w9, w10975; NONEON-NOSVE-NEXT:    lsr w10, w9, #2976; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333977; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333978; NONEON-NOSVE-NEXT:    add w9, w9, w10979; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4980; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f981; NONEON-NOSVE-NEXT:    mul w9, w9, w8982; NONEON-NOSVE-NEXT:    lsr w9, w9, #24983; NONEON-NOSVE-NEXT:    strb w9, [sp, #29]984; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]985; NONEON-NOSVE-NEXT:    lsr w10, w9, #1986; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555987; NONEON-NOSVE-NEXT:    sub w9, w9, w10988; NONEON-NOSVE-NEXT:    lsr w10, w9, #2989; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333990; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333991; NONEON-NOSVE-NEXT:    add w9, w9, w10992; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4993; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f994; NONEON-NOSVE-NEXT:    mul w9, w9, w8995; NONEON-NOSVE-NEXT:    lsr w9, w9, #24996; NONEON-NOSVE-NEXT:    strb w9, [sp, #28]997; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]998; NONEON-NOSVE-NEXT:    lsr w10, w9, #1999; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551000; NONEON-NOSVE-NEXT:    sub w9, w9, w101001; NONEON-NOSVE-NEXT:    lsr w10, w9, #21002; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331003; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331004; NONEON-NOSVE-NEXT:    add w9, w9, w101005; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41006; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1007; NONEON-NOSVE-NEXT:    mul w9, w9, w81008; NONEON-NOSVE-NEXT:    lsr w9, w9, #241009; NONEON-NOSVE-NEXT:    strb w9, [sp, #27]1010; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]1011; NONEON-NOSVE-NEXT:    lsr w10, w9, #11012; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551013; NONEON-NOSVE-NEXT:    sub w9, w9, w101014; NONEON-NOSVE-NEXT:    lsr w10, w9, #21015; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331016; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331017; NONEON-NOSVE-NEXT:    add w9, w9, w101018; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41019; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1020; NONEON-NOSVE-NEXT:    mul w9, w9, w81021; NONEON-NOSVE-NEXT:    lsr w9, w9, #241022; NONEON-NOSVE-NEXT:    strb w9, [sp, #26]1023; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]1024; NONEON-NOSVE-NEXT:    lsr w10, w9, #11025; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551026; NONEON-NOSVE-NEXT:    sub w9, w9, w101027; NONEON-NOSVE-NEXT:    lsr w10, w9, #21028; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331029; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331030; NONEON-NOSVE-NEXT:    add w9, w9, w101031; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41032; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1033; NONEON-NOSVE-NEXT:    mul w9, w9, w81034; NONEON-NOSVE-NEXT:    lsr w9, w9, #241035; NONEON-NOSVE-NEXT:    strb w9, [sp, #25]1036; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]1037; NONEON-NOSVE-NEXT:    lsr w10, w9, #11038; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551039; NONEON-NOSVE-NEXT:    sub w9, w9, w101040; NONEON-NOSVE-NEXT:    lsr w10, w9, #21041; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331042; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331043; NONEON-NOSVE-NEXT:    add w9, w9, w101044; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41045; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1046; NONEON-NOSVE-NEXT:    mul w9, w9, w81047; NONEON-NOSVE-NEXT:    lsr w9, w9, #241048; NONEON-NOSVE-NEXT:    strb w9, [sp, #24]1049; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]1050; NONEON-NOSVE-NEXT:    lsr w10, w9, #11051; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551052; NONEON-NOSVE-NEXT:    sub w9, w9, w101053; NONEON-NOSVE-NEXT:    lsr w10, w9, #21054; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331055; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331056; NONEON-NOSVE-NEXT:    add w9, w9, w101057; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41058; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1059; NONEON-NOSVE-NEXT:    mul w9, w9, w81060; NONEON-NOSVE-NEXT:    lsr w9, w9, #241061; NONEON-NOSVE-NEXT:    strb w9, [sp, #23]1062; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #6]1063; NONEON-NOSVE-NEXT:    lsr w10, w9, #11064; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551065; NONEON-NOSVE-NEXT:    sub w9, w9, w101066; NONEON-NOSVE-NEXT:    lsr w10, w9, #21067; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331068; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331069; NONEON-NOSVE-NEXT:    add w9, w9, w101070; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41071; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1072; NONEON-NOSVE-NEXT:    mul w9, w9, w81073; NONEON-NOSVE-NEXT:    lsr w9, w9, #241074; NONEON-NOSVE-NEXT:    strb w9, [sp, #22]1075; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #5]1076; NONEON-NOSVE-NEXT:    lsr w10, w9, #11077; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551078; NONEON-NOSVE-NEXT:    sub w9, w9, w101079; NONEON-NOSVE-NEXT:    lsr w10, w9, #21080; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331081; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331082; NONEON-NOSVE-NEXT:    add w9, w9, w101083; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41084; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1085; NONEON-NOSVE-NEXT:    mul w9, w9, w81086; NONEON-NOSVE-NEXT:    lsr w9, w9, #241087; NONEON-NOSVE-NEXT:    strb w9, [sp, #21]1088; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #4]1089; NONEON-NOSVE-NEXT:    lsr w10, w9, #11090; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551091; NONEON-NOSVE-NEXT:    sub w9, w9, w101092; NONEON-NOSVE-NEXT:    lsr w10, w9, #21093; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331094; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331095; NONEON-NOSVE-NEXT:    add w9, w9, w101096; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41097; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1098; NONEON-NOSVE-NEXT:    mul w9, w9, w81099; NONEON-NOSVE-NEXT:    lsr w9, w9, #241100; NONEON-NOSVE-NEXT:    strb w9, [sp, #20]1101; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]1102; NONEON-NOSVE-NEXT:    lsr w10, w9, #11103; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551104; NONEON-NOSVE-NEXT:    sub w9, w9, w101105; NONEON-NOSVE-NEXT:    lsr w10, w9, #21106; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331107; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331108; NONEON-NOSVE-NEXT:    add w9, w9, w101109; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41110; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1111; NONEON-NOSVE-NEXT:    mul w9, w9, w81112; NONEON-NOSVE-NEXT:    lsr w9, w9, #241113; NONEON-NOSVE-NEXT:    strb w9, [sp, #19]1114; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #2]1115; NONEON-NOSVE-NEXT:    lsr w10, w9, #11116; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551117; NONEON-NOSVE-NEXT:    sub w9, w9, w101118; NONEON-NOSVE-NEXT:    lsr w10, w9, #21119; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331120; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331121; NONEON-NOSVE-NEXT:    add w9, w9, w101122; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41123; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1124; NONEON-NOSVE-NEXT:    mul w9, w9, w81125; NONEON-NOSVE-NEXT:    lsr w9, w9, #241126; NONEON-NOSVE-NEXT:    strb w9, [sp, #18]1127; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1]1128; NONEON-NOSVE-NEXT:    lsr w10, w9, #11129; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551130; NONEON-NOSVE-NEXT:    sub w9, w9, w101131; NONEON-NOSVE-NEXT:    lsr w10, w9, #21132; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331133; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331134; NONEON-NOSVE-NEXT:    add w9, w9, w101135; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41136; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1137; NONEON-NOSVE-NEXT:    mul w9, w9, w81138; NONEON-NOSVE-NEXT:    lsr w9, w9, #241139; NONEON-NOSVE-NEXT:    strb w9, [sp, #17]1140; NONEON-NOSVE-NEXT:    ldrb w9, [sp]1141; NONEON-NOSVE-NEXT:    lsr w10, w9, #11142; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551143; NONEON-NOSVE-NEXT:    sub w9, w9, w101144; NONEON-NOSVE-NEXT:    lsr w10, w9, #21145; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331146; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331147; NONEON-NOSVE-NEXT:    add w9, w9, w101148; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41149; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1150; NONEON-NOSVE-NEXT:    mul w8, w9, w81151; NONEON-NOSVE-NEXT:    lsr w8, w8, #241152; NONEON-NOSVE-NEXT:    strb w8, [sp, #16]1153; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]1154; NONEON-NOSVE-NEXT:    add sp, sp, #321155; NONEON-NOSVE-NEXT:    ret1156  %res = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %op)1157  ret <16 x i8> %res1158}1159 1160define void @ctpop_v32i8(ptr %a) {1161; CHECK-LABEL: ctpop_v32i8:1162; CHECK:       // %bb.0:1163; CHECK-NEXT:    ldp q0, q1, [x0]1164; CHECK-NEXT:    ptrue p0.b, vl161165; CHECK-NEXT:    cnt z0.b, p0/m, z0.b1166; CHECK-NEXT:    cnt z1.b, p0/m, z1.b1167; CHECK-NEXT:    stp q0, q1, [x0]1168; CHECK-NEXT:    ret1169;1170; NONEON-NOSVE-LABEL: ctpop_v32i8:1171; NONEON-NOSVE:       // %bb.0:1172; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]1173; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x10101011174; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!1175; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 641176; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #31]1177; NONEON-NOSVE-NEXT:    lsr w10, w9, #11178; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551179; NONEON-NOSVE-NEXT:    sub w9, w9, w101180; NONEON-NOSVE-NEXT:    lsr w10, w9, #21181; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331182; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331183; NONEON-NOSVE-NEXT:    add w9, w9, w101184; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41185; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1186; NONEON-NOSVE-NEXT:    mul w9, w9, w81187; NONEON-NOSVE-NEXT:    lsr w9, w9, #241188; NONEON-NOSVE-NEXT:    strb w9, [sp, #63]1189; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #30]1190; NONEON-NOSVE-NEXT:    lsr w10, w9, #11191; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551192; NONEON-NOSVE-NEXT:    sub w9, w9, w101193; NONEON-NOSVE-NEXT:    lsr w10, w9, #21194; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331195; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331196; NONEON-NOSVE-NEXT:    add w9, w9, w101197; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41198; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1199; NONEON-NOSVE-NEXT:    mul w9, w9, w81200; NONEON-NOSVE-NEXT:    lsr w9, w9, #241201; NONEON-NOSVE-NEXT:    strb w9, [sp, #62]1202; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #29]1203; NONEON-NOSVE-NEXT:    lsr w10, w9, #11204; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551205; NONEON-NOSVE-NEXT:    sub w9, w9, w101206; NONEON-NOSVE-NEXT:    lsr w10, w9, #21207; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331208; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331209; NONEON-NOSVE-NEXT:    add w9, w9, w101210; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41211; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1212; NONEON-NOSVE-NEXT:    mul w9, w9, w81213; NONEON-NOSVE-NEXT:    lsr w9, w9, #241214; NONEON-NOSVE-NEXT:    strb w9, [sp, #61]1215; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #28]1216; NONEON-NOSVE-NEXT:    lsr w10, w9, #11217; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551218; NONEON-NOSVE-NEXT:    sub w9, w9, w101219; NONEON-NOSVE-NEXT:    lsr w10, w9, #21220; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331221; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331222; NONEON-NOSVE-NEXT:    add w9, w9, w101223; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41224; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1225; NONEON-NOSVE-NEXT:    mul w9, w9, w81226; NONEON-NOSVE-NEXT:    lsr w9, w9, #241227; NONEON-NOSVE-NEXT:    strb w9, [sp, #60]1228; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #27]1229; NONEON-NOSVE-NEXT:    lsr w10, w9, #11230; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551231; NONEON-NOSVE-NEXT:    sub w9, w9, w101232; NONEON-NOSVE-NEXT:    lsr w10, w9, #21233; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331234; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331235; NONEON-NOSVE-NEXT:    add w9, w9, w101236; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41237; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1238; NONEON-NOSVE-NEXT:    mul w9, w9, w81239; NONEON-NOSVE-NEXT:    lsr w9, w9, #241240; NONEON-NOSVE-NEXT:    strb w9, [sp, #59]1241; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #26]1242; NONEON-NOSVE-NEXT:    lsr w10, w9, #11243; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551244; NONEON-NOSVE-NEXT:    sub w9, w9, w101245; NONEON-NOSVE-NEXT:    lsr w10, w9, #21246; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331247; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331248; NONEON-NOSVE-NEXT:    add w9, w9, w101249; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41250; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1251; NONEON-NOSVE-NEXT:    mul w9, w9, w81252; NONEON-NOSVE-NEXT:    lsr w9, w9, #241253; NONEON-NOSVE-NEXT:    strb w9, [sp, #58]1254; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #25]1255; NONEON-NOSVE-NEXT:    lsr w10, w9, #11256; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551257; NONEON-NOSVE-NEXT:    sub w9, w9, w101258; NONEON-NOSVE-NEXT:    lsr w10, w9, #21259; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331260; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331261; NONEON-NOSVE-NEXT:    add w9, w9, w101262; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41263; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1264; NONEON-NOSVE-NEXT:    mul w9, w9, w81265; NONEON-NOSVE-NEXT:    lsr w9, w9, #241266; NONEON-NOSVE-NEXT:    strb w9, [sp, #57]1267; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #24]1268; NONEON-NOSVE-NEXT:    lsr w10, w9, #11269; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551270; NONEON-NOSVE-NEXT:    sub w9, w9, w101271; NONEON-NOSVE-NEXT:    lsr w10, w9, #21272; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331273; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331274; NONEON-NOSVE-NEXT:    add w9, w9, w101275; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41276; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1277; NONEON-NOSVE-NEXT:    mul w9, w9, w81278; NONEON-NOSVE-NEXT:    lsr w9, w9, #241279; NONEON-NOSVE-NEXT:    strb w9, [sp, #56]1280; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #23]1281; NONEON-NOSVE-NEXT:    lsr w10, w9, #11282; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551283; NONEON-NOSVE-NEXT:    sub w9, w9, w101284; NONEON-NOSVE-NEXT:    lsr w10, w9, #21285; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331286; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331287; NONEON-NOSVE-NEXT:    add w9, w9, w101288; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41289; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1290; NONEON-NOSVE-NEXT:    mul w9, w9, w81291; NONEON-NOSVE-NEXT:    lsr w9, w9, #241292; NONEON-NOSVE-NEXT:    strb w9, [sp, #55]1293; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #22]1294; NONEON-NOSVE-NEXT:    lsr w10, w9, #11295; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551296; NONEON-NOSVE-NEXT:    sub w9, w9, w101297; NONEON-NOSVE-NEXT:    lsr w10, w9, #21298; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331299; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331300; NONEON-NOSVE-NEXT:    add w9, w9, w101301; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41302; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1303; NONEON-NOSVE-NEXT:    mul w9, w9, w81304; NONEON-NOSVE-NEXT:    lsr w9, w9, #241305; NONEON-NOSVE-NEXT:    strb w9, [sp, #54]1306; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #21]1307; NONEON-NOSVE-NEXT:    lsr w10, w9, #11308; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551309; NONEON-NOSVE-NEXT:    sub w9, w9, w101310; NONEON-NOSVE-NEXT:    lsr w10, w9, #21311; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331312; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331313; NONEON-NOSVE-NEXT:    add w9, w9, w101314; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41315; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1316; NONEON-NOSVE-NEXT:    mul w9, w9, w81317; NONEON-NOSVE-NEXT:    lsr w9, w9, #241318; NONEON-NOSVE-NEXT:    strb w9, [sp, #53]1319; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #20]1320; NONEON-NOSVE-NEXT:    lsr w10, w9, #11321; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551322; NONEON-NOSVE-NEXT:    sub w9, w9, w101323; NONEON-NOSVE-NEXT:    lsr w10, w9, #21324; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331325; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331326; NONEON-NOSVE-NEXT:    add w9, w9, w101327; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41328; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1329; NONEON-NOSVE-NEXT:    mul w9, w9, w81330; NONEON-NOSVE-NEXT:    lsr w9, w9, #241331; NONEON-NOSVE-NEXT:    strb w9, [sp, #52]1332; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]1333; NONEON-NOSVE-NEXT:    lsr w10, w9, #11334; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551335; NONEON-NOSVE-NEXT:    sub w9, w9, w101336; NONEON-NOSVE-NEXT:    lsr w10, w9, #21337; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331338; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331339; NONEON-NOSVE-NEXT:    add w9, w9, w101340; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41341; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1342; NONEON-NOSVE-NEXT:    mul w9, w9, w81343; NONEON-NOSVE-NEXT:    lsr w9, w9, #241344; NONEON-NOSVE-NEXT:    strb w9, [sp, #51]1345; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #18]1346; NONEON-NOSVE-NEXT:    lsr w10, w9, #11347; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551348; NONEON-NOSVE-NEXT:    sub w9, w9, w101349; NONEON-NOSVE-NEXT:    lsr w10, w9, #21350; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331351; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331352; NONEON-NOSVE-NEXT:    add w9, w9, w101353; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41354; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1355; NONEON-NOSVE-NEXT:    mul w9, w9, w81356; NONEON-NOSVE-NEXT:    lsr w9, w9, #241357; NONEON-NOSVE-NEXT:    strb w9, [sp, #50]1358; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #17]1359; NONEON-NOSVE-NEXT:    lsr w10, w9, #11360; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551361; NONEON-NOSVE-NEXT:    sub w9, w9, w101362; NONEON-NOSVE-NEXT:    lsr w10, w9, #21363; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331364; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331365; NONEON-NOSVE-NEXT:    add w9, w9, w101366; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41367; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1368; NONEON-NOSVE-NEXT:    mul w9, w9, w81369; NONEON-NOSVE-NEXT:    lsr w9, w9, #241370; NONEON-NOSVE-NEXT:    strb w9, [sp, #49]1371; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #16]1372; NONEON-NOSVE-NEXT:    lsr w10, w9, #11373; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551374; NONEON-NOSVE-NEXT:    sub w9, w9, w101375; NONEON-NOSVE-NEXT:    lsr w10, w9, #21376; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331377; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331378; NONEON-NOSVE-NEXT:    add w9, w9, w101379; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41380; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1381; NONEON-NOSVE-NEXT:    mul w9, w9, w81382; NONEON-NOSVE-NEXT:    lsr w9, w9, #241383; NONEON-NOSVE-NEXT:    strb w9, [sp, #48]1384; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]1385; NONEON-NOSVE-NEXT:    lsr w10, w9, #11386; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551387; NONEON-NOSVE-NEXT:    sub w9, w9, w101388; NONEON-NOSVE-NEXT:    lsr w10, w9, #21389; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331390; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331391; NONEON-NOSVE-NEXT:    add w9, w9, w101392; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41393; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1394; NONEON-NOSVE-NEXT:    mul w9, w9, w81395; NONEON-NOSVE-NEXT:    lsr w9, w9, #241396; NONEON-NOSVE-NEXT:    strb w9, [sp, #47]1397; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]1398; NONEON-NOSVE-NEXT:    lsr w10, w9, #11399; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551400; NONEON-NOSVE-NEXT:    sub w9, w9, w101401; NONEON-NOSVE-NEXT:    lsr w10, w9, #21402; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331403; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331404; NONEON-NOSVE-NEXT:    add w9, w9, w101405; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41406; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1407; NONEON-NOSVE-NEXT:    mul w9, w9, w81408; NONEON-NOSVE-NEXT:    lsr w9, w9, #241409; NONEON-NOSVE-NEXT:    strb w9, [sp, #46]1410; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]1411; NONEON-NOSVE-NEXT:    lsr w10, w9, #11412; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551413; NONEON-NOSVE-NEXT:    sub w9, w9, w101414; NONEON-NOSVE-NEXT:    lsr w10, w9, #21415; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331416; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331417; NONEON-NOSVE-NEXT:    add w9, w9, w101418; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41419; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1420; NONEON-NOSVE-NEXT:    mul w9, w9, w81421; NONEON-NOSVE-NEXT:    lsr w9, w9, #241422; NONEON-NOSVE-NEXT:    strb w9, [sp, #45]1423; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]1424; NONEON-NOSVE-NEXT:    lsr w10, w9, #11425; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551426; NONEON-NOSVE-NEXT:    sub w9, w9, w101427; NONEON-NOSVE-NEXT:    lsr w10, w9, #21428; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331429; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331430; NONEON-NOSVE-NEXT:    add w9, w9, w101431; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41432; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1433; NONEON-NOSVE-NEXT:    mul w9, w9, w81434; NONEON-NOSVE-NEXT:    lsr w9, w9, #241435; NONEON-NOSVE-NEXT:    strb w9, [sp, #44]1436; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]1437; NONEON-NOSVE-NEXT:    lsr w10, w9, #11438; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551439; NONEON-NOSVE-NEXT:    sub w9, w9, w101440; NONEON-NOSVE-NEXT:    lsr w10, w9, #21441; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331442; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331443; NONEON-NOSVE-NEXT:    add w9, w9, w101444; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41445; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1446; NONEON-NOSVE-NEXT:    mul w9, w9, w81447; NONEON-NOSVE-NEXT:    lsr w9, w9, #241448; NONEON-NOSVE-NEXT:    strb w9, [sp, #43]1449; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]1450; NONEON-NOSVE-NEXT:    lsr w10, w9, #11451; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551452; NONEON-NOSVE-NEXT:    sub w9, w9, w101453; NONEON-NOSVE-NEXT:    lsr w10, w9, #21454; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331455; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331456; NONEON-NOSVE-NEXT:    add w9, w9, w101457; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41458; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1459; NONEON-NOSVE-NEXT:    mul w9, w9, w81460; NONEON-NOSVE-NEXT:    lsr w9, w9, #241461; NONEON-NOSVE-NEXT:    strb w9, [sp, #42]1462; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]1463; NONEON-NOSVE-NEXT:    lsr w10, w9, #11464; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551465; NONEON-NOSVE-NEXT:    sub w9, w9, w101466; NONEON-NOSVE-NEXT:    lsr w10, w9, #21467; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331468; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331469; NONEON-NOSVE-NEXT:    add w9, w9, w101470; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41471; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1472; NONEON-NOSVE-NEXT:    mul w9, w9, w81473; NONEON-NOSVE-NEXT:    lsr w9, w9, #241474; NONEON-NOSVE-NEXT:    strb w9, [sp, #41]1475; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]1476; NONEON-NOSVE-NEXT:    lsr w10, w9, #11477; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551478; NONEON-NOSVE-NEXT:    sub w9, w9, w101479; NONEON-NOSVE-NEXT:    lsr w10, w9, #21480; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331481; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331482; NONEON-NOSVE-NEXT:    add w9, w9, w101483; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41484; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1485; NONEON-NOSVE-NEXT:    mul w9, w9, w81486; NONEON-NOSVE-NEXT:    lsr w9, w9, #241487; NONEON-NOSVE-NEXT:    strb w9, [sp, #40]1488; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]1489; NONEON-NOSVE-NEXT:    lsr w10, w9, #11490; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551491; NONEON-NOSVE-NEXT:    sub w9, w9, w101492; NONEON-NOSVE-NEXT:    lsr w10, w9, #21493; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331494; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331495; NONEON-NOSVE-NEXT:    add w9, w9, w101496; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41497; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1498; NONEON-NOSVE-NEXT:    mul w9, w9, w81499; NONEON-NOSVE-NEXT:    lsr w9, w9, #241500; NONEON-NOSVE-NEXT:    strb w9, [sp, #39]1501; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #6]1502; NONEON-NOSVE-NEXT:    lsr w10, w9, #11503; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551504; NONEON-NOSVE-NEXT:    sub w9, w9, w101505; NONEON-NOSVE-NEXT:    lsr w10, w9, #21506; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331507; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331508; NONEON-NOSVE-NEXT:    add w9, w9, w101509; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41510; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1511; NONEON-NOSVE-NEXT:    mul w9, w9, w81512; NONEON-NOSVE-NEXT:    lsr w9, w9, #241513; NONEON-NOSVE-NEXT:    strb w9, [sp, #38]1514; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #5]1515; NONEON-NOSVE-NEXT:    lsr w10, w9, #11516; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551517; NONEON-NOSVE-NEXT:    sub w9, w9, w101518; NONEON-NOSVE-NEXT:    lsr w10, w9, #21519; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331520; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331521; NONEON-NOSVE-NEXT:    add w9, w9, w101522; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41523; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1524; NONEON-NOSVE-NEXT:    mul w9, w9, w81525; NONEON-NOSVE-NEXT:    lsr w9, w9, #241526; NONEON-NOSVE-NEXT:    strb w9, [sp, #37]1527; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #4]1528; NONEON-NOSVE-NEXT:    lsr w10, w9, #11529; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551530; NONEON-NOSVE-NEXT:    sub w9, w9, w101531; NONEON-NOSVE-NEXT:    lsr w10, w9, #21532; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331533; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331534; NONEON-NOSVE-NEXT:    add w9, w9, w101535; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41536; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1537; NONEON-NOSVE-NEXT:    mul w9, w9, w81538; NONEON-NOSVE-NEXT:    lsr w9, w9, #241539; NONEON-NOSVE-NEXT:    strb w9, [sp, #36]1540; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]1541; NONEON-NOSVE-NEXT:    lsr w10, w9, #11542; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551543; NONEON-NOSVE-NEXT:    sub w9, w9, w101544; NONEON-NOSVE-NEXT:    lsr w10, w9, #21545; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331546; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331547; NONEON-NOSVE-NEXT:    add w9, w9, w101548; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41549; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1550; NONEON-NOSVE-NEXT:    mul w9, w9, w81551; NONEON-NOSVE-NEXT:    lsr w9, w9, #241552; NONEON-NOSVE-NEXT:    strb w9, [sp, #35]1553; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #2]1554; NONEON-NOSVE-NEXT:    lsr w10, w9, #11555; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551556; NONEON-NOSVE-NEXT:    sub w9, w9, w101557; NONEON-NOSVE-NEXT:    lsr w10, w9, #21558; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331559; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331560; NONEON-NOSVE-NEXT:    add w9, w9, w101561; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41562; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1563; NONEON-NOSVE-NEXT:    mul w9, w9, w81564; NONEON-NOSVE-NEXT:    lsr w9, w9, #241565; NONEON-NOSVE-NEXT:    strb w9, [sp, #34]1566; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1]1567; NONEON-NOSVE-NEXT:    lsr w10, w9, #11568; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551569; NONEON-NOSVE-NEXT:    sub w9, w9, w101570; NONEON-NOSVE-NEXT:    lsr w10, w9, #21571; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331572; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331573; NONEON-NOSVE-NEXT:    add w9, w9, w101574; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41575; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1576; NONEON-NOSVE-NEXT:    mul w9, w9, w81577; NONEON-NOSVE-NEXT:    lsr w9, w9, #241578; NONEON-NOSVE-NEXT:    strb w9, [sp, #33]1579; NONEON-NOSVE-NEXT:    ldrb w9, [sp]1580; NONEON-NOSVE-NEXT:    lsr w10, w9, #11581; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551582; NONEON-NOSVE-NEXT:    sub w9, w9, w101583; NONEON-NOSVE-NEXT:    lsr w10, w9, #21584; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331585; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331586; NONEON-NOSVE-NEXT:    add w9, w9, w101587; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41588; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1589; NONEON-NOSVE-NEXT:    mul w8, w9, w81590; NONEON-NOSVE-NEXT:    lsr w8, w8, #241591; NONEON-NOSVE-NEXT:    strb w8, [sp, #32]1592; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]1593; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]1594; NONEON-NOSVE-NEXT:    add sp, sp, #641595; NONEON-NOSVE-NEXT:    ret1596  %op = load <32 x i8>, ptr %a1597  %res = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> %op)1598  store <32 x i8> %res, ptr %a1599  ret void1600}1601 1602define <2 x i16> @ctpop_v2i16(<2 x i16> %op) {1603; CHECK-LABEL: ctpop_v2i16:1604; CHECK:       // %bb.0:1605; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z01606; CHECK-NEXT:    ptrue p0.s, vl21607; CHECK-NEXT:    and z0.s, z0.s, #0xffff1608; CHECK-NEXT:    cnt z0.s, p0/m, z0.s1609; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z01610; CHECK-NEXT:    ret1611;1612; NONEON-NOSVE-LABEL: ctpop_v2i16:1613; NONEON-NOSVE:       // %bb.0:1614; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!1615; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 161616; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]1617; NONEON-NOSVE-NEXT:    ldrh w10, [sp]1618; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x10101011619; NONEON-NOSVE-NEXT:    lsr w11, w9, #11620; NONEON-NOSVE-NEXT:    lsr w12, w10, #11621; NONEON-NOSVE-NEXT:    and w11, w11, #0x555555551622; NONEON-NOSVE-NEXT:    and w12, w12, #0x555555551623; NONEON-NOSVE-NEXT:    sub w9, w9, w111624; NONEON-NOSVE-NEXT:    sub w10, w10, w121625; NONEON-NOSVE-NEXT:    lsr w11, w9, #21626; NONEON-NOSVE-NEXT:    lsr w12, w10, #21627; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331628; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331629; NONEON-NOSVE-NEXT:    and w11, w11, #0x333333331630; NONEON-NOSVE-NEXT:    add w9, w9, w111631; NONEON-NOSVE-NEXT:    and w11, w12, #0x333333331632; NONEON-NOSVE-NEXT:    add w10, w10, w111633; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41634; NONEON-NOSVE-NEXT:    add w10, w10, w10, lsr #41635; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1636; NONEON-NOSVE-NEXT:    and w10, w10, #0xf0f0f0f1637; NONEON-NOSVE-NEXT:    mul w9, w9, w81638; NONEON-NOSVE-NEXT:    mul w8, w10, w81639; NONEON-NOSVE-NEXT:    lsr w9, w9, #241640; NONEON-NOSVE-NEXT:    lsr w8, w8, #241641; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8]1642; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]1643; NONEON-NOSVE-NEXT:    add sp, sp, #161644; NONEON-NOSVE-NEXT:    ret1645  %res = call <2 x i16> @llvm.ctpop.v2i16(<2 x i16> %op)1646  ret <2 x i16> %res1647}1648 1649define <4 x i16> @ctpop_v4i16(<4 x i16> %op) {1650; CHECK-LABEL: ctpop_v4i16:1651; CHECK:       // %bb.0:1652; CHECK-NEXT:    ptrue p0.h, vl41653; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z01654; CHECK-NEXT:    cnt z0.h, p0/m, z0.h1655; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z01656; CHECK-NEXT:    ret1657;1658; NONEON-NOSVE-LABEL: ctpop_v4i16:1659; NONEON-NOSVE:       // %bb.0:1660; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!1661; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 161662; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #6]1663; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x10101011664; NONEON-NOSVE-NEXT:    lsr w10, w9, #11665; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551666; NONEON-NOSVE-NEXT:    sub w9, w9, w101667; NONEON-NOSVE-NEXT:    lsr w10, w9, #21668; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331669; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331670; NONEON-NOSVE-NEXT:    add w9, w9, w101671; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41672; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1673; NONEON-NOSVE-NEXT:    mul w9, w9, w81674; NONEON-NOSVE-NEXT:    lsr w9, w9, #241675; NONEON-NOSVE-NEXT:    strh w9, [sp, #14]1676; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]1677; NONEON-NOSVE-NEXT:    lsr w10, w9, #11678; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551679; NONEON-NOSVE-NEXT:    sub w9, w9, w101680; NONEON-NOSVE-NEXT:    lsr w10, w9, #21681; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331682; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331683; NONEON-NOSVE-NEXT:    add w9, w9, w101684; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41685; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1686; NONEON-NOSVE-NEXT:    mul w9, w9, w81687; NONEON-NOSVE-NEXT:    lsr w9, w9, #241688; NONEON-NOSVE-NEXT:    strh w9, [sp, #12]1689; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #2]1690; NONEON-NOSVE-NEXT:    lsr w10, w9, #11691; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551692; NONEON-NOSVE-NEXT:    sub w9, w9, w101693; NONEON-NOSVE-NEXT:    lsr w10, w9, #21694; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331695; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331696; NONEON-NOSVE-NEXT:    add w9, w9, w101697; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41698; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1699; NONEON-NOSVE-NEXT:    mul w9, w9, w81700; NONEON-NOSVE-NEXT:    lsr w9, w9, #241701; NONEON-NOSVE-NEXT:    strh w9, [sp, #10]1702; NONEON-NOSVE-NEXT:    ldrh w9, [sp]1703; NONEON-NOSVE-NEXT:    lsr w10, w9, #11704; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551705; NONEON-NOSVE-NEXT:    sub w9, w9, w101706; NONEON-NOSVE-NEXT:    lsr w10, w9, #21707; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331708; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331709; NONEON-NOSVE-NEXT:    add w9, w9, w101710; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41711; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1712; NONEON-NOSVE-NEXT:    mul w8, w9, w81713; NONEON-NOSVE-NEXT:    lsr w8, w8, #241714; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]1715; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]1716; NONEON-NOSVE-NEXT:    add sp, sp, #161717; NONEON-NOSVE-NEXT:    ret1718  %res = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %op)1719  ret <4 x i16> %res1720}1721 1722define <8 x i16> @ctpop_v8i16(<8 x i16> %op) {1723; CHECK-LABEL: ctpop_v8i16:1724; CHECK:       // %bb.0:1725; CHECK-NEXT:    ptrue p0.h, vl81726; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z01727; CHECK-NEXT:    cnt z0.h, p0/m, z0.h1728; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z01729; CHECK-NEXT:    ret1730;1731; NONEON-NOSVE-LABEL: ctpop_v8i16:1732; NONEON-NOSVE:       // %bb.0:1733; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!1734; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 321735; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]1736; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x10101011737; NONEON-NOSVE-NEXT:    lsr w10, w9, #11738; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551739; NONEON-NOSVE-NEXT:    sub w9, w9, w101740; NONEON-NOSVE-NEXT:    lsr w10, w9, #21741; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331742; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331743; NONEON-NOSVE-NEXT:    add w9, w9, w101744; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41745; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1746; NONEON-NOSVE-NEXT:    mul w9, w9, w81747; NONEON-NOSVE-NEXT:    lsr w9, w9, #241748; NONEON-NOSVE-NEXT:    strh w9, [sp, #30]1749; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]1750; NONEON-NOSVE-NEXT:    lsr w10, w9, #11751; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551752; NONEON-NOSVE-NEXT:    sub w9, w9, w101753; NONEON-NOSVE-NEXT:    lsr w10, w9, #21754; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331755; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331756; NONEON-NOSVE-NEXT:    add w9, w9, w101757; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41758; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1759; NONEON-NOSVE-NEXT:    mul w9, w9, w81760; NONEON-NOSVE-NEXT:    lsr w9, w9, #241761; NONEON-NOSVE-NEXT:    strh w9, [sp, #28]1762; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]1763; NONEON-NOSVE-NEXT:    lsr w10, w9, #11764; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551765; NONEON-NOSVE-NEXT:    sub w9, w9, w101766; NONEON-NOSVE-NEXT:    lsr w10, w9, #21767; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331768; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331769; NONEON-NOSVE-NEXT:    add w9, w9, w101770; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41771; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1772; NONEON-NOSVE-NEXT:    mul w9, w9, w81773; NONEON-NOSVE-NEXT:    lsr w9, w9, #241774; NONEON-NOSVE-NEXT:    strh w9, [sp, #26]1775; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]1776; NONEON-NOSVE-NEXT:    lsr w10, w9, #11777; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551778; NONEON-NOSVE-NEXT:    sub w9, w9, w101779; NONEON-NOSVE-NEXT:    lsr w10, w9, #21780; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331781; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331782; NONEON-NOSVE-NEXT:    add w9, w9, w101783; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41784; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1785; NONEON-NOSVE-NEXT:    mul w9, w9, w81786; NONEON-NOSVE-NEXT:    lsr w9, w9, #241787; NONEON-NOSVE-NEXT:    strh w9, [sp, #24]1788; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #6]1789; NONEON-NOSVE-NEXT:    lsr w10, w9, #11790; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551791; NONEON-NOSVE-NEXT:    sub w9, w9, w101792; NONEON-NOSVE-NEXT:    lsr w10, w9, #21793; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331794; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331795; NONEON-NOSVE-NEXT:    add w9, w9, w101796; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41797; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1798; NONEON-NOSVE-NEXT:    mul w9, w9, w81799; NONEON-NOSVE-NEXT:    lsr w9, w9, #241800; NONEON-NOSVE-NEXT:    strh w9, [sp, #22]1801; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]1802; NONEON-NOSVE-NEXT:    lsr w10, w9, #11803; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551804; NONEON-NOSVE-NEXT:    sub w9, w9, w101805; NONEON-NOSVE-NEXT:    lsr w10, w9, #21806; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331807; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331808; NONEON-NOSVE-NEXT:    add w9, w9, w101809; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41810; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1811; NONEON-NOSVE-NEXT:    mul w9, w9, w81812; NONEON-NOSVE-NEXT:    lsr w9, w9, #241813; NONEON-NOSVE-NEXT:    strh w9, [sp, #20]1814; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #2]1815; NONEON-NOSVE-NEXT:    lsr w10, w9, #11816; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551817; NONEON-NOSVE-NEXT:    sub w9, w9, w101818; NONEON-NOSVE-NEXT:    lsr w10, w9, #21819; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331820; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331821; NONEON-NOSVE-NEXT:    add w9, w9, w101822; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41823; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1824; NONEON-NOSVE-NEXT:    mul w9, w9, w81825; NONEON-NOSVE-NEXT:    lsr w9, w9, #241826; NONEON-NOSVE-NEXT:    strh w9, [sp, #18]1827; NONEON-NOSVE-NEXT:    ldrh w9, [sp]1828; NONEON-NOSVE-NEXT:    lsr w10, w9, #11829; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551830; NONEON-NOSVE-NEXT:    sub w9, w9, w101831; NONEON-NOSVE-NEXT:    lsr w10, w9, #21832; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331833; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331834; NONEON-NOSVE-NEXT:    add w9, w9, w101835; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41836; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1837; NONEON-NOSVE-NEXT:    mul w8, w9, w81838; NONEON-NOSVE-NEXT:    lsr w8, w8, #241839; NONEON-NOSVE-NEXT:    strh w8, [sp, #16]1840; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]1841; NONEON-NOSVE-NEXT:    add sp, sp, #321842; NONEON-NOSVE-NEXT:    ret1843  %res = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %op)1844  ret <8 x i16> %res1845}1846 1847define void @ctpop_v16i16(ptr %a) {1848; CHECK-LABEL: ctpop_v16i16:1849; CHECK:       // %bb.0:1850; CHECK-NEXT:    ldp q0, q1, [x0]1851; CHECK-NEXT:    ptrue p0.h, vl81852; CHECK-NEXT:    cnt z0.h, p0/m, z0.h1853; CHECK-NEXT:    cnt z1.h, p0/m, z1.h1854; CHECK-NEXT:    stp q0, q1, [x0]1855; CHECK-NEXT:    ret1856;1857; NONEON-NOSVE-LABEL: ctpop_v16i16:1858; NONEON-NOSVE:       // %bb.0:1859; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]1860; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x10101011861; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!1862; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 641863; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #30]1864; NONEON-NOSVE-NEXT:    lsr w10, w9, #11865; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551866; NONEON-NOSVE-NEXT:    sub w9, w9, w101867; NONEON-NOSVE-NEXT:    lsr w10, w9, #21868; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331869; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331870; NONEON-NOSVE-NEXT:    add w9, w9, w101871; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41872; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1873; NONEON-NOSVE-NEXT:    mul w9, w9, w81874; NONEON-NOSVE-NEXT:    lsr w9, w9, #241875; NONEON-NOSVE-NEXT:    strh w9, [sp, #62]1876; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #28]1877; NONEON-NOSVE-NEXT:    lsr w10, w9, #11878; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551879; NONEON-NOSVE-NEXT:    sub w9, w9, w101880; NONEON-NOSVE-NEXT:    lsr w10, w9, #21881; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331882; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331883; NONEON-NOSVE-NEXT:    add w9, w9, w101884; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41885; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1886; NONEON-NOSVE-NEXT:    mul w9, w9, w81887; NONEON-NOSVE-NEXT:    lsr w9, w9, #241888; NONEON-NOSVE-NEXT:    strh w9, [sp, #60]1889; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #26]1890; NONEON-NOSVE-NEXT:    lsr w10, w9, #11891; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551892; NONEON-NOSVE-NEXT:    sub w9, w9, w101893; NONEON-NOSVE-NEXT:    lsr w10, w9, #21894; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331895; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331896; NONEON-NOSVE-NEXT:    add w9, w9, w101897; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41898; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1899; NONEON-NOSVE-NEXT:    mul w9, w9, w81900; NONEON-NOSVE-NEXT:    lsr w9, w9, #241901; NONEON-NOSVE-NEXT:    strh w9, [sp, #58]1902; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #24]1903; NONEON-NOSVE-NEXT:    lsr w10, w9, #11904; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551905; NONEON-NOSVE-NEXT:    sub w9, w9, w101906; NONEON-NOSVE-NEXT:    lsr w10, w9, #21907; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331908; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331909; NONEON-NOSVE-NEXT:    add w9, w9, w101910; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41911; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1912; NONEON-NOSVE-NEXT:    mul w9, w9, w81913; NONEON-NOSVE-NEXT:    lsr w9, w9, #241914; NONEON-NOSVE-NEXT:    strh w9, [sp, #56]1915; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #22]1916; NONEON-NOSVE-NEXT:    lsr w10, w9, #11917; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551918; NONEON-NOSVE-NEXT:    sub w9, w9, w101919; NONEON-NOSVE-NEXT:    lsr w10, w9, #21920; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331921; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331922; NONEON-NOSVE-NEXT:    add w9, w9, w101923; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41924; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1925; NONEON-NOSVE-NEXT:    mul w9, w9, w81926; NONEON-NOSVE-NEXT:    lsr w9, w9, #241927; NONEON-NOSVE-NEXT:    strh w9, [sp, #54]1928; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #20]1929; NONEON-NOSVE-NEXT:    lsr w10, w9, #11930; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551931; NONEON-NOSVE-NEXT:    sub w9, w9, w101932; NONEON-NOSVE-NEXT:    lsr w10, w9, #21933; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331934; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331935; NONEON-NOSVE-NEXT:    add w9, w9, w101936; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41937; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1938; NONEON-NOSVE-NEXT:    mul w9, w9, w81939; NONEON-NOSVE-NEXT:    lsr w9, w9, #241940; NONEON-NOSVE-NEXT:    strh w9, [sp, #52]1941; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #18]1942; NONEON-NOSVE-NEXT:    lsr w10, w9, #11943; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551944; NONEON-NOSVE-NEXT:    sub w9, w9, w101945; NONEON-NOSVE-NEXT:    lsr w10, w9, #21946; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331947; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331948; NONEON-NOSVE-NEXT:    add w9, w9, w101949; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41950; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1951; NONEON-NOSVE-NEXT:    mul w9, w9, w81952; NONEON-NOSVE-NEXT:    lsr w9, w9, #241953; NONEON-NOSVE-NEXT:    strh w9, [sp, #50]1954; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #16]1955; NONEON-NOSVE-NEXT:    lsr w10, w9, #11956; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551957; NONEON-NOSVE-NEXT:    sub w9, w9, w101958; NONEON-NOSVE-NEXT:    lsr w10, w9, #21959; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331960; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331961; NONEON-NOSVE-NEXT:    add w9, w9, w101962; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41963; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1964; NONEON-NOSVE-NEXT:    mul w9, w9, w81965; NONEON-NOSVE-NEXT:    lsr w9, w9, #241966; NONEON-NOSVE-NEXT:    strh w9, [sp, #48]1967; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]1968; NONEON-NOSVE-NEXT:    lsr w10, w9, #11969; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551970; NONEON-NOSVE-NEXT:    sub w9, w9, w101971; NONEON-NOSVE-NEXT:    lsr w10, w9, #21972; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331973; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331974; NONEON-NOSVE-NEXT:    add w9, w9, w101975; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41976; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1977; NONEON-NOSVE-NEXT:    mul w9, w9, w81978; NONEON-NOSVE-NEXT:    lsr w9, w9, #241979; NONEON-NOSVE-NEXT:    strh w9, [sp, #46]1980; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]1981; NONEON-NOSVE-NEXT:    lsr w10, w9, #11982; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551983; NONEON-NOSVE-NEXT:    sub w9, w9, w101984; NONEON-NOSVE-NEXT:    lsr w10, w9, #21985; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331986; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333331987; NONEON-NOSVE-NEXT:    add w9, w9, w101988; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #41989; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f1990; NONEON-NOSVE-NEXT:    mul w9, w9, w81991; NONEON-NOSVE-NEXT:    lsr w9, w9, #241992; NONEON-NOSVE-NEXT:    strh w9, [sp, #44]1993; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]1994; NONEON-NOSVE-NEXT:    lsr w10, w9, #11995; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555551996; NONEON-NOSVE-NEXT:    sub w9, w9, w101997; NONEON-NOSVE-NEXT:    lsr w10, w9, #21998; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333331999; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332000; NONEON-NOSVE-NEXT:    add w9, w9, w102001; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42002; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2003; NONEON-NOSVE-NEXT:    mul w9, w9, w82004; NONEON-NOSVE-NEXT:    lsr w9, w9, #242005; NONEON-NOSVE-NEXT:    strh w9, [sp, #42]2006; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]2007; NONEON-NOSVE-NEXT:    lsr w10, w9, #12008; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552009; NONEON-NOSVE-NEXT:    sub w9, w9, w102010; NONEON-NOSVE-NEXT:    lsr w10, w9, #22011; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332012; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332013; NONEON-NOSVE-NEXT:    add w9, w9, w102014; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42015; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2016; NONEON-NOSVE-NEXT:    mul w9, w9, w82017; NONEON-NOSVE-NEXT:    lsr w9, w9, #242018; NONEON-NOSVE-NEXT:    strh w9, [sp, #40]2019; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #6]2020; NONEON-NOSVE-NEXT:    lsr w10, w9, #12021; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552022; NONEON-NOSVE-NEXT:    sub w9, w9, w102023; NONEON-NOSVE-NEXT:    lsr w10, w9, #22024; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332025; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332026; NONEON-NOSVE-NEXT:    add w9, w9, w102027; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42028; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2029; NONEON-NOSVE-NEXT:    mul w9, w9, w82030; NONEON-NOSVE-NEXT:    lsr w9, w9, #242031; NONEON-NOSVE-NEXT:    strh w9, [sp, #38]2032; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]2033; NONEON-NOSVE-NEXT:    lsr w10, w9, #12034; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552035; NONEON-NOSVE-NEXT:    sub w9, w9, w102036; NONEON-NOSVE-NEXT:    lsr w10, w9, #22037; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332038; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332039; NONEON-NOSVE-NEXT:    add w9, w9, w102040; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42041; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2042; NONEON-NOSVE-NEXT:    mul w9, w9, w82043; NONEON-NOSVE-NEXT:    lsr w9, w9, #242044; NONEON-NOSVE-NEXT:    strh w9, [sp, #36]2045; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #2]2046; NONEON-NOSVE-NEXT:    lsr w10, w9, #12047; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552048; NONEON-NOSVE-NEXT:    sub w9, w9, w102049; NONEON-NOSVE-NEXT:    lsr w10, w9, #22050; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332051; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332052; NONEON-NOSVE-NEXT:    add w9, w9, w102053; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42054; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2055; NONEON-NOSVE-NEXT:    mul w9, w9, w82056; NONEON-NOSVE-NEXT:    lsr w9, w9, #242057; NONEON-NOSVE-NEXT:    strh w9, [sp, #34]2058; NONEON-NOSVE-NEXT:    ldrh w9, [sp]2059; NONEON-NOSVE-NEXT:    lsr w10, w9, #12060; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552061; NONEON-NOSVE-NEXT:    sub w9, w9, w102062; NONEON-NOSVE-NEXT:    lsr w10, w9, #22063; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332064; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332065; NONEON-NOSVE-NEXT:    add w9, w9, w102066; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42067; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2068; NONEON-NOSVE-NEXT:    mul w8, w9, w82069; NONEON-NOSVE-NEXT:    lsr w8, w8, #242070; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]2071; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]2072; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]2073; NONEON-NOSVE-NEXT:    add sp, sp, #642074; NONEON-NOSVE-NEXT:    ret2075  %op = load <16 x i16>, ptr %a2076  %res = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> %op)2077  store <16 x i16> %res, ptr %a2078  ret void2079}2080 2081define <2 x i32> @ctpop_v2i32(<2 x i32> %op) {2082; CHECK-LABEL: ctpop_v2i32:2083; CHECK:       // %bb.0:2084; CHECK-NEXT:    ptrue p0.s, vl22085; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z02086; CHECK-NEXT:    cnt z0.s, p0/m, z0.s2087; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z02088; CHECK-NEXT:    ret2089;2090; NONEON-NOSVE-LABEL: ctpop_v2i32:2091; NONEON-NOSVE:       // %bb.0:2092; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!2093; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 162094; NONEON-NOSVE-NEXT:    ldr w9, [sp, #4]2095; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x10101012096; NONEON-NOSVE-NEXT:    lsr w10, w9, #12097; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552098; NONEON-NOSVE-NEXT:    sub w9, w9, w102099; NONEON-NOSVE-NEXT:    lsr w10, w9, #22100; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332101; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332102; NONEON-NOSVE-NEXT:    add w9, w9, w102103; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42104; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2105; NONEON-NOSVE-NEXT:    mul w9, w9, w82106; NONEON-NOSVE-NEXT:    lsr w11, w9, #242107; NONEON-NOSVE-NEXT:    ldr w9, [sp]2108; NONEON-NOSVE-NEXT:    lsr w10, w9, #12109; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552110; NONEON-NOSVE-NEXT:    sub w9, w9, w102111; NONEON-NOSVE-NEXT:    lsr w10, w9, #22112; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332113; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332114; NONEON-NOSVE-NEXT:    add w9, w9, w102115; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42116; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2117; NONEON-NOSVE-NEXT:    mul w8, w9, w82118; NONEON-NOSVE-NEXT:    lsr w8, w8, #242119; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #8]2120; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]2121; NONEON-NOSVE-NEXT:    add sp, sp, #162122; NONEON-NOSVE-NEXT:    ret2123  %res = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %op)2124  ret <2 x i32> %res2125}2126 2127define <4 x i32> @ctpop_v4i32(<4 x i32> %op) {2128; CHECK-LABEL: ctpop_v4i32:2129; CHECK:       // %bb.0:2130; CHECK-NEXT:    ptrue p0.s, vl42131; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z02132; CHECK-NEXT:    cnt z0.s, p0/m, z0.s2133; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z02134; CHECK-NEXT:    ret2135;2136; NONEON-NOSVE-LABEL: ctpop_v4i32:2137; NONEON-NOSVE:       // %bb.0:2138; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!2139; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 322140; NONEON-NOSVE-NEXT:    ldr w9, [sp, #12]2141; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x10101012142; NONEON-NOSVE-NEXT:    lsr w10, w9, #12143; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552144; NONEON-NOSVE-NEXT:    sub w9, w9, w102145; NONEON-NOSVE-NEXT:    lsr w10, w9, #22146; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332147; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332148; NONEON-NOSVE-NEXT:    add w9, w9, w102149; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42150; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2151; NONEON-NOSVE-NEXT:    mul w9, w9, w82152; NONEON-NOSVE-NEXT:    lsr w11, w9, #242153; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8]2154; NONEON-NOSVE-NEXT:    lsr w10, w9, #12155; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552156; NONEON-NOSVE-NEXT:    sub w9, w9, w102157; NONEON-NOSVE-NEXT:    lsr w10, w9, #22158; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332159; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332160; NONEON-NOSVE-NEXT:    add w9, w9, w102161; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42162; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2163; NONEON-NOSVE-NEXT:    mul w9, w9, w82164; NONEON-NOSVE-NEXT:    lsr w9, w9, #242165; NONEON-NOSVE-NEXT:    stp w9, w11, [sp, #24]2166; NONEON-NOSVE-NEXT:    ldr w9, [sp, #4]2167; NONEON-NOSVE-NEXT:    lsr w10, w9, #12168; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552169; NONEON-NOSVE-NEXT:    sub w9, w9, w102170; NONEON-NOSVE-NEXT:    lsr w10, w9, #22171; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332172; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332173; NONEON-NOSVE-NEXT:    add w9, w9, w102174; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42175; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2176; NONEON-NOSVE-NEXT:    mul w9, w9, w82177; NONEON-NOSVE-NEXT:    lsr w11, w9, #242178; NONEON-NOSVE-NEXT:    ldr w9, [sp]2179; NONEON-NOSVE-NEXT:    lsr w10, w9, #12180; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552181; NONEON-NOSVE-NEXT:    sub w9, w9, w102182; NONEON-NOSVE-NEXT:    lsr w10, w9, #22183; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332184; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332185; NONEON-NOSVE-NEXT:    add w9, w9, w102186; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42187; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2188; NONEON-NOSVE-NEXT:    mul w8, w9, w82189; NONEON-NOSVE-NEXT:    lsr w8, w8, #242190; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #16]2191; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]2192; NONEON-NOSVE-NEXT:    add sp, sp, #322193; NONEON-NOSVE-NEXT:    ret2194  %res = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %op)2195  ret <4 x i32> %res2196}2197 2198define void @ctpop_v8i32(ptr %a) {2199; CHECK-LABEL: ctpop_v8i32:2200; CHECK:       // %bb.0:2201; CHECK-NEXT:    ldp q0, q1, [x0]2202; CHECK-NEXT:    ptrue p0.s, vl42203; CHECK-NEXT:    cnt z0.s, p0/m, z0.s2204; CHECK-NEXT:    cnt z1.s, p0/m, z1.s2205; CHECK-NEXT:    stp q0, q1, [x0]2206; CHECK-NEXT:    ret2207;2208; NONEON-NOSVE-LABEL: ctpop_v8i32:2209; NONEON-NOSVE:       // %bb.0:2210; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]2211; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x10101012212; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!2213; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 642214; NONEON-NOSVE-NEXT:    ldr w9, [sp, #28]2215; NONEON-NOSVE-NEXT:    lsr w10, w9, #12216; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552217; NONEON-NOSVE-NEXT:    sub w9, w9, w102218; NONEON-NOSVE-NEXT:    lsr w10, w9, #22219; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332220; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332221; NONEON-NOSVE-NEXT:    add w9, w9, w102222; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42223; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2224; NONEON-NOSVE-NEXT:    mul w9, w9, w82225; NONEON-NOSVE-NEXT:    lsr w11, w9, #242226; NONEON-NOSVE-NEXT:    ldr w9, [sp, #24]2227; NONEON-NOSVE-NEXT:    lsr w10, w9, #12228; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552229; NONEON-NOSVE-NEXT:    sub w9, w9, w102230; NONEON-NOSVE-NEXT:    lsr w10, w9, #22231; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332232; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332233; NONEON-NOSVE-NEXT:    add w9, w9, w102234; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42235; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2236; NONEON-NOSVE-NEXT:    mul w9, w9, w82237; NONEON-NOSVE-NEXT:    lsr w9, w9, #242238; NONEON-NOSVE-NEXT:    stp w9, w11, [sp, #56]2239; NONEON-NOSVE-NEXT:    ldr w9, [sp, #20]2240; NONEON-NOSVE-NEXT:    lsr w10, w9, #12241; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552242; NONEON-NOSVE-NEXT:    sub w9, w9, w102243; NONEON-NOSVE-NEXT:    lsr w10, w9, #22244; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332245; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332246; NONEON-NOSVE-NEXT:    add w9, w9, w102247; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42248; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2249; NONEON-NOSVE-NEXT:    mul w9, w9, w82250; NONEON-NOSVE-NEXT:    lsr w11, w9, #242251; NONEON-NOSVE-NEXT:    ldr w9, [sp, #16]2252; NONEON-NOSVE-NEXT:    lsr w10, w9, #12253; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552254; NONEON-NOSVE-NEXT:    sub w9, w9, w102255; NONEON-NOSVE-NEXT:    lsr w10, w9, #22256; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332257; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332258; NONEON-NOSVE-NEXT:    add w9, w9, w102259; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42260; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2261; NONEON-NOSVE-NEXT:    mul w9, w9, w82262; NONEON-NOSVE-NEXT:    lsr w9, w9, #242263; NONEON-NOSVE-NEXT:    stp w9, w11, [sp, #48]2264; NONEON-NOSVE-NEXT:    ldr w9, [sp, #12]2265; NONEON-NOSVE-NEXT:    lsr w10, w9, #12266; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552267; NONEON-NOSVE-NEXT:    sub w9, w9, w102268; NONEON-NOSVE-NEXT:    lsr w10, w9, #22269; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332270; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332271; NONEON-NOSVE-NEXT:    add w9, w9, w102272; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42273; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2274; NONEON-NOSVE-NEXT:    mul w9, w9, w82275; NONEON-NOSVE-NEXT:    lsr w11, w9, #242276; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8]2277; NONEON-NOSVE-NEXT:    lsr w10, w9, #12278; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552279; NONEON-NOSVE-NEXT:    sub w9, w9, w102280; NONEON-NOSVE-NEXT:    lsr w10, w9, #22281; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332282; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332283; NONEON-NOSVE-NEXT:    add w9, w9, w102284; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42285; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2286; NONEON-NOSVE-NEXT:    mul w9, w9, w82287; NONEON-NOSVE-NEXT:    lsr w9, w9, #242288; NONEON-NOSVE-NEXT:    stp w9, w11, [sp, #40]2289; NONEON-NOSVE-NEXT:    ldr w9, [sp, #4]2290; NONEON-NOSVE-NEXT:    lsr w10, w9, #12291; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552292; NONEON-NOSVE-NEXT:    sub w9, w9, w102293; NONEON-NOSVE-NEXT:    lsr w10, w9, #22294; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332295; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332296; NONEON-NOSVE-NEXT:    add w9, w9, w102297; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42298; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2299; NONEON-NOSVE-NEXT:    mul w9, w9, w82300; NONEON-NOSVE-NEXT:    lsr w11, w9, #242301; NONEON-NOSVE-NEXT:    ldr w9, [sp]2302; NONEON-NOSVE-NEXT:    lsr w10, w9, #12303; NONEON-NOSVE-NEXT:    and w10, w10, #0x555555552304; NONEON-NOSVE-NEXT:    sub w9, w9, w102305; NONEON-NOSVE-NEXT:    lsr w10, w9, #22306; NONEON-NOSVE-NEXT:    and w9, w9, #0x333333332307; NONEON-NOSVE-NEXT:    and w10, w10, #0x333333332308; NONEON-NOSVE-NEXT:    add w9, w9, w102309; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #42310; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f2311; NONEON-NOSVE-NEXT:    mul w8, w9, w82312; NONEON-NOSVE-NEXT:    lsr w8, w8, #242313; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #32]2314; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]2315; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]2316; NONEON-NOSVE-NEXT:    add sp, sp, #642317; NONEON-NOSVE-NEXT:    ret2318  %op = load <8 x i32>, ptr %a2319  %res = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %op)2320  store <8 x i32> %res, ptr %a2321  ret void2322}2323 2324define <1 x i64> @ctpop_v1i64(<1 x i64> %op) {2325; CHECK-LABEL: ctpop_v1i64:2326; CHECK:       // %bb.0:2327; CHECK-NEXT:    ptrue p0.d, vl12328; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z02329; CHECK-NEXT:    cnt z0.d, p0/m, z0.d2330; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z02331; CHECK-NEXT:    ret2332;2333; NONEON-NOSVE-LABEL: ctpop_v1i64:2334; NONEON-NOSVE:       // %bb.0:2335; NONEON-NOSVE-NEXT:    sub sp, sp, #162336; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 162337; NONEON-NOSVE-NEXT:    fmov x9, d02338; NONEON-NOSVE-NEXT:    mov x8, #72340172838076673 // =0x1010101010101012339; NONEON-NOSVE-NEXT:    lsr x10, x9, #12340; NONEON-NOSVE-NEXT:    and x10, x10, #0x55555555555555552341; NONEON-NOSVE-NEXT:    sub x9, x9, x102342; NONEON-NOSVE-NEXT:    lsr x10, x9, #22343; NONEON-NOSVE-NEXT:    and x9, x9, #0x33333333333333332344; NONEON-NOSVE-NEXT:    and x10, x10, #0x33333333333333332345; NONEON-NOSVE-NEXT:    add x9, x9, x102346; NONEON-NOSVE-NEXT:    add x9, x9, x9, lsr #42347; NONEON-NOSVE-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f2348; NONEON-NOSVE-NEXT:    mul x8, x9, x82349; NONEON-NOSVE-NEXT:    lsr x8, x8, #562350; NONEON-NOSVE-NEXT:    str x8, [sp, #8]2351; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]2352; NONEON-NOSVE-NEXT:    add sp, sp, #162353; NONEON-NOSVE-NEXT:    ret2354  %res = call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %op)2355  ret <1 x i64> %res2356}2357 2358define <2 x i64> @ctpop_v2i64(<2 x i64> %op) {2359; CHECK-LABEL: ctpop_v2i64:2360; CHECK:       // %bb.0:2361; CHECK-NEXT:    ptrue p0.d, vl22362; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z02363; CHECK-NEXT:    cnt z0.d, p0/m, z0.d2364; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z02365; CHECK-NEXT:    ret2366;2367; NONEON-NOSVE-LABEL: ctpop_v2i64:2368; NONEON-NOSVE:       // %bb.0:2369; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!2370; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 322371; NONEON-NOSVE-NEXT:    ldr x9, [sp, #8]2372; NONEON-NOSVE-NEXT:    mov x8, #72340172838076673 // =0x1010101010101012373; NONEON-NOSVE-NEXT:    lsr x10, x9, #12374; NONEON-NOSVE-NEXT:    and x10, x10, #0x55555555555555552375; NONEON-NOSVE-NEXT:    sub x9, x9, x102376; NONEON-NOSVE-NEXT:    lsr x10, x9, #22377; NONEON-NOSVE-NEXT:    and x9, x9, #0x33333333333333332378; NONEON-NOSVE-NEXT:    and x10, x10, #0x33333333333333332379; NONEON-NOSVE-NEXT:    add x9, x9, x102380; NONEON-NOSVE-NEXT:    add x9, x9, x9, lsr #42381; NONEON-NOSVE-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f2382; NONEON-NOSVE-NEXT:    mul x9, x9, x82383; NONEON-NOSVE-NEXT:    lsr x11, x9, #562384; NONEON-NOSVE-NEXT:    ldr x9, [sp]2385; NONEON-NOSVE-NEXT:    lsr x10, x9, #12386; NONEON-NOSVE-NEXT:    and x10, x10, #0x55555555555555552387; NONEON-NOSVE-NEXT:    sub x9, x9, x102388; NONEON-NOSVE-NEXT:    lsr x10, x9, #22389; NONEON-NOSVE-NEXT:    and x9, x9, #0x33333333333333332390; NONEON-NOSVE-NEXT:    and x10, x10, #0x33333333333333332391; NONEON-NOSVE-NEXT:    add x9, x9, x102392; NONEON-NOSVE-NEXT:    add x9, x9, x9, lsr #42393; NONEON-NOSVE-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f2394; NONEON-NOSVE-NEXT:    mul x8, x9, x82395; NONEON-NOSVE-NEXT:    lsr x8, x8, #562396; NONEON-NOSVE-NEXT:    stp x8, x11, [sp, #16]2397; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]2398; NONEON-NOSVE-NEXT:    add sp, sp, #322399; NONEON-NOSVE-NEXT:    ret2400  %res = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %op)2401  ret <2 x i64> %res2402}2403 2404define void @ctpop_v4i64(ptr %a) {2405; CHECK-LABEL: ctpop_v4i64:2406; CHECK:       // %bb.0:2407; CHECK-NEXT:    ldp q0, q1, [x0]2408; CHECK-NEXT:    ptrue p0.d, vl22409; CHECK-NEXT:    cnt z0.d, p0/m, z0.d2410; CHECK-NEXT:    cnt z1.d, p0/m, z1.d2411; CHECK-NEXT:    stp q0, q1, [x0]2412; CHECK-NEXT:    ret2413;2414; NONEON-NOSVE-LABEL: ctpop_v4i64:2415; NONEON-NOSVE:       // %bb.0:2416; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]2417; NONEON-NOSVE-NEXT:    mov x8, #72340172838076673 // =0x1010101010101012418; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!2419; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 642420; NONEON-NOSVE-NEXT:    ldr x9, [sp, #24]2421; NONEON-NOSVE-NEXT:    lsr x10, x9, #12422; NONEON-NOSVE-NEXT:    and x10, x10, #0x55555555555555552423; NONEON-NOSVE-NEXT:    sub x9, x9, x102424; NONEON-NOSVE-NEXT:    lsr x10, x9, #22425; NONEON-NOSVE-NEXT:    and x9, x9, #0x33333333333333332426; NONEON-NOSVE-NEXT:    and x10, x10, #0x33333333333333332427; NONEON-NOSVE-NEXT:    add x9, x9, x102428; NONEON-NOSVE-NEXT:    add x9, x9, x9, lsr #42429; NONEON-NOSVE-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f2430; NONEON-NOSVE-NEXT:    mul x9, x9, x82431; NONEON-NOSVE-NEXT:    lsr x11, x9, #562432; NONEON-NOSVE-NEXT:    ldr x9, [sp, #16]2433; NONEON-NOSVE-NEXT:    lsr x10, x9, #12434; NONEON-NOSVE-NEXT:    and x10, x10, #0x55555555555555552435; NONEON-NOSVE-NEXT:    sub x9, x9, x102436; NONEON-NOSVE-NEXT:    lsr x10, x9, #22437; NONEON-NOSVE-NEXT:    and x9, x9, #0x33333333333333332438; NONEON-NOSVE-NEXT:    and x10, x10, #0x33333333333333332439; NONEON-NOSVE-NEXT:    add x9, x9, x102440; NONEON-NOSVE-NEXT:    add x9, x9, x9, lsr #42441; NONEON-NOSVE-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f2442; NONEON-NOSVE-NEXT:    mul x9, x9, x82443; NONEON-NOSVE-NEXT:    lsr x9, x9, #562444; NONEON-NOSVE-NEXT:    stp x9, x11, [sp, #48]2445; NONEON-NOSVE-NEXT:    ldr x9, [sp, #8]2446; NONEON-NOSVE-NEXT:    lsr x10, x9, #12447; NONEON-NOSVE-NEXT:    and x10, x10, #0x55555555555555552448; NONEON-NOSVE-NEXT:    sub x9, x9, x102449; NONEON-NOSVE-NEXT:    lsr x10, x9, #22450; NONEON-NOSVE-NEXT:    and x9, x9, #0x33333333333333332451; NONEON-NOSVE-NEXT:    and x10, x10, #0x33333333333333332452; NONEON-NOSVE-NEXT:    add x9, x9, x102453; NONEON-NOSVE-NEXT:    add x9, x9, x9, lsr #42454; NONEON-NOSVE-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f2455; NONEON-NOSVE-NEXT:    mul x9, x9, x82456; NONEON-NOSVE-NEXT:    lsr x11, x9, #562457; NONEON-NOSVE-NEXT:    ldr x9, [sp]2458; NONEON-NOSVE-NEXT:    lsr x10, x9, #12459; NONEON-NOSVE-NEXT:    and x10, x10, #0x55555555555555552460; NONEON-NOSVE-NEXT:    sub x9, x9, x102461; NONEON-NOSVE-NEXT:    lsr x10, x9, #22462; NONEON-NOSVE-NEXT:    and x9, x9, #0x33333333333333332463; NONEON-NOSVE-NEXT:    and x10, x10, #0x33333333333333332464; NONEON-NOSVE-NEXT:    add x9, x9, x102465; NONEON-NOSVE-NEXT:    add x9, x9, x9, lsr #42466; NONEON-NOSVE-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f2467; NONEON-NOSVE-NEXT:    mul x8, x9, x82468; NONEON-NOSVE-NEXT:    lsr x8, x8, #562469; NONEON-NOSVE-NEXT:    stp x8, x11, [sp, #32]2470; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]2471; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]2472; NONEON-NOSVE-NEXT:    add sp, sp, #642473; NONEON-NOSVE-NEXT:    ret2474  %op = load <4 x i64>, ptr %a2475  %res = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %op)2476  store <4 x i64> %res, ptr %a2477  ret void2478}2479 2480;2481; Count trailing zeros2482;2483 2484define <4 x i8> @cttz_v4i8(<4 x i8> %op) {2485; CHECK-LABEL: cttz_v4i8:2486; CHECK:       // %bb.0:2487; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z02488; CHECK-NEXT:    ptrue p0.h, vl42489; CHECK-NEXT:    orr z0.h, z0.h, #0x1002490; CHECK-NEXT:    rbit z0.h, p0/m, z0.h2491; CHECK-NEXT:    clz z0.h, p0/m, z0.h2492; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z02493; CHECK-NEXT:    ret2494;2495; NONEON-NOSVE-LABEL: cttz_v4i8:2496; NONEON-NOSVE:       // %bb.0:2497; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!2498; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 162499; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #6]2500; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002501; NONEON-NOSVE-NEXT:    rbit w8, w82502; NONEON-NOSVE-NEXT:    clz w8, w82503; NONEON-NOSVE-NEXT:    strh w8, [sp, #14]2504; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]2505; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002506; NONEON-NOSVE-NEXT:    rbit w8, w82507; NONEON-NOSVE-NEXT:    clz w8, w82508; NONEON-NOSVE-NEXT:    strh w8, [sp, #12]2509; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]2510; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002511; NONEON-NOSVE-NEXT:    rbit w8, w82512; NONEON-NOSVE-NEXT:    clz w8, w82513; NONEON-NOSVE-NEXT:    strh w8, [sp, #10]2514; NONEON-NOSVE-NEXT:    ldrh w8, [sp]2515; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002516; NONEON-NOSVE-NEXT:    rbit w8, w82517; NONEON-NOSVE-NEXT:    clz w8, w82518; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]2519; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]2520; NONEON-NOSVE-NEXT:    add sp, sp, #162521; NONEON-NOSVE-NEXT:    ret2522  %res = call <4 x i8> @llvm.cttz.v4i8(<4 x i8> %op)2523  ret <4 x i8> %res2524}2525 2526define <8 x i8> @cttz_v8i8(<8 x i8> %op) {2527; CHECK-LABEL: cttz_v8i8:2528; CHECK:       // %bb.0:2529; CHECK-NEXT:    ptrue p0.b, vl82530; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z02531; CHECK-NEXT:    rbit z0.b, p0/m, z0.b2532; CHECK-NEXT:    clz z0.b, p0/m, z0.b2533; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z02534; CHECK-NEXT:    ret2535;2536; NONEON-NOSVE-LABEL: cttz_v8i8:2537; NONEON-NOSVE:       // %bb.0:2538; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!2539; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 162540; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #7]2541; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002542; NONEON-NOSVE-NEXT:    rbit w8, w82543; NONEON-NOSVE-NEXT:    clz w8, w82544; NONEON-NOSVE-NEXT:    strb w8, [sp, #15]2545; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #6]2546; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002547; NONEON-NOSVE-NEXT:    rbit w8, w82548; NONEON-NOSVE-NEXT:    clz w8, w82549; NONEON-NOSVE-NEXT:    strb w8, [sp, #14]2550; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #5]2551; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002552; NONEON-NOSVE-NEXT:    rbit w8, w82553; NONEON-NOSVE-NEXT:    clz w8, w82554; NONEON-NOSVE-NEXT:    strb w8, [sp, #13]2555; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #4]2556; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002557; NONEON-NOSVE-NEXT:    rbit w8, w82558; NONEON-NOSVE-NEXT:    clz w8, w82559; NONEON-NOSVE-NEXT:    strb w8, [sp, #12]2560; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #3]2561; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002562; NONEON-NOSVE-NEXT:    rbit w8, w82563; NONEON-NOSVE-NEXT:    clz w8, w82564; NONEON-NOSVE-NEXT:    strb w8, [sp, #11]2565; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]2566; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002567; NONEON-NOSVE-NEXT:    rbit w8, w82568; NONEON-NOSVE-NEXT:    clz w8, w82569; NONEON-NOSVE-NEXT:    strb w8, [sp, #10]2570; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #1]2571; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002572; NONEON-NOSVE-NEXT:    rbit w8, w82573; NONEON-NOSVE-NEXT:    clz w8, w82574; NONEON-NOSVE-NEXT:    strb w8, [sp, #9]2575; NONEON-NOSVE-NEXT:    ldrb w8, [sp]2576; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002577; NONEON-NOSVE-NEXT:    rbit w8, w82578; NONEON-NOSVE-NEXT:    clz w8, w82579; NONEON-NOSVE-NEXT:    strb w8, [sp, #8]2580; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]2581; NONEON-NOSVE-NEXT:    add sp, sp, #162582; NONEON-NOSVE-NEXT:    ret2583  %res = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %op)2584  ret <8 x i8> %res2585}2586 2587define <16 x i8> @cttz_v16i8(<16 x i8> %op) {2588; CHECK-LABEL: cttz_v16i8:2589; CHECK:       // %bb.0:2590; CHECK-NEXT:    ptrue p0.b, vl162591; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z02592; CHECK-NEXT:    rbit z0.b, p0/m, z0.b2593; CHECK-NEXT:    clz z0.b, p0/m, z0.b2594; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z02595; CHECK-NEXT:    ret2596;2597; NONEON-NOSVE-LABEL: cttz_v16i8:2598; NONEON-NOSVE:       // %bb.0:2599; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!2600; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 322601; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #15]2602; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002603; NONEON-NOSVE-NEXT:    rbit w8, w82604; NONEON-NOSVE-NEXT:    clz w8, w82605; NONEON-NOSVE-NEXT:    strb w8, [sp, #31]2606; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]2607; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002608; NONEON-NOSVE-NEXT:    rbit w8, w82609; NONEON-NOSVE-NEXT:    clz w8, w82610; NONEON-NOSVE-NEXT:    strb w8, [sp, #30]2611; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #13]2612; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002613; NONEON-NOSVE-NEXT:    rbit w8, w82614; NONEON-NOSVE-NEXT:    clz w8, w82615; NONEON-NOSVE-NEXT:    strb w8, [sp, #29]2616; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]2617; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002618; NONEON-NOSVE-NEXT:    rbit w8, w82619; NONEON-NOSVE-NEXT:    clz w8, w82620; NONEON-NOSVE-NEXT:    strb w8, [sp, #28]2621; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #11]2622; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002623; NONEON-NOSVE-NEXT:    rbit w8, w82624; NONEON-NOSVE-NEXT:    clz w8, w82625; NONEON-NOSVE-NEXT:    strb w8, [sp, #27]2626; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #10]2627; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002628; NONEON-NOSVE-NEXT:    rbit w8, w82629; NONEON-NOSVE-NEXT:    clz w8, w82630; NONEON-NOSVE-NEXT:    strb w8, [sp, #26]2631; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #9]2632; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002633; NONEON-NOSVE-NEXT:    rbit w8, w82634; NONEON-NOSVE-NEXT:    clz w8, w82635; NONEON-NOSVE-NEXT:    strb w8, [sp, #25]2636; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]2637; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002638; NONEON-NOSVE-NEXT:    rbit w8, w82639; NONEON-NOSVE-NEXT:    clz w8, w82640; NONEON-NOSVE-NEXT:    strb w8, [sp, #24]2641; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #7]2642; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002643; NONEON-NOSVE-NEXT:    rbit w8, w82644; NONEON-NOSVE-NEXT:    clz w8, w82645; NONEON-NOSVE-NEXT:    strb w8, [sp, #23]2646; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #6]2647; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002648; NONEON-NOSVE-NEXT:    rbit w8, w82649; NONEON-NOSVE-NEXT:    clz w8, w82650; NONEON-NOSVE-NEXT:    strb w8, [sp, #22]2651; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #5]2652; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002653; NONEON-NOSVE-NEXT:    rbit w8, w82654; NONEON-NOSVE-NEXT:    clz w8, w82655; NONEON-NOSVE-NEXT:    strb w8, [sp, #21]2656; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #4]2657; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002658; NONEON-NOSVE-NEXT:    rbit w8, w82659; NONEON-NOSVE-NEXT:    clz w8, w82660; NONEON-NOSVE-NEXT:    strb w8, [sp, #20]2661; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #3]2662; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002663; NONEON-NOSVE-NEXT:    rbit w8, w82664; NONEON-NOSVE-NEXT:    clz w8, w82665; NONEON-NOSVE-NEXT:    strb w8, [sp, #19]2666; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]2667; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002668; NONEON-NOSVE-NEXT:    rbit w8, w82669; NONEON-NOSVE-NEXT:    clz w8, w82670; NONEON-NOSVE-NEXT:    strb w8, [sp, #18]2671; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #1]2672; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002673; NONEON-NOSVE-NEXT:    rbit w8, w82674; NONEON-NOSVE-NEXT:    clz w8, w82675; NONEON-NOSVE-NEXT:    strb w8, [sp, #17]2676; NONEON-NOSVE-NEXT:    ldrb w8, [sp]2677; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002678; NONEON-NOSVE-NEXT:    rbit w8, w82679; NONEON-NOSVE-NEXT:    clz w8, w82680; NONEON-NOSVE-NEXT:    strb w8, [sp, #16]2681; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]2682; NONEON-NOSVE-NEXT:    add sp, sp, #322683; NONEON-NOSVE-NEXT:    ret2684  %res = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %op)2685  ret <16 x i8> %res2686}2687 2688define void @cttz_v32i8(ptr %a) {2689; CHECK-LABEL: cttz_v32i8:2690; CHECK:       // %bb.0:2691; CHECK-NEXT:    ldp q0, q1, [x0]2692; CHECK-NEXT:    ptrue p0.b, vl162693; CHECK-NEXT:    rbit z0.b, p0/m, z0.b2694; CHECK-NEXT:    rbit z1.b, p0/m, z1.b2695; CHECK-NEXT:    clz z0.b, p0/m, z0.b2696; CHECK-NEXT:    clz z1.b, p0/m, z1.b2697; CHECK-NEXT:    stp q0, q1, [x0]2698; CHECK-NEXT:    ret2699;2700; NONEON-NOSVE-LABEL: cttz_v32i8:2701; NONEON-NOSVE:       // %bb.0:2702; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]2703; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!2704; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 642705; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]2706; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002707; NONEON-NOSVE-NEXT:    rbit w8, w82708; NONEON-NOSVE-NEXT:    clz w8, w82709; NONEON-NOSVE-NEXT:    strb w8, [sp, #63]2710; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]2711; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002712; NONEON-NOSVE-NEXT:    rbit w8, w82713; NONEON-NOSVE-NEXT:    clz w8, w82714; NONEON-NOSVE-NEXT:    strb w8, [sp, #62]2715; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]2716; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002717; NONEON-NOSVE-NEXT:    rbit w8, w82718; NONEON-NOSVE-NEXT:    clz w8, w82719; NONEON-NOSVE-NEXT:    strb w8, [sp, #61]2720; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]2721; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002722; NONEON-NOSVE-NEXT:    rbit w8, w82723; NONEON-NOSVE-NEXT:    clz w8, w82724; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]2725; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]2726; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002727; NONEON-NOSVE-NEXT:    rbit w8, w82728; NONEON-NOSVE-NEXT:    clz w8, w82729; NONEON-NOSVE-NEXT:    strb w8, [sp, #59]2730; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]2731; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002732; NONEON-NOSVE-NEXT:    rbit w8, w82733; NONEON-NOSVE-NEXT:    clz w8, w82734; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]2735; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]2736; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002737; NONEON-NOSVE-NEXT:    rbit w8, w82738; NONEON-NOSVE-NEXT:    clz w8, w82739; NONEON-NOSVE-NEXT:    strb w8, [sp, #57]2740; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]2741; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002742; NONEON-NOSVE-NEXT:    rbit w8, w82743; NONEON-NOSVE-NEXT:    clz w8, w82744; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]2745; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]2746; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002747; NONEON-NOSVE-NEXT:    rbit w8, w82748; NONEON-NOSVE-NEXT:    clz w8, w82749; NONEON-NOSVE-NEXT:    strb w8, [sp, #55]2750; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]2751; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002752; NONEON-NOSVE-NEXT:    rbit w8, w82753; NONEON-NOSVE-NEXT:    clz w8, w82754; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]2755; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]2756; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002757; NONEON-NOSVE-NEXT:    rbit w8, w82758; NONEON-NOSVE-NEXT:    clz w8, w82759; NONEON-NOSVE-NEXT:    strb w8, [sp, #53]2760; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]2761; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002762; NONEON-NOSVE-NEXT:    rbit w8, w82763; NONEON-NOSVE-NEXT:    clz w8, w82764; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]2765; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]2766; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002767; NONEON-NOSVE-NEXT:    rbit w8, w82768; NONEON-NOSVE-NEXT:    clz w8, w82769; NONEON-NOSVE-NEXT:    strb w8, [sp, #51]2770; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]2771; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002772; NONEON-NOSVE-NEXT:    rbit w8, w82773; NONEON-NOSVE-NEXT:    clz w8, w82774; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]2775; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]2776; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002777; NONEON-NOSVE-NEXT:    rbit w8, w82778; NONEON-NOSVE-NEXT:    clz w8, w82779; NONEON-NOSVE-NEXT:    strb w8, [sp, #49]2780; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]2781; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002782; NONEON-NOSVE-NEXT:    rbit w8, w82783; NONEON-NOSVE-NEXT:    clz w8, w82784; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]2785; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #15]2786; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002787; NONEON-NOSVE-NEXT:    rbit w8, w82788; NONEON-NOSVE-NEXT:    clz w8, w82789; NONEON-NOSVE-NEXT:    strb w8, [sp, #47]2790; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]2791; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002792; NONEON-NOSVE-NEXT:    rbit w8, w82793; NONEON-NOSVE-NEXT:    clz w8, w82794; NONEON-NOSVE-NEXT:    strb w8, [sp, #46]2795; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #13]2796; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002797; NONEON-NOSVE-NEXT:    rbit w8, w82798; NONEON-NOSVE-NEXT:    clz w8, w82799; NONEON-NOSVE-NEXT:    strb w8, [sp, #45]2800; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]2801; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002802; NONEON-NOSVE-NEXT:    rbit w8, w82803; NONEON-NOSVE-NEXT:    clz w8, w82804; NONEON-NOSVE-NEXT:    strb w8, [sp, #44]2805; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #11]2806; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002807; NONEON-NOSVE-NEXT:    rbit w8, w82808; NONEON-NOSVE-NEXT:    clz w8, w82809; NONEON-NOSVE-NEXT:    strb w8, [sp, #43]2810; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #10]2811; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002812; NONEON-NOSVE-NEXT:    rbit w8, w82813; NONEON-NOSVE-NEXT:    clz w8, w82814; NONEON-NOSVE-NEXT:    strb w8, [sp, #42]2815; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #9]2816; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002817; NONEON-NOSVE-NEXT:    rbit w8, w82818; NONEON-NOSVE-NEXT:    clz w8, w82819; NONEON-NOSVE-NEXT:    strb w8, [sp, #41]2820; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]2821; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002822; NONEON-NOSVE-NEXT:    rbit w8, w82823; NONEON-NOSVE-NEXT:    clz w8, w82824; NONEON-NOSVE-NEXT:    strb w8, [sp, #40]2825; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #7]2826; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002827; NONEON-NOSVE-NEXT:    rbit w8, w82828; NONEON-NOSVE-NEXT:    clz w8, w82829; NONEON-NOSVE-NEXT:    strb w8, [sp, #39]2830; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #6]2831; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002832; NONEON-NOSVE-NEXT:    rbit w8, w82833; NONEON-NOSVE-NEXT:    clz w8, w82834; NONEON-NOSVE-NEXT:    strb w8, [sp, #38]2835; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #5]2836; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002837; NONEON-NOSVE-NEXT:    rbit w8, w82838; NONEON-NOSVE-NEXT:    clz w8, w82839; NONEON-NOSVE-NEXT:    strb w8, [sp, #37]2840; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #4]2841; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002842; NONEON-NOSVE-NEXT:    rbit w8, w82843; NONEON-NOSVE-NEXT:    clz w8, w82844; NONEON-NOSVE-NEXT:    strb w8, [sp, #36]2845; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #3]2846; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002847; NONEON-NOSVE-NEXT:    rbit w8, w82848; NONEON-NOSVE-NEXT:    clz w8, w82849; NONEON-NOSVE-NEXT:    strb w8, [sp, #35]2850; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]2851; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002852; NONEON-NOSVE-NEXT:    rbit w8, w82853; NONEON-NOSVE-NEXT:    clz w8, w82854; NONEON-NOSVE-NEXT:    strb w8, [sp, #34]2855; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #1]2856; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002857; NONEON-NOSVE-NEXT:    rbit w8, w82858; NONEON-NOSVE-NEXT:    clz w8, w82859; NONEON-NOSVE-NEXT:    strb w8, [sp, #33]2860; NONEON-NOSVE-NEXT:    ldrb w8, [sp]2861; NONEON-NOSVE-NEXT:    orr w8, w8, #0x1002862; NONEON-NOSVE-NEXT:    rbit w8, w82863; NONEON-NOSVE-NEXT:    clz w8, w82864; NONEON-NOSVE-NEXT:    strb w8, [sp, #32]2865; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]2866; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]2867; NONEON-NOSVE-NEXT:    add sp, sp, #642868; NONEON-NOSVE-NEXT:    ret2869  %op = load <32 x i8>, ptr %a2870  %res = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %op)2871  store <32 x i8> %res, ptr %a2872  ret void2873}2874 2875define <2 x i16> @cttz_v2i16(<2 x i16> %op) {2876; CHECK-LABEL: cttz_v2i16:2877; CHECK:       // %bb.0:2878; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z02879; CHECK-NEXT:    ptrue p0.s, vl22880; CHECK-NEXT:    orr z0.s, z0.s, #0x100002881; CHECK-NEXT:    rbit z0.s, p0/m, z0.s2882; CHECK-NEXT:    clz z0.s, p0/m, z0.s2883; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z02884; CHECK-NEXT:    ret2885;2886; NONEON-NOSVE-LABEL: cttz_v2i16:2887; NONEON-NOSVE:       // %bb.0:2888; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!2889; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 162890; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]2891; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100002892; NONEON-NOSVE-NEXT:    rbit w8, w82893; NONEON-NOSVE-NEXT:    clz w9, w82894; NONEON-NOSVE-NEXT:    ldr w8, [sp]2895; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100002896; NONEON-NOSVE-NEXT:    rbit w8, w82897; NONEON-NOSVE-NEXT:    clz w8, w82898; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8]2899; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]2900; NONEON-NOSVE-NEXT:    add sp, sp, #162901; NONEON-NOSVE-NEXT:    ret2902  %res = call <2 x i16> @llvm.cttz.v2i16(<2 x i16> %op)2903  ret <2 x i16> %res2904}2905 2906define <4 x i16> @cttz_v4i16(<4 x i16> %op) {2907; CHECK-LABEL: cttz_v4i16:2908; CHECK:       // %bb.0:2909; CHECK-NEXT:    ptrue p0.h, vl42910; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z02911; CHECK-NEXT:    rbit z0.h, p0/m, z0.h2912; CHECK-NEXT:    clz z0.h, p0/m, z0.h2913; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z02914; CHECK-NEXT:    ret2915;2916; NONEON-NOSVE-LABEL: cttz_v4i16:2917; NONEON-NOSVE:       // %bb.0:2918; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!2919; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 162920; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #6]2921; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100002922; NONEON-NOSVE-NEXT:    rbit w8, w82923; NONEON-NOSVE-NEXT:    clz w8, w82924; NONEON-NOSVE-NEXT:    strh w8, [sp, #14]2925; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]2926; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100002927; NONEON-NOSVE-NEXT:    rbit w8, w82928; NONEON-NOSVE-NEXT:    clz w8, w82929; NONEON-NOSVE-NEXT:    strh w8, [sp, #12]2930; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]2931; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100002932; NONEON-NOSVE-NEXT:    rbit w8, w82933; NONEON-NOSVE-NEXT:    clz w8, w82934; NONEON-NOSVE-NEXT:    strh w8, [sp, #10]2935; NONEON-NOSVE-NEXT:    ldrh w8, [sp]2936; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100002937; NONEON-NOSVE-NEXT:    rbit w8, w82938; NONEON-NOSVE-NEXT:    clz w8, w82939; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]2940; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]2941; NONEON-NOSVE-NEXT:    add sp, sp, #162942; NONEON-NOSVE-NEXT:    ret2943  %res = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %op)2944  ret <4 x i16> %res2945}2946 2947define <8 x i16> @cttz_v8i16(<8 x i16> %op) {2948; CHECK-LABEL: cttz_v8i16:2949; CHECK:       // %bb.0:2950; CHECK-NEXT:    ptrue p0.h, vl82951; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z02952; CHECK-NEXT:    rbit z0.h, p0/m, z0.h2953; CHECK-NEXT:    clz z0.h, p0/m, z0.h2954; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z02955; CHECK-NEXT:    ret2956;2957; NONEON-NOSVE-LABEL: cttz_v8i16:2958; NONEON-NOSVE:       // %bb.0:2959; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!2960; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 322961; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #14]2962; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100002963; NONEON-NOSVE-NEXT:    rbit w8, w82964; NONEON-NOSVE-NEXT:    clz w8, w82965; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]2966; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]2967; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100002968; NONEON-NOSVE-NEXT:    rbit w8, w82969; NONEON-NOSVE-NEXT:    clz w8, w82970; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]2971; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #10]2972; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100002973; NONEON-NOSVE-NEXT:    rbit w8, w82974; NONEON-NOSVE-NEXT:    clz w8, w82975; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]2976; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #8]2977; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100002978; NONEON-NOSVE-NEXT:    rbit w8, w82979; NONEON-NOSVE-NEXT:    clz w8, w82980; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]2981; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #6]2982; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100002983; NONEON-NOSVE-NEXT:    rbit w8, w82984; NONEON-NOSVE-NEXT:    clz w8, w82985; NONEON-NOSVE-NEXT:    strh w8, [sp, #22]2986; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]2987; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100002988; NONEON-NOSVE-NEXT:    rbit w8, w82989; NONEON-NOSVE-NEXT:    clz w8, w82990; NONEON-NOSVE-NEXT:    strh w8, [sp, #20]2991; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]2992; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100002993; NONEON-NOSVE-NEXT:    rbit w8, w82994; NONEON-NOSVE-NEXT:    clz w8, w82995; NONEON-NOSVE-NEXT:    strh w8, [sp, #18]2996; NONEON-NOSVE-NEXT:    ldrh w8, [sp]2997; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100002998; NONEON-NOSVE-NEXT:    rbit w8, w82999; NONEON-NOSVE-NEXT:    clz w8, w83000; NONEON-NOSVE-NEXT:    strh w8, [sp, #16]3001; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]3002; NONEON-NOSVE-NEXT:    add sp, sp, #323003; NONEON-NOSVE-NEXT:    ret3004  %res = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %op)3005  ret <8 x i16> %res3006}3007 3008define void @cttz_v16i16(ptr %a) {3009; CHECK-LABEL: cttz_v16i16:3010; CHECK:       // %bb.0:3011; CHECK-NEXT:    ldp q0, q1, [x0]3012; CHECK-NEXT:    ptrue p0.h, vl83013; CHECK-NEXT:    rbit z0.h, p0/m, z0.h3014; CHECK-NEXT:    rbit z1.h, p0/m, z1.h3015; CHECK-NEXT:    clz z0.h, p0/m, z0.h3016; CHECK-NEXT:    clz z1.h, p0/m, z1.h3017; CHECK-NEXT:    stp q0, q1, [x0]3018; CHECK-NEXT:    ret3019;3020; NONEON-NOSVE-LABEL: cttz_v16i16:3021; NONEON-NOSVE:       // %bb.0:3022; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]3023; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!3024; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 643025; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #30]3026; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003027; NONEON-NOSVE-NEXT:    rbit w8, w83028; NONEON-NOSVE-NEXT:    clz w8, w83029; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]3030; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]3031; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003032; NONEON-NOSVE-NEXT:    rbit w8, w83033; NONEON-NOSVE-NEXT:    clz w8, w83034; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]3035; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #26]3036; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003037; NONEON-NOSVE-NEXT:    rbit w8, w83038; NONEON-NOSVE-NEXT:    clz w8, w83039; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]3040; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]3041; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003042; NONEON-NOSVE-NEXT:    rbit w8, w83043; NONEON-NOSVE-NEXT:    clz w8, w83044; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]3045; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]3046; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003047; NONEON-NOSVE-NEXT:    rbit w8, w83048; NONEON-NOSVE-NEXT:    clz w8, w83049; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]3050; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]3051; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003052; NONEON-NOSVE-NEXT:    rbit w8, w83053; NONEON-NOSVE-NEXT:    clz w8, w83054; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]3055; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]3056; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003057; NONEON-NOSVE-NEXT:    rbit w8, w83058; NONEON-NOSVE-NEXT:    clz w8, w83059; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]3060; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]3061; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003062; NONEON-NOSVE-NEXT:    rbit w8, w83063; NONEON-NOSVE-NEXT:    clz w8, w83064; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]3065; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #14]3066; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003067; NONEON-NOSVE-NEXT:    rbit w8, w83068; NONEON-NOSVE-NEXT:    clz w8, w83069; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]3070; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]3071; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003072; NONEON-NOSVE-NEXT:    rbit w8, w83073; NONEON-NOSVE-NEXT:    clz w8, w83074; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]3075; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #10]3076; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003077; NONEON-NOSVE-NEXT:    rbit w8, w83078; NONEON-NOSVE-NEXT:    clz w8, w83079; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]3080; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #8]3081; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003082; NONEON-NOSVE-NEXT:    rbit w8, w83083; NONEON-NOSVE-NEXT:    clz w8, w83084; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]3085; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #6]3086; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003087; NONEON-NOSVE-NEXT:    rbit w8, w83088; NONEON-NOSVE-NEXT:    clz w8, w83089; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]3090; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]3091; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003092; NONEON-NOSVE-NEXT:    rbit w8, w83093; NONEON-NOSVE-NEXT:    clz w8, w83094; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]3095; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]3096; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003097; NONEON-NOSVE-NEXT:    rbit w8, w83098; NONEON-NOSVE-NEXT:    clz w8, w83099; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]3100; NONEON-NOSVE-NEXT:    ldrh w8, [sp]3101; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100003102; NONEON-NOSVE-NEXT:    rbit w8, w83103; NONEON-NOSVE-NEXT:    clz w8, w83104; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]3105; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]3106; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]3107; NONEON-NOSVE-NEXT:    add sp, sp, #643108; NONEON-NOSVE-NEXT:    ret3109  %op = load <16 x i16>, ptr %a3110  %res = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %op)3111  store <16 x i16> %res, ptr %a3112  ret void3113}3114 3115define <2 x i32> @cttz_v2i32(<2 x i32> %op) {3116; CHECK-LABEL: cttz_v2i32:3117; CHECK:       // %bb.0:3118; CHECK-NEXT:    ptrue p0.s, vl23119; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z03120; CHECK-NEXT:    rbit z0.s, p0/m, z0.s3121; CHECK-NEXT:    clz z0.s, p0/m, z0.s3122; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z03123; CHECK-NEXT:    ret3124;3125; NONEON-NOSVE-LABEL: cttz_v2i32:3126; NONEON-NOSVE:       // %bb.0:3127; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!3128; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 163129; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]3130; NONEON-NOSVE-NEXT:    rbit w8, w83131; NONEON-NOSVE-NEXT:    clz w9, w83132; NONEON-NOSVE-NEXT:    ldr w8, [sp]3133; NONEON-NOSVE-NEXT:    rbit w8, w83134; NONEON-NOSVE-NEXT:    clz w8, w83135; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8]3136; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]3137; NONEON-NOSVE-NEXT:    add sp, sp, #163138; NONEON-NOSVE-NEXT:    ret3139  %res = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %op)3140  ret <2 x i32> %res3141}3142 3143define <4 x i32> @cttz_v4i32(<4 x i32> %op) {3144; CHECK-LABEL: cttz_v4i32:3145; CHECK:       // %bb.0:3146; CHECK-NEXT:    ptrue p0.s, vl43147; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z03148; CHECK-NEXT:    rbit z0.s, p0/m, z0.s3149; CHECK-NEXT:    clz z0.s, p0/m, z0.s3150; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z03151; CHECK-NEXT:    ret3152;3153; NONEON-NOSVE-LABEL: cttz_v4i32:3154; NONEON-NOSVE:       // %bb.0:3155; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!3156; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 323157; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12]3158; NONEON-NOSVE-NEXT:    rbit w8, w83159; NONEON-NOSVE-NEXT:    clz w9, w83160; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]3161; NONEON-NOSVE-NEXT:    rbit w8, w83162; NONEON-NOSVE-NEXT:    clz w8, w83163; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]3164; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]3165; NONEON-NOSVE-NEXT:    rbit w8, w83166; NONEON-NOSVE-NEXT:    clz w9, w83167; NONEON-NOSVE-NEXT:    ldr w8, [sp]3168; NONEON-NOSVE-NEXT:    rbit w8, w83169; NONEON-NOSVE-NEXT:    clz w8, w83170; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]3171; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]3172; NONEON-NOSVE-NEXT:    add sp, sp, #323173; NONEON-NOSVE-NEXT:    ret3174  %res = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %op)3175  ret <4 x i32> %res3176}3177 3178define void @cttz_v8i32(ptr %a) {3179; CHECK-LABEL: cttz_v8i32:3180; CHECK:       // %bb.0:3181; CHECK-NEXT:    ldp q0, q1, [x0]3182; CHECK-NEXT:    ptrue p0.s, vl43183; CHECK-NEXT:    rbit z0.s, p0/m, z0.s3184; CHECK-NEXT:    rbit z1.s, p0/m, z1.s3185; CHECK-NEXT:    clz z0.s, p0/m, z0.s3186; CHECK-NEXT:    clz z1.s, p0/m, z1.s3187; CHECK-NEXT:    stp q0, q1, [x0]3188; CHECK-NEXT:    ret3189;3190; NONEON-NOSVE-LABEL: cttz_v8i32:3191; NONEON-NOSVE:       // %bb.0:3192; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]3193; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!3194; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 643195; NONEON-NOSVE-NEXT:    ldr w8, [sp, #28]3196; NONEON-NOSVE-NEXT:    rbit w8, w83197; NONEON-NOSVE-NEXT:    clz w9, w83198; NONEON-NOSVE-NEXT:    ldr w8, [sp, #24]3199; NONEON-NOSVE-NEXT:    rbit w8, w83200; NONEON-NOSVE-NEXT:    clz w8, w83201; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]3202; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]3203; NONEON-NOSVE-NEXT:    rbit w8, w83204; NONEON-NOSVE-NEXT:    clz w9, w83205; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]3206; NONEON-NOSVE-NEXT:    rbit w8, w83207; NONEON-NOSVE-NEXT:    clz w8, w83208; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]3209; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12]3210; NONEON-NOSVE-NEXT:    rbit w8, w83211; NONEON-NOSVE-NEXT:    clz w9, w83212; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]3213; NONEON-NOSVE-NEXT:    rbit w8, w83214; NONEON-NOSVE-NEXT:    clz w8, w83215; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]3216; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]3217; NONEON-NOSVE-NEXT:    rbit w8, w83218; NONEON-NOSVE-NEXT:    clz w9, w83219; NONEON-NOSVE-NEXT:    ldr w8, [sp]3220; NONEON-NOSVE-NEXT:    rbit w8, w83221; NONEON-NOSVE-NEXT:    clz w8, w83222; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]3223; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]3224; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]3225; NONEON-NOSVE-NEXT:    add sp, sp, #643226; NONEON-NOSVE-NEXT:    ret3227  %op = load <8 x i32>, ptr %a3228  %res = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %op)3229  store <8 x i32> %res, ptr %a3230  ret void3231}3232 3233define <1 x i64> @cttz_v1i64(<1 x i64> %op) {3234; CHECK-LABEL: cttz_v1i64:3235; CHECK:       // %bb.0:3236; CHECK-NEXT:    ptrue p0.d, vl13237; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z03238; CHECK-NEXT:    rbit z0.d, p0/m, z0.d3239; CHECK-NEXT:    clz z0.d, p0/m, z0.d3240; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z03241; CHECK-NEXT:    ret3242;3243; NONEON-NOSVE-LABEL: cttz_v1i64:3244; NONEON-NOSVE:       // %bb.0:3245; NONEON-NOSVE-NEXT:    sub sp, sp, #163246; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 163247; NONEON-NOSVE-NEXT:    fmov x8, d03248; NONEON-NOSVE-NEXT:    rbit x8, x83249; NONEON-NOSVE-NEXT:    clz x8, x83250; NONEON-NOSVE-NEXT:    str x8, [sp, #8]3251; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]3252; NONEON-NOSVE-NEXT:    add sp, sp, #163253; NONEON-NOSVE-NEXT:    ret3254  %res = call <1 x i64> @llvm.cttz.v1i64(<1 x i64> %op)3255  ret <1 x i64> %res3256}3257 3258define <2 x i64> @cttz_v2i64(<2 x i64> %op) {3259; CHECK-LABEL: cttz_v2i64:3260; CHECK:       // %bb.0:3261; CHECK-NEXT:    ptrue p0.d, vl23262; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z03263; CHECK-NEXT:    rbit z0.d, p0/m, z0.d3264; CHECK-NEXT:    clz z0.d, p0/m, z0.d3265; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z03266; CHECK-NEXT:    ret3267;3268; NONEON-NOSVE-LABEL: cttz_v2i64:3269; NONEON-NOSVE:       // %bb.0:3270; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!3271; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 323272; NONEON-NOSVE-NEXT:    ldr x8, [sp, #8]3273; NONEON-NOSVE-NEXT:    rbit x8, x83274; NONEON-NOSVE-NEXT:    clz x9, x83275; NONEON-NOSVE-NEXT:    ldr x8, [sp]3276; NONEON-NOSVE-NEXT:    rbit x8, x83277; NONEON-NOSVE-NEXT:    clz x8, x83278; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #16]3279; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]3280; NONEON-NOSVE-NEXT:    add sp, sp, #323281; NONEON-NOSVE-NEXT:    ret3282  %res = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %op)3283  ret <2 x i64> %res3284}3285 3286define void @cttz_v4i64(ptr %a) {3287; CHECK-LABEL: cttz_v4i64:3288; CHECK:       // %bb.0:3289; CHECK-NEXT:    ldp q0, q1, [x0]3290; CHECK-NEXT:    ptrue p0.d, vl23291; CHECK-NEXT:    rbit z0.d, p0/m, z0.d3292; CHECK-NEXT:    rbit z1.d, p0/m, z1.d3293; CHECK-NEXT:    clz z0.d, p0/m, z0.d3294; CHECK-NEXT:    clz z1.d, p0/m, z1.d3295; CHECK-NEXT:    stp q0, q1, [x0]3296; CHECK-NEXT:    ret3297;3298; NONEON-NOSVE-LABEL: cttz_v4i64:3299; NONEON-NOSVE:       // %bb.0:3300; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]3301; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!3302; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 643303; NONEON-NOSVE-NEXT:    ldr x8, [sp, #24]3304; NONEON-NOSVE-NEXT:    rbit x8, x83305; NONEON-NOSVE-NEXT:    clz x9, x83306; NONEON-NOSVE-NEXT:    ldr x8, [sp, #16]3307; NONEON-NOSVE-NEXT:    rbit x8, x83308; NONEON-NOSVE-NEXT:    clz x8, x83309; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #48]3310; NONEON-NOSVE-NEXT:    ldr x8, [sp, #8]3311; NONEON-NOSVE-NEXT:    rbit x8, x83312; NONEON-NOSVE-NEXT:    clz x9, x83313; NONEON-NOSVE-NEXT:    ldr x8, [sp]3314; NONEON-NOSVE-NEXT:    rbit x8, x83315; NONEON-NOSVE-NEXT:    clz x8, x83316; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #32]3317; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]3318; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]3319; NONEON-NOSVE-NEXT:    add sp, sp, #643320; NONEON-NOSVE-NEXT:    ret3321  %op = load <4 x i64>, ptr %a3322  %res = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %op)3323  store <4 x i64> %res, ptr %a3324  ret void3325}3326 3327 3328declare <4 x i8> @llvm.ctlz.v4i8(<4 x i8>)3329declare <8 x i8> @llvm.ctlz.v8i8(<8 x i8>)3330declare <16 x i8> @llvm.ctlz.v16i8(<16 x i8>)3331declare <32 x i8> @llvm.ctlz.v32i8(<32 x i8>)3332declare <2 x i16> @llvm.ctlz.v2i16(<2 x i16>)3333declare <4 x i16> @llvm.ctlz.v4i16(<4 x i16>)3334declare <8 x i16> @llvm.ctlz.v8i16(<8 x i16>)3335declare <16 x i16> @llvm.ctlz.v16i16(<16 x i16>)3336declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>)3337declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>)3338declare <8 x i32> @llvm.ctlz.v8i32(<8 x i32>)3339declare <1 x i64> @llvm.ctlz.v1i64(<1 x i64>)3340declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>)3341declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>)3342 3343declare <4 x i8> @llvm.ctpop.v4i8(<4 x i8>)3344declare <8 x i8> @llvm.ctpop.v8i8(<8 x i8>)3345declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8>)3346declare <32 x i8> @llvm.ctpop.v32i8(<32 x i8>)3347declare <2 x i16> @llvm.ctpop.v2i16(<2 x i16>)3348declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>)3349declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)3350declare <16 x i16> @llvm.ctpop.v16i16(<16 x i16>)3351declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)3352declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)3353declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>)3354declare <1 x i64> @llvm.ctpop.v1i64(<1 x i64>)3355declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)3356declare <4 x i64> @llvm.ctpop.v4i64(<4 x i64>)3357 3358declare <4 x i8> @llvm.cttz.v4i8(<4 x i8>)3359declare <8 x i8> @llvm.cttz.v8i8(<8 x i8>)3360declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>)3361declare <32 x i8> @llvm.cttz.v32i8(<32 x i8>)3362declare <2 x i16> @llvm.cttz.v2i16(<2 x i16>)3363declare <4 x i16> @llvm.cttz.v4i16(<4 x i16>)3364declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>)3365declare <16 x i16> @llvm.cttz.v16i16(<16 x i16>)3366declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>)3367declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>)3368declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>)3369declare <1 x i64> @llvm.cttz.v1i64(<1 x i64>)3370declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>)3371declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>)3372