626 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 42; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,NONSTREAMING3; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,STREAMING4 5; WITH VSCALE RANGE6 7define i32 @ctz_nxv32i1(<vscale x 32 x i1> %a) #0 {8; CHECK-LABEL: ctz_nxv32i1:9; CHECK: // %bb.0:10; CHECK-NEXT: index z0.h, #0, #-111; CHECK-NEXT: cnth x812; CHECK-NEXT: punpklo p2.h, p0.b13; CHECK-NEXT: neg x8, x814; CHECK-NEXT: punpklo p3.h, p1.b15; CHECK-NEXT: rdvl x9, #216; CHECK-NEXT: mov z1.h, w817; CHECK-NEXT: rdvl x8, #-118; CHECK-NEXT: punpkhi p0.h, p0.b19; CHECK-NEXT: mov z2.h, w820; CHECK-NEXT: punpkhi p1.h, p1.b21; CHECK-NEXT: mov z3.h, p2/z, #-1 // =0xffffffffffffffff22; CHECK-NEXT: inch z0.h, all, mul #423; CHECK-NEXT: mov z5.h, p3/z, #-1 // =0xffffffffffffffff24; CHECK-NEXT: mov z6.h, p0/z, #-1 // =0xffffffffffffffff25; CHECK-NEXT: mov z7.h, p1/z, #-1 // =0xffffffffffffffff26; CHECK-NEXT: ptrue p0.h27; CHECK-NEXT: add z1.h, z0.h, z1.h28; CHECK-NEXT: add z4.h, z0.h, z2.h29; CHECK-NEXT: and z0.d, z0.d, z3.d30; CHECK-NEXT: add z2.h, z1.h, z2.h31; CHECK-NEXT: and z3.d, z4.d, z5.d32; CHECK-NEXT: and z1.d, z1.d, z6.d33; CHECK-NEXT: and z2.d, z2.d, z7.d34; CHECK-NEXT: umax z0.h, p0/m, z0.h, z3.h35; CHECK-NEXT: umax z1.h, p0/m, z1.h, z2.h36; CHECK-NEXT: umax z0.h, p0/m, z0.h, z1.h37; CHECK-NEXT: umaxv h0, p0, z0.h38; CHECK-NEXT: fmov w8, s039; CHECK-NEXT: sub w8, w9, w840; CHECK-NEXT: and w0, w8, #0xffff41; CHECK-NEXT: ret42 %res = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> %a, i1 0)43 ret i32 %res44}45 46define i32 @ctz_nxv4i32(<vscale x 4 x i32> %a) #0 {47; CHECK-LABEL: ctz_nxv4i32:48; CHECK: // %bb.0:49; CHECK-NEXT: ptrue p0.s50; CHECK-NEXT: cntw x951; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #052; CHECK-NEXT: index z0.s, #0, #-153; CHECK-NEXT: mov z1.s, p1/z, #-1 // =0xffffffffffffffff54; CHECK-NEXT: incw z0.s55; CHECK-NEXT: and z0.d, z0.d, z1.d56; CHECK-NEXT: and z0.s, z0.s, #0xff57; CHECK-NEXT: umaxv s0, p0, z0.s58; CHECK-NEXT: fmov w8, s059; CHECK-NEXT: sub w8, w9, w860; CHECK-NEXT: and w0, w8, #0xff61; CHECK-NEXT: ret62 %res = call i32 @llvm.experimental.cttz.elts.i32.nxv4i32(<vscale x 4 x i32> %a, i1 0)63 ret i32 %res64}65 66; VSCALE RANGE, ZERO IS POISON67 68define i64 @vscale_4096(<vscale x 16 x i8> %a) #1 {69; CHECK-LABEL: vscale_4096:70; CHECK: // %bb.0:71; CHECK-NEXT: ptrue p0.b72; CHECK-NEXT: index z1.s, #0, #-173; CHECK-NEXT: cntw x874; CHECK-NEXT: neg x8, x875; CHECK-NEXT: rdvl x9, #176; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #077; CHECK-NEXT: mov z0.s, w878; CHECK-NEXT: cnth x879; CHECK-NEXT: neg x8, x880; CHECK-NEXT: incw z1.s, all, mul #481; CHECK-NEXT: mov z2.s, w882; CHECK-NEXT: punpklo p1.h, p0.b83; CHECK-NEXT: punpkhi p0.h, p0.b84; CHECK-NEXT: add z0.s, z1.s, z0.s85; CHECK-NEXT: add z4.s, z1.s, z2.s86; CHECK-NEXT: punpkhi p2.h, p1.b87; CHECK-NEXT: punpkhi p3.h, p0.b88; CHECK-NEXT: punpklo p0.h, p0.b89; CHECK-NEXT: mov z3.s, p2/z, #-1 // =0xffffffffffffffff90; CHECK-NEXT: add z2.s, z0.s, z2.s91; CHECK-NEXT: punpklo p1.h, p1.b92; CHECK-NEXT: mov z5.s, p3/z, #-1 // =0xffffffffffffffff93; CHECK-NEXT: mov z6.s, p0/z, #-1 // =0xffffffffffffffff94; CHECK-NEXT: ptrue p0.s95; CHECK-NEXT: mov z7.s, p1/z, #-1 // =0xffffffffffffffff96; CHECK-NEXT: and z0.d, z0.d, z3.d97; CHECK-NEXT: and z2.d, z2.d, z5.d98; CHECK-NEXT: and z3.d, z4.d, z6.d99; CHECK-NEXT: and z1.d, z1.d, z7.d100; CHECK-NEXT: umax z0.s, p0/m, z0.s, z2.s101; CHECK-NEXT: umax z1.s, p0/m, z1.s, z3.s102; CHECK-NEXT: umax z0.s, p0/m, z0.s, z1.s103; CHECK-NEXT: umaxv s0, p0, z0.s104; CHECK-NEXT: fmov w8, s0105; CHECK-NEXT: sub w0, w9, w8106; CHECK-NEXT: ret107 %res = call i64 @llvm.experimental.cttz.elts.i64.nxv16i8(<vscale x 16 x i8> %a, i1 0)108 ret i64 %res109}110 111define i64 @vscale_4096_poison(<vscale x 16 x i8> %a) #1 {112; CHECK-LABEL: vscale_4096_poison:113; CHECK: // %bb.0:114; CHECK-NEXT: ptrue p0.b115; CHECK-NEXT: index z1.h, #0, #-1116; CHECK-NEXT: cnth x8117; CHECK-NEXT: neg x8, x8118; CHECK-NEXT: rdvl x9, #1119; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0120; CHECK-NEXT: mov z0.h, w8121; CHECK-NEXT: inch z1.h, all, mul #2122; CHECK-NEXT: punpkhi p1.h, p0.b123; CHECK-NEXT: punpklo p0.h, p0.b124; CHECK-NEXT: add z0.h, z1.h, z0.h125; CHECK-NEXT: mov z2.h, p1/z, #-1 // =0xffffffffffffffff126; CHECK-NEXT: mov z3.h, p0/z, #-1 // =0xffffffffffffffff127; CHECK-NEXT: ptrue p0.h128; CHECK-NEXT: and z0.d, z0.d, z2.d129; CHECK-NEXT: and z1.d, z1.d, z3.d130; CHECK-NEXT: umax z0.h, p0/m, z0.h, z1.h131; CHECK-NEXT: umaxv h0, p0, z0.h132; CHECK-NEXT: fmov w8, s0133; CHECK-NEXT: sub w8, w9, w8134; CHECK-NEXT: and x0, x8, #0xffff135; CHECK-NEXT: ret136 %res = call i64 @llvm.experimental.cttz.elts.i64.nxv16i8(<vscale x 16 x i8> %a, i1 1)137 ret i64 %res138}139 140; EFFICIENT LOWERING USING BRKB141 142define i32 @ctz_nxv2i1(<vscale x 2 x i1> %a) {143; CHECK-LABEL: ctz_nxv2i1:144; CHECK: // %bb.0:145; CHECK-NEXT: ptrue p1.d146; CHECK-NEXT: brkb p0.b, p1/z, p0.b147; CHECK-NEXT: cntp x0, p0, p0.d148; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0149; CHECK-NEXT: ret150 %res = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> %a, i1 0)151 ret i32 %res152}153 154define i32 @ctz_nxv2i1_poison(<vscale x 2 x i1> %a) {155; CHECK-LABEL: ctz_nxv2i1_poison:156; CHECK: // %bb.0:157; CHECK-NEXT: ptrue p1.d158; CHECK-NEXT: brkb p0.b, p1/z, p0.b159; CHECK-NEXT: cntp x0, p0, p0.d160; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0161; CHECK-NEXT: ret162 %res = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> %a, i1 1)163 ret i32 %res164}165 166define i64 @add_i64_ctz_nxv2i1_poison(<vscale x 2 x i1> %a, i64 %b) {167; CHECK-LABEL: add_i64_ctz_nxv2i1_poison:168; CHECK: // %bb.0:169; CHECK-NEXT: ptrue p1.d170; CHECK-NEXT: brkb p0.b, p1/z, p0.b171; CHECK-NEXT: incp x0, p0.d172; CHECK-NEXT: ret173 %res = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> %a, i1 1)174 %add = add i64 %res, %b175 ret i64 %add176}177 178define i32 @add_i32_ctz_nxv2i1_poison(<vscale x 2 x i1> %a, i32 %b) {179; CHECK-LABEL: add_i32_ctz_nxv2i1_poison:180; CHECK: // %bb.0:181; CHECK-NEXT: ptrue p1.d182; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0183; CHECK-NEXT: brkb p0.b, p1/z, p0.b184; CHECK-NEXT: incp x0, p0.d185; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0186; CHECK-NEXT: ret187 %res = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> %a, i1 1)188 %trunc = trunc i64 %res to i32189 %add = add i32 %trunc, %b190 ret i32 %add191}192 193define i32 @ctz_nxv4i1(<vscale x 4 x i1> %a) {194; CHECK-LABEL: ctz_nxv4i1:195; CHECK: // %bb.0:196; CHECK-NEXT: ptrue p1.s197; CHECK-NEXT: brkb p0.b, p1/z, p0.b198; CHECK-NEXT: cntp x0, p0, p0.s199; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0200; CHECK-NEXT: ret201 %res = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> %a, i1 0)202 ret i32 %res203}204 205define i32 @ctz_nxv4i1_poison(<vscale x 4 x i1> %a) {206; CHECK-LABEL: ctz_nxv4i1_poison:207; CHECK: // %bb.0:208; CHECK-NEXT: ptrue p1.s209; CHECK-NEXT: brkb p0.b, p1/z, p0.b210; CHECK-NEXT: cntp x0, p0, p0.s211; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0212; CHECK-NEXT: ret213 %res = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> %a, i1 1)214 ret i32 %res215}216 217define i64 @add_i64_ctz_nxv4i1_poison(<vscale x 4 x i1> %a, i64 %b) {218; CHECK-LABEL: add_i64_ctz_nxv4i1_poison:219; CHECK: // %bb.0:220; CHECK-NEXT: ptrue p1.s221; CHECK-NEXT: brkb p0.b, p1/z, p0.b222; CHECK-NEXT: incp x0, p0.s223; CHECK-NEXT: ret224 %res = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> %a, i1 1)225 %add = add i64 %res, %b226 ret i64 %add227}228 229define i32 @add_i32_ctz_nxv4i1_poison(<vscale x 4 x i1> %a, i32 %b) {230; CHECK-LABEL: add_i32_ctz_nxv4i1_poison:231; CHECK: // %bb.0:232; CHECK-NEXT: ptrue p1.s233; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0234; CHECK-NEXT: brkb p0.b, p1/z, p0.b235; CHECK-NEXT: incp x0, p0.s236; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0237; CHECK-NEXT: ret238 %res = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> %a, i1 1)239 %trunc = trunc i64 %res to i32240 %add = add i32 %trunc, %b241 ret i32 %add242}243 244define i32 @ctz_nxv8i1(<vscale x 8 x i1> %a) {245; CHECK-LABEL: ctz_nxv8i1:246; CHECK: // %bb.0:247; CHECK-NEXT: ptrue p1.h248; CHECK-NEXT: brkb p0.b, p1/z, p0.b249; CHECK-NEXT: cntp x0, p0, p0.h250; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0251; CHECK-NEXT: ret252 %res = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> %a, i1 0)253 ret i32 %res254}255 256define i32 @ctz_nxv8i1_poison(<vscale x 8 x i1> %a) {257; CHECK-LABEL: ctz_nxv8i1_poison:258; CHECK: // %bb.0:259; CHECK-NEXT: ptrue p1.h260; CHECK-NEXT: brkb p0.b, p1/z, p0.b261; CHECK-NEXT: cntp x0, p0, p0.h262; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0263; CHECK-NEXT: ret264 %res = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> %a, i1 1)265 ret i32 %res266}267 268define i64 @add_i64_ctz_nxv8i1_poison(<vscale x 8 x i1> %a, i64 %b) {269; CHECK-LABEL: add_i64_ctz_nxv8i1_poison:270; CHECK: // %bb.0:271; CHECK-NEXT: ptrue p1.h272; CHECK-NEXT: brkb p0.b, p1/z, p0.b273; CHECK-NEXT: incp x0, p0.h274; CHECK-NEXT: ret275 %res = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> %a, i1 1)276 %add = add i64 %res, %b277 ret i64 %add278}279 280define i32 @add_i32_ctz_nxv8i1_poison(<vscale x 8 x i1> %a, i32 %b) {281; CHECK-LABEL: add_i32_ctz_nxv8i1_poison:282; CHECK: // %bb.0:283; CHECK-NEXT: ptrue p1.h284; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0285; CHECK-NEXT: brkb p0.b, p1/z, p0.b286; CHECK-NEXT: incp x0, p0.h287; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0288; CHECK-NEXT: ret289 %res = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> %a, i1 1)290 %trunc = trunc i64 %res to i32291 %add = add i32 %trunc, %b292 ret i32 %add293}294 295define i32 @ctz_nxv16i1(<vscale x 16 x i1> %a) {296; CHECK-LABEL: ctz_nxv16i1:297; CHECK: // %bb.0:298; CHECK-NEXT: ptrue p1.b299; CHECK-NEXT: brkb p0.b, p1/z, p0.b300; CHECK-NEXT: cntp x0, p0, p0.b301; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0302; CHECK-NEXT: ret303 %res = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> %a, i1 0)304 ret i32 %res305}306 307define i32 @ctz_nxv16i1_poison(<vscale x 16 x i1> %a) {308; CHECK-LABEL: ctz_nxv16i1_poison:309; CHECK: // %bb.0:310; CHECK-NEXT: ptrue p1.b311; CHECK-NEXT: brkb p0.b, p1/z, p0.b312; CHECK-NEXT: cntp x0, p0, p0.b313; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0314; CHECK-NEXT: ret315 %res = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> %a, i1 1)316 ret i32 %res317}318 319define i32 @ctz_and_nxv16i1(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {320; CHECK-LABEL: ctz_and_nxv16i1:321; CHECK: // %bb.0:322; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, z1.b323; CHECK-NEXT: ptrue p1.b324; CHECK-NEXT: brkb p0.b, p1/z, p0.b325; CHECK-NEXT: cntp x0, p0, p0.b326; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0327; CHECK-NEXT: ret328 %cmp = icmp ne <vscale x 16 x i8> %a, %b329 %select = select <vscale x 16 x i1> %pg, <vscale x 16 x i1> %cmp, <vscale x 16 x i1> zeroinitializer330 %and = and <vscale x 16 x i1> %pg, %select331 %res = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> %and, i1 0)332 ret i32 %res333}334 335define i64 @add_i64_ctz_nxv16i1_poison(<vscale x 16 x i1> %a, i64 %b) {336; CHECK-LABEL: add_i64_ctz_nxv16i1_poison:337; CHECK: // %bb.0:338; CHECK-NEXT: ptrue p1.b339; CHECK-NEXT: brkb p0.b, p1/z, p0.b340; CHECK-NEXT: incp x0, p0.b341; CHECK-NEXT: ret342 %res = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> %a, i1 1)343 %add = add i64 %res, %b344 ret i64 %add345}346 347define i32 @add_i32_ctz_nxv16i1_poison(<vscale x 16 x i1> %a, i32 %b) {348; CHECK-LABEL: add_i32_ctz_nxv16i1_poison:349; CHECK: // %bb.0:350; CHECK-NEXT: ptrue p1.b351; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0352; CHECK-NEXT: brkb p0.b, p1/z, p0.b353; CHECK-NEXT: incp x0, p0.b354; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0355; CHECK-NEXT: ret356 %res = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> %a, i1 1)357 %trunc = trunc i64 %res to i32358 %add = add i32 %trunc, %b359 ret i32 %add360}361 362; FIXED-WIDTH VECTOR TYPES363 364define i32 @ctz_v16i1(<16 x i1> %a) {365; NONSTREAMING-LABEL: ctz_v16i1:366; NONSTREAMING: // %bb.0:367; NONSTREAMING-NEXT: shl v0.16b, v0.16b, #7368; NONSTREAMING-NEXT: ptrue p0.b, vl16369; NONSTREAMING-NEXT: ptrue p1.b370; NONSTREAMING-NEXT: cmpne p0.b, p0/z, z0.b, #0371; NONSTREAMING-NEXT: brkb p0.b, p1/z, p0.b372; NONSTREAMING-NEXT: cntp x0, p0, p0.b373; NONSTREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0374; NONSTREAMING-NEXT: ret375;376; STREAMING-LABEL: ctz_v16i1:377; STREAMING: // %bb.0:378; STREAMING-NEXT: // kill: def $q0 killed $q0 def $z0379; STREAMING-NEXT: ptrue p0.b, vl16380; STREAMING-NEXT: lsl z0.b, z0.b, #7381; STREAMING-NEXT: ptrue p1.b382; STREAMING-NEXT: asr z0.b, z0.b, #7383; STREAMING-NEXT: cmpne p0.b, p0/z, z0.b, #0384; STREAMING-NEXT: brkb p0.b, p1/z, p0.b385; STREAMING-NEXT: cntp x0, p0, p0.b386; STREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0387; STREAMING-NEXT: ret388 %res = call i32 @llvm.experimental.cttz.elts.i32.v16i1(<16 x i1> %a, i1 0)389 ret i32 %res390}391 392define i32 @ctz_v16i1_poison(<16 x i1> %a) {393; NONSTREAMING-LABEL: ctz_v16i1_poison:394; NONSTREAMING: // %bb.0:395; NONSTREAMING-NEXT: shl v0.16b, v0.16b, #7396; NONSTREAMING-NEXT: ptrue p0.b, vl16397; NONSTREAMING-NEXT: ptrue p1.b398; NONSTREAMING-NEXT: cmpne p0.b, p0/z, z0.b, #0399; NONSTREAMING-NEXT: brkb p0.b, p1/z, p0.b400; NONSTREAMING-NEXT: cntp x0, p0, p0.b401; NONSTREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0402; NONSTREAMING-NEXT: ret403;404; STREAMING-LABEL: ctz_v16i1_poison:405; STREAMING: // %bb.0:406; STREAMING-NEXT: // kill: def $q0 killed $q0 def $z0407; STREAMING-NEXT: ptrue p0.b, vl16408; STREAMING-NEXT: lsl z0.b, z0.b, #7409; STREAMING-NEXT: ptrue p1.b410; STREAMING-NEXT: asr z0.b, z0.b, #7411; STREAMING-NEXT: cmpne p0.b, p0/z, z0.b, #0412; STREAMING-NEXT: brkb p0.b, p1/z, p0.b413; STREAMING-NEXT: cntp x0, p0, p0.b414; STREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0415; STREAMING-NEXT: ret416 %res = call i32 @llvm.experimental.cttz.elts.i32.v16i1(<16 x i1> %a, i1 1)417 ret i32 %res418}419 420define i64 @add_i64_ctz_v16i1_poison(<16 x i1> %a, i64 %b) {421; NONSTREAMING-LABEL: add_i64_ctz_v16i1_poison:422; NONSTREAMING: // %bb.0:423; NONSTREAMING-NEXT: shl v0.16b, v0.16b, #7424; NONSTREAMING-NEXT: ptrue p0.b, vl16425; NONSTREAMING-NEXT: ptrue p1.b426; NONSTREAMING-NEXT: cmpne p0.b, p0/z, z0.b, #0427; NONSTREAMING-NEXT: brkb p0.b, p1/z, p0.b428; NONSTREAMING-NEXT: incp x0, p0.b429; NONSTREAMING-NEXT: ret430;431; STREAMING-LABEL: add_i64_ctz_v16i1_poison:432; STREAMING: // %bb.0:433; STREAMING-NEXT: // kill: def $q0 killed $q0 def $z0434; STREAMING-NEXT: ptrue p0.b, vl16435; STREAMING-NEXT: lsl z0.b, z0.b, #7436; STREAMING-NEXT: ptrue p1.b437; STREAMING-NEXT: asr z0.b, z0.b, #7438; STREAMING-NEXT: cmpne p0.b, p0/z, z0.b, #0439; STREAMING-NEXT: brkb p0.b, p1/z, p0.b440; STREAMING-NEXT: incp x0, p0.b441; STREAMING-NEXT: ret442 %res = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> %a, i1 1)443 %add = add i64 %res, %b444 ret i64 %add445}446 447define i32 @ctz_v8i1(<8 x i1> %a) {448; NONSTREAMING-LABEL: ctz_v8i1:449; NONSTREAMING: // %bb.0:450; NONSTREAMING-NEXT: shl v0.8b, v0.8b, #7451; NONSTREAMING-NEXT: ptrue p0.b, vl8452; NONSTREAMING-NEXT: ptrue p1.b453; NONSTREAMING-NEXT: cmpne p0.b, p0/z, z0.b, #0454; NONSTREAMING-NEXT: brkb p0.b, p1/z, p0.b455; NONSTREAMING-NEXT: cntp x0, p0, p0.b456; NONSTREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0457; NONSTREAMING-NEXT: ret458;459; STREAMING-LABEL: ctz_v8i1:460; STREAMING: // %bb.0:461; STREAMING-NEXT: // kill: def $d0 killed $d0 def $z0462; STREAMING-NEXT: ptrue p0.b, vl8463; STREAMING-NEXT: lsl z0.b, z0.b, #7464; STREAMING-NEXT: ptrue p1.b465; STREAMING-NEXT: asr z0.b, z0.b, #7466; STREAMING-NEXT: cmpne p0.b, p0/z, z0.b, #0467; STREAMING-NEXT: brkb p0.b, p1/z, p0.b468; STREAMING-NEXT: cntp x0, p0, p0.b469; STREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0470; STREAMING-NEXT: ret471 %res = call i32 @llvm.experimental.cttz.elts.i32.v8i1(<8 x i1> %a, i1 0)472 ret i32 %res473}474 475define i32 @ctz_v8i1_poison(<8 x i1> %a) {476; NONSTREAMING-LABEL: ctz_v8i1_poison:477; NONSTREAMING: // %bb.0:478; NONSTREAMING-NEXT: shl v0.8b, v0.8b, #7479; NONSTREAMING-NEXT: ptrue p0.b, vl8480; NONSTREAMING-NEXT: ptrue p1.b481; NONSTREAMING-NEXT: cmpne p0.b, p0/z, z0.b, #0482; NONSTREAMING-NEXT: brkb p0.b, p1/z, p0.b483; NONSTREAMING-NEXT: cntp x0, p0, p0.b484; NONSTREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0485; NONSTREAMING-NEXT: ret486;487; STREAMING-LABEL: ctz_v8i1_poison:488; STREAMING: // %bb.0:489; STREAMING-NEXT: // kill: def $d0 killed $d0 def $z0490; STREAMING-NEXT: ptrue p0.b, vl8491; STREAMING-NEXT: lsl z0.b, z0.b, #7492; STREAMING-NEXT: ptrue p1.b493; STREAMING-NEXT: asr z0.b, z0.b, #7494; STREAMING-NEXT: cmpne p0.b, p0/z, z0.b, #0495; STREAMING-NEXT: brkb p0.b, p1/z, p0.b496; STREAMING-NEXT: cntp x0, p0, p0.b497; STREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0498; STREAMING-NEXT: ret499 %res = call i32 @llvm.experimental.cttz.elts.i32.v8i1(<8 x i1> %a, i1 1)500 ret i32 %res501}502 503define i32 @ctz_v4i1(<4 x i1> %a) {504; NONSTREAMING-LABEL: ctz_v4i1:505; NONSTREAMING: // %bb.0:506; NONSTREAMING-NEXT: shl v0.4h, v0.4h, #15507; NONSTREAMING-NEXT: ptrue p0.h, vl4508; NONSTREAMING-NEXT: ptrue p1.h509; NONSTREAMING-NEXT: cmpne p0.h, p0/z, z0.h, #0510; NONSTREAMING-NEXT: brkb p0.b, p1/z, p0.b511; NONSTREAMING-NEXT: cntp x0, p0, p0.h512; NONSTREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0513; NONSTREAMING-NEXT: ret514;515; STREAMING-LABEL: ctz_v4i1:516; STREAMING: // %bb.0:517; STREAMING-NEXT: // kill: def $d0 killed $d0 def $z0518; STREAMING-NEXT: ptrue p0.h, vl4519; STREAMING-NEXT: lsl z0.h, z0.h, #15520; STREAMING-NEXT: ptrue p1.h521; STREAMING-NEXT: asr z0.h, z0.h, #15522; STREAMING-NEXT: cmpne p0.h, p0/z, z0.h, #0523; STREAMING-NEXT: brkb p0.b, p1/z, p0.b524; STREAMING-NEXT: cntp x0, p0, p0.h525; STREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0526; STREAMING-NEXT: ret527 %res = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> %a, i1 0)528 ret i32 %res529}530 531define i32 @ctz_v4i1_poison(<4 x i1> %a) {532; NONSTREAMING-LABEL: ctz_v4i1_poison:533; NONSTREAMING: // %bb.0:534; NONSTREAMING-NEXT: shl v0.4h, v0.4h, #15535; NONSTREAMING-NEXT: ptrue p0.h, vl4536; NONSTREAMING-NEXT: ptrue p1.h537; NONSTREAMING-NEXT: cmpne p0.h, p0/z, z0.h, #0538; NONSTREAMING-NEXT: brkb p0.b, p1/z, p0.b539; NONSTREAMING-NEXT: cntp x0, p0, p0.h540; NONSTREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0541; NONSTREAMING-NEXT: ret542;543; STREAMING-LABEL: ctz_v4i1_poison:544; STREAMING: // %bb.0:545; STREAMING-NEXT: // kill: def $d0 killed $d0 def $z0546; STREAMING-NEXT: ptrue p0.h, vl4547; STREAMING-NEXT: lsl z0.h, z0.h, #15548; STREAMING-NEXT: ptrue p1.h549; STREAMING-NEXT: asr z0.h, z0.h, #15550; STREAMING-NEXT: cmpne p0.h, p0/z, z0.h, #0551; STREAMING-NEXT: brkb p0.b, p1/z, p0.b552; STREAMING-NEXT: cntp x0, p0, p0.h553; STREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0554; STREAMING-NEXT: ret555 %res = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> %a, i1 1)556 ret i32 %res557}558 559define i32 @ctz_v2i1(<2 x i1> %a) {560; NONSTREAMING-LABEL: ctz_v2i1:561; NONSTREAMING: // %bb.0:562; NONSTREAMING-NEXT: shl v0.2s, v0.2s, #31563; NONSTREAMING-NEXT: ptrue p0.s, vl2564; NONSTREAMING-NEXT: ptrue p1.s565; NONSTREAMING-NEXT: cmpne p0.s, p0/z, z0.s, #0566; NONSTREAMING-NEXT: brkb p0.b, p1/z, p0.b567; NONSTREAMING-NEXT: cntp x0, p0, p0.s568; NONSTREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0569; NONSTREAMING-NEXT: ret570;571; STREAMING-LABEL: ctz_v2i1:572; STREAMING: // %bb.0:573; STREAMING-NEXT: // kill: def $d0 killed $d0 def $z0574; STREAMING-NEXT: ptrue p0.s, vl2575; STREAMING-NEXT: lsl z0.s, z0.s, #31576; STREAMING-NEXT: ptrue p1.s577; STREAMING-NEXT: asr z0.s, z0.s, #31578; STREAMING-NEXT: cmpne p0.s, p0/z, z0.s, #0579; STREAMING-NEXT: brkb p0.b, p1/z, p0.b580; STREAMING-NEXT: cntp x0, p0, p0.s581; STREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0582; STREAMING-NEXT: ret583 %res = call i32 @llvm.experimental.cttz.elts.i32.v2i1(<2 x i1> %a, i1 0)584 ret i32 %res585}586 587define i32 @ctz_v2i1_poison(<2 x i1> %a) {588; NONSTREAMING-LABEL: ctz_v2i1_poison:589; NONSTREAMING: // %bb.0:590; NONSTREAMING-NEXT: shl v0.2s, v0.2s, #31591; NONSTREAMING-NEXT: ptrue p0.s, vl2592; NONSTREAMING-NEXT: ptrue p1.s593; NONSTREAMING-NEXT: cmpne p0.s, p0/z, z0.s, #0594; NONSTREAMING-NEXT: brkb p0.b, p1/z, p0.b595; NONSTREAMING-NEXT: cntp x0, p0, p0.s596; NONSTREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0597; NONSTREAMING-NEXT: ret598;599; STREAMING-LABEL: ctz_v2i1_poison:600; STREAMING: // %bb.0:601; STREAMING-NEXT: // kill: def $d0 killed $d0 def $z0602; STREAMING-NEXT: ptrue p0.s, vl2603; STREAMING-NEXT: lsl z0.s, z0.s, #31604; STREAMING-NEXT: ptrue p1.s605; STREAMING-NEXT: asr z0.s, z0.s, #31606; STREAMING-NEXT: cmpne p0.s, p0/z, z0.s, #0607; STREAMING-NEXT: brkb p0.b, p1/z, p0.b608; STREAMING-NEXT: cntp x0, p0, p0.s609; STREAMING-NEXT: // kill: def $w0 killed $w0 killed $x0610; STREAMING-NEXT: ret611 %res = call i32 @llvm.experimental.cttz.elts.i32.v2i1(<2 x i1> %a, i1 1)612 ret i32 %res613}614 615declare i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1>, i1)616declare i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1>, i1)617declare i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1>, i1)618declare i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1>, i1)619declare i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1>, i1)620declare i32 @llvm.experimental.cttz.elts.i32.nxv4i32(<vscale x 4 x i32>, i1)621 622declare i64 @llvm.experimental.cttz.elts.i64.nxv16i8(<vscale x 16 x i8>, i1)623 624attributes #0 = { vscale_range(1,16) }625attributes #1 = { vscale_range(1,4096) }626