1212 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2563; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5124; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5125 6target triple = "aarch64-unknown-linux-gnu"7 8;9; CLZ10;11 12; Don't use SVE for 64-bit vectors.13define <8 x i8> @ctlz_v8i8(<8 x i8> %op) vscale_range(2,0) #0 {14; CHECK-LABEL: ctlz_v8i8:15; CHECK: // %bb.0:16; CHECK-NEXT: clz v0.8b, v0.8b17; CHECK-NEXT: ret18 %res = call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %op)19 ret <8 x i8> %res20}21 22; Don't use SVE for 128-bit vectors.23define <16 x i8> @ctlz_v16i8(<16 x i8> %op) vscale_range(2,0) #0 {24; CHECK-LABEL: ctlz_v16i8:25; CHECK: // %bb.0:26; CHECK-NEXT: clz v0.16b, v0.16b27; CHECK-NEXT: ret28 %res = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %op)29 ret <16 x i8> %res30}31 32define void @ctlz_v32i8(ptr %a) vscale_range(2,0) #0 {33; CHECK-LABEL: ctlz_v32i8:34; CHECK: // %bb.0:35; CHECK-NEXT: ptrue p0.b, vl3236; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]37; CHECK-NEXT: clz z0.b, p0/m, z0.b38; CHECK-NEXT: st1b { z0.b }, p0, [x0]39; CHECK-NEXT: ret40 %op = load <32 x i8>, ptr %a41 %res = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %op)42 store <32 x i8> %res, ptr %a43 ret void44}45 46define void @ctlz_v64i8(ptr %a) #0 {47; VBITS_GE_256-LABEL: ctlz_v64i8:48; VBITS_GE_256: // %bb.0:49; VBITS_GE_256-NEXT: ptrue p0.b, vl3250; VBITS_GE_256-NEXT: mov w8, #32 // =0x2051; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]52; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]53; VBITS_GE_256-NEXT: clz z0.b, p0/m, z0.b54; VBITS_GE_256-NEXT: clz z1.b, p0/m, z1.b55; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]56; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0]57; VBITS_GE_256-NEXT: ret58;59; VBITS_GE_512-LABEL: ctlz_v64i8:60; VBITS_GE_512: // %bb.0:61; VBITS_GE_512-NEXT: ptrue p0.b, vl6462; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]63; VBITS_GE_512-NEXT: clz z0.b, p0/m, z0.b64; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]65; VBITS_GE_512-NEXT: ret66 %op = load <64 x i8>, ptr %a67 %res = call <64 x i8> @llvm.ctlz.v64i8(<64 x i8> %op)68 store <64 x i8> %res, ptr %a69 ret void70}71 72define void @ctlz_v128i8(ptr %a) vscale_range(8,0) #0 {73; CHECK-LABEL: ctlz_v128i8:74; CHECK: // %bb.0:75; CHECK-NEXT: ptrue p0.b, vl12876; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]77; CHECK-NEXT: clz z0.b, p0/m, z0.b78; CHECK-NEXT: st1b { z0.b }, p0, [x0]79; CHECK-NEXT: ret80 %op = load <128 x i8>, ptr %a81 %res = call <128 x i8> @llvm.ctlz.v128i8(<128 x i8> %op)82 store <128 x i8> %res, ptr %a83 ret void84}85 86define void @ctlz_v256i8(ptr %a) vscale_range(16,0) #0 {87; CHECK-LABEL: ctlz_v256i8:88; CHECK: // %bb.0:89; CHECK-NEXT: ptrue p0.b, vl25690; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]91; CHECK-NEXT: clz z0.b, p0/m, z0.b92; CHECK-NEXT: st1b { z0.b }, p0, [x0]93; CHECK-NEXT: ret94 %op = load <256 x i8>, ptr %a95 %res = call <256 x i8> @llvm.ctlz.v256i8(<256 x i8> %op)96 store <256 x i8> %res, ptr %a97 ret void98}99 100; Don't use SVE for 64-bit vectors.101define <4 x i16> @ctlz_v4i16(<4 x i16> %op) vscale_range(2,0) #0 {102; CHECK-LABEL: ctlz_v4i16:103; CHECK: // %bb.0:104; CHECK-NEXT: clz v0.4h, v0.4h105; CHECK-NEXT: ret106 %res = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %op)107 ret <4 x i16> %res108}109 110; Don't use SVE for 128-bit vectors.111define <8 x i16> @ctlz_v8i16(<8 x i16> %op) vscale_range(2,0) #0 {112; CHECK-LABEL: ctlz_v8i16:113; CHECK: // %bb.0:114; CHECK-NEXT: clz v0.8h, v0.8h115; CHECK-NEXT: ret116 %res = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %op)117 ret <8 x i16> %res118}119 120define void @ctlz_v16i16(ptr %a) vscale_range(2,0) #0 {121; CHECK-LABEL: ctlz_v16i16:122; CHECK: // %bb.0:123; CHECK-NEXT: ptrue p0.h, vl16124; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]125; CHECK-NEXT: clz z0.h, p0/m, z0.h126; CHECK-NEXT: st1h { z0.h }, p0, [x0]127; CHECK-NEXT: ret128 %op = load <16 x i16>, ptr %a129 %res = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %op)130 store <16 x i16> %res, ptr %a131 ret void132}133 134define void @ctlz_v32i16(ptr %a) #0 {135; VBITS_GE_256-LABEL: ctlz_v32i16:136; VBITS_GE_256: // %bb.0:137; VBITS_GE_256-NEXT: ptrue p0.h, vl16138; VBITS_GE_256-NEXT: mov x8, #16 // =0x10139; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]140; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]141; VBITS_GE_256-NEXT: clz z0.h, p0/m, z0.h142; VBITS_GE_256-NEXT: clz z1.h, p0/m, z1.h143; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]144; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]145; VBITS_GE_256-NEXT: ret146;147; VBITS_GE_512-LABEL: ctlz_v32i16:148; VBITS_GE_512: // %bb.0:149; VBITS_GE_512-NEXT: ptrue p0.h, vl32150; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]151; VBITS_GE_512-NEXT: clz z0.h, p0/m, z0.h152; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]153; VBITS_GE_512-NEXT: ret154 %op = load <32 x i16>, ptr %a155 %res = call <32 x i16> @llvm.ctlz.v32i16(<32 x i16> %op)156 store <32 x i16> %res, ptr %a157 ret void158}159 160define void @ctlz_v64i16(ptr %a) vscale_range(8,0) #0 {161; CHECK-LABEL: ctlz_v64i16:162; CHECK: // %bb.0:163; CHECK-NEXT: ptrue p0.h, vl64164; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]165; CHECK-NEXT: clz z0.h, p0/m, z0.h166; CHECK-NEXT: st1h { z0.h }, p0, [x0]167; CHECK-NEXT: ret168 %op = load <64 x i16>, ptr %a169 %res = call <64 x i16> @llvm.ctlz.v64i16(<64 x i16> %op)170 store <64 x i16> %res, ptr %a171 ret void172}173 174define void @ctlz_v128i16(ptr %a) vscale_range(16,0) #0 {175; CHECK-LABEL: ctlz_v128i16:176; CHECK: // %bb.0:177; CHECK-NEXT: ptrue p0.h, vl128178; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]179; CHECK-NEXT: clz z0.h, p0/m, z0.h180; CHECK-NEXT: st1h { z0.h }, p0, [x0]181; CHECK-NEXT: ret182 %op = load <128 x i16>, ptr %a183 %res = call <128 x i16> @llvm.ctlz.v128i16(<128 x i16> %op)184 store <128 x i16> %res, ptr %a185 ret void186}187 188; Don't use SVE for 64-bit vectors.189define <2 x i32> @ctlz_v2i32(<2 x i32> %op) vscale_range(2,0) #0 {190; CHECK-LABEL: ctlz_v2i32:191; CHECK: // %bb.0:192; CHECK-NEXT: clz v0.2s, v0.2s193; CHECK-NEXT: ret194 %res = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %op)195 ret <2 x i32> %res196}197 198; Don't use SVE for 128-bit vectors.199define <4 x i32> @ctlz_v4i32(<4 x i32> %op) vscale_range(2,0) #0 {200; CHECK-LABEL: ctlz_v4i32:201; CHECK: // %bb.0:202; CHECK-NEXT: clz v0.4s, v0.4s203; CHECK-NEXT: ret204 %res = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %op)205 ret <4 x i32> %res206}207 208define void @ctlz_v8i32(ptr %a) vscale_range(2,0) #0 {209; CHECK-LABEL: ctlz_v8i32:210; CHECK: // %bb.0:211; CHECK-NEXT: ptrue p0.s, vl8212; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]213; CHECK-NEXT: clz z0.s, p0/m, z0.s214; CHECK-NEXT: st1w { z0.s }, p0, [x0]215; CHECK-NEXT: ret216 %op = load <8 x i32>, ptr %a217 %res = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %op)218 store <8 x i32> %res, ptr %a219 ret void220}221 222define void @ctlz_v16i32(ptr %a) #0 {223; VBITS_GE_256-LABEL: ctlz_v16i32:224; VBITS_GE_256: // %bb.0:225; VBITS_GE_256-NEXT: ptrue p0.s, vl8226; VBITS_GE_256-NEXT: mov x8, #8 // =0x8227; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]228; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]229; VBITS_GE_256-NEXT: clz z0.s, p0/m, z0.s230; VBITS_GE_256-NEXT: clz z1.s, p0/m, z1.s231; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]232; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]233; VBITS_GE_256-NEXT: ret234;235; VBITS_GE_512-LABEL: ctlz_v16i32:236; VBITS_GE_512: // %bb.0:237; VBITS_GE_512-NEXT: ptrue p0.s, vl16238; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]239; VBITS_GE_512-NEXT: clz z0.s, p0/m, z0.s240; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]241; VBITS_GE_512-NEXT: ret242 %op = load <16 x i32>, ptr %a243 %res = call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> %op)244 store <16 x i32> %res, ptr %a245 ret void246}247 248define void @ctlz_v32i32(ptr %a) vscale_range(8,0) #0 {249; CHECK-LABEL: ctlz_v32i32:250; CHECK: // %bb.0:251; CHECK-NEXT: ptrue p0.s, vl32252; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]253; CHECK-NEXT: clz z0.s, p0/m, z0.s254; CHECK-NEXT: st1w { z0.s }, p0, [x0]255; CHECK-NEXT: ret256 %op = load <32 x i32>, ptr %a257 %res = call <32 x i32> @llvm.ctlz.v32i32(<32 x i32> %op)258 store <32 x i32> %res, ptr %a259 ret void260}261 262define void @ctlz_v64i32(ptr %a) vscale_range(16,0) #0 {263; CHECK-LABEL: ctlz_v64i32:264; CHECK: // %bb.0:265; CHECK-NEXT: ptrue p0.s, vl64266; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]267; CHECK-NEXT: clz z0.s, p0/m, z0.s268; CHECK-NEXT: st1w { z0.s }, p0, [x0]269; CHECK-NEXT: ret270 %op = load <64 x i32>, ptr %a271 %res = call <64 x i32> @llvm.ctlz.v64i32(<64 x i32> %op)272 store <64 x i32> %res, ptr %a273 ret void274}275 276define <1 x i64> @ctlz_v1i64(<1 x i64> %op) vscale_range(2,0) #0 {277; CHECK-LABEL: ctlz_v1i64:278; CHECK: // %bb.0:279; CHECK-NEXT: ptrue p0.d, vl1280; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0281; CHECK-NEXT: clz z0.d, p0/m, z0.d282; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0283; CHECK-NEXT: ret284 %res = call <1 x i64> @llvm.ctlz.v1i64(<1 x i64> %op)285 ret <1 x i64> %res286}287 288define <2 x i64> @ctlz_v2i64(<2 x i64> %op) vscale_range(2,0) #0 {289; CHECK-LABEL: ctlz_v2i64:290; CHECK: // %bb.0:291; CHECK-NEXT: ptrue p0.d, vl2292; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0293; CHECK-NEXT: clz z0.d, p0/m, z0.d294; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0295; CHECK-NEXT: ret296 %res = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %op)297 ret <2 x i64> %res298}299 300define void @ctlz_v4i64(ptr %a) vscale_range(2,0) #0 {301; CHECK-LABEL: ctlz_v4i64:302; CHECK: // %bb.0:303; CHECK-NEXT: ptrue p0.d, vl4304; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]305; CHECK-NEXT: clz z0.d, p0/m, z0.d306; CHECK-NEXT: st1d { z0.d }, p0, [x0]307; CHECK-NEXT: ret308 %op = load <4 x i64>, ptr %a309 %res = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %op)310 store <4 x i64> %res, ptr %a311 ret void312}313 314define void @ctlz_v8i64(ptr %a) #0 {315; VBITS_GE_256-LABEL: ctlz_v8i64:316; VBITS_GE_256: // %bb.0:317; VBITS_GE_256-NEXT: ptrue p0.d, vl4318; VBITS_GE_256-NEXT: mov x8, #4 // =0x4319; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]320; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]321; VBITS_GE_256-NEXT: clz z0.d, p0/m, z0.d322; VBITS_GE_256-NEXT: clz z1.d, p0/m, z1.d323; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]324; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]325; VBITS_GE_256-NEXT: ret326;327; VBITS_GE_512-LABEL: ctlz_v8i64:328; VBITS_GE_512: // %bb.0:329; VBITS_GE_512-NEXT: ptrue p0.d, vl8330; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]331; VBITS_GE_512-NEXT: clz z0.d, p0/m, z0.d332; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]333; VBITS_GE_512-NEXT: ret334 %op = load <8 x i64>, ptr %a335 %res = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %op)336 store <8 x i64> %res, ptr %a337 ret void338}339 340define void @ctlz_v16i64(ptr %a) vscale_range(8,0) #0 {341; CHECK-LABEL: ctlz_v16i64:342; CHECK: // %bb.0:343; CHECK-NEXT: ptrue p0.d, vl16344; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]345; CHECK-NEXT: clz z0.d, p0/m, z0.d346; CHECK-NEXT: st1d { z0.d }, p0, [x0]347; CHECK-NEXT: ret348 %op = load <16 x i64>, ptr %a349 %res = call <16 x i64> @llvm.ctlz.v16i64(<16 x i64> %op)350 store <16 x i64> %res, ptr %a351 ret void352}353 354define void @ctlz_v32i64(ptr %a) vscale_range(16,0) #0 {355; CHECK-LABEL: ctlz_v32i64:356; CHECK: // %bb.0:357; CHECK-NEXT: ptrue p0.d, vl32358; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]359; CHECK-NEXT: clz z0.d, p0/m, z0.d360; CHECK-NEXT: st1d { z0.d }, p0, [x0]361; CHECK-NEXT: ret362 %op = load <32 x i64>, ptr %a363 %res = call <32 x i64> @llvm.ctlz.v32i64(<32 x i64> %op)364 store <32 x i64> %res, ptr %a365 ret void366}367 368;369; CNT370;371 372; Don't use SVE for 64-bit vectors.373define <8 x i8> @ctpop_v8i8(<8 x i8> %op) vscale_range(2,0) #0 {374; CHECK-LABEL: ctpop_v8i8:375; CHECK: // %bb.0:376; CHECK-NEXT: cnt v0.8b, v0.8b377; CHECK-NEXT: ret378 %res = call <8 x i8> @llvm.ctpop.v8i8(<8 x i8> %op)379 ret <8 x i8> %res380}381 382; Don't use SVE for 128-bit vectors.383define <16 x i8> @ctpop_v16i8(<16 x i8> %op) vscale_range(2,0) #0 {384; CHECK-LABEL: ctpop_v16i8:385; CHECK: // %bb.0:386; CHECK-NEXT: cnt v0.16b, v0.16b387; CHECK-NEXT: ret388 %res = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %op)389 ret <16 x i8> %res390}391 392define void @ctpop_v32i8(ptr %a) vscale_range(2,0) #0 {393; CHECK-LABEL: ctpop_v32i8:394; CHECK: // %bb.0:395; CHECK-NEXT: ptrue p0.b, vl32396; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]397; CHECK-NEXT: cnt z0.b, p0/m, z0.b398; CHECK-NEXT: st1b { z0.b }, p0, [x0]399; CHECK-NEXT: ret400 %op = load <32 x i8>, ptr %a401 %res = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> %op)402 store <32 x i8> %res, ptr %a403 ret void404}405 406define void @ctpop_v64i8(ptr %a) #0 {407; VBITS_GE_256-LABEL: ctpop_v64i8:408; VBITS_GE_256: // %bb.0:409; VBITS_GE_256-NEXT: ptrue p0.b, vl32410; VBITS_GE_256-NEXT: mov w8, #32 // =0x20411; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]412; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]413; VBITS_GE_256-NEXT: cnt z0.b, p0/m, z0.b414; VBITS_GE_256-NEXT: cnt z1.b, p0/m, z1.b415; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]416; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0]417; VBITS_GE_256-NEXT: ret418;419; VBITS_GE_512-LABEL: ctpop_v64i8:420; VBITS_GE_512: // %bb.0:421; VBITS_GE_512-NEXT: ptrue p0.b, vl64422; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]423; VBITS_GE_512-NEXT: cnt z0.b, p0/m, z0.b424; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]425; VBITS_GE_512-NEXT: ret426 %op = load <64 x i8>, ptr %a427 %res = call <64 x i8> @llvm.ctpop.v64i8(<64 x i8> %op)428 store <64 x i8> %res, ptr %a429 ret void430}431 432define void @ctpop_v128i8(ptr %a) vscale_range(8,0) #0 {433; CHECK-LABEL: ctpop_v128i8:434; CHECK: // %bb.0:435; CHECK-NEXT: ptrue p0.b, vl128436; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]437; CHECK-NEXT: cnt z0.b, p0/m, z0.b438; CHECK-NEXT: st1b { z0.b }, p0, [x0]439; CHECK-NEXT: ret440 %op = load <128 x i8>, ptr %a441 %res = call <128 x i8> @llvm.ctpop.v128i8(<128 x i8> %op)442 store <128 x i8> %res, ptr %a443 ret void444}445 446define void @ctpop_v256i8(ptr %a) vscale_range(16,0) #0 {447; CHECK-LABEL: ctpop_v256i8:448; CHECK: // %bb.0:449; CHECK-NEXT: ptrue p0.b, vl256450; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]451; CHECK-NEXT: cnt z0.b, p0/m, z0.b452; CHECK-NEXT: st1b { z0.b }, p0, [x0]453; CHECK-NEXT: ret454 %op = load <256 x i8>, ptr %a455 %res = call <256 x i8> @llvm.ctpop.v256i8(<256 x i8> %op)456 store <256 x i8> %res, ptr %a457 ret void458}459 460define <4 x i16> @ctpop_v4i16(<4 x i16> %op) vscale_range(2,0) #0 {461; CHECK-LABEL: ctpop_v4i16:462; CHECK: // %bb.0:463; CHECK-NEXT: ptrue p0.h, vl4464; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0465; CHECK-NEXT: cnt z0.h, p0/m, z0.h466; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0467; CHECK-NEXT: ret468 %res = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %op)469 ret <4 x i16> %res470}471 472define <8 x i16> @ctpop_v8i16(<8 x i16> %op) vscale_range(2,0) #0 {473; CHECK-LABEL: ctpop_v8i16:474; CHECK: // %bb.0:475; CHECK-NEXT: ptrue p0.h, vl8476; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0477; CHECK-NEXT: cnt z0.h, p0/m, z0.h478; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0479; CHECK-NEXT: ret480 %res = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %op)481 ret <8 x i16> %res482}483 484define void @ctpop_v16i16(ptr %a) vscale_range(2,0) #0 {485; CHECK-LABEL: ctpop_v16i16:486; CHECK: // %bb.0:487; CHECK-NEXT: ptrue p0.h, vl16488; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]489; CHECK-NEXT: cnt z0.h, p0/m, z0.h490; CHECK-NEXT: st1h { z0.h }, p0, [x0]491; CHECK-NEXT: ret492 %op = load <16 x i16>, ptr %a493 %res = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> %op)494 store <16 x i16> %res, ptr %a495 ret void496}497 498define void @ctpop_v32i16(ptr %a) #0 {499; VBITS_GE_256-LABEL: ctpop_v32i16:500; VBITS_GE_256: // %bb.0:501; VBITS_GE_256-NEXT: ptrue p0.h, vl16502; VBITS_GE_256-NEXT: mov x8, #16 // =0x10503; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]504; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]505; VBITS_GE_256-NEXT: cnt z0.h, p0/m, z0.h506; VBITS_GE_256-NEXT: cnt z1.h, p0/m, z1.h507; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]508; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]509; VBITS_GE_256-NEXT: ret510;511; VBITS_GE_512-LABEL: ctpop_v32i16:512; VBITS_GE_512: // %bb.0:513; VBITS_GE_512-NEXT: ptrue p0.h, vl32514; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]515; VBITS_GE_512-NEXT: cnt z0.h, p0/m, z0.h516; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]517; VBITS_GE_512-NEXT: ret518 %op = load <32 x i16>, ptr %a519 %res = call <32 x i16> @llvm.ctpop.v32i16(<32 x i16> %op)520 store <32 x i16> %res, ptr %a521 ret void522}523 524define void @ctpop_v64i16(ptr %a) vscale_range(8,0) #0 {525; CHECK-LABEL: ctpop_v64i16:526; CHECK: // %bb.0:527; CHECK-NEXT: ptrue p0.h, vl64528; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]529; CHECK-NEXT: cnt z0.h, p0/m, z0.h530; CHECK-NEXT: st1h { z0.h }, p0, [x0]531; CHECK-NEXT: ret532 %op = load <64 x i16>, ptr %a533 %res = call <64 x i16> @llvm.ctpop.v64i16(<64 x i16> %op)534 store <64 x i16> %res, ptr %a535 ret void536}537 538define void @ctpop_v128i16(ptr %a) vscale_range(16,0) #0 {539; CHECK-LABEL: ctpop_v128i16:540; CHECK: // %bb.0:541; CHECK-NEXT: ptrue p0.h, vl128542; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]543; CHECK-NEXT: cnt z0.h, p0/m, z0.h544; CHECK-NEXT: st1h { z0.h }, p0, [x0]545; CHECK-NEXT: ret546 %op = load <128 x i16>, ptr %a547 %res = call <128 x i16> @llvm.ctpop.v128i16(<128 x i16> %op)548 store <128 x i16> %res, ptr %a549 ret void550}551 552define <2 x i32> @ctpop_v2i32(<2 x i32> %op) vscale_range(2,0) #0 {553; CHECK-LABEL: ctpop_v2i32:554; CHECK: // %bb.0:555; CHECK-NEXT: ptrue p0.s, vl2556; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0557; CHECK-NEXT: cnt z0.s, p0/m, z0.s558; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0559; CHECK-NEXT: ret560 %res = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %op)561 ret <2 x i32> %res562}563 564define <4 x i32> @ctpop_v4i32(<4 x i32> %op) vscale_range(2,0) #0 {565; CHECK-LABEL: ctpop_v4i32:566; CHECK: // %bb.0:567; CHECK-NEXT: ptrue p0.s, vl4568; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0569; CHECK-NEXT: cnt z0.s, p0/m, z0.s570; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0571; CHECK-NEXT: ret572 %res = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %op)573 ret <4 x i32> %res574}575 576define void @ctpop_v8i32(ptr %a) vscale_range(2,0) #0 {577; CHECK-LABEL: ctpop_v8i32:578; CHECK: // %bb.0:579; CHECK-NEXT: ptrue p0.s, vl8580; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]581; CHECK-NEXT: cnt z0.s, p0/m, z0.s582; CHECK-NEXT: st1w { z0.s }, p0, [x0]583; CHECK-NEXT: ret584 %op = load <8 x i32>, ptr %a585 %res = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %op)586 store <8 x i32> %res, ptr %a587 ret void588}589 590define void @ctpop_v16i32(ptr %a) #0 {591; VBITS_GE_256-LABEL: ctpop_v16i32:592; VBITS_GE_256: // %bb.0:593; VBITS_GE_256-NEXT: ptrue p0.s, vl8594; VBITS_GE_256-NEXT: mov x8, #8 // =0x8595; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]596; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]597; VBITS_GE_256-NEXT: cnt z0.s, p0/m, z0.s598; VBITS_GE_256-NEXT: cnt z1.s, p0/m, z1.s599; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]600; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]601; VBITS_GE_256-NEXT: ret602;603; VBITS_GE_512-LABEL: ctpop_v16i32:604; VBITS_GE_512: // %bb.0:605; VBITS_GE_512-NEXT: ptrue p0.s, vl16606; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]607; VBITS_GE_512-NEXT: cnt z0.s, p0/m, z0.s608; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]609; VBITS_GE_512-NEXT: ret610 %op = load <16 x i32>, ptr %a611 %res = call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> %op)612 store <16 x i32> %res, ptr %a613 ret void614}615 616define void @ctpop_v32i32(ptr %a) vscale_range(8,0) #0 {617; CHECK-LABEL: ctpop_v32i32:618; CHECK: // %bb.0:619; CHECK-NEXT: ptrue p0.s, vl32620; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]621; CHECK-NEXT: cnt z0.s, p0/m, z0.s622; CHECK-NEXT: st1w { z0.s }, p0, [x0]623; CHECK-NEXT: ret624 %op = load <32 x i32>, ptr %a625 %res = call <32 x i32> @llvm.ctpop.v32i32(<32 x i32> %op)626 store <32 x i32> %res, ptr %a627 ret void628}629 630define void @ctpop_v64i32(ptr %a) vscale_range(16,0) #0 {631; CHECK-LABEL: ctpop_v64i32:632; CHECK: // %bb.0:633; CHECK-NEXT: ptrue p0.s, vl64634; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]635; CHECK-NEXT: cnt z0.s, p0/m, z0.s636; CHECK-NEXT: st1w { z0.s }, p0, [x0]637; CHECK-NEXT: ret638 %op = load <64 x i32>, ptr %a639 %res = call <64 x i32> @llvm.ctpop.v64i32(<64 x i32> %op)640 store <64 x i32> %res, ptr %a641 ret void642}643 644define <1 x i64> @ctpop_v1i64(<1 x i64> %op) vscale_range(2,0) #0 {645; CHECK-LABEL: ctpop_v1i64:646; CHECK: // %bb.0:647; CHECK-NEXT: ptrue p0.d, vl1648; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0649; CHECK-NEXT: cnt z0.d, p0/m, z0.d650; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0651; CHECK-NEXT: ret652 %res = call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %op)653 ret <1 x i64> %res654}655 656define <2 x i64> @ctpop_v2i64(<2 x i64> %op) vscale_range(2,0) #0 {657; CHECK-LABEL: ctpop_v2i64:658; CHECK: // %bb.0:659; CHECK-NEXT: ptrue p0.d, vl2660; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0661; CHECK-NEXT: cnt z0.d, p0/m, z0.d662; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0663; CHECK-NEXT: ret664 %res = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %op)665 ret <2 x i64> %res666}667 668define void @ctpop_v4i64(ptr %a) vscale_range(2,0) #0 {669; CHECK-LABEL: ctpop_v4i64:670; CHECK: // %bb.0:671; CHECK-NEXT: ptrue p0.d, vl4672; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]673; CHECK-NEXT: cnt z0.d, p0/m, z0.d674; CHECK-NEXT: st1d { z0.d }, p0, [x0]675; CHECK-NEXT: ret676 %op = load <4 x i64>, ptr %a677 %res = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %op)678 store <4 x i64> %res, ptr %a679 ret void680}681 682define void @ctpop_v8i64(ptr %a) #0 {683; VBITS_GE_256-LABEL: ctpop_v8i64:684; VBITS_GE_256: // %bb.0:685; VBITS_GE_256-NEXT: ptrue p0.d, vl4686; VBITS_GE_256-NEXT: mov x8, #4 // =0x4687; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]688; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]689; VBITS_GE_256-NEXT: cnt z0.d, p0/m, z0.d690; VBITS_GE_256-NEXT: cnt z1.d, p0/m, z1.d691; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]692; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]693; VBITS_GE_256-NEXT: ret694;695; VBITS_GE_512-LABEL: ctpop_v8i64:696; VBITS_GE_512: // %bb.0:697; VBITS_GE_512-NEXT: ptrue p0.d, vl8698; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]699; VBITS_GE_512-NEXT: cnt z0.d, p0/m, z0.d700; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]701; VBITS_GE_512-NEXT: ret702 %op = load <8 x i64>, ptr %a703 %res = call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> %op)704 store <8 x i64> %res, ptr %a705 ret void706}707 708define void @ctpop_v16i64(ptr %a) vscale_range(8,0) #0 {709; CHECK-LABEL: ctpop_v16i64:710; CHECK: // %bb.0:711; CHECK-NEXT: ptrue p0.d, vl16712; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]713; CHECK-NEXT: cnt z0.d, p0/m, z0.d714; CHECK-NEXT: st1d { z0.d }, p0, [x0]715; CHECK-NEXT: ret716 %op = load <16 x i64>, ptr %a717 %res = call <16 x i64> @llvm.ctpop.v16i64(<16 x i64> %op)718 store <16 x i64> %res, ptr %a719 ret void720}721 722define void @ctpop_v32i64(ptr %a) vscale_range(16,0) #0 {723; CHECK-LABEL: ctpop_v32i64:724; CHECK: // %bb.0:725; CHECK-NEXT: ptrue p0.d, vl32726; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]727; CHECK-NEXT: cnt z0.d, p0/m, z0.d728; CHECK-NEXT: st1d { z0.d }, p0, [x0]729; CHECK-NEXT: ret730 %op = load <32 x i64>, ptr %a731 %res = call <32 x i64> @llvm.ctpop.v32i64(<32 x i64> %op)732 store <32 x i64> %res, ptr %a733 ret void734}735 736;737; Count trailing zeros738;739 740define <8 x i8> @cttz_v8i8(<8 x i8> %op) vscale_range(2,0) #0 {741; CHECK-LABEL: cttz_v8i8:742; CHECK: // %bb.0:743; CHECK-NEXT: ptrue p0.b, vl8744; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0745; CHECK-NEXT: rbit z0.b, p0/m, z0.b746; CHECK-NEXT: clz v0.8b, v0.8b747; CHECK-NEXT: ret748 %res = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %op)749 ret <8 x i8> %res750}751 752define <16 x i8> @cttz_v16i8(<16 x i8> %op) vscale_range(2,0) #0 {753; CHECK-LABEL: cttz_v16i8:754; CHECK: // %bb.0:755; CHECK-NEXT: ptrue p0.b, vl16756; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0757; CHECK-NEXT: rbit z0.b, p0/m, z0.b758; CHECK-NEXT: clz v0.16b, v0.16b759; CHECK-NEXT: ret760 %res = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %op)761 ret <16 x i8> %res762}763 764define void @cttz_v32i8(ptr %a) vscale_range(2,0) #0 {765; CHECK-LABEL: cttz_v32i8:766; CHECK: // %bb.0:767; CHECK-NEXT: ptrue p0.b, vl32768; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]769; CHECK-NEXT: rbit z0.b, p0/m, z0.b770; CHECK-NEXT: clz z0.b, p0/m, z0.b771; CHECK-NEXT: st1b { z0.b }, p0, [x0]772; CHECK-NEXT: ret773 %op = load <32 x i8>, ptr %a774 %res = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %op)775 store <32 x i8> %res, ptr %a776 ret void777}778 779define void @cttz_v64i8(ptr %a) #0 {780; VBITS_GE_256-LABEL: cttz_v64i8:781; VBITS_GE_256: // %bb.0:782; VBITS_GE_256-NEXT: ptrue p0.b, vl32783; VBITS_GE_256-NEXT: mov w8, #32 // =0x20784; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]785; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]786; VBITS_GE_256-NEXT: rbit z0.b, p0/m, z0.b787; VBITS_GE_256-NEXT: rbit z1.b, p0/m, z1.b788; VBITS_GE_256-NEXT: clz z0.b, p0/m, z0.b789; VBITS_GE_256-NEXT: clz z1.b, p0/m, z1.b790; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]791; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0]792; VBITS_GE_256-NEXT: ret793;794; VBITS_GE_512-LABEL: cttz_v64i8:795; VBITS_GE_512: // %bb.0:796; VBITS_GE_512-NEXT: ptrue p0.b, vl64797; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]798; VBITS_GE_512-NEXT: rbit z0.b, p0/m, z0.b799; VBITS_GE_512-NEXT: clz z0.b, p0/m, z0.b800; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]801; VBITS_GE_512-NEXT: ret802 %op = load <64 x i8>, ptr %a803 %res = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %op)804 store <64 x i8> %res, ptr %a805 ret void806}807 808define void @cttz_v128i8(ptr %a) vscale_range(8,0) #0 {809; CHECK-LABEL: cttz_v128i8:810; CHECK: // %bb.0:811; CHECK-NEXT: ptrue p0.b, vl128812; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]813; CHECK-NEXT: rbit z0.b, p0/m, z0.b814; CHECK-NEXT: clz z0.b, p0/m, z0.b815; CHECK-NEXT: st1b { z0.b }, p0, [x0]816; CHECK-NEXT: ret817 %op = load <128 x i8>, ptr %a818 %res = call <128 x i8> @llvm.cttz.v128i8(<128 x i8> %op)819 store <128 x i8> %res, ptr %a820 ret void821}822 823define void @cttz_v256i8(ptr %a) vscale_range(16,0) #0 {824; CHECK-LABEL: cttz_v256i8:825; CHECK: // %bb.0:826; CHECK-NEXT: ptrue p0.b, vl256827; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]828; CHECK-NEXT: rbit z0.b, p0/m, z0.b829; CHECK-NEXT: clz z0.b, p0/m, z0.b830; CHECK-NEXT: st1b { z0.b }, p0, [x0]831; CHECK-NEXT: ret832 %op = load <256 x i8>, ptr %a833 %res = call <256 x i8> @llvm.cttz.v256i8(<256 x i8> %op)834 store <256 x i8> %res, ptr %a835 ret void836}837 838define <4 x i16> @cttz_v4i16(<4 x i16> %op) vscale_range(2,0) #0 {839; CHECK-LABEL: cttz_v4i16:840; CHECK: // %bb.0:841; CHECK-NEXT: ptrue p0.h, vl4842; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0843; CHECK-NEXT: rbit z0.h, p0/m, z0.h844; CHECK-NEXT: clz v0.4h, v0.4h845; CHECK-NEXT: ret846 %res = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %op)847 ret <4 x i16> %res848}849 850define <8 x i16> @cttz_v8i16(<8 x i16> %op) vscale_range(2,0) #0 {851; CHECK-LABEL: cttz_v8i16:852; CHECK: // %bb.0:853; CHECK-NEXT: ptrue p0.h, vl8854; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0855; CHECK-NEXT: rbit z0.h, p0/m, z0.h856; CHECK-NEXT: clz v0.8h, v0.8h857; CHECK-NEXT: ret858 %res = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %op)859 ret <8 x i16> %res860}861 862define void @cttz_v16i16(ptr %a) vscale_range(2,0) #0 {863; CHECK-LABEL: cttz_v16i16:864; CHECK: // %bb.0:865; CHECK-NEXT: ptrue p0.h, vl16866; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]867; CHECK-NEXT: rbit z0.h, p0/m, z0.h868; CHECK-NEXT: clz z0.h, p0/m, z0.h869; CHECK-NEXT: st1h { z0.h }, p0, [x0]870; CHECK-NEXT: ret871 %op = load <16 x i16>, ptr %a872 %res = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %op)873 store <16 x i16> %res, ptr %a874 ret void875}876 877define void @cttz_v32i16(ptr %a) #0 {878; VBITS_GE_256-LABEL: cttz_v32i16:879; VBITS_GE_256: // %bb.0:880; VBITS_GE_256-NEXT: ptrue p0.h, vl16881; VBITS_GE_256-NEXT: mov x8, #16 // =0x10882; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]883; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]884; VBITS_GE_256-NEXT: rbit z0.h, p0/m, z0.h885; VBITS_GE_256-NEXT: rbit z1.h, p0/m, z1.h886; VBITS_GE_256-NEXT: clz z0.h, p0/m, z0.h887; VBITS_GE_256-NEXT: clz z1.h, p0/m, z1.h888; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]889; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]890; VBITS_GE_256-NEXT: ret891;892; VBITS_GE_512-LABEL: cttz_v32i16:893; VBITS_GE_512: // %bb.0:894; VBITS_GE_512-NEXT: ptrue p0.h, vl32895; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]896; VBITS_GE_512-NEXT: rbit z0.h, p0/m, z0.h897; VBITS_GE_512-NEXT: clz z0.h, p0/m, z0.h898; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]899; VBITS_GE_512-NEXT: ret900 %op = load <32 x i16>, ptr %a901 %res = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %op)902 store <32 x i16> %res, ptr %a903 ret void904}905 906define void @cttz_v64i16(ptr %a) vscale_range(8,0) #0 {907; CHECK-LABEL: cttz_v64i16:908; CHECK: // %bb.0:909; CHECK-NEXT: ptrue p0.h, vl64910; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]911; CHECK-NEXT: rbit z0.h, p0/m, z0.h912; CHECK-NEXT: clz z0.h, p0/m, z0.h913; CHECK-NEXT: st1h { z0.h }, p0, [x0]914; CHECK-NEXT: ret915 %op = load <64 x i16>, ptr %a916 %res = call <64 x i16> @llvm.cttz.v64i16(<64 x i16> %op)917 store <64 x i16> %res, ptr %a918 ret void919}920 921define void @cttz_v128i16(ptr %a) vscale_range(16,0) #0 {922; CHECK-LABEL: cttz_v128i16:923; CHECK: // %bb.0:924; CHECK-NEXT: ptrue p0.h, vl128925; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]926; CHECK-NEXT: rbit z0.h, p0/m, z0.h927; CHECK-NEXT: clz z0.h, p0/m, z0.h928; CHECK-NEXT: st1h { z0.h }, p0, [x0]929; CHECK-NEXT: ret930 %op = load <128 x i16>, ptr %a931 %res = call <128 x i16> @llvm.cttz.v128i16(<128 x i16> %op)932 store <128 x i16> %res, ptr %a933 ret void934}935 936; Don't use SVE for 64-bit vectors.937define <2 x i32> @cttz_v2i32(<2 x i32> %op) vscale_range(2,0) #0 {938; CHECK-LABEL: cttz_v2i32:939; CHECK: // %bb.0:940; CHECK-NEXT: ptrue p0.s, vl2941; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0942; CHECK-NEXT: rbit z0.s, p0/m, z0.s943; CHECK-NEXT: clz v0.2s, v0.2s944; CHECK-NEXT: ret945 %res = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %op)946 ret <2 x i32> %res947}948 949; Don't use SVE for 128-bit vectors.950define <4 x i32> @cttz_v4i32(<4 x i32> %op) vscale_range(2,0) #0 {951; CHECK-LABEL: cttz_v4i32:952; CHECK: // %bb.0:953; CHECK-NEXT: ptrue p0.s, vl4954; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0955; CHECK-NEXT: rbit z0.s, p0/m, z0.s956; CHECK-NEXT: clz v0.4s, v0.4s957; CHECK-NEXT: ret958 %res = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %op)959 ret <4 x i32> %res960}961 962define void @cttz_v8i32(ptr %a) vscale_range(2,0) #0 {963; CHECK-LABEL: cttz_v8i32:964; CHECK: // %bb.0:965; CHECK-NEXT: ptrue p0.s, vl8966; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]967; CHECK-NEXT: rbit z0.s, p0/m, z0.s968; CHECK-NEXT: clz z0.s, p0/m, z0.s969; CHECK-NEXT: st1w { z0.s }, p0, [x0]970; CHECK-NEXT: ret971 %op = load <8 x i32>, ptr %a972 %res = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %op)973 store <8 x i32> %res, ptr %a974 ret void975}976 977define void @cttz_v16i32(ptr %a) #0 {978; VBITS_GE_256-LABEL: cttz_v16i32:979; VBITS_GE_256: // %bb.0:980; VBITS_GE_256-NEXT: ptrue p0.s, vl8981; VBITS_GE_256-NEXT: mov x8, #8 // =0x8982; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]983; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]984; VBITS_GE_256-NEXT: rbit z0.s, p0/m, z0.s985; VBITS_GE_256-NEXT: rbit z1.s, p0/m, z1.s986; VBITS_GE_256-NEXT: clz z0.s, p0/m, z0.s987; VBITS_GE_256-NEXT: clz z1.s, p0/m, z1.s988; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]989; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]990; VBITS_GE_256-NEXT: ret991;992; VBITS_GE_512-LABEL: cttz_v16i32:993; VBITS_GE_512: // %bb.0:994; VBITS_GE_512-NEXT: ptrue p0.s, vl16995; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]996; VBITS_GE_512-NEXT: rbit z0.s, p0/m, z0.s997; VBITS_GE_512-NEXT: clz z0.s, p0/m, z0.s998; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]999; VBITS_GE_512-NEXT: ret1000 %op = load <16 x i32>, ptr %a1001 %res = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %op)1002 store <16 x i32> %res, ptr %a1003 ret void1004}1005 1006define void @cttz_v32i32(ptr %a) vscale_range(8,0) #0 {1007; CHECK-LABEL: cttz_v32i32:1008; CHECK: // %bb.0:1009; CHECK-NEXT: ptrue p0.s, vl321010; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1011; CHECK-NEXT: rbit z0.s, p0/m, z0.s1012; CHECK-NEXT: clz z0.s, p0/m, z0.s1013; CHECK-NEXT: st1w { z0.s }, p0, [x0]1014; CHECK-NEXT: ret1015 %op = load <32 x i32>, ptr %a1016 %res = call <32 x i32> @llvm.cttz.v32i32(<32 x i32> %op)1017 store <32 x i32> %res, ptr %a1018 ret void1019}1020 1021define void @cttz_v64i32(ptr %a) vscale_range(16,0) #0 {1022; CHECK-LABEL: cttz_v64i32:1023; CHECK: // %bb.0:1024; CHECK-NEXT: ptrue p0.s, vl641025; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1026; CHECK-NEXT: rbit z0.s, p0/m, z0.s1027; CHECK-NEXT: clz z0.s, p0/m, z0.s1028; CHECK-NEXT: st1w { z0.s }, p0, [x0]1029; CHECK-NEXT: ret1030 %op = load <64 x i32>, ptr %a1031 %res = call <64 x i32> @llvm.cttz.v64i32(<64 x i32> %op)1032 store <64 x i32> %res, ptr %a1033 ret void1034}1035 1036define <1 x i64> @cttz_v1i64(<1 x i64> %op) vscale_range(2,0) #0 {1037; CHECK-LABEL: cttz_v1i64:1038; CHECK: // %bb.0:1039; CHECK-NEXT: ptrue p0.d, vl11040; CHECK-NEXT: // kill: def $d0 killed $d0 def $z01041; CHECK-NEXT: rbit z0.d, p0/m, z0.d1042; CHECK-NEXT: clz z0.d, p0/m, z0.d1043; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z01044; CHECK-NEXT: ret1045 %res = call <1 x i64> @llvm.cttz.v1i64(<1 x i64> %op)1046 ret <1 x i64> %res1047}1048 1049define <2 x i64> @cttz_v2i64(<2 x i64> %op) vscale_range(2,0) #0 {1050; CHECK-LABEL: cttz_v2i64:1051; CHECK: // %bb.0:1052; CHECK-NEXT: ptrue p0.d, vl21053; CHECK-NEXT: // kill: def $q0 killed $q0 def $z01054; CHECK-NEXT: rbit z0.d, p0/m, z0.d1055; CHECK-NEXT: clz z0.d, p0/m, z0.d1056; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z01057; CHECK-NEXT: ret1058 %res = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %op)1059 ret <2 x i64> %res1060}1061 1062define void @cttz_v4i64(ptr %a) vscale_range(2,0) #0 {1063; CHECK-LABEL: cttz_v4i64:1064; CHECK: // %bb.0:1065; CHECK-NEXT: ptrue p0.d, vl41066; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1067; CHECK-NEXT: rbit z0.d, p0/m, z0.d1068; CHECK-NEXT: clz z0.d, p0/m, z0.d1069; CHECK-NEXT: st1d { z0.d }, p0, [x0]1070; CHECK-NEXT: ret1071 %op = load <4 x i64>, ptr %a1072 %res = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %op)1073 store <4 x i64> %res, ptr %a1074 ret void1075}1076 1077define void @cttz_v8i64(ptr %a) #0 {1078; VBITS_GE_256-LABEL: cttz_v8i64:1079; VBITS_GE_256: // %bb.0:1080; VBITS_GE_256-NEXT: ptrue p0.d, vl41081; VBITS_GE_256-NEXT: mov x8, #4 // =0x41082; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]1083; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]1084; VBITS_GE_256-NEXT: rbit z0.d, p0/m, z0.d1085; VBITS_GE_256-NEXT: rbit z1.d, p0/m, z1.d1086; VBITS_GE_256-NEXT: clz z0.d, p0/m, z0.d1087; VBITS_GE_256-NEXT: clz z1.d, p0/m, z1.d1088; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]1089; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]1090; VBITS_GE_256-NEXT: ret1091;1092; VBITS_GE_512-LABEL: cttz_v8i64:1093; VBITS_GE_512: // %bb.0:1094; VBITS_GE_512-NEXT: ptrue p0.d, vl81095; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]1096; VBITS_GE_512-NEXT: rbit z0.d, p0/m, z0.d1097; VBITS_GE_512-NEXT: clz z0.d, p0/m, z0.d1098; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]1099; VBITS_GE_512-NEXT: ret1100 %op = load <8 x i64>, ptr %a1101 %res = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %op)1102 store <8 x i64> %res, ptr %a1103 ret void1104}1105 1106define void @cttz_v16i64(ptr %a) vscale_range(8,0) #0 {1107; CHECK-LABEL: cttz_v16i64:1108; CHECK: // %bb.0:1109; CHECK-NEXT: ptrue p0.d, vl161110; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1111; CHECK-NEXT: rbit z0.d, p0/m, z0.d1112; CHECK-NEXT: clz z0.d, p0/m, z0.d1113; CHECK-NEXT: st1d { z0.d }, p0, [x0]1114; CHECK-NEXT: ret1115 %op = load <16 x i64>, ptr %a1116 %res = call <16 x i64> @llvm.cttz.v16i64(<16 x i64> %op)1117 store <16 x i64> %res, ptr %a1118 ret void1119}1120 1121define void @cttz_v32i64(ptr %a) vscale_range(16,0) #0 {1122; CHECK-LABEL: cttz_v32i64:1123; CHECK: // %bb.0:1124; CHECK-NEXT: ptrue p0.d, vl321125; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1126; CHECK-NEXT: rbit z0.d, p0/m, z0.d1127; CHECK-NEXT: clz z0.d, p0/m, z0.d1128; CHECK-NEXT: st1d { z0.d }, p0, [x0]1129; CHECK-NEXT: ret1130 %op = load <32 x i64>, ptr %a1131 %res = call <32 x i64> @llvm.cttz.v32i64(<32 x i64> %op)1132 store <32 x i64> %res, ptr %a1133 ret void1134}1135 1136attributes #0 = { "target-features"="+sve" }1137 1138declare <8 x i8> @llvm.ctlz.v8i8(<8 x i8>)1139declare <16 x i8> @llvm.ctlz.v16i8(<16 x i8>)1140declare <32 x i8> @llvm.ctlz.v32i8(<32 x i8>)1141declare <64 x i8> @llvm.ctlz.v64i8(<64 x i8>)1142declare <128 x i8> @llvm.ctlz.v128i8(<128 x i8>)1143declare <256 x i8> @llvm.ctlz.v256i8(<256 x i8>)1144declare <4 x i16> @llvm.ctlz.v4i16(<4 x i16>)1145declare <8 x i16> @llvm.ctlz.v8i16(<8 x i16>)1146declare <16 x i16> @llvm.ctlz.v16i16(<16 x i16>)1147declare <32 x i16> @llvm.ctlz.v32i16(<32 x i16>)1148declare <64 x i16> @llvm.ctlz.v64i16(<64 x i16>)1149declare <128 x i16> @llvm.ctlz.v128i16(<128 x i16>)1150declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>)1151declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>)1152declare <8 x i32> @llvm.ctlz.v8i32(<8 x i32>)1153declare <16 x i32> @llvm.ctlz.v16i32(<16 x i32>)1154declare <32 x i32> @llvm.ctlz.v32i32(<32 x i32>)1155declare <64 x i32> @llvm.ctlz.v64i32(<64 x i32>)1156declare <1 x i64> @llvm.ctlz.v1i64(<1 x i64>)1157declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>)1158declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>)1159declare <8 x i64> @llvm.ctlz.v8i64(<8 x i64>)1160declare <16 x i64> @llvm.ctlz.v16i64(<16 x i64>)1161declare <32 x i64> @llvm.ctlz.v32i64(<32 x i64>)1162 1163declare <8 x i8> @llvm.ctpop.v8i8(<8 x i8>)1164declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8>)1165declare <32 x i8> @llvm.ctpop.v32i8(<32 x i8>)1166declare <64 x i8> @llvm.ctpop.v64i8(<64 x i8>)1167declare <128 x i8> @llvm.ctpop.v128i8(<128 x i8>)1168declare <256 x i8> @llvm.ctpop.v256i8(<256 x i8>)1169declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>)1170declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)1171declare <16 x i16> @llvm.ctpop.v16i16(<16 x i16>)1172declare <32 x i16> @llvm.ctpop.v32i16(<32 x i16>)1173declare <64 x i16> @llvm.ctpop.v64i16(<64 x i16>)1174declare <128 x i16> @llvm.ctpop.v128i16(<128 x i16>)1175declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)1176declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)1177declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>)1178declare <16 x i32> @llvm.ctpop.v16i32(<16 x i32>)1179declare <32 x i32> @llvm.ctpop.v32i32(<32 x i32>)1180declare <64 x i32> @llvm.ctpop.v64i32(<64 x i32>)1181declare <1 x i64> @llvm.ctpop.v1i64(<1 x i64>)1182declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)1183declare <4 x i64> @llvm.ctpop.v4i64(<4 x i64>)1184declare <8 x i64> @llvm.ctpop.v8i64(<8 x i64>)1185declare <16 x i64> @llvm.ctpop.v16i64(<16 x i64>)1186declare <32 x i64> @llvm.ctpop.v32i64(<32 x i64>)1187 1188declare <8 x i8> @llvm.cttz.v8i8(<8 x i8>)1189declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>)1190declare <32 x i8> @llvm.cttz.v32i8(<32 x i8>)1191declare <64 x i8> @llvm.cttz.v64i8(<64 x i8>)1192declare <128 x i8> @llvm.cttz.v128i8(<128 x i8>)1193declare <256 x i8> @llvm.cttz.v256i8(<256 x i8>)1194declare <4 x i16> @llvm.cttz.v4i16(<4 x i16>)1195declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>)1196declare <16 x i16> @llvm.cttz.v16i16(<16 x i16>)1197declare <32 x i16> @llvm.cttz.v32i16(<32 x i16>)1198declare <64 x i16> @llvm.cttz.v64i16(<64 x i16>)1199declare <128 x i16> @llvm.cttz.v128i16(<128 x i16>)1200declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>)1201declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>)1202declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>)1203declare <16 x i32> @llvm.cttz.v16i32(<16 x i32>)1204declare <32 x i32> @llvm.cttz.v32i32(<32 x i32>)1205declare <64 x i32> @llvm.cttz.v64i32(<64 x i32>)1206declare <1 x i64> @llvm.cttz.v1i64(<1 x i64>)1207declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>)1208declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>)1209declare <8 x i64> @llvm.cttz.v8i64(<8 x i64>)1210declare <16 x i64> @llvm.cttz.v16i64(<16 x i64>)1211declare <32 x i64> @llvm.cttz.v32i64(<32 x i64>)1212