907 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -O0 -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefix=CHECKO03; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefixes=CHECK,NEON4; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefixes=CHECK,DOT5; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefixes=CHECK,SVE6; RUN: llc < %s -mtriple=aarch64_be-unknown-unknown | FileCheck %s --check-prefixes=BE7; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefix=GISEL8; RUN: llc < %s -O0 -global-isel -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefix=GISELO09; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefixes=GISEL,NEON-GISEL10; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefixes=GISEL,DOT-GISEL11; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefixes=GISEL,SVE-GISEL12 13 14; Function Attrs: nobuiltin nounwind readonly15define i8 @popcount128(ptr nocapture nonnull readonly %0) {16; CHECKO0-LABEL: popcount128:17; CHECKO0: // %bb.0: // %Entry18; CHECKO0-NEXT: ldr q0, [x0]19; CHECKO0-NEXT: cnt v0.16b, v0.16b20; CHECKO0-NEXT: uaddlv h0, v0.16b21; CHECKO0-NEXT: // kill: def $q0 killed $h022; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q023; CHECKO0-NEXT: fmov w0, s024; CHECKO0-NEXT: ret25;26; NEON-LABEL: popcount128:27; NEON: // %bb.0: // %Entry28; NEON-NEXT: ldr d0, [x0]29; NEON-NEXT: add x8, x0, #830; NEON-NEXT: ld1 { v0.d }[1], [x8]31; NEON-NEXT: cnt v0.16b, v0.16b32; NEON-NEXT: addv b0, v0.16b33; NEON-NEXT: fmov w0, s034; NEON-NEXT: ret35;36; DOT-LABEL: popcount128:37; DOT: // %bb.0: // %Entry38; DOT-NEXT: ldr d0, [x0]39; DOT-NEXT: add x8, x0, #840; DOT-NEXT: ld1 { v0.d }[1], [x8]41; DOT-NEXT: cnt v0.16b, v0.16b42; DOT-NEXT: addv b0, v0.16b43; DOT-NEXT: fmov w0, s044; DOT-NEXT: ret45;46; SVE-LABEL: popcount128:47; SVE: // %bb.0: // %Entry48; SVE-NEXT: ldr d0, [x0]49; SVE-NEXT: add x8, x0, #850; SVE-NEXT: ptrue p0.d51; SVE-NEXT: ld1 { v0.d }[1], [x8]52; SVE-NEXT: cnt z0.d, p0/m, z0.d53; SVE-NEXT: addp d0, v0.2d54; SVE-NEXT: fmov w0, s055; SVE-NEXT: ret56;57; BE-LABEL: popcount128:58; BE: // %bb.0: // %Entry59; BE-NEXT: ldr d0, [x0]60; BE-NEXT: add x8, x0, #861; BE-NEXT: ld1 { v0.d }[1], [x8]62; BE-NEXT: rev64 v0.16b, v0.16b63; BE-NEXT: cnt v0.16b, v0.16b64; BE-NEXT: addv b0, v0.16b65; BE-NEXT: rev64 v0.4s, v0.4s66; BE-NEXT: mov w0, v0.s[1]67; BE-NEXT: ret68;69; GISEL-LABEL: popcount128:70; GISEL: // %bb.0: // %Entry71; GISEL-NEXT: ldr q0, [x0]72; GISEL-NEXT: cnt v0.16b, v0.16b73; GISEL-NEXT: uaddlv h0, v0.16b74; GISEL-NEXT: fmov w0, s075; GISEL-NEXT: ret76;77; GISELO0-LABEL: popcount128:78; GISELO0: // %bb.0: // %Entry79; GISELO0-NEXT: ldr q0, [x0]80; GISELO0-NEXT: cnt v0.16b, v0.16b81; GISELO0-NEXT: uaddlv h0, v0.16b82; GISELO0-NEXT: // kill: def $q0 killed $h083; GISELO0-NEXT: // kill: def $s0 killed $s0 killed $q084; GISELO0-NEXT: fmov w0, s085; GISELO0-NEXT: ret86Entry:87 %1 = load i128, ptr %0, align 1688 %2 = tail call i128 @llvm.ctpop.i128(i128 %1)89 %3 = trunc i128 %2 to i890 ret i8 %391}92 93; Function Attrs: nounwind readnone speculatable willreturn94declare i128 @llvm.ctpop.i128(i128)95 96; Function Attrs: nobuiltin nounwind readonly97define i16 @popcount256(ptr nocapture nonnull readonly %0) {98; CHECKO0-LABEL: popcount256:99; CHECKO0: // %bb.0: // %Entry100; CHECKO0-NEXT: ldr x11, [x0]101; CHECKO0-NEXT: ldr x10, [x0, #8]102; CHECKO0-NEXT: ldr x9, [x0, #16]103; CHECKO0-NEXT: ldr x8, [x0, #24]104; CHECKO0-NEXT: // implicit-def: $q1105; CHECKO0-NEXT: mov v1.d[0], x11106; CHECKO0-NEXT: mov v1.d[1], x10107; CHECKO0-NEXT: // implicit-def: $q0108; CHECKO0-NEXT: mov v0.d[0], x9109; CHECKO0-NEXT: mov v0.d[1], x8110; CHECKO0-NEXT: cnt v1.16b, v1.16b111; CHECKO0-NEXT: uaddlv h1, v1.16b112; CHECKO0-NEXT: // kill: def $q1 killed $h1113; CHECKO0-NEXT: // kill: def $s1 killed $s1 killed $q1114; CHECKO0-NEXT: fmov w0, s1115; CHECKO0-NEXT: mov w10, wzr116; CHECKO0-NEXT: mov w9, w0117; CHECKO0-NEXT: mov w8, w10118; CHECKO0-NEXT: bfi x9, x8, #32, #32119; CHECKO0-NEXT: cnt v0.16b, v0.16b120; CHECKO0-NEXT: uaddlv h0, v0.16b121; CHECKO0-NEXT: // kill: def $q0 killed $h0122; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0123; CHECKO0-NEXT: fmov w0, s0124; CHECKO0-NEXT: mov w8, w0125; CHECKO0-NEXT: // kill: def $x10 killed $w10126; CHECKO0-NEXT: bfi x8, x10, #32, #32127; CHECKO0-NEXT: adds x8, x8, x9128; CHECKO0-NEXT: mov w0, w8129; CHECKO0-NEXT: ret130;131; NEON-LABEL: popcount256:132; NEON: // %bb.0: // %Entry133; NEON-NEXT: ldr d0, [x0, #16]134; NEON-NEXT: ldr d1, [x0]135; NEON-NEXT: add x8, x0, #8136; NEON-NEXT: add x9, x0, #24137; NEON-NEXT: ld1 { v0.d }[1], [x9]138; NEON-NEXT: ld1 { v1.d }[1], [x8]139; NEON-NEXT: cnt v0.16b, v0.16b140; NEON-NEXT: cnt v1.16b, v1.16b141; NEON-NEXT: addv b0, v0.16b142; NEON-NEXT: addv b1, v1.16b143; NEON-NEXT: fmov w8, s0144; NEON-NEXT: fmov w9, s1145; NEON-NEXT: add w0, w9, w8146; NEON-NEXT: ret147;148; DOT-LABEL: popcount256:149; DOT: // %bb.0: // %Entry150; DOT-NEXT: ldr d0, [x0, #16]151; DOT-NEXT: ldr d1, [x0]152; DOT-NEXT: add x8, x0, #8153; DOT-NEXT: add x9, x0, #24154; DOT-NEXT: ld1 { v0.d }[1], [x9]155; DOT-NEXT: ld1 { v1.d }[1], [x8]156; DOT-NEXT: cnt v0.16b, v0.16b157; DOT-NEXT: cnt v1.16b, v1.16b158; DOT-NEXT: addv b0, v0.16b159; DOT-NEXT: addv b1, v1.16b160; DOT-NEXT: fmov w8, s0161; DOT-NEXT: fmov w9, s1162; DOT-NEXT: add w0, w9, w8163; DOT-NEXT: ret164;165; SVE-LABEL: popcount256:166; SVE: // %bb.0: // %Entry167; SVE-NEXT: ldr d0, [x0, #16]168; SVE-NEXT: ldr d1, [x0]169; SVE-NEXT: add x8, x0, #8170; SVE-NEXT: add x9, x0, #24171; SVE-NEXT: ptrue p0.d172; SVE-NEXT: ld1 { v0.d }[1], [x9]173; SVE-NEXT: ld1 { v1.d }[1], [x8]174; SVE-NEXT: cnt z0.d, p0/m, z0.d175; SVE-NEXT: cnt z1.d, p0/m, z1.d176; SVE-NEXT: add v0.2d, v1.2d, v0.2d177; SVE-NEXT: addp d0, v0.2d178; SVE-NEXT: fmov w0, s0179; SVE-NEXT: ret180;181; BE-LABEL: popcount256:182; BE: // %bb.0: // %Entry183; BE-NEXT: ldr d0, [x0]184; BE-NEXT: ldr d1, [x0, #16]185; BE-NEXT: add x8, x0, #24186; BE-NEXT: add x9, x0, #8187; BE-NEXT: ld1 { v0.d }[1], [x9]188; BE-NEXT: ld1 { v1.d }[1], [x8]189; BE-NEXT: rev64 v0.16b, v0.16b190; BE-NEXT: rev64 v1.16b, v1.16b191; BE-NEXT: cnt v0.16b, v0.16b192; BE-NEXT: cnt v1.16b, v1.16b193; BE-NEXT: addv b0, v0.16b194; BE-NEXT: addv b1, v1.16b195; BE-NEXT: rev64 v0.4s, v0.4s196; BE-NEXT: rev64 v1.4s, v1.4s197; BE-NEXT: mov w8, v0.s[1]198; BE-NEXT: mov w9, v1.s[1]199; BE-NEXT: add w0, w9, w8200; BE-NEXT: ret201;202; GISEL-LABEL: popcount256:203; GISEL: // %bb.0: // %Entry204; GISEL-NEXT: ldp x8, x9, [x0]205; GISEL-NEXT: mov v0.d[0], x8206; GISEL-NEXT: ldp x8, x10, [x0, #16]207; GISEL-NEXT: mov v1.d[0], x8208; GISEL-NEXT: mov v0.d[1], x9209; GISEL-NEXT: mov v1.d[1], x10210; GISEL-NEXT: cnt v0.16b, v0.16b211; GISEL-NEXT: cnt v1.16b, v1.16b212; GISEL-NEXT: uaddlv h0, v0.16b213; GISEL-NEXT: uaddlv h1, v1.16b214; GISEL-NEXT: mov w8, v0.s[0]215; GISEL-NEXT: fmov w9, s1216; GISEL-NEXT: add x0, x8, w9, uxtw217; GISEL-NEXT: // kill: def $w0 killed $w0 killed $x0218; GISEL-NEXT: ret219;220; GISELO0-LABEL: popcount256:221; GISELO0: // %bb.0: // %Entry222; GISELO0-NEXT: ldr x11, [x0]223; GISELO0-NEXT: ldr x10, [x0, #8]224; GISELO0-NEXT: ldr x9, [x0, #16]225; GISELO0-NEXT: ldr x8, [x0, #24]226; GISELO0-NEXT: // implicit-def: $q1227; GISELO0-NEXT: mov v1.d[0], x11228; GISELO0-NEXT: mov v1.d[1], x10229; GISELO0-NEXT: // implicit-def: $q0230; GISELO0-NEXT: mov v0.d[0], x9231; GISELO0-NEXT: mov v0.d[1], x8232; GISELO0-NEXT: cnt v1.16b, v1.16b233; GISELO0-NEXT: uaddlv h1, v1.16b234; GISELO0-NEXT: // kill: def $q1 killed $h1235; GISELO0-NEXT: // kill: def $s1 killed $s1 killed $q1236; GISELO0-NEXT: fmov w0, s1237; GISELO0-NEXT: mov w10, wzr238; GISELO0-NEXT: mov w9, w0239; GISELO0-NEXT: mov w8, w10240; GISELO0-NEXT: bfi x9, x8, #32, #32241; GISELO0-NEXT: cnt v0.16b, v0.16b242; GISELO0-NEXT: uaddlv h0, v0.16b243; GISELO0-NEXT: // kill: def $q0 killed $h0244; GISELO0-NEXT: // kill: def $s0 killed $s0 killed $q0245; GISELO0-NEXT: fmov w0, s0246; GISELO0-NEXT: mov w8, w0247; GISELO0-NEXT: // kill: def $x10 killed $w10248; GISELO0-NEXT: bfi x8, x10, #32, #32249; GISELO0-NEXT: adds x8, x8, x9250; GISELO0-NEXT: mov w0, w8251; GISELO0-NEXT: ret252Entry:253 %1 = load i256, ptr %0, align 16254 %2 = tail call i256 @llvm.ctpop.i256(i256 %1)255 %3 = trunc i256 %2 to i16256 ret i16 %3257}258 259; Function Attrs: nounwind readnone speculatable willreturn260declare i256 @llvm.ctpop.i256(i256)261 262define <1 x i128> @popcount1x128(<1 x i128> %0) {263; CHECKO0-LABEL: popcount1x128:264; CHECKO0: // %bb.0: // %Entry265; CHECKO0-NEXT: // implicit-def: $q0266; CHECKO0-NEXT: mov v0.d[0], x0267; CHECKO0-NEXT: mov v0.d[1], x1268; CHECKO0-NEXT: cnt v0.16b, v0.16b269; CHECKO0-NEXT: uaddlv h0, v0.16b270; CHECKO0-NEXT: // kill: def $q0 killed $h0271; CHECKO0-NEXT: mov x1, xzr272; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0273; CHECKO0-NEXT: fmov w0, s0274; CHECKO0-NEXT: mov w8, wzr275; CHECKO0-NEXT: // kill: def $x0 killed $w0276; CHECKO0-NEXT: // kill: def $x8 killed $w8277; CHECKO0-NEXT: bfi x0, x8, #32, #32278; CHECKO0-NEXT: ret279;280; NEON-LABEL: popcount1x128:281; NEON: // %bb.0: // %Entry282; NEON-NEXT: fmov d0, x0283; NEON-NEXT: mov v0.d[1], x1284; NEON-NEXT: mov x1, xzr285; NEON-NEXT: cnt v0.16b, v0.16b286; NEON-NEXT: addv b0, v0.16b287; NEON-NEXT: fmov x0, d0288; NEON-NEXT: ret289;290; DOT-LABEL: popcount1x128:291; DOT: // %bb.0: // %Entry292; DOT-NEXT: fmov d0, x0293; DOT-NEXT: mov v0.d[1], x1294; DOT-NEXT: mov x1, xzr295; DOT-NEXT: cnt v0.16b, v0.16b296; DOT-NEXT: addv b0, v0.16b297; DOT-NEXT: fmov x0, d0298; DOT-NEXT: ret299;300; SVE-LABEL: popcount1x128:301; SVE: // %bb.0: // %Entry302; SVE-NEXT: fmov d0, x0303; SVE-NEXT: ptrue p0.d304; SVE-NEXT: mov v0.d[1], x1305; SVE-NEXT: mov x1, xzr306; SVE-NEXT: cnt z0.d, p0/m, z0.d307; SVE-NEXT: addp d0, v0.2d308; SVE-NEXT: fmov x0, d0309; SVE-NEXT: ret310;311; BE-LABEL: popcount1x128:312; BE: // %bb.0: // %Entry313; BE-NEXT: fmov d0, x0314; BE-NEXT: mov x0, xzr315; BE-NEXT: mov v0.d[1], x1316; BE-NEXT: rev64 v0.16b, v0.16b317; BE-NEXT: cnt v0.16b, v0.16b318; BE-NEXT: addv b0, v0.16b319; BE-NEXT: fmov x1, d0320; BE-NEXT: ret321;322; GISEL-LABEL: popcount1x128:323; GISEL: // %bb.0: // %Entry324; GISEL-NEXT: mov v0.d[0], x0325; GISEL-NEXT: mov v0.d[1], x1326; GISEL-NEXT: mov x1, xzr327; GISEL-NEXT: cnt v0.16b, v0.16b328; GISEL-NEXT: uaddlv h0, v0.16b329; GISEL-NEXT: mov w0, v0.s[0]330; GISEL-NEXT: ret331;332; GISELO0-LABEL: popcount1x128:333; GISELO0: // %bb.0: // %Entry334; GISELO0-NEXT: // implicit-def: $q0335; GISELO0-NEXT: mov v0.d[0], x0336; GISELO0-NEXT: mov v0.d[1], x1337; GISELO0-NEXT: cnt v0.16b, v0.16b338; GISELO0-NEXT: uaddlv h0, v0.16b339; GISELO0-NEXT: // kill: def $q0 killed $h0340; GISELO0-NEXT: mov x1, xzr341; GISELO0-NEXT: // kill: def $s0 killed $s0 killed $q0342; GISELO0-NEXT: fmov w0, s0343; GISELO0-NEXT: mov w8, wzr344; GISELO0-NEXT: // kill: def $x0 killed $w0345; GISELO0-NEXT: // kill: def $x8 killed $w8346; GISELO0-NEXT: bfi x0, x8, #32, #32347; GISELO0-NEXT: ret348Entry:349 %1 = tail call <1 x i128> @llvm.ctpop.v1i128(<1 x i128> %0)350 ret <1 x i128> %1351}352 353declare <1 x i128> @llvm.ctpop.v1i128(<1 x i128>)354 355define <2 x i64> @popcount2x64(<2 x i64> %0) {356; CHECKO0-LABEL: popcount2x64:357; CHECKO0: // %bb.0: // %Entry358; CHECKO0-NEXT: cnt v0.16b, v0.16b359; CHECKO0-NEXT: uaddlp v0.8h, v0.16b360; CHECKO0-NEXT: uaddlp v0.4s, v0.8h361; CHECKO0-NEXT: uaddlp v0.2d, v0.4s362; CHECKO0-NEXT: ret363;364; NEON-LABEL: popcount2x64:365; NEON: // %bb.0: // %Entry366; NEON-NEXT: cnt v0.16b, v0.16b367; NEON-NEXT: uaddlp v0.8h, v0.16b368; NEON-NEXT: uaddlp v0.4s, v0.8h369; NEON-NEXT: uaddlp v0.2d, v0.4s370; NEON-NEXT: ret371;372; DOT-LABEL: popcount2x64:373; DOT: // %bb.0: // %Entry374; DOT-NEXT: movi v1.16b, #1375; DOT-NEXT: cnt v0.16b, v0.16b376; DOT-NEXT: movi v2.2d, #0000000000000000377; DOT-NEXT: udot v2.4s, v1.16b, v0.16b378; DOT-NEXT: uaddlp v0.2d, v2.4s379; DOT-NEXT: ret380;381; SVE-LABEL: popcount2x64:382; SVE: // %bb.0: // %Entry383; SVE-NEXT: ptrue p0.d, vl2384; SVE-NEXT: // kill: def $q0 killed $q0 def $z0385; SVE-NEXT: cnt z0.d, p0/m, z0.d386; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0387; SVE-NEXT: ret388;389; BE-LABEL: popcount2x64:390; BE: // %bb.0: // %Entry391; BE-NEXT: rev64 v0.16b, v0.16b392; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8393; BE-NEXT: cnt v0.16b, v0.16b394; BE-NEXT: uaddlp v0.8h, v0.16b395; BE-NEXT: uaddlp v0.4s, v0.8h396; BE-NEXT: uaddlp v0.2d, v0.4s397; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8398; BE-NEXT: ret399;400; GISELO0-LABEL: popcount2x64:401; GISELO0: // %bb.0: // %Entry402; GISELO0-NEXT: cnt v0.16b, v0.16b403; GISELO0-NEXT: uaddlp v0.8h, v0.16b404; GISELO0-NEXT: uaddlp v0.4s, v0.8h405; GISELO0-NEXT: uaddlp v0.2d, v0.4s406; GISELO0-NEXT: ret407;408; NEON-GISEL-LABEL: popcount2x64:409; NEON-GISEL: // %bb.0: // %Entry410; NEON-GISEL-NEXT: cnt v0.16b, v0.16b411; NEON-GISEL-NEXT: uaddlp v0.8h, v0.16b412; NEON-GISEL-NEXT: uaddlp v0.4s, v0.8h413; NEON-GISEL-NEXT: uaddlp v0.2d, v0.4s414; NEON-GISEL-NEXT: ret415;416; DOT-GISEL-LABEL: popcount2x64:417; DOT-GISEL: // %bb.0: // %Entry418; DOT-GISEL-NEXT: movi v1.2d, #0000000000000000419; DOT-GISEL-NEXT: cnt v0.16b, v0.16b420; DOT-GISEL-NEXT: movi v2.16b, #1421; DOT-GISEL-NEXT: udot v1.4s, v2.16b, v0.16b422; DOT-GISEL-NEXT: uaddlp v0.2d, v1.4s423; DOT-GISEL-NEXT: ret424;425; SVE-GISEL-LABEL: popcount2x64:426; SVE-GISEL: // %bb.0: // %Entry427; SVE-GISEL-NEXT: cnt v0.16b, v0.16b428; SVE-GISEL-NEXT: uaddlp v0.8h, v0.16b429; SVE-GISEL-NEXT: uaddlp v0.4s, v0.8h430; SVE-GISEL-NEXT: uaddlp v0.2d, v0.4s431; SVE-GISEL-NEXT: ret432Entry:433 %1 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)434 ret <2 x i64> %1435}436 437declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)438 439define <1 x i64> @popcount1x64(<1 x i64> %0) {440; CHECKO0-LABEL: popcount1x64:441; CHECKO0: // %bb.0: // %Entry442; CHECKO0-NEXT: fmov x0, d0443; CHECKO0-NEXT: fmov d0, x0444; CHECKO0-NEXT: cnt v0.8b, v0.8b445; CHECKO0-NEXT: uaddlv h0, v0.8b446; CHECKO0-NEXT: // kill: def $q0 killed $h0447; CHECKO0-NEXT: mov w8, v0.s[0]448; CHECKO0-NEXT: // kill: def $x8 killed $w8449; CHECKO0-NEXT: fmov d0, x8450; CHECKO0-NEXT: ret451;452; NEON-LABEL: popcount1x64:453; NEON: // %bb.0: // %Entry454; NEON-NEXT: cnt v0.8b, v0.8b455; NEON-NEXT: uaddlp v0.4h, v0.8b456; NEON-NEXT: uaddlp v0.2s, v0.4h457; NEON-NEXT: uaddlp v0.1d, v0.2s458; NEON-NEXT: ret459;460; DOT-LABEL: popcount1x64:461; DOT: // %bb.0: // %Entry462; DOT-NEXT: cnt v0.8b, v0.8b463; DOT-NEXT: uaddlp v0.4h, v0.8b464; DOT-NEXT: uaddlp v0.2s, v0.4h465; DOT-NEXT: uaddlp v0.1d, v0.2s466; DOT-NEXT: ret467;468; SVE-LABEL: popcount1x64:469; SVE: // %bb.0: // %Entry470; SVE-NEXT: ptrue p0.d, vl1471; SVE-NEXT: // kill: def $d0 killed $d0 def $z0472; SVE-NEXT: cnt z0.d, p0/m, z0.d473; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0474; SVE-NEXT: ret475;476; BE-LABEL: popcount1x64:477; BE: // %bb.0: // %Entry478; BE-NEXT: rev64 v0.8b, v0.8b479; BE-NEXT: cnt v0.8b, v0.8b480; BE-NEXT: uaddlp v0.4h, v0.8b481; BE-NEXT: uaddlp v0.2s, v0.4h482; BE-NEXT: uaddlp v0.1d, v0.2s483; BE-NEXT: ret484;485; GISEL-LABEL: popcount1x64:486; GISEL: // %bb.0: // %Entry487; GISEL-NEXT: cnt v0.8b, v0.8b488; GISEL-NEXT: uaddlv h0, v0.8b489; GISEL-NEXT: mov w8, v0.s[0]490; GISEL-NEXT: fmov d0, x8491; GISEL-NEXT: ret492;493; GISELO0-LABEL: popcount1x64:494; GISELO0: // %bb.0: // %Entry495; GISELO0-NEXT: fmov x0, d0496; GISELO0-NEXT: fmov d0, x0497; GISELO0-NEXT: cnt v0.8b, v0.8b498; GISELO0-NEXT: uaddlv h0, v0.8b499; GISELO0-NEXT: // kill: def $q0 killed $h0500; GISELO0-NEXT: mov w8, v0.s[0]501; GISELO0-NEXT: // kill: def $x8 killed $w8502; GISELO0-NEXT: fmov d0, x8503; GISELO0-NEXT: ret504Entry:505 %1 = tail call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %0)506 ret <1 x i64> %1507}508 509declare <1 x i64> @llvm.ctpop.v1i64(<1 x i64>)510 511define <4 x i32> @popcount4x32(<4 x i32> %0) {512; CHECKO0-LABEL: popcount4x32:513; CHECKO0: // %bb.0: // %Entry514; CHECKO0-NEXT: cnt v0.16b, v0.16b515; CHECKO0-NEXT: uaddlp v0.8h, v0.16b516; CHECKO0-NEXT: uaddlp v0.4s, v0.8h517; CHECKO0-NEXT: ret518;519; NEON-LABEL: popcount4x32:520; NEON: // %bb.0: // %Entry521; NEON-NEXT: cnt v0.16b, v0.16b522; NEON-NEXT: uaddlp v0.8h, v0.16b523; NEON-NEXT: uaddlp v0.4s, v0.8h524; NEON-NEXT: ret525;526; DOT-LABEL: popcount4x32:527; DOT: // %bb.0: // %Entry528; DOT-NEXT: movi v1.16b, #1529; DOT-NEXT: cnt v2.16b, v0.16b530; DOT-NEXT: movi v0.2d, #0000000000000000531; DOT-NEXT: udot v0.4s, v1.16b, v2.16b532; DOT-NEXT: ret533;534; SVE-LABEL: popcount4x32:535; SVE: // %bb.0: // %Entry536; SVE-NEXT: ptrue p0.s, vl4537; SVE-NEXT: // kill: def $q0 killed $q0 def $z0538; SVE-NEXT: cnt z0.s, p0/m, z0.s539; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0540; SVE-NEXT: ret541;542; BE-LABEL: popcount4x32:543; BE: // %bb.0: // %Entry544; BE-NEXT: rev64 v0.16b, v0.16b545; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8546; BE-NEXT: cnt v0.16b, v0.16b547; BE-NEXT: uaddlp v0.8h, v0.16b548; BE-NEXT: uaddlp v0.4s, v0.8h549; BE-NEXT: rev64 v0.4s, v0.4s550; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8551; BE-NEXT: ret552;553; GISELO0-LABEL: popcount4x32:554; GISELO0: // %bb.0: // %Entry555; GISELO0-NEXT: cnt v0.16b, v0.16b556; GISELO0-NEXT: uaddlp v0.8h, v0.16b557; GISELO0-NEXT: uaddlp v0.4s, v0.8h558; GISELO0-NEXT: ret559;560; NEON-GISEL-LABEL: popcount4x32:561; NEON-GISEL: // %bb.0: // %Entry562; NEON-GISEL-NEXT: cnt v0.16b, v0.16b563; NEON-GISEL-NEXT: uaddlp v0.8h, v0.16b564; NEON-GISEL-NEXT: uaddlp v0.4s, v0.8h565; NEON-GISEL-NEXT: ret566;567; DOT-GISEL-LABEL: popcount4x32:568; DOT-GISEL: // %bb.0: // %Entry569; DOT-GISEL-NEXT: movi v1.2d, #0000000000000000570; DOT-GISEL-NEXT: cnt v0.16b, v0.16b571; DOT-GISEL-NEXT: movi v2.16b, #1572; DOT-GISEL-NEXT: udot v1.4s, v2.16b, v0.16b573; DOT-GISEL-NEXT: mov v0.16b, v1.16b574; DOT-GISEL-NEXT: ret575;576; SVE-GISEL-LABEL: popcount4x32:577; SVE-GISEL: // %bb.0: // %Entry578; SVE-GISEL-NEXT: cnt v0.16b, v0.16b579; SVE-GISEL-NEXT: uaddlp v0.8h, v0.16b580; SVE-GISEL-NEXT: uaddlp v0.4s, v0.8h581; SVE-GISEL-NEXT: ret582Entry:583 %1 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)584 ret <4 x i32> %1585}586 587declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)588 589define <2 x i32> @popcount2x32(<2 x i32> %0) {590; CHECKO0-LABEL: popcount2x32:591; CHECKO0: // %bb.0: // %Entry592; CHECKO0-NEXT: cnt v0.8b, v0.8b593; CHECKO0-NEXT: uaddlp v0.4h, v0.8b594; CHECKO0-NEXT: uaddlp v0.2s, v0.4h595; CHECKO0-NEXT: ret596;597; NEON-LABEL: popcount2x32:598; NEON: // %bb.0: // %Entry599; NEON-NEXT: cnt v0.8b, v0.8b600; NEON-NEXT: uaddlp v0.4h, v0.8b601; NEON-NEXT: uaddlp v0.2s, v0.4h602; NEON-NEXT: ret603;604; DOT-LABEL: popcount2x32:605; DOT: // %bb.0: // %Entry606; DOT-NEXT: movi v1.2d, #0000000000000000607; DOT-NEXT: cnt v0.8b, v0.8b608; DOT-NEXT: movi v2.8b, #1609; DOT-NEXT: udot v1.2s, v2.8b, v0.8b610; DOT-NEXT: fmov d0, d1611; DOT-NEXT: ret612;613; SVE-LABEL: popcount2x32:614; SVE: // %bb.0: // %Entry615; SVE-NEXT: ptrue p0.s, vl2616; SVE-NEXT: // kill: def $d0 killed $d0 def $z0617; SVE-NEXT: cnt z0.s, p0/m, z0.s618; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0619; SVE-NEXT: ret620;621; BE-LABEL: popcount2x32:622; BE: // %bb.0: // %Entry623; BE-NEXT: rev64 v0.8b, v0.8b624; BE-NEXT: cnt v0.8b, v0.8b625; BE-NEXT: uaddlp v0.4h, v0.8b626; BE-NEXT: uaddlp v0.2s, v0.4h627; BE-NEXT: rev64 v0.2s, v0.2s628; BE-NEXT: ret629;630; GISELO0-LABEL: popcount2x32:631; GISELO0: // %bb.0: // %Entry632; GISELO0-NEXT: cnt v0.8b, v0.8b633; GISELO0-NEXT: uaddlp v0.4h, v0.8b634; GISELO0-NEXT: uaddlp v0.2s, v0.4h635; GISELO0-NEXT: ret636;637; NEON-GISEL-LABEL: popcount2x32:638; NEON-GISEL: // %bb.0: // %Entry639; NEON-GISEL-NEXT: cnt v0.8b, v0.8b640; NEON-GISEL-NEXT: uaddlp v0.4h, v0.8b641; NEON-GISEL-NEXT: uaddlp v0.2s, v0.4h642; NEON-GISEL-NEXT: ret643;644; DOT-GISEL-LABEL: popcount2x32:645; DOT-GISEL: // %bb.0: // %Entry646; DOT-GISEL-NEXT: movi v1.2d, #0000000000000000647; DOT-GISEL-NEXT: cnt v0.8b, v0.8b648; DOT-GISEL-NEXT: movi v2.8b, #1649; DOT-GISEL-NEXT: udot v1.2s, v2.8b, v0.8b650; DOT-GISEL-NEXT: fmov d0, d1651; DOT-GISEL-NEXT: ret652;653; SVE-GISEL-LABEL: popcount2x32:654; SVE-GISEL: // %bb.0: // %Entry655; SVE-GISEL-NEXT: cnt v0.8b, v0.8b656; SVE-GISEL-NEXT: uaddlp v0.4h, v0.8b657; SVE-GISEL-NEXT: uaddlp v0.2s, v0.4h658; SVE-GISEL-NEXT: ret659Entry:660 %1 = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %0)661 ret <2 x i32> %1662}663 664declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)665 666define <8 x i16> @popcount8x16(<8 x i16> %0) {667; CHECKO0-LABEL: popcount8x16:668; CHECKO0: // %bb.0: // %Entry669; CHECKO0-NEXT: cnt v0.16b, v0.16b670; CHECKO0-NEXT: uaddlp v0.8h, v0.16b671; CHECKO0-NEXT: ret672;673; NEON-LABEL: popcount8x16:674; NEON: // %bb.0: // %Entry675; NEON-NEXT: cnt v0.16b, v0.16b676; NEON-NEXT: uaddlp v0.8h, v0.16b677; NEON-NEXT: ret678;679; DOT-LABEL: popcount8x16:680; DOT: // %bb.0: // %Entry681; DOT-NEXT: cnt v0.16b, v0.16b682; DOT-NEXT: uaddlp v0.8h, v0.16b683; DOT-NEXT: ret684;685; SVE-LABEL: popcount8x16:686; SVE: // %bb.0: // %Entry687; SVE-NEXT: ptrue p0.h, vl8688; SVE-NEXT: // kill: def $q0 killed $q0 def $z0689; SVE-NEXT: cnt z0.h, p0/m, z0.h690; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0691; SVE-NEXT: ret692;693; BE-LABEL: popcount8x16:694; BE: // %bb.0: // %Entry695; BE-NEXT: rev64 v0.16b, v0.16b696; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8697; BE-NEXT: cnt v0.16b, v0.16b698; BE-NEXT: uaddlp v0.8h, v0.16b699; BE-NEXT: rev64 v0.8h, v0.8h700; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8701; BE-NEXT: ret702;703; GISEL-LABEL: popcount8x16:704; GISEL: // %bb.0: // %Entry705; GISEL-NEXT: cnt v0.16b, v0.16b706; GISEL-NEXT: uaddlp v0.8h, v0.16b707; GISEL-NEXT: ret708;709; GISELO0-LABEL: popcount8x16:710; GISELO0: // %bb.0: // %Entry711; GISELO0-NEXT: cnt v0.16b, v0.16b712; GISELO0-NEXT: uaddlp v0.8h, v0.16b713; GISELO0-NEXT: ret714Entry:715 %1 = tail call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %0)716 ret <8 x i16> %1717}718 719declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)720 721define <4 x i16> @popcount4x16(<4 x i16> %0) {722; CHECKO0-LABEL: popcount4x16:723; CHECKO0: // %bb.0: // %Entry724; CHECKO0-NEXT: cnt v0.8b, v0.8b725; CHECKO0-NEXT: uaddlp v0.4h, v0.8b726; CHECKO0-NEXT: ret727;728; NEON-LABEL: popcount4x16:729; NEON: // %bb.0: // %Entry730; NEON-NEXT: cnt v0.8b, v0.8b731; NEON-NEXT: uaddlp v0.4h, v0.8b732; NEON-NEXT: ret733;734; DOT-LABEL: popcount4x16:735; DOT: // %bb.0: // %Entry736; DOT-NEXT: cnt v0.8b, v0.8b737; DOT-NEXT: uaddlp v0.4h, v0.8b738; DOT-NEXT: ret739;740; SVE-LABEL: popcount4x16:741; SVE: // %bb.0: // %Entry742; SVE-NEXT: ptrue p0.h, vl4743; SVE-NEXT: // kill: def $d0 killed $d0 def $z0744; SVE-NEXT: cnt z0.h, p0/m, z0.h745; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0746; SVE-NEXT: ret747;748; BE-LABEL: popcount4x16:749; BE: // %bb.0: // %Entry750; BE-NEXT: rev64 v0.8b, v0.8b751; BE-NEXT: cnt v0.8b, v0.8b752; BE-NEXT: uaddlp v0.4h, v0.8b753; BE-NEXT: rev64 v0.4h, v0.4h754; BE-NEXT: ret755;756; GISEL-LABEL: popcount4x16:757; GISEL: // %bb.0: // %Entry758; GISEL-NEXT: cnt v0.8b, v0.8b759; GISEL-NEXT: uaddlp v0.4h, v0.8b760; GISEL-NEXT: ret761;762; GISELO0-LABEL: popcount4x16:763; GISELO0: // %bb.0: // %Entry764; GISELO0-NEXT: cnt v0.8b, v0.8b765; GISELO0-NEXT: uaddlp v0.4h, v0.8b766; GISELO0-NEXT: ret767Entry:768 %1 = tail call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %0)769 ret <4 x i16> %1770}771 772define i32 @ctpop_into_extract(ptr %p) {773; CHECKO0-LABEL: ctpop_into_extract:774; CHECKO0: // %bb.0:775; CHECKO0-NEXT: mov w8, #-1 // =0xffffffff776; CHECKO0-NEXT: // implicit-def: $d2777; CHECKO0-NEXT: fmov s2, w8778; CHECKO0-NEXT: ldr d0, [x0]779; CHECKO0-NEXT: fmov s1, s0780; CHECKO0-NEXT: fmov w8, s1781; CHECKO0-NEXT: fmov s1, w8782; CHECKO0-NEXT: // kill: def $d1 killed $s1783; CHECKO0-NEXT: cnt v1.8b, v1.8b784; CHECKO0-NEXT: uaddlv h1, v1.8b785; CHECKO0-NEXT: // kill: def $q1 killed $h1786; CHECKO0-NEXT: // kill: def $s1 killed $s1 killed $q1787; CHECKO0-NEXT: fmov w8, s1788; CHECKO0-NEXT: // implicit-def: $q1789; CHECKO0-NEXT: fmov d1, d2790; CHECKO0-NEXT: mov v1.s[1], w8791; CHECKO0-NEXT: // kill: def $d1 killed $d1 killed $q1792; CHECKO0-NEXT: sub v0.2s, v0.2s, v1.2s793; CHECKO0-NEXT: str d0, [x0]794; CHECKO0-NEXT: mov w0, wzr795; CHECKO0-NEXT: ret796;797; NEON-LABEL: ctpop_into_extract:798; NEON: // %bb.0:799; NEON-NEXT: ldr d0, [x0]800; NEON-NEXT: movi v2.2d, #0xffffffffffffffff801; NEON-NEXT: mov x8, x0802; NEON-NEXT: mov w0, wzr803; NEON-NEXT: fmov w9, s0804; NEON-NEXT: fmov s1, w9805; NEON-NEXT: cnt v1.8b, v1.8b806; NEON-NEXT: addv b1, v1.8b807; NEON-NEXT: mov v2.s[1], v1.s[0]808; NEON-NEXT: sub v0.2s, v0.2s, v2.2s809; NEON-NEXT: str d0, [x8]810; NEON-NEXT: ret811;812; DOT-LABEL: ctpop_into_extract:813; DOT: // %bb.0:814; DOT-NEXT: ldr d0, [x0]815; DOT-NEXT: movi v2.2d, #0xffffffffffffffff816; DOT-NEXT: mov x8, x0817; DOT-NEXT: mov w0, wzr818; DOT-NEXT: fmov w9, s0819; DOT-NEXT: fmov s1, w9820; DOT-NEXT: cnt v1.8b, v1.8b821; DOT-NEXT: addv b1, v1.8b822; DOT-NEXT: mov v2.s[1], v1.s[0]823; DOT-NEXT: sub v0.2s, v0.2s, v2.2s824; DOT-NEXT: str d0, [x8]825; DOT-NEXT: ret826;827; SVE-LABEL: ctpop_into_extract:828; SVE: // %bb.0:829; SVE-NEXT: ldr d0, [x0]830; SVE-NEXT: ptrue p0.s831; SVE-NEXT: movi v2.2d, #0xffffffffffffffff832; SVE-NEXT: mov x8, x0833; SVE-NEXT: mov w0, wzr834; SVE-NEXT: movprfx z1, z0835; SVE-NEXT: cnt z1.s, p0/m, z0.s836; SVE-NEXT: mov v2.s[1], v1.s[0]837; SVE-NEXT: sub v0.2s, v0.2s, v2.2s838; SVE-NEXT: str d0, [x8]839; SVE-NEXT: ret840;841; BE-LABEL: ctpop_into_extract:842; BE: // %bb.0:843; BE-NEXT: ld1 { v0.2s }, [x0]844; BE-NEXT: movi v2.2d, #0xffffffffffffffff845; BE-NEXT: mov x8, x0846; BE-NEXT: mov w0, wzr847; BE-NEXT: fmov w9, s0848; BE-NEXT: fmov s1, w9849; BE-NEXT: cnt v1.8b, v1.8b850; BE-NEXT: addv b1, v1.8b851; BE-NEXT: mov v2.s[1], v1.s[0]852; BE-NEXT: sub v0.2s, v0.2s, v2.2s853; BE-NEXT: st1 { v0.2s }, [x8]854; BE-NEXT: ret855;856; GISEL-LABEL: ctpop_into_extract:857; GISEL: // %bb.0:858; GISEL-NEXT: ldr d0, [x0]859; GISEL-NEXT: mov x8, x0860; GISEL-NEXT: mov w0, wzr861; GISEL-NEXT: fmov w9, s0862; GISEL-NEXT: fmov s1, w9863; GISEL-NEXT: mov w9, #-1 // =0xffffffff864; GISEL-NEXT: fmov s2, w9865; GISEL-NEXT: cnt v1.8b, v1.8b866; GISEL-NEXT: uaddlv h1, v1.8b867; GISEL-NEXT: mov v2.s[1], v1.s[0]868; GISEL-NEXT: sub v0.2s, v0.2s, v2.2s869; GISEL-NEXT: str d0, [x8]870; GISEL-NEXT: ret871;872; GISELO0-LABEL: ctpop_into_extract:873; GISELO0: // %bb.0:874; GISELO0-NEXT: mov w8, #-1 // =0xffffffff875; GISELO0-NEXT: // implicit-def: $d2876; GISELO0-NEXT: fmov s2, w8877; GISELO0-NEXT: ldr d0, [x0]878; GISELO0-NEXT: fmov s1, s0879; GISELO0-NEXT: fmov w8, s1880; GISELO0-NEXT: fmov s1, w8881; GISELO0-NEXT: // kill: def $d1 killed $s1882; GISELO0-NEXT: cnt v1.8b, v1.8b883; GISELO0-NEXT: uaddlv h1, v1.8b884; GISELO0-NEXT: // kill: def $q1 killed $h1885; GISELO0-NEXT: // kill: def $s1 killed $s1 killed $q1886; GISELO0-NEXT: fmov w8, s1887; GISELO0-NEXT: // implicit-def: $q1888; GISELO0-NEXT: fmov d1, d2889; GISELO0-NEXT: mov v1.s[1], w8890; GISELO0-NEXT: // kill: def $d1 killed $d1 killed $q1891; GISELO0-NEXT: sub v0.2s, v0.2s, v1.2s892; GISELO0-NEXT: str d0, [x0]893; GISELO0-NEXT: mov w0, wzr894; GISELO0-NEXT: ret895 %1 = load <2 x i32>, ptr %p, align 4896 %2 = extractelement <2 x i32> %1, i64 0897 %3 = call i32 @llvm.ctpop.i32(i32 %2)898 %4 = insertelement <2 x i32> <i32 -1, i32 poison>, i32 %3, i64 1899 %5 = sub <2 x i32> %1, %4900 store <2 x i32> %5, ptr %p, align 4901 ret i32 0902}903 904declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>)905;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:906; CHECK: {{.*}}907