281 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 62; RUN: llc -mtriple=aarch64 -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BASE3; RUN: llc -mtriple=aarch64 -aarch64-sve-vector-bits-min=256 -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,CHECK-VL2564 5;; Full SVE vectors (supported with +sve)6 7define void @test_compressstore_nxv4i32(ptr %p, <vscale x 4 x i32> %vec, <vscale x 4 x i1> %mask) {8; CHECK-LABEL: test_compressstore_nxv4i32:9; CHECK: // %bb.0:10; CHECK-NEXT: cntp x8, p0, p0.s11; CHECK-NEXT: compact z0.s, p0, z0.s12; CHECK-NEXT: whilelo p0.s, xzr, x813; CHECK-NEXT: st1w { z0.s }, p0, [x0]14; CHECK-NEXT: ret15 tail call void @llvm.masked.compressstore.nxv4i32(<vscale x 4 x i32> %vec, ptr align 4 %p, <vscale x 4 x i1> %mask)16 ret void17}18 19define void @test_compressstore_nxv2i64(ptr %p, <vscale x 2 x i64> %vec, <vscale x 2 x i1> %mask) {20; CHECK-LABEL: test_compressstore_nxv2i64:21; CHECK: // %bb.0:22; CHECK-NEXT: cntp x8, p0, p0.d23; CHECK-NEXT: compact z0.d, p0, z0.d24; CHECK-NEXT: whilelo p0.d, xzr, x825; CHECK-NEXT: st1d { z0.d }, p0, [x0]26; CHECK-NEXT: ret27 tail call void @llvm.masked.compressstore.nxv2i64(<vscale x 2 x i64> %vec, ptr align 8 %p, <vscale x 2 x i1> %mask)28 ret void29}30 31define void @test_compressstore_nxv4f32(ptr %p, <vscale x 4 x float> %vec, <vscale x 4 x i1> %mask) {32; CHECK-LABEL: test_compressstore_nxv4f32:33; CHECK: // %bb.0:34; CHECK-NEXT: cntp x8, p0, p0.s35; CHECK-NEXT: compact z0.s, p0, z0.s36; CHECK-NEXT: whilelo p0.s, xzr, x837; CHECK-NEXT: st1w { z0.s }, p0, [x0]38; CHECK-NEXT: ret39 tail call void @llvm.masked.compressstore.nxv4f32(<vscale x 4 x float> %vec, ptr align 4 %p, <vscale x 4 x i1> %mask)40 ret void41}42 43define void @test_compressstore_nxv2f64(ptr %p, <vscale x 2 x double> %vec, <vscale x 2 x i1> %mask) {44; CHECK-LABEL: test_compressstore_nxv2f64:45; CHECK: // %bb.0:46; CHECK-NEXT: cntp x8, p0, p0.d47; CHECK-NEXT: compact z0.d, p0, z0.d48; CHECK-NEXT: whilelo p0.d, xzr, x849; CHECK-NEXT: st1d { z0.d }, p0, [x0]50; CHECK-NEXT: ret51 tail call void @llvm.masked.compressstore.nxv2f64(<vscale x 2 x double> %vec, ptr align 8 %p, <vscale x 2 x i1> %mask)52 ret void53}54 55;; SVE vectors that will be split56 57define void @test_compressstore_nxv8i32(ptr %p, <vscale x 8 x i32> %vec, <vscale x 8 x i1> %mask) {58; CHECK-LABEL: test_compressstore_nxv8i32:59; CHECK: // %bb.0:60; CHECK-NEXT: punpkhi p1.h, p0.b61; CHECK-NEXT: punpklo p0.h, p0.b62; CHECK-NEXT: cntp x8, p1, p1.s63; CHECK-NEXT: compact z1.s, p1, z1.s64; CHECK-NEXT: cntp x9, p0, p0.s65; CHECK-NEXT: compact z0.s, p0, z0.s66; CHECK-NEXT: whilelo p0.s, xzr, x867; CHECK-NEXT: whilelo p1.s, xzr, x968; CHECK-NEXT: st1w { z1.s }, p0, [x0, x9, lsl #2]69; CHECK-NEXT: st1w { z0.s }, p1, [x0]70; CHECK-NEXT: ret71 tail call void @llvm.masked.compressstore.nxv8i32(<vscale x 8 x i32> %vec, ptr align 4 %p, <vscale x 8 x i1> %mask)72 ret void73}74 75;; Unpacked SVE vector types76 77define void @test_compressstore_nxv2f32(ptr %p, <vscale x 2 x float> %vec, <vscale x 2 x i1> %mask) {78; CHECK-LABEL: test_compressstore_nxv2f32:79; CHECK: // %bb.0:80; CHECK-NEXT: cntp x8, p0, p0.d81; CHECK-NEXT: compact z0.d, p0, z0.d82; CHECK-NEXT: whilelo p0.d, xzr, x883; CHECK-NEXT: st1w { z0.d }, p0, [x0]84; CHECK-NEXT: ret85 tail call void @llvm.masked.compressstore.nxv2f32(<vscale x 2 x float> %vec, ptr align 4 %p, <vscale x 2 x i1> %mask)86 ret void87}88 89;; SVE vector types promoted to 32/64-bit (non-exhaustive)90 91define void @test_compressstore_nxv2i8(ptr %p, <vscale x 2 x i8> %vec, <vscale x 2 x i1> %mask) {92; CHECK-LABEL: test_compressstore_nxv2i8:93; CHECK: // %bb.0:94; CHECK-NEXT: cntp x8, p0, p0.d95; CHECK-NEXT: compact z0.d, p0, z0.d96; CHECK-NEXT: whilelo p0.d, xzr, x897; CHECK-NEXT: st1b { z0.d }, p0, [x0]98; CHECK-NEXT: ret99 tail call void @llvm.masked.compressstore.nxv2i8(<vscale x 2 x i8> %vec, ptr align 1 %p, <vscale x 2 x i1> %mask)100 ret void101}102 103define void @test_compressstore_nxv4i16(ptr %p, <vscale x 4 x i16> %vec, <vscale x 4 x i1> %mask) {104; CHECK-LABEL: test_compressstore_nxv4i16:105; CHECK: // %bb.0:106; CHECK-NEXT: cntp x8, p0, p0.s107; CHECK-NEXT: compact z0.s, p0, z0.s108; CHECK-NEXT: whilelo p0.s, xzr, x8109; CHECK-NEXT: st1h { z0.s }, p0, [x0]110; CHECK-NEXT: ret111 tail call void @llvm.masked.compressstore.nxv4i16(<vscale x 4 x i16> %vec, ptr align 2 %p, <vscale x 4 x i1> %mask)112 ret void113}114 115;; NEON vector types (promoted to SVE)116 117define void @test_compressstore_v2f64(ptr %p, <2 x double> %vec, <2 x i1> %mask) {118; CHECK-LABEL: test_compressstore_v2f64:119; CHECK: // %bb.0:120; CHECK-NEXT: ushll v1.2d, v1.2s, #0121; CHECK-NEXT: ptrue p0.d, vl2122; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0123; CHECK-NEXT: shl v1.2d, v1.2d, #63124; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0125; CHECK-NEXT: cntp x8, p0, p0.d126; CHECK-NEXT: compact z0.d, p0, z0.d127; CHECK-NEXT: whilelo p0.d, xzr, x8128; CHECK-NEXT: st1d { z0.d }, p0, [x0]129; CHECK-NEXT: ret130 tail call void @llvm.masked.compressstore.v2f64(<2 x double> %vec, ptr align 8 %p, <2 x i1> %mask)131 ret void132}133 134define void @test_compressstore_v4i32(ptr %p, <4 x i32> %vec, <4 x i1> %mask) {135; CHECK-LABEL: test_compressstore_v4i32:136; CHECK: // %bb.0:137; CHECK-NEXT: ushll v1.4s, v1.4h, #0138; CHECK-NEXT: ptrue p0.s, vl4139; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0140; CHECK-NEXT: shl v1.4s, v1.4s, #31141; CHECK-NEXT: cmpne p0.s, p0/z, z1.s, #0142; CHECK-NEXT: cntp x8, p0, p0.s143; CHECK-NEXT: compact z0.s, p0, z0.s144; CHECK-NEXT: whilelo p0.s, xzr, x8145; CHECK-NEXT: st1w { z0.s }, p0, [x0]146; CHECK-NEXT: ret147 tail call void @llvm.masked.compressstore.v4i32(<4 x i32> %vec, ptr align 4 %p, <4 x i1> %mask)148 ret void149}150 151define void @test_compressstore_v2i64(ptr %p, <2 x i64> %vec, <2 x i1> %mask) {152; CHECK-LABEL: test_compressstore_v2i64:153; CHECK: // %bb.0:154; CHECK-NEXT: ushll v1.2d, v1.2s, #0155; CHECK-NEXT: ptrue p0.d, vl2156; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0157; CHECK-NEXT: shl v1.2d, v1.2d, #63158; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0159; CHECK-NEXT: cntp x8, p0, p0.d160; CHECK-NEXT: compact z0.d, p0, z0.d161; CHECK-NEXT: whilelo p0.d, xzr, x8162; CHECK-NEXT: st1d { z0.d }, p0, [x0]163; CHECK-NEXT: ret164 tail call void @llvm.masked.compressstore.v2i64(<2 x i64> %vec, ptr align 8 %p, <2 x i1> %mask)165 ret void166}167 168define void @test_compressstore_v8i32(ptr %p, <8 x i32> %vec, <8 x i1> %mask) {169; CHECK-BASE-LABEL: test_compressstore_v8i32:170; CHECK-BASE: // %bb.0:171; CHECK-BASE-NEXT: // kill: def $q0 killed $q0 def $z0172; CHECK-BASE-NEXT: zip1 v3.8b, v2.8b, v0.8b173; CHECK-BASE-NEXT: adrp x8, .LCPI11_0174; CHECK-BASE-NEXT: zip2 v2.8b, v2.8b, v0.8b175; CHECK-BASE-NEXT: ldr d5, [x8, :lo12:.LCPI11_0]176; CHECK-BASE-NEXT: ptrue p0.s177; CHECK-BASE-NEXT: // kill: def $q1 killed $q1 def $z1178; CHECK-BASE-NEXT: ptrue p1.s, vl4179; CHECK-BASE-NEXT: shl v4.4h, v3.4h, #15180; CHECK-BASE-NEXT: ushll v2.4s, v2.4h, #0181; CHECK-BASE-NEXT: ushll v3.4s, v3.4h, #0182; CHECK-BASE-NEXT: cmlt v4.4h, v4.4h, #0183; CHECK-BASE-NEXT: shl v2.4s, v2.4s, #31184; CHECK-BASE-NEXT: shl v3.4s, v3.4s, #31185; CHECK-BASE-NEXT: and v4.8b, v4.8b, v5.8b186; CHECK-BASE-NEXT: addv h4, v4.4h187; CHECK-BASE-NEXT: fmov w8, s4188; CHECK-BASE-NEXT: and w8, w8, #0xf189; CHECK-BASE-NEXT: fmov s4, w8190; CHECK-BASE-NEXT: cnt z4.s, p0/m, z4.s191; CHECK-BASE-NEXT: cmpne p0.s, p1/z, z2.s, #0192; CHECK-BASE-NEXT: cmpne p1.s, p1/z, z3.s, #0193; CHECK-BASE-NEXT: cntp x8, p0, p0.s194; CHECK-BASE-NEXT: compact z1.s, p0, z1.s195; CHECK-BASE-NEXT: compact z0.s, p1, z0.s196; CHECK-BASE-NEXT: cntp x9, p1, p1.s197; CHECK-BASE-NEXT: fmov w10, s4198; CHECK-BASE-NEXT: whilelo p0.s, xzr, x8199; CHECK-BASE-NEXT: whilelo p1.s, xzr, x9200; CHECK-BASE-NEXT: st1w { z1.s }, p0, [x0, x10, lsl #2]201; CHECK-BASE-NEXT: st1w { z0.s }, p1, [x0]202; CHECK-BASE-NEXT: ret203;204; CHECK-VL256-LABEL: test_compressstore_v8i32:205; CHECK-VL256: // %bb.0:206; CHECK-VL256-NEXT: // kill: def $d2 killed $d2 def $z2207; CHECK-VL256-NEXT: ptrue p0.s, vl8208; CHECK-VL256-NEXT: // kill: def $q0 killed $q0 def $z0209; CHECK-VL256-NEXT: // kill: def $q1 killed $q1 def $z1210; CHECK-VL256-NEXT: uunpklo z2.h, z2.b211; CHECK-VL256-NEXT: ptrue p1.s, vl4212; CHECK-VL256-NEXT: splice z0.s, p1, z0.s, z1.s213; CHECK-VL256-NEXT: uunpklo z2.s, z2.h214; CHECK-VL256-NEXT: lsl z2.s, z2.s, #31215; CHECK-VL256-NEXT: asr z2.s, z2.s, #31216; CHECK-VL256-NEXT: cmpne p0.s, p0/z, z2.s, #0217; CHECK-VL256-NEXT: cntp x8, p0, p0.s218; CHECK-VL256-NEXT: compact z0.s, p0, z0.s219; CHECK-VL256-NEXT: whilelo p0.s, xzr, x8220; CHECK-VL256-NEXT: st1w { z0.s }, p0, [x0]221; CHECK-VL256-NEXT: ret222 tail call void @llvm.masked.compressstore.v8i32(<8 x i32> %vec, ptr align 4 %p, <8 x i1> %mask)223 ret void224}225 226define void @test_compressstore_v4i64(ptr %p, <4 x i64> %vec, <4 x i1> %mask) {227; CHECK-BASE-LABEL: test_compressstore_v4i64:228; CHECK-BASE: // %bb.0:229; CHECK-BASE-NEXT: ushll v2.4s, v2.4h, #0230; CHECK-BASE-NEXT: index z4.s, #1, #1231; CHECK-BASE-NEXT: ptrue p0.s232; CHECK-BASE-NEXT: ptrue p1.d, vl2233; CHECK-BASE-NEXT: // kill: def $q1 killed $q1 def $z1234; CHECK-BASE-NEXT: // kill: def $q0 killed $q0 def $z0235; CHECK-BASE-NEXT: shl v3.2s, v2.2s, #31236; CHECK-BASE-NEXT: cmlt v3.2s, v3.2s, #0237; CHECK-BASE-NEXT: and v3.8b, v3.8b, v4.8b238; CHECK-BASE-NEXT: ushll2 v4.2d, v2.4s, #0239; CHECK-BASE-NEXT: ushll v2.2d, v2.2s, #0240; CHECK-BASE-NEXT: addp v3.2s, v3.2s, v3.2s241; CHECK-BASE-NEXT: shl v2.2d, v2.2d, #63242; CHECK-BASE-NEXT: fmov w8, s3243; CHECK-BASE-NEXT: shl v3.2d, v4.2d, #63244; CHECK-BASE-NEXT: and w8, w8, #0x3245; CHECK-BASE-NEXT: fmov s4, w8246; CHECK-BASE-NEXT: cnt z4.s, p0/m, z4.s247; CHECK-BASE-NEXT: cmpne p0.d, p1/z, z3.d, #0248; CHECK-BASE-NEXT: cmpne p1.d, p1/z, z2.d, #0249; CHECK-BASE-NEXT: cntp x8, p0, p0.d250; CHECK-BASE-NEXT: compact z1.d, p0, z1.d251; CHECK-BASE-NEXT: compact z0.d, p1, z0.d252; CHECK-BASE-NEXT: cntp x9, p1, p1.d253; CHECK-BASE-NEXT: fmov w10, s4254; CHECK-BASE-NEXT: whilelo p0.d, xzr, x8255; CHECK-BASE-NEXT: whilelo p1.d, xzr, x9256; CHECK-BASE-NEXT: st1d { z1.d }, p0, [x0, x10, lsl #3]257; CHECK-BASE-NEXT: st1d { z0.d }, p1, [x0]258; CHECK-BASE-NEXT: ret259;260; CHECK-VL256-LABEL: test_compressstore_v4i64:261; CHECK-VL256: // %bb.0:262; CHECK-VL256-NEXT: // kill: def $d2 killed $d2 def $z2263; CHECK-VL256-NEXT: ptrue p0.d, vl4264; CHECK-VL256-NEXT: // kill: def $q0 killed $q0 def $z0265; CHECK-VL256-NEXT: // kill: def $q1 killed $q1 def $z1266; CHECK-VL256-NEXT: uunpklo z2.s, z2.h267; CHECK-VL256-NEXT: ptrue p1.d, vl2268; CHECK-VL256-NEXT: splice z0.d, p1, z0.d, z1.d269; CHECK-VL256-NEXT: uunpklo z2.d, z2.s270; CHECK-VL256-NEXT: lsl z2.d, z2.d, #63271; CHECK-VL256-NEXT: asr z2.d, z2.d, #63272; CHECK-VL256-NEXT: cmpne p0.d, p0/z, z2.d, #0273; CHECK-VL256-NEXT: cntp x8, p0, p0.d274; CHECK-VL256-NEXT: compact z0.d, p0, z0.d275; CHECK-VL256-NEXT: whilelo p0.d, xzr, x8276; CHECK-VL256-NEXT: st1d { z0.d }, p0, [x0]277; CHECK-VL256-NEXT: ret278 tail call void @llvm.masked.compressstore.v4i64(<4 x i64> %vec, ptr align 8 %p, <4 x i1> %mask)279 ret void280}281