312 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 42; RUN: llc -mtriple=aarch64 -mattr=+sve < %s | FileCheck %s3 4define <vscale x 2 x i8> @test_compress_nxv2i8(<vscale x 2 x i8> %vec, <vscale x 2 x i1> %mask) {5; CHECK-LABEL: test_compress_nxv2i8:6; CHECK: // %bb.0:7; CHECK-NEXT: compact z0.d, p0, z0.d8; CHECK-NEXT: ret9 %out = call <vscale x 2 x i8> @llvm.experimental.vector.compress(<vscale x 2 x i8> %vec, <vscale x 2 x i1> %mask, <vscale x 2 x i8> poison)10 ret <vscale x 2 x i8> %out11}12 13define <vscale x 2 x i16> @test_compress_nxv2i16(<vscale x 2 x i16> %vec, <vscale x 2 x i1> %mask) {14; CHECK-LABEL: test_compress_nxv2i16:15; CHECK: // %bb.0:16; CHECK-NEXT: compact z0.d, p0, z0.d17; CHECK-NEXT: ret18 %out = call <vscale x 2 x i16> @llvm.experimental.vector.compress(<vscale x 2 x i16> %vec, <vscale x 2 x i1> %mask, <vscale x 2 x i16> poison)19 ret <vscale x 2 x i16> %out20}21 22define <vscale x 2 x i32> @test_compress_nxv2i32(<vscale x 2 x i32> %vec, <vscale x 2 x i1> %mask) {23; CHECK-LABEL: test_compress_nxv2i32:24; CHECK: // %bb.0:25; CHECK-NEXT: compact z0.d, p0, z0.d26; CHECK-NEXT: ret27 %out = call <vscale x 2 x i32> @llvm.experimental.vector.compress(<vscale x 2 x i32> %vec, <vscale x 2 x i1> %mask, <vscale x 2 x i32> poison)28 ret <vscale x 2 x i32> %out29}30 31define <vscale x 2 x i64> @test_compress_nxv2i64(<vscale x 2 x i64> %vec, <vscale x 2 x i1> %mask) {32; CHECK-LABEL: test_compress_nxv2i64:33; CHECK: // %bb.0:34; CHECK-NEXT: compact z0.d, p0, z0.d35; CHECK-NEXT: ret36 %out = call <vscale x 2 x i64> @llvm.experimental.vector.compress(<vscale x 2 x i64> %vec, <vscale x 2 x i1> %mask, <vscale x 2 x i64> poison)37 ret <vscale x 2 x i64> %out38}39 40define <vscale x 2 x float> @test_compress_nxv2f32(<vscale x 2 x float> %vec, <vscale x 2 x i1> %mask) {41; CHECK-LABEL: test_compress_nxv2f32:42; CHECK: // %bb.0:43; CHECK-NEXT: compact z0.d, p0, z0.d44; CHECK-NEXT: ret45 %out = call <vscale x 2 x float> @llvm.experimental.vector.compress(<vscale x 2 x float> %vec, <vscale x 2 x i1> %mask, <vscale x 2 x float> poison)46 ret <vscale x 2 x float> %out47}48 49define <vscale x 2 x double> @test_compress_nxv2f64(<vscale x 2 x double> %vec, <vscale x 2 x i1> %mask) {50; CHECK-LABEL: test_compress_nxv2f64:51; CHECK: // %bb.0:52; CHECK-NEXT: compact z0.d, p0, z0.d53; CHECK-NEXT: ret54 %out = call <vscale x 2 x double> @llvm.experimental.vector.compress(<vscale x 2 x double> %vec, <vscale x 2 x i1> %mask, <vscale x 2 x double> poison)55 ret <vscale x 2 x double> %out56}57 58define <vscale x 4 x i8> @test_compress_nxv4i8(<vscale x 4 x i8> %vec, <vscale x 4 x i1> %mask) {59; CHECK-LABEL: test_compress_nxv4i8:60; CHECK: // %bb.0:61; CHECK-NEXT: compact z0.s, p0, z0.s62; CHECK-NEXT: ret63 %out = call <vscale x 4 x i8> @llvm.experimental.vector.compress(<vscale x 4 x i8> %vec, <vscale x 4 x i1> %mask, <vscale x 4 x i8> poison)64 ret <vscale x 4 x i8> %out65}66 67define <vscale x 4 x i16> @test_compress_nxv4i16(<vscale x 4 x i16> %vec, <vscale x 4 x i1> %mask) {68; CHECK-LABEL: test_compress_nxv4i16:69; CHECK: // %bb.0:70; CHECK-NEXT: compact z0.s, p0, z0.s71; CHECK-NEXT: ret72 %out = call <vscale x 4 x i16> @llvm.experimental.vector.compress(<vscale x 4 x i16> %vec, <vscale x 4 x i1> %mask, <vscale x 4 x i16> poison)73 ret <vscale x 4 x i16> %out74}75 76define <vscale x 4 x i32> @test_compress_nxv4i32(<vscale x 4 x i32> %vec, <vscale x 4 x i1> %mask) {77; CHECK-LABEL: test_compress_nxv4i32:78; CHECK: // %bb.0:79; CHECK-NEXT: compact z0.s, p0, z0.s80; CHECK-NEXT: ret81 %out = call <vscale x 4 x i32> @llvm.experimental.vector.compress(<vscale x 4 x i32> %vec, <vscale x 4 x i1> %mask, <vscale x 4 x i32> poison)82 ret <vscale x 4 x i32> %out83}84 85define <vscale x 4 x float> @test_compress_nxv4f32(<vscale x 4 x float> %vec, <vscale x 4 x i1> %mask) {86; CHECK-LABEL: test_compress_nxv4f32:87; CHECK: // %bb.0:88; CHECK-NEXT: compact z0.s, p0, z0.s89; CHECK-NEXT: ret90 %out = call <vscale x 4 x float> @llvm.experimental.vector.compress(<vscale x 4 x float> %vec, <vscale x 4 x i1> %mask, <vscale x 4 x float> poison)91 ret <vscale x 4 x float> %out92}93 94define <vscale x 4 x i4> @test_compress_illegal_element_type(<vscale x 4 x i4> %vec, <vscale x 4 x i1> %mask) {95; CHECK-LABEL: test_compress_illegal_element_type:96; CHECK: // %bb.0:97; CHECK-NEXT: compact z0.s, p0, z0.s98; CHECK-NEXT: ret99 %out = call <vscale x 4 x i4> @llvm.experimental.vector.compress(<vscale x 4 x i4> %vec, <vscale x 4 x i1> %mask, <vscale x 4 x i4> poison)100 ret <vscale x 4 x i4> %out101}102 103define <vscale x 4 x i32> @test_compress_knownbits_zext(<vscale x 4 x i16> %vec, <vscale x 4 x i1> %mask, <vscale x 4 x i32> %passthru) nounwind {104; CHECK-LABEL: test_compress_knownbits_zext:105; CHECK: // %bb.0:106; CHECK-NEXT: and z0.s, z0.s, #0xffff107; CHECK-NEXT: cntp x8, p0, p0.s108; CHECK-NEXT: and z1.s, z1.s, #0x3109; CHECK-NEXT: compact z0.s, p0, z0.s110; CHECK-NEXT: whilelo p0.s, xzr, x8111; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s112; CHECK-NEXT: ret113 %xvec = zext <vscale x 4 x i16> %vec to <vscale x 4 x i32>114 %xpassthru = and <vscale x 4 x i32> %passthru, splat (i32 3)115 %out = call <vscale x 4 x i32> @llvm.experimental.vector.compress(<vscale x 4 x i32> %xvec, <vscale x 4 x i1> %mask, <vscale x 4 x i32> %xpassthru)116 %res = and <vscale x 4 x i32> %out, splat (i32 65535)117 ret <vscale x 4 x i32> %res118}119 120define <vscale x 4 x i32> @test_compress_numsignbits_sext(<vscale x 4 x i16> %vec, <vscale x 4 x i1> %mask, <vscale x 4 x i32> %passthru) nounwind {121; CHECK-LABEL: test_compress_numsignbits_sext:122; CHECK: // %bb.0:123; CHECK-NEXT: ptrue p1.s124; CHECK-NEXT: and z1.s, z1.s, #0x3125; CHECK-NEXT: cntp x8, p0, p0.s126; CHECK-NEXT: sxth z0.s, p1/m, z0.s127; CHECK-NEXT: compact z0.s, p0, z0.s128; CHECK-NEXT: whilelo p0.s, xzr, x8129; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s130; CHECK-NEXT: ret131 %xvec = sext <vscale x 4 x i16> %vec to <vscale x 4 x i32>132 %xpassthru = and <vscale x 4 x i32> %passthru, splat (i32 3)133 %out = call <vscale x 4 x i32> @llvm.experimental.vector.compress(<vscale x 4 x i32> %xvec, <vscale x 4 x i1> %mask, <vscale x 4 x i32> %xpassthru)134 %shl = shl <vscale x 4 x i32> %out, splat (i32 16)135 %res = ashr <vscale x 4 x i32> %shl, splat (i32 16)136 ret <vscale x 4 x i32> %res137}138 139define <vscale x 8 x i32> @test_compress_large(<vscale x 8 x i32> %vec, <vscale x 8 x i1> %mask) {140; CHECK-LABEL: test_compress_large:141; CHECK: // %bb.0:142; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill143; CHECK-NEXT: addvl sp, sp, #-2144; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG145; CHECK-NEXT: .cfi_offset w29, -16146; CHECK-NEXT: punpklo p1.h, p0.b147; CHECK-NEXT: cnth x9148; CHECK-NEXT: ptrue p2.s149; CHECK-NEXT: sub x9, x9, #1150; CHECK-NEXT: punpkhi p0.h, p0.b151; CHECK-NEXT: compact z0.s, p1, z0.s152; CHECK-NEXT: cntp x8, p2, p1.s153; CHECK-NEXT: compact z1.s, p0, z1.s154; CHECK-NEXT: str z0, [sp]155; CHECK-NEXT: cmp x8, x9156; CHECK-NEXT: csel x8, x8, x9, lo157; CHECK-NEXT: mov x9, sp158; CHECK-NEXT: st1w { z1.s }, p2, [x9, x8, lsl #2]159; CHECK-NEXT: ldr z0, [sp]160; CHECK-NEXT: ldr z1, [sp, #1, mul vl]161; CHECK-NEXT: addvl sp, sp, #2162; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload163; CHECK-NEXT: ret164 %out = call <vscale x 8 x i32> @llvm.experimental.vector.compress(<vscale x 8 x i32> %vec, <vscale x 8 x i1> %mask, <vscale x 8 x i32> poison)165 ret <vscale x 8 x i32> %out166}167 168; We pass a placeholder value for the const_mask* tests to check that they are converted to a no-op by simply copying169; the second vector input register to the ret register or doing nothing.170define <vscale x 4 x i32> @test_compress_const_splat1_mask(<vscale x 4 x i32> %ignore, <vscale x 4 x i32> %vec) {171; CHECK-LABEL: test_compress_const_splat1_mask:172; CHECK: // %bb.0:173; CHECK-NEXT: mov z0.d, z1.d174; CHECK-NEXT: ret175 %out = call <vscale x 4 x i32> @llvm.experimental.vector.compress(<vscale x 4 x i32> %vec, <vscale x 4 x i1> splat (i1 -1), <vscale x 4 x i32> poison)176 ret <vscale x 4 x i32> %out177}178define <vscale x 4 x i32> @test_compress_const_splat0_mask(<vscale x 4 x i32> %ignore, <vscale x 4 x i32> %vec) {179; CHECK-LABEL: test_compress_const_splat0_mask:180; CHECK: // %bb.0:181; CHECK-NEXT: ret182 %out = call <vscale x 4 x i32> @llvm.experimental.vector.compress(<vscale x 4 x i32> %vec, <vscale x 4 x i1> splat (i1 0), <vscale x 4 x i32> poison)183 ret <vscale x 4 x i32> %out184}185define <vscale x 4 x i32> @test_compress_poison_mask(<vscale x 4 x i32> %ignore, <vscale x 4 x i32> %vec) {186; CHECK-LABEL: test_compress_poison_mask:187; CHECK: // %bb.0:188; CHECK-NEXT: ret189 %out = call <vscale x 4 x i32> @llvm.experimental.vector.compress(<vscale x 4 x i32> %vec, <vscale x 4 x i1> poison, <vscale x 4 x i32> poison)190 ret <vscale x 4 x i32> %out191}192 193define <4 x i32> @test_compress_v4i32_with_sve(<4 x i32> %vec, <4 x i1> %mask) {194; CHECK-LABEL: test_compress_v4i32_with_sve:195; CHECK: // %bb.0:196; CHECK-NEXT: ushll v1.4s, v1.4h, #0197; CHECK-NEXT: ptrue p0.s198; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0199; CHECK-NEXT: shl v1.4s, v1.4s, #31200; CHECK-NEXT: cmlt v1.4s, v1.4s, #0201; CHECK-NEXT: and z1.s, z1.s, #0x1202; CHECK-NEXT: cmpne p0.s, p0/z, z1.s, #0203; CHECK-NEXT: compact z0.s, p0, z0.s204; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0205; CHECK-NEXT: ret206 %out = call <4 x i32> @llvm.experimental.vector.compress(<4 x i32> %vec, <4 x i1> %mask, <4 x i32> poison)207 ret <4 x i32> %out208}209 210define <1 x i32> @test_compress_v1i32_with_sve(<1 x i32> %vec, <1 x i1> %mask) {211; CHECK-LABEL: test_compress_v1i32_with_sve:212; CHECK: // %bb.0:213; CHECK-NEXT: movi v1.2d, #0000000000000000214; CHECK-NEXT: sbfx w8, w0, #0, #1215; CHECK-NEXT: ptrue p0.d216; CHECK-NEXT: ushll v0.2d, v0.2s, #0217; CHECK-NEXT: mov v1.s[0], w8218; CHECK-NEXT: ushll v1.2d, v1.2s, #0219; CHECK-NEXT: and z1.d, z1.d, #0x1220; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0221; CHECK-NEXT: compact z0.d, p0, z0.d222; CHECK-NEXT: xtn v0.2s, v0.2d223; CHECK-NEXT: ret224 %out = call <1 x i32> @llvm.experimental.vector.compress(<1 x i32> %vec, <1 x i1> %mask, <1 x i32> poison)225 ret <1 x i32> %out226}227 228define <4 x double> @test_compress_v4f64_with_sve(<4 x double> %vec, <4 x i1> %mask) {229; CHECK-LABEL: test_compress_v4f64_with_sve:230; CHECK: // %bb.0:231; CHECK-NEXT: sub sp, sp, #32232; CHECK-NEXT: .cfi_def_cfa_offset 32233; CHECK-NEXT: ushll v2.4s, v2.4h, #0234; CHECK-NEXT: ptrue p0.d235; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0236; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1237; CHECK-NEXT: ushll v3.2d, v2.2s, #0238; CHECK-NEXT: ushll2 v4.2d, v2.4s, #0239; CHECK-NEXT: fmov x8, d2240; CHECK-NEXT: shl v3.2d, v3.2d, #63241; CHECK-NEXT: shl v4.2d, v4.2d, #63242; CHECK-NEXT: lsr x9, x8, #32243; CHECK-NEXT: eor w8, w8, w9244; CHECK-NEXT: mov x9, sp245; CHECK-NEXT: cmlt v3.2d, v3.2d, #0246; CHECK-NEXT: cmlt v4.2d, v4.2d, #0247; CHECK-NEXT: and x8, x8, #0x3248; CHECK-NEXT: lsl x8, x8, #3249; CHECK-NEXT: and z3.d, z3.d, #0x1250; CHECK-NEXT: and z4.d, z4.d, #0x1251; CHECK-NEXT: cmpne p1.d, p0/z, z3.d, #0252; CHECK-NEXT: cmpne p0.d, p0/z, z4.d, #0253; CHECK-NEXT: compact z0.d, p1, z0.d254; CHECK-NEXT: compact z1.d, p0, z1.d255; CHECK-NEXT: str q0, [sp]256; CHECK-NEXT: str q1, [x9, x8]257; CHECK-NEXT: ldp q0, q1, [sp], #32258; CHECK-NEXT: ret259 %out = call <4 x double> @llvm.experimental.vector.compress(<4 x double> %vec, <4 x i1> %mask, <4 x double> poison)260 ret <4 x double> %out261}262 263define <2 x i16> @test_compress_v2i16_with_sve(<2 x i16> %vec, <2 x i1> %mask) {264; CHECK-LABEL: test_compress_v2i16_with_sve:265; CHECK: // %bb.0:266; CHECK-NEXT: ushll v1.2d, v1.2s, #0267; CHECK-NEXT: ptrue p0.d268; CHECK-NEXT: ushll v0.2d, v0.2s, #0269; CHECK-NEXT: and z1.d, z1.d, #0x1270; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0271; CHECK-NEXT: compact z0.d, p0, z0.d272; CHECK-NEXT: xtn v0.2s, v0.2d273; CHECK-NEXT: ret274 %out = call <2 x i16> @llvm.experimental.vector.compress(<2 x i16> %vec, <2 x i1> %mask, <2 x i16> poison)275 ret <2 x i16> %out276}277 278 279define <vscale x 4 x i32> @test_compress_nxv4i32_with_passthru(<vscale x 4 x i32> %vec, <vscale x 4 x i1> %mask, <vscale x 4 x i32> %passthru) {280; CHECK-LABEL: test_compress_nxv4i32_with_passthru:281; CHECK: // %bb.0:282; CHECK-NEXT: cntp x8, p0, p0.s283; CHECK-NEXT: compact z0.s, p0, z0.s284; CHECK-NEXT: whilelo p0.s, xzr, x8285; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s286; CHECK-NEXT: ret287 %out = call <vscale x 4 x i32> @llvm.experimental.vector.compress(<vscale x 4 x i32> %vec, <vscale x 4 x i1> %mask, <vscale x 4 x i32> %passthru)288 ret <vscale x 4 x i32> %out289}290 291define <vscale x 4 x i32> @test_compress_nxv4i32_with_zero_passthru(<vscale x 4 x i32> %vec, <vscale x 4 x i1> %mask) {292; CHECK-LABEL: test_compress_nxv4i32_with_zero_passthru:293; CHECK: // %bb.0:294; CHECK-NEXT: compact z0.s, p0, z0.s295; CHECK-NEXT: ret296 %out = call <vscale x 4 x i32> @llvm.experimental.vector.compress(<vscale x 4 x i32> %vec, <vscale x 4 x i1> %mask, <vscale x 4 x i32> splat(i32 0))297 ret <vscale x 4 x i32> %out298}299 300define <vscale x 4 x i32> @test_compress_nxv4i32_with_const_passthru(<vscale x 4 x i32> %vec, <vscale x 4 x i1> %mask) {301; CHECK-LABEL: test_compress_nxv4i32_with_const_passthru:302; CHECK: // %bb.0:303; CHECK-NEXT: cntp x8, p0, p0.s304; CHECK-NEXT: compact z0.s, p0, z0.s305; CHECK-NEXT: mov z1.s, #5 // =0x5306; CHECK-NEXT: whilelo p0.s, xzr, x8307; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s308; CHECK-NEXT: ret309 %out = call <vscale x 4 x i32> @llvm.experimental.vector.compress(<vscale x 4 x i32> %vec, <vscale x 4 x i1> %mask, <vscale x 4 x i32> splat(i32 5))310 ret <vscale x 4 x i32> %out311}312