512 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 32; RUN: llc -mtriple=aarch64 -mattr=+bf16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,NEON-FIXED3; RUN: llc -mtriple=aarch64 -mattr=+sve,+bf16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,SVE-FIXED4 5define i8 @extract_last_i8(<16 x i8> %data, <16 x i8> %mask, i8 %passthru) {6; NEON-FIXED-LABEL: extract_last_i8:7; NEON-FIXED: // %bb.0:8; NEON-FIXED-NEXT: sub sp, sp, #169; NEON-FIXED-NEXT: .cfi_def_cfa_offset 1610; NEON-FIXED-NEXT: cmeq v2.16b, v1.16b, #011; NEON-FIXED-NEXT: adrp x8, .LCPI0_012; NEON-FIXED-NEXT: cmtst v1.16b, v1.16b, v1.16b13; NEON-FIXED-NEXT: ldr q3, [x8, :lo12:.LCPI0_0]14; NEON-FIXED-NEXT: mov x9, sp15; NEON-FIXED-NEXT: str q0, [sp]16; NEON-FIXED-NEXT: bic v2.16b, v3.16b, v2.16b17; NEON-FIXED-NEXT: umaxv b1, v1.16b18; NEON-FIXED-NEXT: umaxv b2, v2.16b19; NEON-FIXED-NEXT: fmov w8, s220; NEON-FIXED-NEXT: bfxil x9, x8, #0, #421; NEON-FIXED-NEXT: ldrb w8, [x9]22; NEON-FIXED-NEXT: fmov w9, s123; NEON-FIXED-NEXT: tst w9, #0x124; NEON-FIXED-NEXT: csel w0, w8, w0, ne25; NEON-FIXED-NEXT: add sp, sp, #1626; NEON-FIXED-NEXT: ret27;28; SVE-FIXED-LABEL: extract_last_i8:29; SVE-FIXED: // %bb.0:30; SVE-FIXED-NEXT: sub sp, sp, #1631; SVE-FIXED-NEXT: .cfi_def_cfa_offset 1632; SVE-FIXED-NEXT: index z2.b, #0, #133; SVE-FIXED-NEXT: cmeq v3.16b, v1.16b, #034; SVE-FIXED-NEXT: cmtst v1.16b, v1.16b, v1.16b35; SVE-FIXED-NEXT: mov x9, sp36; SVE-FIXED-NEXT: str q0, [sp]37; SVE-FIXED-NEXT: bic v2.16b, v2.16b, v3.16b38; SVE-FIXED-NEXT: umaxv b1, v1.16b39; SVE-FIXED-NEXT: umaxv b2, v2.16b40; SVE-FIXED-NEXT: fmov w8, s241; SVE-FIXED-NEXT: bfxil x9, x8, #0, #442; SVE-FIXED-NEXT: ldrb w8, [x9]43; SVE-FIXED-NEXT: fmov w9, s144; SVE-FIXED-NEXT: tst w9, #0x145; SVE-FIXED-NEXT: csel w0, w8, w0, ne46; SVE-FIXED-NEXT: add sp, sp, #1647; SVE-FIXED-NEXT: ret48 %notzero = icmp ne <16 x i8> %mask, zeroinitializer49 %res = call i8 @llvm.experimental.vector.extract.last.active.v16i8(<16 x i8> %data, <16 x i1> %notzero, i8 %passthru)50 ret i8 %res51}52 53define i16 @extract_last_i16(<8 x i16> %data, <8 x i16> %mask, i16 %passthru) {54; NEON-FIXED-LABEL: extract_last_i16:55; NEON-FIXED: // %bb.0:56; NEON-FIXED-NEXT: sub sp, sp, #1657; NEON-FIXED-NEXT: .cfi_def_cfa_offset 1658; NEON-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h59; NEON-FIXED-NEXT: adrp x8, .LCPI1_060; NEON-FIXED-NEXT: mov x9, sp61; NEON-FIXED-NEXT: ldr d3, [x8, :lo12:.LCPI1_0]62; NEON-FIXED-NEXT: str q0, [sp]63; NEON-FIXED-NEXT: xtn v2.8b, v1.8h64; NEON-FIXED-NEXT: umaxv h1, v1.8h65; NEON-FIXED-NEXT: and v2.8b, v2.8b, v3.8b66; NEON-FIXED-NEXT: umaxv b2, v2.8b67; NEON-FIXED-NEXT: fmov w8, s268; NEON-FIXED-NEXT: bfi x9, x8, #1, #369; NEON-FIXED-NEXT: ldrh w8, [x9]70; NEON-FIXED-NEXT: fmov w9, s171; NEON-FIXED-NEXT: tst w9, #0x172; NEON-FIXED-NEXT: csel w0, w8, w0, ne73; NEON-FIXED-NEXT: add sp, sp, #1674; NEON-FIXED-NEXT: ret75;76; SVE-FIXED-LABEL: extract_last_i16:77; SVE-FIXED: // %bb.0:78; SVE-FIXED-NEXT: sub sp, sp, #1679; SVE-FIXED-NEXT: .cfi_def_cfa_offset 1680; SVE-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h81; SVE-FIXED-NEXT: index z3.b, #0, #182; SVE-FIXED-NEXT: mov x9, sp83; SVE-FIXED-NEXT: str q0, [sp]84; SVE-FIXED-NEXT: xtn v2.8b, v1.8h85; SVE-FIXED-NEXT: umaxv h1, v1.8h86; SVE-FIXED-NEXT: and v2.8b, v2.8b, v3.8b87; SVE-FIXED-NEXT: umaxv b2, v2.8b88; SVE-FIXED-NEXT: fmov w8, s289; SVE-FIXED-NEXT: bfi x9, x8, #1, #390; SVE-FIXED-NEXT: ldrh w8, [x9]91; SVE-FIXED-NEXT: fmov w9, s192; SVE-FIXED-NEXT: tst w9, #0x193; SVE-FIXED-NEXT: csel w0, w8, w0, ne94; SVE-FIXED-NEXT: add sp, sp, #1695; SVE-FIXED-NEXT: ret96 %notzero = icmp ne <8 x i16> %mask, zeroinitializer97 %res = call i16 @llvm.experimental.vector.extract.last.active.v8i16(<8 x i16> %data, <8 x i1> %notzero, i16 %passthru)98 ret i16 %res99}100 101define i32 @extract_last_i32(<4 x i32> %data, <4 x i32> %mask, i32 %passthru) {102; NEON-FIXED-LABEL: extract_last_i32:103; NEON-FIXED: // %bb.0:104; NEON-FIXED-NEXT: sub sp, sp, #16105; NEON-FIXED-NEXT: .cfi_def_cfa_offset 16106; NEON-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s107; NEON-FIXED-NEXT: adrp x8, .LCPI2_0108; NEON-FIXED-NEXT: mov x9, sp109; NEON-FIXED-NEXT: ldr d3, [x8, :lo12:.LCPI2_0]110; NEON-FIXED-NEXT: str q0, [sp]111; NEON-FIXED-NEXT: xtn v2.4h, v1.4s112; NEON-FIXED-NEXT: umaxv s1, v1.4s113; NEON-FIXED-NEXT: and v2.8b, v2.8b, v3.8b114; NEON-FIXED-NEXT: umaxv h2, v2.4h115; NEON-FIXED-NEXT: fmov w8, s2116; NEON-FIXED-NEXT: bfi x9, x8, #2, #2117; NEON-FIXED-NEXT: ldr w8, [x9]118; NEON-FIXED-NEXT: fmov w9, s1119; NEON-FIXED-NEXT: tst w9, #0x1120; NEON-FIXED-NEXT: csel w0, w8, w0, ne121; NEON-FIXED-NEXT: add sp, sp, #16122; NEON-FIXED-NEXT: ret123;124; SVE-FIXED-LABEL: extract_last_i32:125; SVE-FIXED: // %bb.0:126; SVE-FIXED-NEXT: sub sp, sp, #16127; SVE-FIXED-NEXT: .cfi_def_cfa_offset 16128; SVE-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s129; SVE-FIXED-NEXT: index z3.h, #0, #1130; SVE-FIXED-NEXT: mov x9, sp131; SVE-FIXED-NEXT: str q0, [sp]132; SVE-FIXED-NEXT: xtn v2.4h, v1.4s133; SVE-FIXED-NEXT: umaxv s1, v1.4s134; SVE-FIXED-NEXT: and v2.8b, v2.8b, v3.8b135; SVE-FIXED-NEXT: umaxv h2, v2.4h136; SVE-FIXED-NEXT: fmov w8, s2137; SVE-FIXED-NEXT: bfi x9, x8, #2, #2138; SVE-FIXED-NEXT: ldr w8, [x9]139; SVE-FIXED-NEXT: fmov w9, s1140; SVE-FIXED-NEXT: tst w9, #0x1141; SVE-FIXED-NEXT: csel w0, w8, w0, ne142; SVE-FIXED-NEXT: add sp, sp, #16143; SVE-FIXED-NEXT: ret144 %notzero = icmp ne <4 x i32> %mask, zeroinitializer145 %res = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> %data, <4 x i1> %notzero, i32 %passthru)146 ret i32 %res147}148 149define i64 @extract_last_i64(<2 x i64> %data, <2 x i64> %mask, i64 %passthru) {150; NEON-FIXED-LABEL: extract_last_i64:151; NEON-FIXED: // %bb.0:152; NEON-FIXED-NEXT: sub sp, sp, #16153; NEON-FIXED-NEXT: .cfi_def_cfa_offset 16154; NEON-FIXED-NEXT: cmtst v1.2d, v1.2d, v1.2d155; NEON-FIXED-NEXT: adrp x8, .LCPI3_0156; NEON-FIXED-NEXT: mov x9, sp157; NEON-FIXED-NEXT: ldr d3, [x8, :lo12:.LCPI3_0]158; NEON-FIXED-NEXT: str q0, [sp]159; NEON-FIXED-NEXT: xtn v2.2s, v1.2d160; NEON-FIXED-NEXT: umaxv s1, v1.4s161; NEON-FIXED-NEXT: and v2.8b, v2.8b, v3.8b162; NEON-FIXED-NEXT: umaxp v2.2s, v2.2s, v2.2s163; NEON-FIXED-NEXT: fmov w8, s2164; NEON-FIXED-NEXT: bfi x9, x8, #3, #1165; NEON-FIXED-NEXT: ldr x8, [x9]166; NEON-FIXED-NEXT: fmov w9, s1167; NEON-FIXED-NEXT: tst w9, #0x1168; NEON-FIXED-NEXT: csel x0, x8, x0, ne169; NEON-FIXED-NEXT: add sp, sp, #16170; NEON-FIXED-NEXT: ret171;172; SVE-FIXED-LABEL: extract_last_i64:173; SVE-FIXED: // %bb.0:174; SVE-FIXED-NEXT: sub sp, sp, #16175; SVE-FIXED-NEXT: .cfi_def_cfa_offset 16176; SVE-FIXED-NEXT: cmtst v1.2d, v1.2d, v1.2d177; SVE-FIXED-NEXT: index z3.s, #0, #1178; SVE-FIXED-NEXT: mov x9, sp179; SVE-FIXED-NEXT: str q0, [sp]180; SVE-FIXED-NEXT: xtn v2.2s, v1.2d181; SVE-FIXED-NEXT: umaxv s1, v1.4s182; SVE-FIXED-NEXT: and v2.8b, v2.8b, v3.8b183; SVE-FIXED-NEXT: umaxp v2.2s, v2.2s, v2.2s184; SVE-FIXED-NEXT: fmov w8, s2185; SVE-FIXED-NEXT: bfi x9, x8, #3, #1186; SVE-FIXED-NEXT: ldr x8, [x9]187; SVE-FIXED-NEXT: fmov w9, s1188; SVE-FIXED-NEXT: tst w9, #0x1189; SVE-FIXED-NEXT: csel x0, x8, x0, ne190; SVE-FIXED-NEXT: add sp, sp, #16191; SVE-FIXED-NEXT: ret192 %notzero = icmp ne <2 x i64> %mask, zeroinitializer193 %res = call i64 @llvm.experimental.vector.extract.last.active.v2i64(<2 x i64> %data, <2 x i1> %notzero, i64 %passthru)194 ret i64 %res195}196 197define half @extract_last_half(<8 x half> %data, <8 x i16> %mask, half %passthru) {198; NEON-FIXED-LABEL: extract_last_half:199; NEON-FIXED: // %bb.0:200; NEON-FIXED-NEXT: sub sp, sp, #16201; NEON-FIXED-NEXT: .cfi_def_cfa_offset 16202; NEON-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h203; NEON-FIXED-NEXT: adrp x8, .LCPI4_0204; NEON-FIXED-NEXT: mov x9, sp205; NEON-FIXED-NEXT: ldr d4, [x8, :lo12:.LCPI4_0]206; NEON-FIXED-NEXT: str q0, [sp]207; NEON-FIXED-NEXT: // kill: def $h2 killed $h2 def $s2208; NEON-FIXED-NEXT: xtn v3.8b, v1.8h209; NEON-FIXED-NEXT: umaxv h1, v1.8h210; NEON-FIXED-NEXT: and v3.8b, v3.8b, v4.8b211; NEON-FIXED-NEXT: umaxv b3, v3.8b212; NEON-FIXED-NEXT: fmov w8, s3213; NEON-FIXED-NEXT: bfi x9, x8, #1, #3214; NEON-FIXED-NEXT: fmov w8, s1215; NEON-FIXED-NEXT: ldr h0, [x9]216; NEON-FIXED-NEXT: tst w8, #0x1217; NEON-FIXED-NEXT: fcsel s0, s0, s2, ne218; NEON-FIXED-NEXT: // kill: def $h0 killed $h0 killed $s0219; NEON-FIXED-NEXT: add sp, sp, #16220; NEON-FIXED-NEXT: ret221;222; SVE-FIXED-LABEL: extract_last_half:223; SVE-FIXED: // %bb.0:224; SVE-FIXED-NEXT: sub sp, sp, #16225; SVE-FIXED-NEXT: .cfi_def_cfa_offset 16226; SVE-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h227; SVE-FIXED-NEXT: index z4.b, #0, #1228; SVE-FIXED-NEXT: mov x9, sp229; SVE-FIXED-NEXT: str q0, [sp]230; SVE-FIXED-NEXT: xtn v3.8b, v1.8h231; SVE-FIXED-NEXT: umaxv h1, v1.8h232; SVE-FIXED-NEXT: and v3.8b, v3.8b, v4.8b233; SVE-FIXED-NEXT: umaxv b3, v3.8b234; SVE-FIXED-NEXT: fmov w8, s3235; SVE-FIXED-NEXT: bfi x9, x8, #1, #3236; SVE-FIXED-NEXT: fmov w8, s1237; SVE-FIXED-NEXT: ldr h0, [x9]238; SVE-FIXED-NEXT: tst w8, #0x1239; SVE-FIXED-NEXT: fcsel h0, h0, h2, ne240; SVE-FIXED-NEXT: add sp, sp, #16241; SVE-FIXED-NEXT: ret242 %notzero = icmp ne <8 x i16> %mask, zeroinitializer243 %res = call half @llvm.experimental.vector.extract.last.active.v8f16(<8 x half> %data, <8 x i1> %notzero, half %passthru)244 ret half %res245}246 247define bfloat @extract_last_bfloat(<8 x bfloat> %data, <8 x i16> %mask, bfloat %passthru) {248; NEON-FIXED-LABEL: extract_last_bfloat:249; NEON-FIXED: // %bb.0:250; NEON-FIXED-NEXT: sub sp, sp, #16251; NEON-FIXED-NEXT: .cfi_def_cfa_offset 16252; NEON-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h253; NEON-FIXED-NEXT: adrp x8, .LCPI5_0254; NEON-FIXED-NEXT: mov x9, sp255; NEON-FIXED-NEXT: ldr d4, [x8, :lo12:.LCPI5_0]256; NEON-FIXED-NEXT: str q0, [sp]257; NEON-FIXED-NEXT: // kill: def $h2 killed $h2 def $s2258; NEON-FIXED-NEXT: xtn v3.8b, v1.8h259; NEON-FIXED-NEXT: umaxv h1, v1.8h260; NEON-FIXED-NEXT: and v3.8b, v3.8b, v4.8b261; NEON-FIXED-NEXT: umaxv b3, v3.8b262; NEON-FIXED-NEXT: fmov w8, s3263; NEON-FIXED-NEXT: bfi x9, x8, #1, #3264; NEON-FIXED-NEXT: fmov w8, s1265; NEON-FIXED-NEXT: ldr h0, [x9]266; NEON-FIXED-NEXT: tst w8, #0x1267; NEON-FIXED-NEXT: fcsel s0, s0, s2, ne268; NEON-FIXED-NEXT: // kill: def $h0 killed $h0 killed $s0269; NEON-FIXED-NEXT: add sp, sp, #16270; NEON-FIXED-NEXT: ret271;272; SVE-FIXED-LABEL: extract_last_bfloat:273; SVE-FIXED: // %bb.0:274; SVE-FIXED-NEXT: sub sp, sp, #16275; SVE-FIXED-NEXT: .cfi_def_cfa_offset 16276; SVE-FIXED-NEXT: cmtst v1.8h, v1.8h, v1.8h277; SVE-FIXED-NEXT: index z4.b, #0, #1278; SVE-FIXED-NEXT: mov x9, sp279; SVE-FIXED-NEXT: str q0, [sp]280; SVE-FIXED-NEXT: xtn v3.8b, v1.8h281; SVE-FIXED-NEXT: umaxv h1, v1.8h282; SVE-FIXED-NEXT: and v3.8b, v3.8b, v4.8b283; SVE-FIXED-NEXT: umaxv b3, v3.8b284; SVE-FIXED-NEXT: fmov w8, s3285; SVE-FIXED-NEXT: bfi x9, x8, #1, #3286; SVE-FIXED-NEXT: fmov w8, s1287; SVE-FIXED-NEXT: ldr h0, [x9]288; SVE-FIXED-NEXT: tst w8, #0x1289; SVE-FIXED-NEXT: fcsel h0, h0, h2, ne290; SVE-FIXED-NEXT: add sp, sp, #16291; SVE-FIXED-NEXT: ret292 %notzero = icmp ne <8 x i16> %mask, zeroinitializer293 %res = call bfloat @llvm.experimental.vector.extract.last.active.v8bf16(<8 x bfloat> %data, <8 x i1> %notzero, bfloat %passthru)294 ret bfloat %res295}296 297define float @extract_last_float(<4 x float> %data, <4 x i32> %mask, float %passthru) {298; NEON-FIXED-LABEL: extract_last_float:299; NEON-FIXED: // %bb.0:300; NEON-FIXED-NEXT: sub sp, sp, #16301; NEON-FIXED-NEXT: .cfi_def_cfa_offset 16302; NEON-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s303; NEON-FIXED-NEXT: adrp x8, .LCPI6_0304; NEON-FIXED-NEXT: mov x9, sp305; NEON-FIXED-NEXT: ldr d4, [x8, :lo12:.LCPI6_0]306; NEON-FIXED-NEXT: str q0, [sp]307; NEON-FIXED-NEXT: xtn v3.4h, v1.4s308; NEON-FIXED-NEXT: umaxv s1, v1.4s309; NEON-FIXED-NEXT: and v3.8b, v3.8b, v4.8b310; NEON-FIXED-NEXT: umaxv h3, v3.4h311; NEON-FIXED-NEXT: fmov w8, s3312; NEON-FIXED-NEXT: bfi x9, x8, #2, #2313; NEON-FIXED-NEXT: fmov w8, s1314; NEON-FIXED-NEXT: ldr s0, [x9]315; NEON-FIXED-NEXT: tst w8, #0x1316; NEON-FIXED-NEXT: fcsel s0, s0, s2, ne317; NEON-FIXED-NEXT: add sp, sp, #16318; NEON-FIXED-NEXT: ret319;320; SVE-FIXED-LABEL: extract_last_float:321; SVE-FIXED: // %bb.0:322; SVE-FIXED-NEXT: sub sp, sp, #16323; SVE-FIXED-NEXT: .cfi_def_cfa_offset 16324; SVE-FIXED-NEXT: cmtst v1.4s, v1.4s, v1.4s325; SVE-FIXED-NEXT: index z4.h, #0, #1326; SVE-FIXED-NEXT: mov x9, sp327; SVE-FIXED-NEXT: str q0, [sp]328; SVE-FIXED-NEXT: xtn v3.4h, v1.4s329; SVE-FIXED-NEXT: umaxv s1, v1.4s330; SVE-FIXED-NEXT: and v3.8b, v3.8b, v4.8b331; SVE-FIXED-NEXT: umaxv h3, v3.4h332; SVE-FIXED-NEXT: fmov w8, s3333; SVE-FIXED-NEXT: bfi x9, x8, #2, #2334; SVE-FIXED-NEXT: fmov w8, s1335; SVE-FIXED-NEXT: ldr s0, [x9]336; SVE-FIXED-NEXT: tst w8, #0x1337; SVE-FIXED-NEXT: fcsel s0, s0, s2, ne338; SVE-FIXED-NEXT: add sp, sp, #16339; SVE-FIXED-NEXT: ret340 %notzero = icmp ne <4 x i32> %mask, zeroinitializer341 %res = call float @llvm.experimental.vector.extract.last.active.v4f32(<4 x float> %data, <4 x i1> %notzero, float %passthru)342 ret float %res343}344 345define double @extract_last_double(<2 x double> %data, <2 x i64> %mask, double %passthru) {346; NEON-FIXED-LABEL: extract_last_double:347; NEON-FIXED: // %bb.0:348; NEON-FIXED-NEXT: sub sp, sp, #16349; NEON-FIXED-NEXT: .cfi_def_cfa_offset 16350; NEON-FIXED-NEXT: cmtst v1.2d, v1.2d, v1.2d351; NEON-FIXED-NEXT: adrp x8, .LCPI7_0352; NEON-FIXED-NEXT: mov x9, sp353; NEON-FIXED-NEXT: ldr d4, [x8, :lo12:.LCPI7_0]354; NEON-FIXED-NEXT: str q0, [sp]355; NEON-FIXED-NEXT: xtn v3.2s, v1.2d356; NEON-FIXED-NEXT: umaxv s1, v1.4s357; NEON-FIXED-NEXT: and v3.8b, v3.8b, v4.8b358; NEON-FIXED-NEXT: umaxp v3.2s, v3.2s, v3.2s359; NEON-FIXED-NEXT: fmov w8, s3360; NEON-FIXED-NEXT: bfi x9, x8, #3, #1361; NEON-FIXED-NEXT: fmov w8, s1362; NEON-FIXED-NEXT: ldr d0, [x9]363; NEON-FIXED-NEXT: tst w8, #0x1364; NEON-FIXED-NEXT: fcsel d0, d0, d2, ne365; NEON-FIXED-NEXT: add sp, sp, #16366; NEON-FIXED-NEXT: ret367;368; SVE-FIXED-LABEL: extract_last_double:369; SVE-FIXED: // %bb.0:370; SVE-FIXED-NEXT: sub sp, sp, #16371; SVE-FIXED-NEXT: .cfi_def_cfa_offset 16372; SVE-FIXED-NEXT: cmtst v1.2d, v1.2d, v1.2d373; SVE-FIXED-NEXT: index z4.s, #0, #1374; SVE-FIXED-NEXT: mov x9, sp375; SVE-FIXED-NEXT: str q0, [sp]376; SVE-FIXED-NEXT: xtn v3.2s, v1.2d377; SVE-FIXED-NEXT: umaxv s1, v1.4s378; SVE-FIXED-NEXT: and v3.8b, v3.8b, v4.8b379; SVE-FIXED-NEXT: umaxp v3.2s, v3.2s, v3.2s380; SVE-FIXED-NEXT: fmov w8, s3381; SVE-FIXED-NEXT: bfi x9, x8, #3, #1382; SVE-FIXED-NEXT: fmov w8, s1383; SVE-FIXED-NEXT: ldr d0, [x9]384; SVE-FIXED-NEXT: tst w8, #0x1385; SVE-FIXED-NEXT: fcsel d0, d0, d2, ne386; SVE-FIXED-NEXT: add sp, sp, #16387; SVE-FIXED-NEXT: ret388 %notzero = icmp ne <2 x i64> %mask, zeroinitializer389 %res = call double @llvm.experimental.vector.extract.last.active.v2f64(<2 x double> %data, <2 x i1> %notzero, double %passthru)390 ret double %res391}392 393define i8 @extract_last_i8_scalable(<vscale x 16 x i8> %data, <vscale x 16 x i1> %mask, i8 %passthru) #0 {394; CHECK-LABEL: extract_last_i8_scalable:395; CHECK: // %bb.0:396; CHECK-NEXT: clastb w0, p0, w0, z0.b397; CHECK-NEXT: ret398 %res = call i8 @llvm.experimental.vector.extract.last.active.nxv16i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %mask, i8 %passthru)399 ret i8 %res400}401 402define i16 @extract_last_i16_scalable(<vscale x 8 x i16> %data, <vscale x 8 x i1> %mask, i16 %passthru) #0 {403; CHECK-LABEL: extract_last_i16_scalable:404; CHECK: // %bb.0:405; CHECK-NEXT: clastb w0, p0, w0, z0.h406; CHECK-NEXT: ret407 %res = call i16 @llvm.experimental.vector.extract.last.active.nxv8i16(<vscale x 8 x i16> %data, <vscale x 8 x i1> %mask, i16 %passthru)408 ret i16 %res409}410 411define i32 @extract_last_i32_scalable(<vscale x 4 x i32> %data, <vscale x 4 x i1> %mask, i32 %passthru) #0 {412; CHECK-LABEL: extract_last_i32_scalable:413; CHECK: // %bb.0:414; CHECK-NEXT: clastb w0, p0, w0, z0.s415; CHECK-NEXT: ret416 %res = call i32 @llvm.experimental.vector.extract.last.active.nxv4i32(<vscale x 4 x i32> %data, <vscale x 4 x i1> %mask, i32 %passthru)417 ret i32 %res418}419 420define i64 @extract_last_i64_scalable(<vscale x 2 x i64> %data, <vscale x 2 x i1> %mask, i64 %passthru) #0 {421; CHECK-LABEL: extract_last_i64_scalable:422; CHECK: // %bb.0:423; CHECK-NEXT: clastb x0, p0, x0, z0.d424; CHECK-NEXT: ret425 %res = call i64 @llvm.experimental.vector.extract.last.active.nxv2i64(<vscale x 2 x i64> %data, <vscale x 2 x i1> %mask, i64 %passthru)426 ret i64 %res427}428 429define half @extract_last_half_scalable(<vscale x 8 x half> %data, <vscale x 8 x i1> %mask, half %passthru) #0 {430; CHECK-LABEL: extract_last_half_scalable:431; CHECK: // %bb.0:432; CHECK-NEXT: clastb h1, p0, h1, z0.h433; CHECK-NEXT: fmov s0, s1434; CHECK-NEXT: ret435 %res = call half @llvm.experimental.vector.extract.last.active.nxv8f16(<vscale x 8 x half> %data, <vscale x 8 x i1> %mask, half %passthru)436 ret half %res437}438 439define bfloat @extract_last_bfloat_scalable(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %mask, bfloat %passthru) #0 {440; CHECK-LABEL: extract_last_bfloat_scalable:441; CHECK: // %bb.0:442; CHECK-NEXT: clastb h1, p0, h1, z0.h443; CHECK-NEXT: fmov s0, s1444; CHECK-NEXT: ret445 %res = call bfloat @llvm.experimental.vector.extract.last.active.nxv8bf16(<vscale x 8 x bfloat> %data, <vscale x 8 x i1> %mask, bfloat %passthru)446 ret bfloat %res447}448 449define float @extract_last_float_scalable(<vscale x 4 x float> %data, <vscale x 4 x i1> %mask, float %passthru) #0 {450; CHECK-LABEL: extract_last_float_scalable:451; CHECK: // %bb.0:452; CHECK-NEXT: clastb s1, p0, s1, z0.s453; CHECK-NEXT: fmov s0, s1454; CHECK-NEXT: ret455 %res = call float @llvm.experimental.vector.extract.last.active.nxv4f32(<vscale x 4 x float> %data, <vscale x 4 x i1> %mask, float %passthru)456 ret float %res457}458 459define double @extract_last_double_scalable(<vscale x 2 x double> %data, <vscale x 2 x i1> %mask, double %passthru) #0 {460; CHECK-LABEL: extract_last_double_scalable:461; CHECK: // %bb.0:462; CHECK-NEXT: clastb d1, p0, d1, z0.d463; CHECK-NEXT: fmov d0, d1464; CHECK-NEXT: ret465 %res = call double @llvm.experimental.vector.extract.last.active.nxv2f64(<vscale x 2 x double> %data, <vscale x 2 x i1> %mask, double %passthru)466 ret double %res467}468 469;; If the passthru parameter is poison, we shouldn't see a select at the end.470define i8 @extract_last_i8_scalable_poison_passthru(<vscale x 16 x i8> %data, <vscale x 16 x i1> %mask) #0 {471; CHECK-LABEL: extract_last_i8_scalable_poison_passthru:472; CHECK: // %bb.0:473; CHECK-NEXT: lastb w0, p0, z0.b474; CHECK-NEXT: ret475 %res = call i8 @llvm.experimental.vector.extract.last.active.nxv16i8(<vscale x 16 x i8> %data, <vscale x 16 x i1> %mask, i8 poison)476 ret i8 %res477}478 479;; (c)lastb doesn't exist for predicate types; check we get functional codegen480define i1 @extract_last_i1_scalable(<vscale x 16 x i1> %data, <vscale x 16 x i1> %mask) #0 {481; CHECK-LABEL: extract_last_i1_scalable:482; CHECK: // %bb.0:483; CHECK-NEXT: mov z0.b, p0/z, #1 // =0x1484; CHECK-NEXT: ptest p1, p1.b485; CHECK-NEXT: cset w9, ne486; CHECK-NEXT: lastb w8, p1, z0.b487; CHECK-NEXT: and w0, w9, w8488; CHECK-NEXT: ret489 %res = call i1 @llvm.experimental.vector.extract.last.active.nxv16i1(<vscale x 16 x i1> %data, <vscale x 16 x i1> %mask, i1 false)490 ret i1 %res491}492 493declare i8 @llvm.experimental.vector.extract.last.active.v16i8(<16 x i8>, <16 x i1>, i8)494declare i16 @llvm.experimental.vector.extract.last.active.v8i16(<8 x i16>, <8 x i1>, i16)495declare i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32>, <4 x i1>, i32)496declare i64 @llvm.experimental.vector.extract.last.active.v2i64(<2 x i64>, <2 x i1>, i64)497declare half @llvm.experimental.vector.extract.last.active.v8f16(<8 x half>, <8 x i1>, half)498declare bfloat @llvm.experimental.vector.extract.last.active.v8bf16(<8 x bfloat>, <8 x i1>, bfloat)499declare float @llvm.experimental.vector.extract.last.active.v4f32(<4 x float>, <4 x i1>, float)500declare double @llvm.experimental.vector.extract.last.active.v2f64(<2 x double>, <2 x i1>, double)501declare i8 @llvm.experimental.vector.extract.last.active.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8)502declare i16 @llvm.experimental.vector.extract.last.active.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16)503declare i32 @llvm.experimental.vector.extract.last.active.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32)504declare i64 @llvm.experimental.vector.extract.last.active.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64)505declare half @llvm.experimental.vector.extract.last.active.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half)506declare bfloat @llvm.experimental.vector.extract.last.active.nxv8bf16(<vscale x 8 x bfloat>, <vscale x 8 x i1>, bfloat)507declare float @llvm.experimental.vector.extract.last.active.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float)508declare double @llvm.experimental.vector.extract.last.active.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double)509declare i1 @llvm.experimental.vector.extract.last.active.nxv16i1(<vscale x 16 x i1>, <vscale x 16 x i1>, i1)510 511attributes #0 = { "target-features"="+sve" vscale_range(1, 16) }512