1877 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2563; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5124; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5125 6target triple = "aarch64-unknown-linux-gnu"7 8;9; UADDV10;11 12; Don't use SVE for 64-bit vectors.13define i8 @uaddv_v8i8(<8 x i8> %a) vscale_range(2,0) #0 {14; CHECK-LABEL: uaddv_v8i8:15; CHECK: // %bb.0:16; CHECK-NEXT: addv b0, v0.8b17; CHECK-NEXT: fmov w0, s018; CHECK-NEXT: ret19 %res = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %a)20 ret i8 %res21}22 23; Don't use SVE for 128-bit vectors.24define i8 @uaddv_v16i8(<16 x i8> %a) vscale_range(2,0) #0 {25; CHECK-LABEL: uaddv_v16i8:26; CHECK: // %bb.0:27; CHECK-NEXT: addv b0, v0.16b28; CHECK-NEXT: fmov w0, s029; CHECK-NEXT: ret30 %res = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %a)31 ret i8 %res32}33 34define i8 @uaddv_v32i8(ptr %a) vscale_range(2,0) #0 {35; CHECK-LABEL: uaddv_v32i8:36; CHECK: // %bb.0:37; CHECK-NEXT: ptrue p0.b, vl3238; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]39; CHECK-NEXT: uaddv d0, p0, z0.b40; CHECK-NEXT: fmov w0, s041; CHECK-NEXT: ret42 %op = load <32 x i8>, ptr %a43 %res = call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> %op)44 ret i8 %res45}46 47define i8 @uaddv_v64i8(ptr %a) #0 {48; VBITS_GE_256-LABEL: uaddv_v64i8:49; VBITS_GE_256: // %bb.0:50; VBITS_GE_256-NEXT: ptrue p0.b, vl3251; VBITS_GE_256-NEXT: mov w8, #32 // =0x2052; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]53; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]54; VBITS_GE_256-NEXT: add z0.b, z1.b, z0.b55; VBITS_GE_256-NEXT: uaddv d0, p0, z0.b56; VBITS_GE_256-NEXT: fmov w0, s057; VBITS_GE_256-NEXT: ret58;59; VBITS_GE_512-LABEL: uaddv_v64i8:60; VBITS_GE_512: // %bb.0:61; VBITS_GE_512-NEXT: ptrue p0.b, vl6462; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]63; VBITS_GE_512-NEXT: uaddv d0, p0, z0.b64; VBITS_GE_512-NEXT: fmov w0, s065; VBITS_GE_512-NEXT: ret66 %op = load <64 x i8>, ptr %a67 %res = call i8 @llvm.vector.reduce.add.v64i8(<64 x i8> %op)68 ret i8 %res69}70 71define i8 @uaddv_v128i8(ptr %a) vscale_range(8,0) #0 {72; CHECK-LABEL: uaddv_v128i8:73; CHECK: // %bb.0:74; CHECK-NEXT: ptrue p0.b, vl12875; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]76; CHECK-NEXT: uaddv d0, p0, z0.b77; CHECK-NEXT: fmov w0, s078; CHECK-NEXT: ret79 %op = load <128 x i8>, ptr %a80 %res = call i8 @llvm.vector.reduce.add.v128i8(<128 x i8> %op)81 ret i8 %res82}83 84define i8 @uaddv_v256i8(ptr %a) vscale_range(16,0) #0 {85; CHECK-LABEL: uaddv_v256i8:86; CHECK: // %bb.0:87; CHECK-NEXT: ptrue p0.b, vl25688; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]89; CHECK-NEXT: uaddv d0, p0, z0.b90; CHECK-NEXT: fmov w0, s091; CHECK-NEXT: ret92 %op = load <256 x i8>, ptr %a93 %res = call i8 @llvm.vector.reduce.add.v256i8(<256 x i8> %op)94 ret i8 %res95}96 97; Don't use SVE for 64-bit vectors.98define i16 @uaddv_v4i16(<4 x i16> %a) vscale_range(2,0) #0 {99; CHECK-LABEL: uaddv_v4i16:100; CHECK: // %bb.0:101; CHECK-NEXT: addv h0, v0.4h102; CHECK-NEXT: fmov w0, s0103; CHECK-NEXT: ret104 %res = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %a)105 ret i16 %res106}107 108; Don't use SVE for 128-bit vectors.109define i16 @uaddv_v8i16(<8 x i16> %a) vscale_range(2,0) #0 {110; CHECK-LABEL: uaddv_v8i16:111; CHECK: // %bb.0:112; CHECK-NEXT: addv h0, v0.8h113; CHECK-NEXT: fmov w0, s0114; CHECK-NEXT: ret115 %res = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %a)116 ret i16 %res117}118 119define i16 @uaddv_v16i16(ptr %a) vscale_range(2,0) #0 {120; CHECK-LABEL: uaddv_v16i16:121; CHECK: // %bb.0:122; CHECK-NEXT: ptrue p0.h, vl16123; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]124; CHECK-NEXT: uaddv d0, p0, z0.h125; CHECK-NEXT: fmov w0, s0126; CHECK-NEXT: ret127 %op = load <16 x i16>, ptr %a128 %res = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %op)129 ret i16 %res130}131 132define i16 @uaddv_v32i16(ptr %a) #0 {133; VBITS_GE_256-LABEL: uaddv_v32i16:134; VBITS_GE_256: // %bb.0:135; VBITS_GE_256-NEXT: ptrue p0.h, vl16136; VBITS_GE_256-NEXT: mov x8, #16 // =0x10137; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]138; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]139; VBITS_GE_256-NEXT: add z0.h, z1.h, z0.h140; VBITS_GE_256-NEXT: uaddv d0, p0, z0.h141; VBITS_GE_256-NEXT: fmov w0, s0142; VBITS_GE_256-NEXT: ret143;144; VBITS_GE_512-LABEL: uaddv_v32i16:145; VBITS_GE_512: // %bb.0:146; VBITS_GE_512-NEXT: ptrue p0.h, vl32147; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]148; VBITS_GE_512-NEXT: uaddv d0, p0, z0.h149; VBITS_GE_512-NEXT: fmov w0, s0150; VBITS_GE_512-NEXT: ret151 %op = load <32 x i16>, ptr %a152 %res = call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> %op)153 ret i16 %res154}155 156define i16 @uaddv_v64i16(ptr %a) vscale_range(8,0) #0 {157; CHECK-LABEL: uaddv_v64i16:158; CHECK: // %bb.0:159; CHECK-NEXT: ptrue p0.h, vl64160; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]161; CHECK-NEXT: uaddv d0, p0, z0.h162; CHECK-NEXT: fmov w0, s0163; CHECK-NEXT: ret164 %op = load <64 x i16>, ptr %a165 %res = call i16 @llvm.vector.reduce.add.v64i16(<64 x i16> %op)166 ret i16 %res167}168 169define i16 @uaddv_v128i16(ptr %a) vscale_range(16,0) #0 {170; CHECK-LABEL: uaddv_v128i16:171; CHECK: // %bb.0:172; CHECK-NEXT: ptrue p0.h, vl128173; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]174; CHECK-NEXT: uaddv d0, p0, z0.h175; CHECK-NEXT: fmov w0, s0176; CHECK-NEXT: ret177 %op = load <128 x i16>, ptr %a178 %res = call i16 @llvm.vector.reduce.add.v128i16(<128 x i16> %op)179 ret i16 %res180}181 182; Don't use SVE for 64-bit vectors.183define i32 @uaddv_v2i32(<2 x i32> %a) vscale_range(2,0) #0 {184; CHECK-LABEL: uaddv_v2i32:185; CHECK: // %bb.0:186; CHECK-NEXT: addp v0.2s, v0.2s, v0.2s187; CHECK-NEXT: fmov w0, s0188; CHECK-NEXT: ret189 %res = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a)190 ret i32 %res191}192 193; Don't use SVE for 128-bit vectors.194define i32 @uaddv_v4i32(<4 x i32> %a) vscale_range(2,0) #0 {195; CHECK-LABEL: uaddv_v4i32:196; CHECK: // %bb.0:197; CHECK-NEXT: addv s0, v0.4s198; CHECK-NEXT: fmov w0, s0199; CHECK-NEXT: ret200 %res = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)201 ret i32 %res202}203 204define i32 @uaddv_v8i32(ptr %a) vscale_range(2,0) #0 {205; CHECK-LABEL: uaddv_v8i32:206; CHECK: // %bb.0:207; CHECK-NEXT: ptrue p0.s, vl8208; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]209; CHECK-NEXT: uaddv d0, p0, z0.s210; CHECK-NEXT: fmov w0, s0211; CHECK-NEXT: ret212 %op = load <8 x i32>, ptr %a213 %res = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %op)214 ret i32 %res215}216 217define i32 @uaddv_v16i32(ptr %a) #0 {218; VBITS_GE_256-LABEL: uaddv_v16i32:219; VBITS_GE_256: // %bb.0:220; VBITS_GE_256-NEXT: ptrue p0.s, vl8221; VBITS_GE_256-NEXT: mov x8, #8 // =0x8222; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]223; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]224; VBITS_GE_256-NEXT: add z0.s, z1.s, z0.s225; VBITS_GE_256-NEXT: uaddv d0, p0, z0.s226; VBITS_GE_256-NEXT: fmov w0, s0227; VBITS_GE_256-NEXT: ret228;229; VBITS_GE_512-LABEL: uaddv_v16i32:230; VBITS_GE_512: // %bb.0:231; VBITS_GE_512-NEXT: ptrue p0.s, vl16232; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]233; VBITS_GE_512-NEXT: uaddv d0, p0, z0.s234; VBITS_GE_512-NEXT: fmov w0, s0235; VBITS_GE_512-NEXT: ret236 %op = load <16 x i32>, ptr %a237 %res = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %op)238 ret i32 %res239}240 241define i32 @uaddv_v32i32(ptr %a) vscale_range(8,0) #0 {242; CHECK-LABEL: uaddv_v32i32:243; CHECK: // %bb.0:244; CHECK-NEXT: ptrue p0.s, vl32245; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]246; CHECK-NEXT: uaddv d0, p0, z0.s247; CHECK-NEXT: fmov w0, s0248; CHECK-NEXT: ret249 %op = load <32 x i32>, ptr %a250 %res = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %op)251 ret i32 %res252}253 254define i32 @uaddv_v64i32(ptr %a) vscale_range(16,0) #0 {255; CHECK-LABEL: uaddv_v64i32:256; CHECK: // %bb.0:257; CHECK-NEXT: ptrue p0.s, vl64258; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]259; CHECK-NEXT: uaddv d0, p0, z0.s260; CHECK-NEXT: fmov w0, s0261; CHECK-NEXT: ret262 %op = load <64 x i32>, ptr %a263 %res = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %op)264 ret i32 %res265}266 267; Nothing to do for single element vectors.268define i64 @uaddv_v1i64(<1 x i64> %a) vscale_range(2,0) #0 {269; CHECK-LABEL: uaddv_v1i64:270; CHECK: // %bb.0:271; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0272; CHECK-NEXT: fmov x0, d0273; CHECK-NEXT: ret274 %res = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a)275 ret i64 %res276}277 278; Don't use SVE for 128-bit vectors.279define i64 @uaddv_v2i64(<2 x i64> %a) vscale_range(2,0) #0 {280; CHECK-LABEL: uaddv_v2i64:281; CHECK: // %bb.0:282; CHECK-NEXT: addp d0, v0.2d283; CHECK-NEXT: fmov x0, d0284; CHECK-NEXT: ret285 %res = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a)286 ret i64 %res287}288 289define i64 @uaddv_v4i64(ptr %a) vscale_range(2,0) #0 {290; CHECK-LABEL: uaddv_v4i64:291; CHECK: // %bb.0:292; CHECK-NEXT: ptrue p0.d, vl4293; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]294; CHECK-NEXT: uaddv d0, p0, z0.d295; CHECK-NEXT: fmov x0, d0296; CHECK-NEXT: ret297 %op = load <4 x i64>, ptr %a298 %res = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %op)299 ret i64 %res300}301 302define i64 @uaddv_v8i64(ptr %a) #0 {303; VBITS_GE_256-LABEL: uaddv_v8i64:304; VBITS_GE_256: // %bb.0:305; VBITS_GE_256-NEXT: ptrue p0.d, vl4306; VBITS_GE_256-NEXT: mov x8, #4 // =0x4307; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]308; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]309; VBITS_GE_256-NEXT: add z0.d, z1.d, z0.d310; VBITS_GE_256-NEXT: uaddv d0, p0, z0.d311; VBITS_GE_256-NEXT: fmov x0, d0312; VBITS_GE_256-NEXT: ret313;314; VBITS_GE_512-LABEL: uaddv_v8i64:315; VBITS_GE_512: // %bb.0:316; VBITS_GE_512-NEXT: ptrue p0.d, vl8317; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]318; VBITS_GE_512-NEXT: uaddv d0, p0, z0.d319; VBITS_GE_512-NEXT: fmov x0, d0320; VBITS_GE_512-NEXT: ret321 %op = load <8 x i64>, ptr %a322 %res = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %op)323 ret i64 %res324}325 326define i64 @uaddv_v16i64(ptr %a) vscale_range(8,0) #0 {327; CHECK-LABEL: uaddv_v16i64:328; CHECK: // %bb.0:329; CHECK-NEXT: ptrue p0.d, vl16330; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]331; CHECK-NEXT: uaddv d0, p0, z0.d332; CHECK-NEXT: fmov x0, d0333; CHECK-NEXT: ret334 %op = load <16 x i64>, ptr %a335 %res = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %op)336 ret i64 %res337}338 339define i64 @uaddv_v32i64(ptr %a) vscale_range(16,0) #0 {340; CHECK-LABEL: uaddv_v32i64:341; CHECK: // %bb.0:342; CHECK-NEXT: ptrue p0.d, vl32343; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]344; CHECK-NEXT: uaddv d0, p0, z0.d345; CHECK-NEXT: fmov x0, d0346; CHECK-NEXT: ret347 %op = load <32 x i64>, ptr %a348 %res = call i64 @llvm.vector.reduce.add.v32i64(<32 x i64> %op)349 ret i64 %res350}351 352;353; SMAXV354;355 356; Don't use SVE for 64-bit vectors.357define i8 @smaxv_v8i8(<8 x i8> %a) vscale_range(2,0) #0 {358; CHECK-LABEL: smaxv_v8i8:359; CHECK: // %bb.0:360; CHECK-NEXT: smaxv b0, v0.8b361; CHECK-NEXT: fmov w0, s0362; CHECK-NEXT: ret363 %res = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> %a)364 ret i8 %res365}366 367; Don't use SVE for 128-bit vectors.368define i8 @smaxv_v16i8(<16 x i8> %a) vscale_range(2,0) #0 {369; CHECK-LABEL: smaxv_v16i8:370; CHECK: // %bb.0:371; CHECK-NEXT: smaxv b0, v0.16b372; CHECK-NEXT: fmov w0, s0373; CHECK-NEXT: ret374 %res = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> %a)375 ret i8 %res376}377 378define i8 @smaxv_v32i8(ptr %a) vscale_range(2,0) #0 {379; CHECK-LABEL: smaxv_v32i8:380; CHECK: // %bb.0:381; CHECK-NEXT: ptrue p0.b, vl32382; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]383; CHECK-NEXT: smaxv b0, p0, z0.b384; CHECK-NEXT: fmov w0, s0385; CHECK-NEXT: ret386 %op = load <32 x i8>, ptr %a387 %res = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> %op)388 ret i8 %res389}390 391define i8 @smaxv_v64i8(ptr %a) #0 {392; VBITS_GE_256-LABEL: smaxv_v64i8:393; VBITS_GE_256: // %bb.0:394; VBITS_GE_256-NEXT: ptrue p0.b, vl32395; VBITS_GE_256-NEXT: mov w8, #32 // =0x20396; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]397; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]398; VBITS_GE_256-NEXT: smax z0.b, p0/m, z0.b, z1.b399; VBITS_GE_256-NEXT: smaxv b0, p0, z0.b400; VBITS_GE_256-NEXT: fmov w0, s0401; VBITS_GE_256-NEXT: ret402;403; VBITS_GE_512-LABEL: smaxv_v64i8:404; VBITS_GE_512: // %bb.0:405; VBITS_GE_512-NEXT: ptrue p0.b, vl64406; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]407; VBITS_GE_512-NEXT: smaxv b0, p0, z0.b408; VBITS_GE_512-NEXT: fmov w0, s0409; VBITS_GE_512-NEXT: ret410 %op = load <64 x i8>, ptr %a411 %res = call i8 @llvm.vector.reduce.smax.v64i8(<64 x i8> %op)412 ret i8 %res413}414 415define i8 @smaxv_v128i8(ptr %a) vscale_range(8,0) #0 {416; CHECK-LABEL: smaxv_v128i8:417; CHECK: // %bb.0:418; CHECK-NEXT: ptrue p0.b, vl128419; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]420; CHECK-NEXT: smaxv b0, p0, z0.b421; CHECK-NEXT: fmov w0, s0422; CHECK-NEXT: ret423 %op = load <128 x i8>, ptr %a424 %res = call i8 @llvm.vector.reduce.smax.v128i8(<128 x i8> %op)425 ret i8 %res426}427 428define i8 @smaxv_v256i8(ptr %a) vscale_range(16,0) #0 {429; CHECK-LABEL: smaxv_v256i8:430; CHECK: // %bb.0:431; CHECK-NEXT: ptrue p0.b, vl256432; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]433; CHECK-NEXT: smaxv b0, p0, z0.b434; CHECK-NEXT: fmov w0, s0435; CHECK-NEXT: ret436 %op = load <256 x i8>, ptr %a437 %res = call i8 @llvm.vector.reduce.smax.v256i8(<256 x i8> %op)438 ret i8 %res439}440 441; Don't use SVE for 64-bit vectors.442define i16 @smaxv_v4i16(<4 x i16> %a) vscale_range(2,0) #0 {443; CHECK-LABEL: smaxv_v4i16:444; CHECK: // %bb.0:445; CHECK-NEXT: smaxv h0, v0.4h446; CHECK-NEXT: fmov w0, s0447; CHECK-NEXT: ret448 %res = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> %a)449 ret i16 %res450}451 452; Don't use SVE for 128-bit vectors.453define i16 @smaxv_v8i16(<8 x i16> %a) vscale_range(2,0) #0 {454; CHECK-LABEL: smaxv_v8i16:455; CHECK: // %bb.0:456; CHECK-NEXT: smaxv h0, v0.8h457; CHECK-NEXT: fmov w0, s0458; CHECK-NEXT: ret459 %res = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> %a)460 ret i16 %res461}462 463define i16 @smaxv_v16i16(ptr %a) vscale_range(2,0) #0 {464; CHECK-LABEL: smaxv_v16i16:465; CHECK: // %bb.0:466; CHECK-NEXT: ptrue p0.h, vl16467; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]468; CHECK-NEXT: smaxv h0, p0, z0.h469; CHECK-NEXT: fmov w0, s0470; CHECK-NEXT: ret471 %op = load <16 x i16>, ptr %a472 %res = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> %op)473 ret i16 %res474}475 476define i16 @smaxv_v32i16(ptr %a) #0 {477; VBITS_GE_256-LABEL: smaxv_v32i16:478; VBITS_GE_256: // %bb.0:479; VBITS_GE_256-NEXT: ptrue p0.h, vl16480; VBITS_GE_256-NEXT: mov x8, #16 // =0x10481; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]482; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]483; VBITS_GE_256-NEXT: smax z0.h, p0/m, z0.h, z1.h484; VBITS_GE_256-NEXT: smaxv h0, p0, z0.h485; VBITS_GE_256-NEXT: fmov w0, s0486; VBITS_GE_256-NEXT: ret487;488; VBITS_GE_512-LABEL: smaxv_v32i16:489; VBITS_GE_512: // %bb.0:490; VBITS_GE_512-NEXT: ptrue p0.h, vl32491; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]492; VBITS_GE_512-NEXT: smaxv h0, p0, z0.h493; VBITS_GE_512-NEXT: fmov w0, s0494; VBITS_GE_512-NEXT: ret495 %op = load <32 x i16>, ptr %a496 %res = call i16 @llvm.vector.reduce.smax.v32i16(<32 x i16> %op)497 ret i16 %res498}499 500define i16 @smaxv_v64i16(ptr %a) vscale_range(8,0) #0 {501; CHECK-LABEL: smaxv_v64i16:502; CHECK: // %bb.0:503; CHECK-NEXT: ptrue p0.h, vl64504; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]505; CHECK-NEXT: smaxv h0, p0, z0.h506; CHECK-NEXT: fmov w0, s0507; CHECK-NEXT: ret508 %op = load <64 x i16>, ptr %a509 %res = call i16 @llvm.vector.reduce.smax.v64i16(<64 x i16> %op)510 ret i16 %res511}512 513define i16 @smaxv_v128i16(ptr %a) vscale_range(16,0) #0 {514; CHECK-LABEL: smaxv_v128i16:515; CHECK: // %bb.0:516; CHECK-NEXT: ptrue p0.h, vl128517; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]518; CHECK-NEXT: smaxv h0, p0, z0.h519; CHECK-NEXT: fmov w0, s0520; CHECK-NEXT: ret521 %op = load <128 x i16>, ptr %a522 %res = call i16 @llvm.vector.reduce.smax.v128i16(<128 x i16> %op)523 ret i16 %res524}525 526; Don't use SVE for 64-bit vectors.527define i32 @smaxv_v2i32(<2 x i32> %a) vscale_range(2,0) #0 {528; CHECK-LABEL: smaxv_v2i32:529; CHECK: // %bb.0:530; CHECK-NEXT: smaxp v0.2s, v0.2s, v0.2s531; CHECK-NEXT: fmov w0, s0532; CHECK-NEXT: ret533 %res = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> %a)534 ret i32 %res535}536 537; Don't use SVE for 128-bit vectors.538define i32 @smaxv_v4i32(<4 x i32> %a) vscale_range(2,0) #0 {539; CHECK-LABEL: smaxv_v4i32:540; CHECK: // %bb.0:541; CHECK-NEXT: smaxv s0, v0.4s542; CHECK-NEXT: fmov w0, s0543; CHECK-NEXT: ret544 %res = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> %a)545 ret i32 %res546}547 548define i32 @smaxv_v8i32(ptr %a) vscale_range(2,0) #0 {549; CHECK-LABEL: smaxv_v8i32:550; CHECK: // %bb.0:551; CHECK-NEXT: ptrue p0.s, vl8552; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]553; CHECK-NEXT: smaxv s0, p0, z0.s554; CHECK-NEXT: fmov w0, s0555; CHECK-NEXT: ret556 %op = load <8 x i32>, ptr %a557 %res = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> %op)558 ret i32 %res559}560 561define i32 @smaxv_v16i32(ptr %a) #0 {562; VBITS_GE_256-LABEL: smaxv_v16i32:563; VBITS_GE_256: // %bb.0:564; VBITS_GE_256-NEXT: ptrue p0.s, vl8565; VBITS_GE_256-NEXT: mov x8, #8 // =0x8566; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]567; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]568; VBITS_GE_256-NEXT: smax z0.s, p0/m, z0.s, z1.s569; VBITS_GE_256-NEXT: smaxv s0, p0, z0.s570; VBITS_GE_256-NEXT: fmov w0, s0571; VBITS_GE_256-NEXT: ret572;573; VBITS_GE_512-LABEL: smaxv_v16i32:574; VBITS_GE_512: // %bb.0:575; VBITS_GE_512-NEXT: ptrue p0.s, vl16576; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]577; VBITS_GE_512-NEXT: smaxv s0, p0, z0.s578; VBITS_GE_512-NEXT: fmov w0, s0579; VBITS_GE_512-NEXT: ret580 %op = load <16 x i32>, ptr %a581 %res = call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> %op)582 ret i32 %res583}584 585define i32 @smaxv_v32i32(ptr %a) vscale_range(8,0) #0 {586; CHECK-LABEL: smaxv_v32i32:587; CHECK: // %bb.0:588; CHECK-NEXT: ptrue p0.s, vl32589; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]590; CHECK-NEXT: smaxv s0, p0, z0.s591; CHECK-NEXT: fmov w0, s0592; CHECK-NEXT: ret593 %op = load <32 x i32>, ptr %a594 %res = call i32 @llvm.vector.reduce.smax.v32i32(<32 x i32> %op)595 ret i32 %res596}597 598define i32 @smaxv_v64i32(ptr %a) vscale_range(16,0) #0 {599; CHECK-LABEL: smaxv_v64i32:600; CHECK: // %bb.0:601; CHECK-NEXT: ptrue p0.s, vl64602; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]603; CHECK-NEXT: smaxv s0, p0, z0.s604; CHECK-NEXT: fmov w0, s0605; CHECK-NEXT: ret606 %op = load <64 x i32>, ptr %a607 %res = call i32 @llvm.vector.reduce.smax.v64i32(<64 x i32> %op)608 ret i32 %res609}610 611; Nothing to do for single element vectors.612define i64 @smaxv_v1i64(<1 x i64> %a) vscale_range(2,0) #0 {613; CHECK-LABEL: smaxv_v1i64:614; CHECK: // %bb.0:615; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0616; CHECK-NEXT: fmov x0, d0617; CHECK-NEXT: ret618 %res = call i64 @llvm.vector.reduce.smax.v1i64(<1 x i64> %a)619 ret i64 %res620}621 622; No NEON 64-bit vector SMAXV support. Use SVE.623define i64 @smaxv_v2i64(<2 x i64> %a) vscale_range(2,0) #0 {624; CHECK-LABEL: smaxv_v2i64:625; CHECK: // %bb.0:626; CHECK-NEXT: ptrue p0.d, vl2627; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0628; CHECK-NEXT: smaxv d0, p0, z0.d629; CHECK-NEXT: fmov x0, d0630; CHECK-NEXT: ret631 %res = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> %a)632 ret i64 %res633}634 635define i64 @smaxv_v4i64(ptr %a) vscale_range(2,0) #0 {636; CHECK-LABEL: smaxv_v4i64:637; CHECK: // %bb.0:638; CHECK-NEXT: ptrue p0.d, vl4639; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]640; CHECK-NEXT: smaxv d0, p0, z0.d641; CHECK-NEXT: fmov x0, d0642; CHECK-NEXT: ret643 %op = load <4 x i64>, ptr %a644 %res = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> %op)645 ret i64 %res646}647 648define i64 @smaxv_v8i64(ptr %a) #0 {649; VBITS_GE_256-LABEL: smaxv_v8i64:650; VBITS_GE_256: // %bb.0:651; VBITS_GE_256-NEXT: ptrue p0.d, vl4652; VBITS_GE_256-NEXT: mov x8, #4 // =0x4653; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]654; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]655; VBITS_GE_256-NEXT: smax z0.d, p0/m, z0.d, z1.d656; VBITS_GE_256-NEXT: smaxv d0, p0, z0.d657; VBITS_GE_256-NEXT: fmov x0, d0658; VBITS_GE_256-NEXT: ret659;660; VBITS_GE_512-LABEL: smaxv_v8i64:661; VBITS_GE_512: // %bb.0:662; VBITS_GE_512-NEXT: ptrue p0.d, vl8663; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]664; VBITS_GE_512-NEXT: smaxv d0, p0, z0.d665; VBITS_GE_512-NEXT: fmov x0, d0666; VBITS_GE_512-NEXT: ret667 %op = load <8 x i64>, ptr %a668 %res = call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> %op)669 ret i64 %res670}671 672define i64 @smaxv_v16i64(ptr %a) vscale_range(8,0) #0 {673; CHECK-LABEL: smaxv_v16i64:674; CHECK: // %bb.0:675; CHECK-NEXT: ptrue p0.d, vl16676; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]677; CHECK-NEXT: smaxv d0, p0, z0.d678; CHECK-NEXT: fmov x0, d0679; CHECK-NEXT: ret680 %op = load <16 x i64>, ptr %a681 %res = call i64 @llvm.vector.reduce.smax.v16i64(<16 x i64> %op)682 ret i64 %res683}684 685define i64 @smaxv_v32i64(ptr %a) vscale_range(16,0) #0 {686; CHECK-LABEL: smaxv_v32i64:687; CHECK: // %bb.0:688; CHECK-NEXT: ptrue p0.d, vl32689; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]690; CHECK-NEXT: smaxv d0, p0, z0.d691; CHECK-NEXT: fmov x0, d0692; CHECK-NEXT: ret693 %op = load <32 x i64>, ptr %a694 %res = call i64 @llvm.vector.reduce.smax.v32i64(<32 x i64> %op)695 ret i64 %res696}697 698;699; SMINV700;701 702; Don't use SVE for 64-bit vectors.703define i8 @sminv_v8i8(<8 x i8> %a) vscale_range(2,0) #0 {704; CHECK-LABEL: sminv_v8i8:705; CHECK: // %bb.0:706; CHECK-NEXT: sminv b0, v0.8b707; CHECK-NEXT: fmov w0, s0708; CHECK-NEXT: ret709 %res = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> %a)710 ret i8 %res711}712 713; Don't use SVE for 128-bit vectors.714define i8 @sminv_v16i8(<16 x i8> %a) vscale_range(2,0) #0 {715; CHECK-LABEL: sminv_v16i8:716; CHECK: // %bb.0:717; CHECK-NEXT: sminv b0, v0.16b718; CHECK-NEXT: fmov w0, s0719; CHECK-NEXT: ret720 %res = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> %a)721 ret i8 %res722}723 724define i8 @sminv_v32i8(ptr %a) vscale_range(2,0) #0 {725; CHECK-LABEL: sminv_v32i8:726; CHECK: // %bb.0:727; CHECK-NEXT: ptrue p0.b, vl32728; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]729; CHECK-NEXT: sminv b0, p0, z0.b730; CHECK-NEXT: fmov w0, s0731; CHECK-NEXT: ret732 %op = load <32 x i8>, ptr %a733 %res = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> %op)734 ret i8 %res735}736 737define i8 @sminv_v64i8(ptr %a) #0 {738; VBITS_GE_256-LABEL: sminv_v64i8:739; VBITS_GE_256: // %bb.0:740; VBITS_GE_256-NEXT: ptrue p0.b, vl32741; VBITS_GE_256-NEXT: mov w8, #32 // =0x20742; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]743; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]744; VBITS_GE_256-NEXT: smin z0.b, p0/m, z0.b, z1.b745; VBITS_GE_256-NEXT: sminv b0, p0, z0.b746; VBITS_GE_256-NEXT: fmov w0, s0747; VBITS_GE_256-NEXT: ret748;749; VBITS_GE_512-LABEL: sminv_v64i8:750; VBITS_GE_512: // %bb.0:751; VBITS_GE_512-NEXT: ptrue p0.b, vl64752; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]753; VBITS_GE_512-NEXT: sminv b0, p0, z0.b754; VBITS_GE_512-NEXT: fmov w0, s0755; VBITS_GE_512-NEXT: ret756 %op = load <64 x i8>, ptr %a757 %res = call i8 @llvm.vector.reduce.smin.v64i8(<64 x i8> %op)758 ret i8 %res759}760 761define i8 @sminv_v128i8(ptr %a) vscale_range(8,0) #0 {762; CHECK-LABEL: sminv_v128i8:763; CHECK: // %bb.0:764; CHECK-NEXT: ptrue p0.b, vl128765; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]766; CHECK-NEXT: sminv b0, p0, z0.b767; CHECK-NEXT: fmov w0, s0768; CHECK-NEXT: ret769 %op = load <128 x i8>, ptr %a770 %res = call i8 @llvm.vector.reduce.smin.v128i8(<128 x i8> %op)771 ret i8 %res772}773 774define i8 @sminv_v256i8(ptr %a) vscale_range(16,0) #0 {775; CHECK-LABEL: sminv_v256i8:776; CHECK: // %bb.0:777; CHECK-NEXT: ptrue p0.b, vl256778; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]779; CHECK-NEXT: sminv b0, p0, z0.b780; CHECK-NEXT: fmov w0, s0781; CHECK-NEXT: ret782 %op = load <256 x i8>, ptr %a783 %res = call i8 @llvm.vector.reduce.smin.v256i8(<256 x i8> %op)784 ret i8 %res785}786 787; Don't use SVE for 64-bit vectors.788define i16 @sminv_v4i16(<4 x i16> %a) vscale_range(2,0) #0 {789; CHECK-LABEL: sminv_v4i16:790; CHECK: // %bb.0:791; CHECK-NEXT: sminv h0, v0.4h792; CHECK-NEXT: fmov w0, s0793; CHECK-NEXT: ret794 %res = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> %a)795 ret i16 %res796}797 798; Don't use SVE for 128-bit vectors.799define i16 @sminv_v8i16(<8 x i16> %a) vscale_range(2,0) #0 {800; CHECK-LABEL: sminv_v8i16:801; CHECK: // %bb.0:802; CHECK-NEXT: sminv h0, v0.8h803; CHECK-NEXT: fmov w0, s0804; CHECK-NEXT: ret805 %res = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> %a)806 ret i16 %res807}808 809define i16 @sminv_v16i16(ptr %a) vscale_range(2,0) #0 {810; CHECK-LABEL: sminv_v16i16:811; CHECK: // %bb.0:812; CHECK-NEXT: ptrue p0.h, vl16813; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]814; CHECK-NEXT: sminv h0, p0, z0.h815; CHECK-NEXT: fmov w0, s0816; CHECK-NEXT: ret817 %op = load <16 x i16>, ptr %a818 %res = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> %op)819 ret i16 %res820}821 822define i16 @sminv_v32i16(ptr %a) #0 {823; VBITS_GE_256-LABEL: sminv_v32i16:824; VBITS_GE_256: // %bb.0:825; VBITS_GE_256-NEXT: ptrue p0.h, vl16826; VBITS_GE_256-NEXT: mov x8, #16 // =0x10827; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]828; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]829; VBITS_GE_256-NEXT: smin z0.h, p0/m, z0.h, z1.h830; VBITS_GE_256-NEXT: sminv h0, p0, z0.h831; VBITS_GE_256-NEXT: fmov w0, s0832; VBITS_GE_256-NEXT: ret833;834; VBITS_GE_512-LABEL: sminv_v32i16:835; VBITS_GE_512: // %bb.0:836; VBITS_GE_512-NEXT: ptrue p0.h, vl32837; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]838; VBITS_GE_512-NEXT: sminv h0, p0, z0.h839; VBITS_GE_512-NEXT: fmov w0, s0840; VBITS_GE_512-NEXT: ret841 %op = load <32 x i16>, ptr %a842 %res = call i16 @llvm.vector.reduce.smin.v32i16(<32 x i16> %op)843 ret i16 %res844}845 846define i16 @sminv_v64i16(ptr %a) vscale_range(8,0) #0 {847; CHECK-LABEL: sminv_v64i16:848; CHECK: // %bb.0:849; CHECK-NEXT: ptrue p0.h, vl64850; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]851; CHECK-NEXT: sminv h0, p0, z0.h852; CHECK-NEXT: fmov w0, s0853; CHECK-NEXT: ret854 %op = load <64 x i16>, ptr %a855 %res = call i16 @llvm.vector.reduce.smin.v64i16(<64 x i16> %op)856 ret i16 %res857}858 859define i16 @sminv_v128i16(ptr %a) vscale_range(16,0) #0 {860; CHECK-LABEL: sminv_v128i16:861; CHECK: // %bb.0:862; CHECK-NEXT: ptrue p0.h, vl128863; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]864; CHECK-NEXT: sminv h0, p0, z0.h865; CHECK-NEXT: fmov w0, s0866; CHECK-NEXT: ret867 %op = load <128 x i16>, ptr %a868 %res = call i16 @llvm.vector.reduce.smin.v128i16(<128 x i16> %op)869 ret i16 %res870}871 872; Don't use SVE for 64-bit vectors.873define i32 @sminv_v2i32(<2 x i32> %a) vscale_range(2,0) #0 {874; CHECK-LABEL: sminv_v2i32:875; CHECK: // %bb.0:876; CHECK-NEXT: sminp v0.2s, v0.2s, v0.2s877; CHECK-NEXT: fmov w0, s0878; CHECK-NEXT: ret879 %res = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> %a)880 ret i32 %res881}882 883; Don't use SVE for 128-bit vectors.884define i32 @sminv_v4i32(<4 x i32> %a) vscale_range(2,0) #0 {885; CHECK-LABEL: sminv_v4i32:886; CHECK: // %bb.0:887; CHECK-NEXT: sminv s0, v0.4s888; CHECK-NEXT: fmov w0, s0889; CHECK-NEXT: ret890 %res = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> %a)891 ret i32 %res892}893 894define i32 @sminv_v8i32(ptr %a) vscale_range(2,0) #0 {895; CHECK-LABEL: sminv_v8i32:896; CHECK: // %bb.0:897; CHECK-NEXT: ptrue p0.s, vl8898; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]899; CHECK-NEXT: sminv s0, p0, z0.s900; CHECK-NEXT: fmov w0, s0901; CHECK-NEXT: ret902 %op = load <8 x i32>, ptr %a903 %res = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> %op)904 ret i32 %res905}906 907define i32 @sminv_v16i32(ptr %a) #0 {908; VBITS_GE_256-LABEL: sminv_v16i32:909; VBITS_GE_256: // %bb.0:910; VBITS_GE_256-NEXT: ptrue p0.s, vl8911; VBITS_GE_256-NEXT: mov x8, #8 // =0x8912; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]913; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]914; VBITS_GE_256-NEXT: smin z0.s, p0/m, z0.s, z1.s915; VBITS_GE_256-NEXT: sminv s0, p0, z0.s916; VBITS_GE_256-NEXT: fmov w0, s0917; VBITS_GE_256-NEXT: ret918;919; VBITS_GE_512-LABEL: sminv_v16i32:920; VBITS_GE_512: // %bb.0:921; VBITS_GE_512-NEXT: ptrue p0.s, vl16922; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]923; VBITS_GE_512-NEXT: sminv s0, p0, z0.s924; VBITS_GE_512-NEXT: fmov w0, s0925; VBITS_GE_512-NEXT: ret926 %op = load <16 x i32>, ptr %a927 %res = call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> %op)928 ret i32 %res929}930 931define i32 @sminv_v32i32(ptr %a) vscale_range(8,0) #0 {932; CHECK-LABEL: sminv_v32i32:933; CHECK: // %bb.0:934; CHECK-NEXT: ptrue p0.s, vl32935; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]936; CHECK-NEXT: sminv s0, p0, z0.s937; CHECK-NEXT: fmov w0, s0938; CHECK-NEXT: ret939 %op = load <32 x i32>, ptr %a940 %res = call i32 @llvm.vector.reduce.smin.v32i32(<32 x i32> %op)941 ret i32 %res942}943 944define i32 @sminv_v64i32(ptr %a) vscale_range(16,0) #0 {945; CHECK-LABEL: sminv_v64i32:946; CHECK: // %bb.0:947; CHECK-NEXT: ptrue p0.s, vl64948; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]949; CHECK-NEXT: sminv s0, p0, z0.s950; CHECK-NEXT: fmov w0, s0951; CHECK-NEXT: ret952 %op = load <64 x i32>, ptr %a953 %res = call i32 @llvm.vector.reduce.smin.v64i32(<64 x i32> %op)954 ret i32 %res955}956 957; Nothing to do for single element vectors.958define i64 @sminv_v1i64(<1 x i64> %a) vscale_range(2,0) #0 {959; CHECK-LABEL: sminv_v1i64:960; CHECK: // %bb.0:961; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0962; CHECK-NEXT: fmov x0, d0963; CHECK-NEXT: ret964 %res = call i64 @llvm.vector.reduce.smin.v1i64(<1 x i64> %a)965 ret i64 %res966}967 968; No NEON 64-bit vector SMINV support. Use SVE.969define i64 @sminv_v2i64(<2 x i64> %a) vscale_range(2,0) #0 {970; CHECK-LABEL: sminv_v2i64:971; CHECK: // %bb.0:972; CHECK-NEXT: ptrue p0.d, vl2973; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0974; CHECK-NEXT: sminv d0, p0, z0.d975; CHECK-NEXT: fmov x0, d0976; CHECK-NEXT: ret977 %res = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> %a)978 ret i64 %res979}980 981define i64 @sminv_v4i64(ptr %a) vscale_range(2,0) #0 {982; CHECK-LABEL: sminv_v4i64:983; CHECK: // %bb.0:984; CHECK-NEXT: ptrue p0.d, vl4985; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]986; CHECK-NEXT: sminv d0, p0, z0.d987; CHECK-NEXT: fmov x0, d0988; CHECK-NEXT: ret989 %op = load <4 x i64>, ptr %a990 %res = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> %op)991 ret i64 %res992}993 994define i64 @sminv_v8i64(ptr %a) #0 {995; VBITS_GE_256-LABEL: sminv_v8i64:996; VBITS_GE_256: // %bb.0:997; VBITS_GE_256-NEXT: ptrue p0.d, vl4998; VBITS_GE_256-NEXT: mov x8, #4 // =0x4999; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]1000; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]1001; VBITS_GE_256-NEXT: smin z0.d, p0/m, z0.d, z1.d1002; VBITS_GE_256-NEXT: sminv d0, p0, z0.d1003; VBITS_GE_256-NEXT: fmov x0, d01004; VBITS_GE_256-NEXT: ret1005;1006; VBITS_GE_512-LABEL: sminv_v8i64:1007; VBITS_GE_512: // %bb.0:1008; VBITS_GE_512-NEXT: ptrue p0.d, vl81009; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]1010; VBITS_GE_512-NEXT: sminv d0, p0, z0.d1011; VBITS_GE_512-NEXT: fmov x0, d01012; VBITS_GE_512-NEXT: ret1013 %op = load <8 x i64>, ptr %a1014 %res = call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> %op)1015 ret i64 %res1016}1017 1018define i64 @sminv_v16i64(ptr %a) vscale_range(8,0) #0 {1019; CHECK-LABEL: sminv_v16i64:1020; CHECK: // %bb.0:1021; CHECK-NEXT: ptrue p0.d, vl161022; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1023; CHECK-NEXT: sminv d0, p0, z0.d1024; CHECK-NEXT: fmov x0, d01025; CHECK-NEXT: ret1026 %op = load <16 x i64>, ptr %a1027 %res = call i64 @llvm.vector.reduce.smin.v16i64(<16 x i64> %op)1028 ret i64 %res1029}1030 1031define i64 @sminv_v32i64(ptr %a) vscale_range(16,0) #0 {1032; CHECK-LABEL: sminv_v32i64:1033; CHECK: // %bb.0:1034; CHECK-NEXT: ptrue p0.d, vl321035; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1036; CHECK-NEXT: sminv d0, p0, z0.d1037; CHECK-NEXT: fmov x0, d01038; CHECK-NEXT: ret1039 %op = load <32 x i64>, ptr %a1040 %res = call i64 @llvm.vector.reduce.smin.v32i64(<32 x i64> %op)1041 ret i64 %res1042}1043 1044;1045; UMAXV1046;1047 1048; Don't use SVE for 64-bit vectors.1049define i8 @umaxv_v8i8(<8 x i8> %a) vscale_range(2,0) #0 {1050; CHECK-LABEL: umaxv_v8i8:1051; CHECK: // %bb.0:1052; CHECK-NEXT: umaxv b0, v0.8b1053; CHECK-NEXT: fmov w0, s01054; CHECK-NEXT: ret1055 %res = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> %a)1056 ret i8 %res1057}1058 1059; Don't use SVE for 128-bit vectors.1060define i8 @umaxv_v16i8(<16 x i8> %a) vscale_range(2,0) #0 {1061; CHECK-LABEL: umaxv_v16i8:1062; CHECK: // %bb.0:1063; CHECK-NEXT: umaxv b0, v0.16b1064; CHECK-NEXT: fmov w0, s01065; CHECK-NEXT: ret1066 %res = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> %a)1067 ret i8 %res1068}1069 1070define i8 @umaxv_v32i8(ptr %a) vscale_range(2,0) #0 {1071; CHECK-LABEL: umaxv_v32i8:1072; CHECK: // %bb.0:1073; CHECK-NEXT: ptrue p0.b, vl321074; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]1075; CHECK-NEXT: umaxv b0, p0, z0.b1076; CHECK-NEXT: fmov w0, s01077; CHECK-NEXT: ret1078 %op = load <32 x i8>, ptr %a1079 %res = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> %op)1080 ret i8 %res1081}1082 1083define i8 @umaxv_v64i8(ptr %a) #0 {1084; VBITS_GE_256-LABEL: umaxv_v64i8:1085; VBITS_GE_256: // %bb.0:1086; VBITS_GE_256-NEXT: ptrue p0.b, vl321087; VBITS_GE_256-NEXT: mov w8, #32 // =0x201088; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]1089; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]1090; VBITS_GE_256-NEXT: umax z0.b, p0/m, z0.b, z1.b1091; VBITS_GE_256-NEXT: umaxv b0, p0, z0.b1092; VBITS_GE_256-NEXT: fmov w0, s01093; VBITS_GE_256-NEXT: ret1094;1095; VBITS_GE_512-LABEL: umaxv_v64i8:1096; VBITS_GE_512: // %bb.0:1097; VBITS_GE_512-NEXT: ptrue p0.b, vl641098; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]1099; VBITS_GE_512-NEXT: umaxv b0, p0, z0.b1100; VBITS_GE_512-NEXT: fmov w0, s01101; VBITS_GE_512-NEXT: ret1102 %op = load <64 x i8>, ptr %a1103 %res = call i8 @llvm.vector.reduce.umax.v64i8(<64 x i8> %op)1104 ret i8 %res1105}1106 1107define i8 @umaxv_v128i8(ptr %a) vscale_range(8,0) #0 {1108; CHECK-LABEL: umaxv_v128i8:1109; CHECK: // %bb.0:1110; CHECK-NEXT: ptrue p0.b, vl1281111; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]1112; CHECK-NEXT: umaxv b0, p0, z0.b1113; CHECK-NEXT: fmov w0, s01114; CHECK-NEXT: ret1115 %op = load <128 x i8>, ptr %a1116 %res = call i8 @llvm.vector.reduce.umax.v128i8(<128 x i8> %op)1117 ret i8 %res1118}1119 1120define i8 @umaxv_v256i8(ptr %a) vscale_range(16,0) #0 {1121; CHECK-LABEL: umaxv_v256i8:1122; CHECK: // %bb.0:1123; CHECK-NEXT: ptrue p0.b, vl2561124; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]1125; CHECK-NEXT: umaxv b0, p0, z0.b1126; CHECK-NEXT: fmov w0, s01127; CHECK-NEXT: ret1128 %op = load <256 x i8>, ptr %a1129 %res = call i8 @llvm.vector.reduce.umax.v256i8(<256 x i8> %op)1130 ret i8 %res1131}1132 1133; Don't use SVE for 64-bit vectors.1134define i16 @umaxv_v4i16(<4 x i16> %a) vscale_range(2,0) #0 {1135; CHECK-LABEL: umaxv_v4i16:1136; CHECK: // %bb.0:1137; CHECK-NEXT: umaxv h0, v0.4h1138; CHECK-NEXT: fmov w0, s01139; CHECK-NEXT: ret1140 %res = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> %a)1141 ret i16 %res1142}1143 1144; Don't use SVE for 128-bit vectors.1145define i16 @umaxv_v8i16(<8 x i16> %a) vscale_range(2,0) #0 {1146; CHECK-LABEL: umaxv_v8i16:1147; CHECK: // %bb.0:1148; CHECK-NEXT: umaxv h0, v0.8h1149; CHECK-NEXT: fmov w0, s01150; CHECK-NEXT: ret1151 %res = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> %a)1152 ret i16 %res1153}1154 1155define i16 @umaxv_v16i16(ptr %a) vscale_range(2,0) #0 {1156; CHECK-LABEL: umaxv_v16i16:1157; CHECK: // %bb.0:1158; CHECK-NEXT: ptrue p0.h, vl161159; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1160; CHECK-NEXT: umaxv h0, p0, z0.h1161; CHECK-NEXT: fmov w0, s01162; CHECK-NEXT: ret1163 %op = load <16 x i16>, ptr %a1164 %res = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> %op)1165 ret i16 %res1166}1167 1168define i16 @umaxv_v32i16(ptr %a) #0 {1169; VBITS_GE_256-LABEL: umaxv_v32i16:1170; VBITS_GE_256: // %bb.0:1171; VBITS_GE_256-NEXT: ptrue p0.h, vl161172; VBITS_GE_256-NEXT: mov x8, #16 // =0x101173; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]1174; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]1175; VBITS_GE_256-NEXT: umax z0.h, p0/m, z0.h, z1.h1176; VBITS_GE_256-NEXT: umaxv h0, p0, z0.h1177; VBITS_GE_256-NEXT: fmov w0, s01178; VBITS_GE_256-NEXT: ret1179;1180; VBITS_GE_512-LABEL: umaxv_v32i16:1181; VBITS_GE_512: // %bb.0:1182; VBITS_GE_512-NEXT: ptrue p0.h, vl321183; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]1184; VBITS_GE_512-NEXT: umaxv h0, p0, z0.h1185; VBITS_GE_512-NEXT: fmov w0, s01186; VBITS_GE_512-NEXT: ret1187 %op = load <32 x i16>, ptr %a1188 %res = call i16 @llvm.vector.reduce.umax.v32i16(<32 x i16> %op)1189 ret i16 %res1190}1191 1192define i16 @umaxv_v64i16(ptr %a) vscale_range(8,0) #0 {1193; CHECK-LABEL: umaxv_v64i16:1194; CHECK: // %bb.0:1195; CHECK-NEXT: ptrue p0.h, vl641196; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1197; CHECK-NEXT: umaxv h0, p0, z0.h1198; CHECK-NEXT: fmov w0, s01199; CHECK-NEXT: ret1200 %op = load <64 x i16>, ptr %a1201 %res = call i16 @llvm.vector.reduce.umax.v64i16(<64 x i16> %op)1202 ret i16 %res1203}1204 1205define i16 @umaxv_v128i16(ptr %a) vscale_range(16,0) #0 {1206; CHECK-LABEL: umaxv_v128i16:1207; CHECK: // %bb.0:1208; CHECK-NEXT: ptrue p0.h, vl1281209; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1210; CHECK-NEXT: umaxv h0, p0, z0.h1211; CHECK-NEXT: fmov w0, s01212; CHECK-NEXT: ret1213 %op = load <128 x i16>, ptr %a1214 %res = call i16 @llvm.vector.reduce.umax.v128i16(<128 x i16> %op)1215 ret i16 %res1216}1217 1218; Don't use SVE for 64-bit vectors.1219define i32 @umaxv_v2i32(<2 x i32> %a) vscale_range(2,0) #0 {1220; CHECK-LABEL: umaxv_v2i32:1221; CHECK: // %bb.0:1222; CHECK-NEXT: umaxp v0.2s, v0.2s, v0.2s1223; CHECK-NEXT: fmov w0, s01224; CHECK-NEXT: ret1225 %res = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> %a)1226 ret i32 %res1227}1228 1229; Don't use SVE for 128-bit vectors.1230define i32 @umaxv_v4i32(<4 x i32> %a) vscale_range(2,0) #0 {1231; CHECK-LABEL: umaxv_v4i32:1232; CHECK: // %bb.0:1233; CHECK-NEXT: umaxv s0, v0.4s1234; CHECK-NEXT: fmov w0, s01235; CHECK-NEXT: ret1236 %res = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> %a)1237 ret i32 %res1238}1239 1240define i32 @umaxv_v8i32(ptr %a) vscale_range(2,0) #0 {1241; CHECK-LABEL: umaxv_v8i32:1242; CHECK: // %bb.0:1243; CHECK-NEXT: ptrue p0.s, vl81244; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1245; CHECK-NEXT: umaxv s0, p0, z0.s1246; CHECK-NEXT: fmov w0, s01247; CHECK-NEXT: ret1248 %op = load <8 x i32>, ptr %a1249 %res = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> %op)1250 ret i32 %res1251}1252 1253define i32 @umaxv_v16i32(ptr %a) #0 {1254; VBITS_GE_256-LABEL: umaxv_v16i32:1255; VBITS_GE_256: // %bb.0:1256; VBITS_GE_256-NEXT: ptrue p0.s, vl81257; VBITS_GE_256-NEXT: mov x8, #8 // =0x81258; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]1259; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]1260; VBITS_GE_256-NEXT: umax z0.s, p0/m, z0.s, z1.s1261; VBITS_GE_256-NEXT: umaxv s0, p0, z0.s1262; VBITS_GE_256-NEXT: fmov w0, s01263; VBITS_GE_256-NEXT: ret1264;1265; VBITS_GE_512-LABEL: umaxv_v16i32:1266; VBITS_GE_512: // %bb.0:1267; VBITS_GE_512-NEXT: ptrue p0.s, vl161268; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]1269; VBITS_GE_512-NEXT: umaxv s0, p0, z0.s1270; VBITS_GE_512-NEXT: fmov w0, s01271; VBITS_GE_512-NEXT: ret1272 %op = load <16 x i32>, ptr %a1273 %res = call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> %op)1274 ret i32 %res1275}1276 1277define i32 @umaxv_v32i32(ptr %a) vscale_range(8,0) #0 {1278; CHECK-LABEL: umaxv_v32i32:1279; CHECK: // %bb.0:1280; CHECK-NEXT: ptrue p0.s, vl321281; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1282; CHECK-NEXT: umaxv s0, p0, z0.s1283; CHECK-NEXT: fmov w0, s01284; CHECK-NEXT: ret1285 %op = load <32 x i32>, ptr %a1286 %res = call i32 @llvm.vector.reduce.umax.v32i32(<32 x i32> %op)1287 ret i32 %res1288}1289 1290define i32 @umaxv_v64i32(ptr %a) vscale_range(16,0) #0 {1291; CHECK-LABEL: umaxv_v64i32:1292; CHECK: // %bb.0:1293; CHECK-NEXT: ptrue p0.s, vl641294; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1295; CHECK-NEXT: umaxv s0, p0, z0.s1296; CHECK-NEXT: fmov w0, s01297; CHECK-NEXT: ret1298 %op = load <64 x i32>, ptr %a1299 %res = call i32 @llvm.vector.reduce.umax.v64i32(<64 x i32> %op)1300 ret i32 %res1301}1302 1303; Nothing to do for single element vectors.1304define i64 @umaxv_v1i64(<1 x i64> %a) vscale_range(2,0) #0 {1305; CHECK-LABEL: umaxv_v1i64:1306; CHECK: // %bb.0:1307; CHECK-NEXT: // kill: def $d0 killed $d0 def $q01308; CHECK-NEXT: fmov x0, d01309; CHECK-NEXT: ret1310 %res = call i64 @llvm.vector.reduce.umax.v1i64(<1 x i64> %a)1311 ret i64 %res1312}1313 1314; No NEON 64-bit vector UMAXV support. Use SVE.1315define i64 @umaxv_v2i64(<2 x i64> %a) vscale_range(2,0) #0 {1316; CHECK-LABEL: umaxv_v2i64:1317; CHECK: // %bb.0:1318; CHECK-NEXT: ptrue p0.d, vl21319; CHECK-NEXT: // kill: def $q0 killed $q0 def $z01320; CHECK-NEXT: umaxv d0, p0, z0.d1321; CHECK-NEXT: fmov x0, d01322; CHECK-NEXT: ret1323 %res = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> %a)1324 ret i64 %res1325}1326 1327define i64 @umaxv_v4i64(ptr %a) vscale_range(2,0) #0 {1328; CHECK-LABEL: umaxv_v4i64:1329; CHECK: // %bb.0:1330; CHECK-NEXT: ptrue p0.d, vl41331; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1332; CHECK-NEXT: umaxv d0, p0, z0.d1333; CHECK-NEXT: fmov x0, d01334; CHECK-NEXT: ret1335 %op = load <4 x i64>, ptr %a1336 %res = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> %op)1337 ret i64 %res1338}1339 1340define i64 @umaxv_v8i64(ptr %a) #0 {1341; VBITS_GE_256-LABEL: umaxv_v8i64:1342; VBITS_GE_256: // %bb.0:1343; VBITS_GE_256-NEXT: ptrue p0.d, vl41344; VBITS_GE_256-NEXT: mov x8, #4 // =0x41345; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]1346; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]1347; VBITS_GE_256-NEXT: umax z0.d, p0/m, z0.d, z1.d1348; VBITS_GE_256-NEXT: umaxv d0, p0, z0.d1349; VBITS_GE_256-NEXT: fmov x0, d01350; VBITS_GE_256-NEXT: ret1351;1352; VBITS_GE_512-LABEL: umaxv_v8i64:1353; VBITS_GE_512: // %bb.0:1354; VBITS_GE_512-NEXT: ptrue p0.d, vl81355; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]1356; VBITS_GE_512-NEXT: umaxv d0, p0, z0.d1357; VBITS_GE_512-NEXT: fmov x0, d01358; VBITS_GE_512-NEXT: ret1359 %op = load <8 x i64>, ptr %a1360 %res = call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> %op)1361 ret i64 %res1362}1363 1364define i64 @umaxv_v16i64(ptr %a) vscale_range(8,0) #0 {1365; CHECK-LABEL: umaxv_v16i64:1366; CHECK: // %bb.0:1367; CHECK-NEXT: ptrue p0.d, vl161368; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1369; CHECK-NEXT: umaxv d0, p0, z0.d1370; CHECK-NEXT: fmov x0, d01371; CHECK-NEXT: ret1372 %op = load <16 x i64>, ptr %a1373 %res = call i64 @llvm.vector.reduce.umax.v16i64(<16 x i64> %op)1374 ret i64 %res1375}1376 1377define i64 @umaxv_v32i64(ptr %a) vscale_range(16,0) #0 {1378; CHECK-LABEL: umaxv_v32i64:1379; CHECK: // %bb.0:1380; CHECK-NEXT: ptrue p0.d, vl321381; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1382; CHECK-NEXT: umaxv d0, p0, z0.d1383; CHECK-NEXT: fmov x0, d01384; CHECK-NEXT: ret1385 %op = load <32 x i64>, ptr %a1386 %res = call i64 @llvm.vector.reduce.umax.v32i64(<32 x i64> %op)1387 ret i64 %res1388}1389 1390;1391; UMINV1392;1393 1394; Don't use SVE for 64-bit vectors.1395define i8 @uminv_v8i8(<8 x i8> %a) vscale_range(2,0) #0 {1396; CHECK-LABEL: uminv_v8i8:1397; CHECK: // %bb.0:1398; CHECK-NEXT: uminv b0, v0.8b1399; CHECK-NEXT: fmov w0, s01400; CHECK-NEXT: ret1401 %res = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> %a)1402 ret i8 %res1403}1404 1405; Don't use SVE for 128-bit vectors.1406define i8 @uminv_v16i8(<16 x i8> %a) vscale_range(2,0) #0 {1407; CHECK-LABEL: uminv_v16i8:1408; CHECK: // %bb.0:1409; CHECK-NEXT: uminv b0, v0.16b1410; CHECK-NEXT: fmov w0, s01411; CHECK-NEXT: ret1412 %res = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> %a)1413 ret i8 %res1414}1415 1416define i8 @uminv_v32i8(ptr %a) vscale_range(2,0) #0 {1417; CHECK-LABEL: uminv_v32i8:1418; CHECK: // %bb.0:1419; CHECK-NEXT: ptrue p0.b, vl321420; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]1421; CHECK-NEXT: uminv b0, p0, z0.b1422; CHECK-NEXT: fmov w0, s01423; CHECK-NEXT: ret1424 %op = load <32 x i8>, ptr %a1425 %res = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> %op)1426 ret i8 %res1427}1428 1429define i8 @uminv_v64i8(ptr %a) #0 {1430; VBITS_GE_256-LABEL: uminv_v64i8:1431; VBITS_GE_256: // %bb.0:1432; VBITS_GE_256-NEXT: ptrue p0.b, vl321433; VBITS_GE_256-NEXT: mov w8, #32 // =0x201434; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]1435; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]1436; VBITS_GE_256-NEXT: umin z0.b, p0/m, z0.b, z1.b1437; VBITS_GE_256-NEXT: uminv b0, p0, z0.b1438; VBITS_GE_256-NEXT: fmov w0, s01439; VBITS_GE_256-NEXT: ret1440;1441; VBITS_GE_512-LABEL: uminv_v64i8:1442; VBITS_GE_512: // %bb.0:1443; VBITS_GE_512-NEXT: ptrue p0.b, vl641444; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]1445; VBITS_GE_512-NEXT: uminv b0, p0, z0.b1446; VBITS_GE_512-NEXT: fmov w0, s01447; VBITS_GE_512-NEXT: ret1448 %op = load <64 x i8>, ptr %a1449 %res = call i8 @llvm.vector.reduce.umin.v64i8(<64 x i8> %op)1450 ret i8 %res1451}1452 1453define i8 @uminv_v128i8(ptr %a) vscale_range(8,0) #0 {1454; CHECK-LABEL: uminv_v128i8:1455; CHECK: // %bb.0:1456; CHECK-NEXT: ptrue p0.b, vl1281457; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]1458; CHECK-NEXT: uminv b0, p0, z0.b1459; CHECK-NEXT: fmov w0, s01460; CHECK-NEXT: ret1461 %op = load <128 x i8>, ptr %a1462 %res = call i8 @llvm.vector.reduce.umin.v128i8(<128 x i8> %op)1463 ret i8 %res1464}1465 1466define i8 @uminv_v256i8(ptr %a) vscale_range(16,0) #0 {1467; CHECK-LABEL: uminv_v256i8:1468; CHECK: // %bb.0:1469; CHECK-NEXT: ptrue p0.b, vl2561470; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]1471; CHECK-NEXT: uminv b0, p0, z0.b1472; CHECK-NEXT: fmov w0, s01473; CHECK-NEXT: ret1474 %op = load <256 x i8>, ptr %a1475 %res = call i8 @llvm.vector.reduce.umin.v256i8(<256 x i8> %op)1476 ret i8 %res1477}1478 1479; Don't use SVE for 64-bit vectors.1480define i16 @uminv_v4i16(<4 x i16> %a) vscale_range(2,0) #0 {1481; CHECK-LABEL: uminv_v4i16:1482; CHECK: // %bb.0:1483; CHECK-NEXT: uminv h0, v0.4h1484; CHECK-NEXT: fmov w0, s01485; CHECK-NEXT: ret1486 %res = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> %a)1487 ret i16 %res1488}1489 1490; Don't use SVE for 128-bit vectors.1491define i16 @uminv_v8i16(<8 x i16> %a) vscale_range(2,0) #0 {1492; CHECK-LABEL: uminv_v8i16:1493; CHECK: // %bb.0:1494; CHECK-NEXT: uminv h0, v0.8h1495; CHECK-NEXT: fmov w0, s01496; CHECK-NEXT: ret1497 %res = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> %a)1498 ret i16 %res1499}1500 1501define i16 @uminv_v16i16(ptr %a) vscale_range(2,0) #0 {1502; CHECK-LABEL: uminv_v16i16:1503; CHECK: // %bb.0:1504; CHECK-NEXT: ptrue p0.h, vl161505; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1506; CHECK-NEXT: uminv h0, p0, z0.h1507; CHECK-NEXT: fmov w0, s01508; CHECK-NEXT: ret1509 %op = load <16 x i16>, ptr %a1510 %res = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> %op)1511 ret i16 %res1512}1513 1514define i16 @uminv_v32i16(ptr %a) #0 {1515; VBITS_GE_256-LABEL: uminv_v32i16:1516; VBITS_GE_256: // %bb.0:1517; VBITS_GE_256-NEXT: ptrue p0.h, vl161518; VBITS_GE_256-NEXT: mov x8, #16 // =0x101519; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]1520; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]1521; VBITS_GE_256-NEXT: umin z0.h, p0/m, z0.h, z1.h1522; VBITS_GE_256-NEXT: uminv h0, p0, z0.h1523; VBITS_GE_256-NEXT: fmov w0, s01524; VBITS_GE_256-NEXT: ret1525;1526; VBITS_GE_512-LABEL: uminv_v32i16:1527; VBITS_GE_512: // %bb.0:1528; VBITS_GE_512-NEXT: ptrue p0.h, vl321529; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]1530; VBITS_GE_512-NEXT: uminv h0, p0, z0.h1531; VBITS_GE_512-NEXT: fmov w0, s01532; VBITS_GE_512-NEXT: ret1533 %op = load <32 x i16>, ptr %a1534 %res = call i16 @llvm.vector.reduce.umin.v32i16(<32 x i16> %op)1535 ret i16 %res1536}1537 1538define i16 @uminv_v64i16(ptr %a) vscale_range(8,0) #0 {1539; CHECK-LABEL: uminv_v64i16:1540; CHECK: // %bb.0:1541; CHECK-NEXT: ptrue p0.h, vl641542; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1543; CHECK-NEXT: uminv h0, p0, z0.h1544; CHECK-NEXT: fmov w0, s01545; CHECK-NEXT: ret1546 %op = load <64 x i16>, ptr %a1547 %res = call i16 @llvm.vector.reduce.umin.v64i16(<64 x i16> %op)1548 ret i16 %res1549}1550 1551define i16 @uminv_v128i16(ptr %a) vscale_range(16,0) #0 {1552; CHECK-LABEL: uminv_v128i16:1553; CHECK: // %bb.0:1554; CHECK-NEXT: ptrue p0.h, vl1281555; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1556; CHECK-NEXT: uminv h0, p0, z0.h1557; CHECK-NEXT: fmov w0, s01558; CHECK-NEXT: ret1559 %op = load <128 x i16>, ptr %a1560 %res = call i16 @llvm.vector.reduce.umin.v128i16(<128 x i16> %op)1561 ret i16 %res1562}1563 1564; Don't use SVE for 64-bit vectors.1565define i32 @uminv_v2i32(<2 x i32> %a) vscale_range(2,0) #0 {1566; CHECK-LABEL: uminv_v2i32:1567; CHECK: // %bb.0:1568; CHECK-NEXT: uminp v0.2s, v0.2s, v0.2s1569; CHECK-NEXT: fmov w0, s01570; CHECK-NEXT: ret1571 %res = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> %a)1572 ret i32 %res1573}1574 1575; Don't use SVE for 128-bit vectors.1576define i32 @uminv_v4i32(<4 x i32> %a) vscale_range(2,0) #0 {1577; CHECK-LABEL: uminv_v4i32:1578; CHECK: // %bb.0:1579; CHECK-NEXT: uminv s0, v0.4s1580; CHECK-NEXT: fmov w0, s01581; CHECK-NEXT: ret1582 %res = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> %a)1583 ret i32 %res1584}1585 1586define i32 @uminv_v8i32(ptr %a) vscale_range(2,0) #0 {1587; CHECK-LABEL: uminv_v8i32:1588; CHECK: // %bb.0:1589; CHECK-NEXT: ptrue p0.s, vl81590; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1591; CHECK-NEXT: uminv s0, p0, z0.s1592; CHECK-NEXT: fmov w0, s01593; CHECK-NEXT: ret1594 %op = load <8 x i32>, ptr %a1595 %res = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> %op)1596 ret i32 %res1597}1598 1599define i32 @uminv_v16i32(ptr %a) #0 {1600; VBITS_GE_256-LABEL: uminv_v16i32:1601; VBITS_GE_256: // %bb.0:1602; VBITS_GE_256-NEXT: ptrue p0.s, vl81603; VBITS_GE_256-NEXT: mov x8, #8 // =0x81604; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]1605; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]1606; VBITS_GE_256-NEXT: umin z0.s, p0/m, z0.s, z1.s1607; VBITS_GE_256-NEXT: uminv s0, p0, z0.s1608; VBITS_GE_256-NEXT: fmov w0, s01609; VBITS_GE_256-NEXT: ret1610;1611; VBITS_GE_512-LABEL: uminv_v16i32:1612; VBITS_GE_512: // %bb.0:1613; VBITS_GE_512-NEXT: ptrue p0.s, vl161614; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]1615; VBITS_GE_512-NEXT: uminv s0, p0, z0.s1616; VBITS_GE_512-NEXT: fmov w0, s01617; VBITS_GE_512-NEXT: ret1618 %op = load <16 x i32>, ptr %a1619 %res = call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> %op)1620 ret i32 %res1621}1622 1623define i32 @uminv_v32i32(ptr %a) vscale_range(8,0) #0 {1624; CHECK-LABEL: uminv_v32i32:1625; CHECK: // %bb.0:1626; CHECK-NEXT: ptrue p0.s, vl321627; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1628; CHECK-NEXT: uminv s0, p0, z0.s1629; CHECK-NEXT: fmov w0, s01630; CHECK-NEXT: ret1631 %op = load <32 x i32>, ptr %a1632 %res = call i32 @llvm.vector.reduce.umin.v32i32(<32 x i32> %op)1633 ret i32 %res1634}1635 1636define i32 @uminv_v64i32(ptr %a) vscale_range(16,0) #0 {1637; CHECK-LABEL: uminv_v64i32:1638; CHECK: // %bb.0:1639; CHECK-NEXT: ptrue p0.s, vl641640; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1641; CHECK-NEXT: uminv s0, p0, z0.s1642; CHECK-NEXT: fmov w0, s01643; CHECK-NEXT: ret1644 %op = load <64 x i32>, ptr %a1645 %res = call i32 @llvm.vector.reduce.umin.v64i32(<64 x i32> %op)1646 ret i32 %res1647}1648 1649; Nothing to do for single element vectors.1650define i64 @uminv_v1i64(<1 x i64> %a) vscale_range(2,0) #0 {1651; CHECK-LABEL: uminv_v1i64:1652; CHECK: // %bb.0:1653; CHECK-NEXT: // kill: def $d0 killed $d0 def $q01654; CHECK-NEXT: fmov x0, d01655; CHECK-NEXT: ret1656 %res = call i64 @llvm.vector.reduce.umin.v1i64(<1 x i64> %a)1657 ret i64 %res1658}1659 1660; No NEON 64-bit vector UMINV support. Use SVE.1661define i64 @uminv_v2i64(<2 x i64> %a) vscale_range(2,0) #0 {1662; CHECK-LABEL: uminv_v2i64:1663; CHECK: // %bb.0:1664; CHECK-NEXT: ptrue p0.d, vl21665; CHECK-NEXT: // kill: def $q0 killed $q0 def $z01666; CHECK-NEXT: uminv d0, p0, z0.d1667; CHECK-NEXT: fmov x0, d01668; CHECK-NEXT: ret1669 %res = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> %a)1670 ret i64 %res1671}1672 1673define i64 @uminv_v4i64(ptr %a) vscale_range(2,0) #0 {1674; CHECK-LABEL: uminv_v4i64:1675; CHECK: // %bb.0:1676; CHECK-NEXT: ptrue p0.d, vl41677; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1678; CHECK-NEXT: uminv d0, p0, z0.d1679; CHECK-NEXT: fmov x0, d01680; CHECK-NEXT: ret1681 %op = load <4 x i64>, ptr %a1682 %res = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> %op)1683 ret i64 %res1684}1685 1686define i64 @uminv_v8i64(ptr %a) #0 {1687; VBITS_GE_256-LABEL: uminv_v8i64:1688; VBITS_GE_256: // %bb.0:1689; VBITS_GE_256-NEXT: ptrue p0.d, vl41690; VBITS_GE_256-NEXT: mov x8, #4 // =0x41691; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]1692; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]1693; VBITS_GE_256-NEXT: umin z0.d, p0/m, z0.d, z1.d1694; VBITS_GE_256-NEXT: uminv d0, p0, z0.d1695; VBITS_GE_256-NEXT: fmov x0, d01696; VBITS_GE_256-NEXT: ret1697;1698; VBITS_GE_512-LABEL: uminv_v8i64:1699; VBITS_GE_512: // %bb.0:1700; VBITS_GE_512-NEXT: ptrue p0.d, vl81701; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]1702; VBITS_GE_512-NEXT: uminv d0, p0, z0.d1703; VBITS_GE_512-NEXT: fmov x0, d01704; VBITS_GE_512-NEXT: ret1705 %op = load <8 x i64>, ptr %a1706 %res = call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> %op)1707 ret i64 %res1708}1709 1710define i64 @uminv_v16i64(ptr %a) vscale_range(8,0) #0 {1711; CHECK-LABEL: uminv_v16i64:1712; CHECK: // %bb.0:1713; CHECK-NEXT: ptrue p0.d, vl161714; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1715; CHECK-NEXT: uminv d0, p0, z0.d1716; CHECK-NEXT: fmov x0, d01717; CHECK-NEXT: ret1718 %op = load <16 x i64>, ptr %a1719 %res = call i64 @llvm.vector.reduce.umin.v16i64(<16 x i64> %op)1720 ret i64 %res1721}1722 1723define i64 @uminv_v32i64(ptr %a) vscale_range(16,0) #0 {1724; CHECK-LABEL: uminv_v32i64:1725; CHECK: // %bb.0:1726; CHECK-NEXT: ptrue p0.d, vl321727; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1728; CHECK-NEXT: uminv d0, p0, z0.d1729; CHECK-NEXT: fmov x0, d01730; CHECK-NEXT: ret1731 %op = load <32 x i64>, ptr %a1732 %res = call i64 @llvm.vector.reduce.umin.v32i64(<32 x i64> %op)1733 ret i64 %res1734}1735 1736attributes #0 = { "target-features"="+sve" }1737 1738declare i8 @llvm.vector.reduce.add.v8i8(<8 x i8>)1739declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)1740declare i8 @llvm.vector.reduce.add.v32i8(<32 x i8>)1741declare i8 @llvm.vector.reduce.add.v64i8(<64 x i8>)1742declare i8 @llvm.vector.reduce.add.v128i8(<128 x i8>)1743declare i8 @llvm.vector.reduce.add.v256i8(<256 x i8>)1744 1745declare i16 @llvm.vector.reduce.add.v4i16(<4 x i16>)1746declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)1747declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>)1748declare i16 @llvm.vector.reduce.add.v32i16(<32 x i16>)1749declare i16 @llvm.vector.reduce.add.v64i16(<64 x i16>)1750declare i16 @llvm.vector.reduce.add.v128i16(<128 x i16>)1751 1752declare i32 @llvm.vector.reduce.add.v2i32(<2 x i32>)1753declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)1754declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)1755declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)1756declare i32 @llvm.vector.reduce.add.v32i32(<32 x i32>)1757declare i32 @llvm.vector.reduce.add.v64i32(<64 x i32>)1758 1759declare i64 @llvm.vector.reduce.add.v1i64(<1 x i64>)1760declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>)1761declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)1762declare i64 @llvm.vector.reduce.add.v8i64(<8 x i64>)1763declare i64 @llvm.vector.reduce.add.v16i64(<16 x i64>)1764declare i64 @llvm.vector.reduce.add.v32i64(<32 x i64>)1765 1766declare i8 @llvm.vector.reduce.smax.v8i8(<8 x i8>)1767declare i8 @llvm.vector.reduce.smax.v16i8(<16 x i8>)1768declare i8 @llvm.vector.reduce.smax.v32i8(<32 x i8>)1769declare i8 @llvm.vector.reduce.smax.v64i8(<64 x i8>)1770declare i8 @llvm.vector.reduce.smax.v128i8(<128 x i8>)1771declare i8 @llvm.vector.reduce.smax.v256i8(<256 x i8>)1772 1773declare i16 @llvm.vector.reduce.smax.v4i16(<4 x i16>)1774declare i16 @llvm.vector.reduce.smax.v8i16(<8 x i16>)1775declare i16 @llvm.vector.reduce.smax.v16i16(<16 x i16>)1776declare i16 @llvm.vector.reduce.smax.v32i16(<32 x i16>)1777declare i16 @llvm.vector.reduce.smax.v64i16(<64 x i16>)1778declare i16 @llvm.vector.reduce.smax.v128i16(<128 x i16>)1779 1780declare i32 @llvm.vector.reduce.smax.v2i32(<2 x i32>)1781declare i32 @llvm.vector.reduce.smax.v4i32(<4 x i32>)1782declare i32 @llvm.vector.reduce.smax.v8i32(<8 x i32>)1783declare i32 @llvm.vector.reduce.smax.v16i32(<16 x i32>)1784declare i32 @llvm.vector.reduce.smax.v32i32(<32 x i32>)1785declare i32 @llvm.vector.reduce.smax.v64i32(<64 x i32>)1786 1787declare i64 @llvm.vector.reduce.smax.v1i64(<1 x i64>)1788declare i64 @llvm.vector.reduce.smax.v2i64(<2 x i64>)1789declare i64 @llvm.vector.reduce.smax.v4i64(<4 x i64>)1790declare i64 @llvm.vector.reduce.smax.v8i64(<8 x i64>)1791declare i64 @llvm.vector.reduce.smax.v16i64(<16 x i64>)1792declare i64 @llvm.vector.reduce.smax.v32i64(<32 x i64>)1793 1794declare i8 @llvm.vector.reduce.smin.v8i8(<8 x i8>)1795declare i8 @llvm.vector.reduce.smin.v16i8(<16 x i8>)1796declare i8 @llvm.vector.reduce.smin.v32i8(<32 x i8>)1797declare i8 @llvm.vector.reduce.smin.v64i8(<64 x i8>)1798declare i8 @llvm.vector.reduce.smin.v128i8(<128 x i8>)1799declare i8 @llvm.vector.reduce.smin.v256i8(<256 x i8>)1800 1801declare i16 @llvm.vector.reduce.smin.v4i16(<4 x i16>)1802declare i16 @llvm.vector.reduce.smin.v8i16(<8 x i16>)1803declare i16 @llvm.vector.reduce.smin.v16i16(<16 x i16>)1804declare i16 @llvm.vector.reduce.smin.v32i16(<32 x i16>)1805declare i16 @llvm.vector.reduce.smin.v64i16(<64 x i16>)1806declare i16 @llvm.vector.reduce.smin.v128i16(<128 x i16>)1807 1808declare i32 @llvm.vector.reduce.smin.v2i32(<2 x i32>)1809declare i32 @llvm.vector.reduce.smin.v4i32(<4 x i32>)1810declare i32 @llvm.vector.reduce.smin.v8i32(<8 x i32>)1811declare i32 @llvm.vector.reduce.smin.v16i32(<16 x i32>)1812declare i32 @llvm.vector.reduce.smin.v32i32(<32 x i32>)1813declare i32 @llvm.vector.reduce.smin.v64i32(<64 x i32>)1814 1815declare i64 @llvm.vector.reduce.smin.v1i64(<1 x i64>)1816declare i64 @llvm.vector.reduce.smin.v2i64(<2 x i64>)1817declare i64 @llvm.vector.reduce.smin.v4i64(<4 x i64>)1818declare i64 @llvm.vector.reduce.smin.v8i64(<8 x i64>)1819declare i64 @llvm.vector.reduce.smin.v16i64(<16 x i64>)1820declare i64 @llvm.vector.reduce.smin.v32i64(<32 x i64>)1821 1822declare i8 @llvm.vector.reduce.umax.v8i8(<8 x i8>)1823declare i8 @llvm.vector.reduce.umax.v16i8(<16 x i8>)1824declare i8 @llvm.vector.reduce.umax.v32i8(<32 x i8>)1825declare i8 @llvm.vector.reduce.umax.v64i8(<64 x i8>)1826declare i8 @llvm.vector.reduce.umax.v128i8(<128 x i8>)1827declare i8 @llvm.vector.reduce.umax.v256i8(<256 x i8>)1828 1829declare i16 @llvm.vector.reduce.umax.v4i16(<4 x i16>)1830declare i16 @llvm.vector.reduce.umax.v8i16(<8 x i16>)1831declare i16 @llvm.vector.reduce.umax.v16i16(<16 x i16>)1832declare i16 @llvm.vector.reduce.umax.v32i16(<32 x i16>)1833declare i16 @llvm.vector.reduce.umax.v64i16(<64 x i16>)1834declare i16 @llvm.vector.reduce.umax.v128i16(<128 x i16>)1835 1836declare i32 @llvm.vector.reduce.umax.v2i32(<2 x i32>)1837declare i32 @llvm.vector.reduce.umax.v4i32(<4 x i32>)1838declare i32 @llvm.vector.reduce.umax.v8i32(<8 x i32>)1839declare i32 @llvm.vector.reduce.umax.v16i32(<16 x i32>)1840declare i32 @llvm.vector.reduce.umax.v32i32(<32 x i32>)1841declare i32 @llvm.vector.reduce.umax.v64i32(<64 x i32>)1842 1843declare i64 @llvm.vector.reduce.umax.v1i64(<1 x i64>)1844declare i64 @llvm.vector.reduce.umax.v2i64(<2 x i64>)1845declare i64 @llvm.vector.reduce.umax.v4i64(<4 x i64>)1846declare i64 @llvm.vector.reduce.umax.v8i64(<8 x i64>)1847declare i64 @llvm.vector.reduce.umax.v16i64(<16 x i64>)1848declare i64 @llvm.vector.reduce.umax.v32i64(<32 x i64>)1849 1850declare i8 @llvm.vector.reduce.umin.v8i8(<8 x i8>)1851declare i8 @llvm.vector.reduce.umin.v16i8(<16 x i8>)1852declare i8 @llvm.vector.reduce.umin.v32i8(<32 x i8>)1853declare i8 @llvm.vector.reduce.umin.v64i8(<64 x i8>)1854declare i8 @llvm.vector.reduce.umin.v128i8(<128 x i8>)1855declare i8 @llvm.vector.reduce.umin.v256i8(<256 x i8>)1856 1857declare i16 @llvm.vector.reduce.umin.v4i16(<4 x i16>)1858declare i16 @llvm.vector.reduce.umin.v8i16(<8 x i16>)1859declare i16 @llvm.vector.reduce.umin.v16i16(<16 x i16>)1860declare i16 @llvm.vector.reduce.umin.v32i16(<32 x i16>)1861declare i16 @llvm.vector.reduce.umin.v64i16(<64 x i16>)1862declare i16 @llvm.vector.reduce.umin.v128i16(<128 x i16>)1863 1864declare i32 @llvm.vector.reduce.umin.v2i32(<2 x i32>)1865declare i32 @llvm.vector.reduce.umin.v4i32(<4 x i32>)1866declare i32 @llvm.vector.reduce.umin.v8i32(<8 x i32>)1867declare i32 @llvm.vector.reduce.umin.v16i32(<16 x i32>)1868declare i32 @llvm.vector.reduce.umin.v32i32(<32 x i32>)1869declare i32 @llvm.vector.reduce.umin.v64i32(<64 x i32>)1870 1871declare i64 @llvm.vector.reduce.umin.v1i64(<1 x i64>)1872declare i64 @llvm.vector.reduce.umin.v2i64(<2 x i64>)1873declare i64 @llvm.vector.reduce.umin.v4i64(<4 x i64>)1874declare i64 @llvm.vector.reduce.umin.v8i64(<8 x i64>)1875declare i64 @llvm.vector.reduce.umin.v16i64(<16 x i64>)1876declare i64 @llvm.vector.reduce.umin.v32i64(<32 x i64>)1877