2395 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2563; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5124; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5125 6target triple = "aarch64-unknown-linux-gnu"7 8;9; FADD10;11 12; Don't use SVE for 64-bit vectors.13define <4 x half> @fadd_v4f16(<4 x half> %op1, <4 x half> %op2) vscale_range(2,0) #0 {14; CHECK-LABEL: fadd_v4f16:15; CHECK: // %bb.0:16; CHECK-NEXT: fadd v0.4h, v0.4h, v1.4h17; CHECK-NEXT: ret18 %res = fadd <4 x half> %op1, %op219 ret <4 x half> %res20}21 22; Don't use SVE for 128-bit vectors.23define <8 x half> @fadd_v8f16(<8 x half> %op1, <8 x half> %op2) vscale_range(2,0) #0 {24; CHECK-LABEL: fadd_v8f16:25; CHECK: // %bb.0:26; CHECK-NEXT: fadd v0.8h, v0.8h, v1.8h27; CHECK-NEXT: ret28 %res = fadd <8 x half> %op1, %op229 ret <8 x half> %res30}31 32define void @fadd_v16f16(ptr %a, ptr %b) vscale_range(2,0) #0 {33; CHECK-LABEL: fadd_v16f16:34; CHECK: // %bb.0:35; CHECK-NEXT: ptrue p0.h, vl1636; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]37; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]38; CHECK-NEXT: fadd z0.h, p0/m, z0.h, z1.h39; CHECK-NEXT: st1h { z0.h }, p0, [x0]40; CHECK-NEXT: ret41 %op1 = load <16 x half>, ptr %a42 %op2 = load <16 x half>, ptr %b43 %res = fadd <16 x half> %op1, %op244 store <16 x half> %res, ptr %a45 ret void46}47 48define void @fadd_v32f16(ptr %a, ptr %b) #0 {49; VBITS_GE_256-LABEL: fadd_v32f16:50; VBITS_GE_256: // %bb.0:51; VBITS_GE_256-NEXT: ptrue p0.h, vl1652; VBITS_GE_256-NEXT: mov x8, #16 // =0x1053; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]54; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]55; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0]56; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1]57; VBITS_GE_256-NEXT: fadd z0.h, p0/m, z0.h, z1.h58; VBITS_GE_256-NEXT: fadd z2.h, p0/m, z2.h, z3.h59; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]60; VBITS_GE_256-NEXT: st1h { z2.h }, p0, [x0]61; VBITS_GE_256-NEXT: ret62;63; VBITS_GE_512-LABEL: fadd_v32f16:64; VBITS_GE_512: // %bb.0:65; VBITS_GE_512-NEXT: ptrue p0.h, vl3266; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]67; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]68; VBITS_GE_512-NEXT: fadd z0.h, p0/m, z0.h, z1.h69; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]70; VBITS_GE_512-NEXT: ret71 %op1 = load <32 x half>, ptr %a72 %op2 = load <32 x half>, ptr %b73 %res = fadd <32 x half> %op1, %op274 store <32 x half> %res, ptr %a75 ret void76}77 78define void @fadd_v64f16(ptr %a, ptr %b) vscale_range(8,0) #0 {79; CHECK-LABEL: fadd_v64f16:80; CHECK: // %bb.0:81; CHECK-NEXT: ptrue p0.h, vl6482; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]83; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]84; CHECK-NEXT: fadd z0.h, p0/m, z0.h, z1.h85; CHECK-NEXT: st1h { z0.h }, p0, [x0]86; CHECK-NEXT: ret87 %op1 = load <64 x half>, ptr %a88 %op2 = load <64 x half>, ptr %b89 %res = fadd <64 x half> %op1, %op290 store <64 x half> %res, ptr %a91 ret void92}93 94define void @fadd_v128f16(ptr %a, ptr %b) vscale_range(16,0) #0 {95; CHECK-LABEL: fadd_v128f16:96; CHECK: // %bb.0:97; CHECK-NEXT: ptrue p0.h, vl12898; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]99; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]100; CHECK-NEXT: fadd z0.h, p0/m, z0.h, z1.h101; CHECK-NEXT: st1h { z0.h }, p0, [x0]102; CHECK-NEXT: ret103 %op1 = load <128 x half>, ptr %a104 %op2 = load <128 x half>, ptr %b105 %res = fadd <128 x half> %op1, %op2106 store <128 x half> %res, ptr %a107 ret void108}109 110; Don't use SVE for 64-bit vectors.111define <2 x float> @fadd_v2f32(<2 x float> %op1, <2 x float> %op2) vscale_range(2,0) #0 {112; CHECK-LABEL: fadd_v2f32:113; CHECK: // %bb.0:114; CHECK-NEXT: fadd v0.2s, v0.2s, v1.2s115; CHECK-NEXT: ret116 %res = fadd <2 x float> %op1, %op2117 ret <2 x float> %res118}119 120; Don't use SVE for 128-bit vectors.121define <4 x float> @fadd_v4f32(<4 x float> %op1, <4 x float> %op2) vscale_range(2,0) #0 {122; CHECK-LABEL: fadd_v4f32:123; CHECK: // %bb.0:124; CHECK-NEXT: fadd v0.4s, v0.4s, v1.4s125; CHECK-NEXT: ret126 %res = fadd <4 x float> %op1, %op2127 ret <4 x float> %res128}129 130define void @fadd_v8f32(ptr %a, ptr %b) vscale_range(2,0) #0 {131; CHECK-LABEL: fadd_v8f32:132; CHECK: // %bb.0:133; CHECK-NEXT: ptrue p0.s, vl8134; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]135; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]136; CHECK-NEXT: fadd z0.s, p0/m, z0.s, z1.s137; CHECK-NEXT: st1w { z0.s }, p0, [x0]138; CHECK-NEXT: ret139 %op1 = load <8 x float>, ptr %a140 %op2 = load <8 x float>, ptr %b141 %res = fadd <8 x float> %op1, %op2142 store <8 x float> %res, ptr %a143 ret void144}145 146define void @fadd_v16f32(ptr %a, ptr %b) #0 {147; VBITS_GE_256-LABEL: fadd_v16f32:148; VBITS_GE_256: // %bb.0:149; VBITS_GE_256-NEXT: ptrue p0.s, vl8150; VBITS_GE_256-NEXT: mov x8, #8 // =0x8151; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]152; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]153; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]154; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]155; VBITS_GE_256-NEXT: fadd z0.s, p0/m, z0.s, z1.s156; VBITS_GE_256-NEXT: fadd z2.s, p0/m, z2.s, z3.s157; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]158; VBITS_GE_256-NEXT: st1w { z2.s }, p0, [x0]159; VBITS_GE_256-NEXT: ret160;161; VBITS_GE_512-LABEL: fadd_v16f32:162; VBITS_GE_512: // %bb.0:163; VBITS_GE_512-NEXT: ptrue p0.s, vl16164; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]165; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]166; VBITS_GE_512-NEXT: fadd z0.s, p0/m, z0.s, z1.s167; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]168; VBITS_GE_512-NEXT: ret169 %op1 = load <16 x float>, ptr %a170 %op2 = load <16 x float>, ptr %b171 %res = fadd <16 x float> %op1, %op2172 store <16 x float> %res, ptr %a173 ret void174}175 176define void @fadd_v32f32(ptr %a, ptr %b) vscale_range(8,0) #0 {177; CHECK-LABEL: fadd_v32f32:178; CHECK: // %bb.0:179; CHECK-NEXT: ptrue p0.s, vl32180; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]181; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]182; CHECK-NEXT: fadd z0.s, p0/m, z0.s, z1.s183; CHECK-NEXT: st1w { z0.s }, p0, [x0]184; CHECK-NEXT: ret185 %op1 = load <32 x float>, ptr %a186 %op2 = load <32 x float>, ptr %b187 %res = fadd <32 x float> %op1, %op2188 store <32 x float> %res, ptr %a189 ret void190}191 192define void @fadd_v64f32(ptr %a, ptr %b) vscale_range(16,0) #0 {193; CHECK-LABEL: fadd_v64f32:194; CHECK: // %bb.0:195; CHECK-NEXT: ptrue p0.s, vl64196; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]197; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]198; CHECK-NEXT: fadd z0.s, p0/m, z0.s, z1.s199; CHECK-NEXT: st1w { z0.s }, p0, [x0]200; CHECK-NEXT: ret201 %op1 = load <64 x float>, ptr %a202 %op2 = load <64 x float>, ptr %b203 %res = fadd <64 x float> %op1, %op2204 store <64 x float> %res, ptr %a205 ret void206}207 208; Don't use SVE for 64-bit vectors.209define <1 x double> @fadd_v1f64(<1 x double> %op1, <1 x double> %op2) vscale_range(2,0) #0 {210; CHECK-LABEL: fadd_v1f64:211; CHECK: // %bb.0:212; CHECK-NEXT: fadd d0, d0, d1213; CHECK-NEXT: ret214 %res = fadd <1 x double> %op1, %op2215 ret <1 x double> %res216}217 218; Don't use SVE for 128-bit vectors.219define <2 x double> @fadd_v2f64(<2 x double> %op1, <2 x double> %op2) vscale_range(2,0) #0 {220; CHECK-LABEL: fadd_v2f64:221; CHECK: // %bb.0:222; CHECK-NEXT: fadd v0.2d, v0.2d, v1.2d223; CHECK-NEXT: ret224 %res = fadd <2 x double> %op1, %op2225 ret <2 x double> %res226}227 228define void @fadd_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 {229; CHECK-LABEL: fadd_v4f64:230; CHECK: // %bb.0:231; CHECK-NEXT: ptrue p0.d, vl4232; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]233; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]234; CHECK-NEXT: fadd z0.d, p0/m, z0.d, z1.d235; CHECK-NEXT: st1d { z0.d }, p0, [x0]236; CHECK-NEXT: ret237 %op1 = load <4 x double>, ptr %a238 %op2 = load <4 x double>, ptr %b239 %res = fadd <4 x double> %op1, %op2240 store <4 x double> %res, ptr %a241 ret void242}243 244define void @fadd_v8f64(ptr %a, ptr %b) #0 {245; VBITS_GE_256-LABEL: fadd_v8f64:246; VBITS_GE_256: // %bb.0:247; VBITS_GE_256-NEXT: ptrue p0.d, vl4248; VBITS_GE_256-NEXT: mov x8, #4 // =0x4249; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]250; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]251; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]252; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]253; VBITS_GE_256-NEXT: fadd z0.d, p0/m, z0.d, z1.d254; VBITS_GE_256-NEXT: fadd z2.d, p0/m, z2.d, z3.d255; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]256; VBITS_GE_256-NEXT: st1d { z2.d }, p0, [x0]257; VBITS_GE_256-NEXT: ret258;259; VBITS_GE_512-LABEL: fadd_v8f64:260; VBITS_GE_512: // %bb.0:261; VBITS_GE_512-NEXT: ptrue p0.d, vl8262; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]263; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]264; VBITS_GE_512-NEXT: fadd z0.d, p0/m, z0.d, z1.d265; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]266; VBITS_GE_512-NEXT: ret267 %op1 = load <8 x double>, ptr %a268 %op2 = load <8 x double>, ptr %b269 %res = fadd <8 x double> %op1, %op2270 store <8 x double> %res, ptr %a271 ret void272}273 274define void @fadd_v16f64(ptr %a, ptr %b) vscale_range(8,0) #0 {275; CHECK-LABEL: fadd_v16f64:276; CHECK: // %bb.0:277; CHECK-NEXT: ptrue p0.d, vl16278; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]279; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]280; CHECK-NEXT: fadd z0.d, p0/m, z0.d, z1.d281; CHECK-NEXT: st1d { z0.d }, p0, [x0]282; CHECK-NEXT: ret283 %op1 = load <16 x double>, ptr %a284 %op2 = load <16 x double>, ptr %b285 %res = fadd <16 x double> %op1, %op2286 store <16 x double> %res, ptr %a287 ret void288}289 290define void @fadd_v32f64(ptr %a, ptr %b) vscale_range(16,0) #0 {291; CHECK-LABEL: fadd_v32f64:292; CHECK: // %bb.0:293; CHECK-NEXT: ptrue p0.d, vl32294; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]295; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]296; CHECK-NEXT: fadd z0.d, p0/m, z0.d, z1.d297; CHECK-NEXT: st1d { z0.d }, p0, [x0]298; CHECK-NEXT: ret299 %op1 = load <32 x double>, ptr %a300 %op2 = load <32 x double>, ptr %b301 %res = fadd <32 x double> %op1, %op2302 store <32 x double> %res, ptr %a303 ret void304}305 306;307; FDIV308;309 310; Don't use SVE for 64-bit vectors.311define <4 x half> @fdiv_v4f16(<4 x half> %op1, <4 x half> %op2) vscale_range(2,0) #0 {312; CHECK-LABEL: fdiv_v4f16:313; CHECK: // %bb.0:314; CHECK-NEXT: fdiv v0.4h, v0.4h, v1.4h315; CHECK-NEXT: ret316 %res = fdiv <4 x half> %op1, %op2317 ret <4 x half> %res318}319 320; Don't use SVE for 128-bit vectors.321define <8 x half> @fdiv_v8f16(<8 x half> %op1, <8 x half> %op2) vscale_range(2,0) #0 {322; CHECK-LABEL: fdiv_v8f16:323; CHECK: // %bb.0:324; CHECK-NEXT: fdiv v0.8h, v0.8h, v1.8h325; CHECK-NEXT: ret326 %res = fdiv <8 x half> %op1, %op2327 ret <8 x half> %res328}329 330define void @fdiv_v16f16(ptr %a, ptr %b) vscale_range(2,0) #0 {331; CHECK-LABEL: fdiv_v16f16:332; CHECK: // %bb.0:333; CHECK-NEXT: ptrue p0.h, vl16334; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]335; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]336; CHECK-NEXT: fdiv z0.h, p0/m, z0.h, z1.h337; CHECK-NEXT: st1h { z0.h }, p0, [x0]338; CHECK-NEXT: ret339 %op1 = load <16 x half>, ptr %a340 %op2 = load <16 x half>, ptr %b341 %res = fdiv <16 x half> %op1, %op2342 store <16 x half> %res, ptr %a343 ret void344}345 346define void @fdiv_v32f16(ptr %a, ptr %b) #0 {347; VBITS_GE_256-LABEL: fdiv_v32f16:348; VBITS_GE_256: // %bb.0:349; VBITS_GE_256-NEXT: ptrue p0.h, vl16350; VBITS_GE_256-NEXT: mov x8, #16 // =0x10351; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]352; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]353; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x1]354; VBITS_GE_256-NEXT: fdiv z0.h, p0/m, z0.h, z1.h355; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]356; VBITS_GE_256-NEXT: fdiv z1.h, p0/m, z1.h, z2.h357; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]358; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]359; VBITS_GE_256-NEXT: ret360;361; VBITS_GE_512-LABEL: fdiv_v32f16:362; VBITS_GE_512: // %bb.0:363; VBITS_GE_512-NEXT: ptrue p0.h, vl32364; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]365; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]366; VBITS_GE_512-NEXT: fdiv z0.h, p0/m, z0.h, z1.h367; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]368; VBITS_GE_512-NEXT: ret369 %op1 = load <32 x half>, ptr %a370 %op2 = load <32 x half>, ptr %b371 %res = fdiv <32 x half> %op1, %op2372 store <32 x half> %res, ptr %a373 ret void374}375 376define void @fdiv_v64f16(ptr %a, ptr %b) vscale_range(8,0) #0 {377; CHECK-LABEL: fdiv_v64f16:378; CHECK: // %bb.0:379; CHECK-NEXT: ptrue p0.h, vl64380; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]381; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]382; CHECK-NEXT: fdiv z0.h, p0/m, z0.h, z1.h383; CHECK-NEXT: st1h { z0.h }, p0, [x0]384; CHECK-NEXT: ret385 %op1 = load <64 x half>, ptr %a386 %op2 = load <64 x half>, ptr %b387 %res = fdiv <64 x half> %op1, %op2388 store <64 x half> %res, ptr %a389 ret void390}391 392define void @fdiv_v128f16(ptr %a, ptr %b) vscale_range(16,0) #0 {393; CHECK-LABEL: fdiv_v128f16:394; CHECK: // %bb.0:395; CHECK-NEXT: ptrue p0.h, vl128396; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]397; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]398; CHECK-NEXT: fdiv z0.h, p0/m, z0.h, z1.h399; CHECK-NEXT: st1h { z0.h }, p0, [x0]400; CHECK-NEXT: ret401 %op1 = load <128 x half>, ptr %a402 %op2 = load <128 x half>, ptr %b403 %res = fdiv <128 x half> %op1, %op2404 store <128 x half> %res, ptr %a405 ret void406}407 408; Don't use SVE for 64-bit vectors.409define <2 x float> @fdiv_v2f32(<2 x float> %op1, <2 x float> %op2) vscale_range(2,0) #0 {410; CHECK-LABEL: fdiv_v2f32:411; CHECK: // %bb.0:412; CHECK-NEXT: fdiv v0.2s, v0.2s, v1.2s413; CHECK-NEXT: ret414 %res = fdiv <2 x float> %op1, %op2415 ret <2 x float> %res416}417 418; Don't use SVE for 128-bit vectors.419define <4 x float> @fdiv_v4f32(<4 x float> %op1, <4 x float> %op2) vscale_range(2,0) #0 {420; CHECK-LABEL: fdiv_v4f32:421; CHECK: // %bb.0:422; CHECK-NEXT: fdiv v0.4s, v0.4s, v1.4s423; CHECK-NEXT: ret424 %res = fdiv <4 x float> %op1, %op2425 ret <4 x float> %res426}427 428define void @fdiv_v8f32(ptr %a, ptr %b) vscale_range(2,0) #0 {429; CHECK-LABEL: fdiv_v8f32:430; CHECK: // %bb.0:431; CHECK-NEXT: ptrue p0.s, vl8432; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]433; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]434; CHECK-NEXT: fdiv z0.s, p0/m, z0.s, z1.s435; CHECK-NEXT: st1w { z0.s }, p0, [x0]436; CHECK-NEXT: ret437 %op1 = load <8 x float>, ptr %a438 %op2 = load <8 x float>, ptr %b439 %res = fdiv <8 x float> %op1, %op2440 store <8 x float> %res, ptr %a441 ret void442}443 444define void @fdiv_v16f32(ptr %a, ptr %b) #0 {445; VBITS_GE_256-LABEL: fdiv_v16f32:446; VBITS_GE_256: // %bb.0:447; VBITS_GE_256-NEXT: ptrue p0.s, vl8448; VBITS_GE_256-NEXT: mov x8, #8 // =0x8449; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]450; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]451; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x1]452; VBITS_GE_256-NEXT: fdiv z0.s, p0/m, z0.s, z1.s453; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]454; VBITS_GE_256-NEXT: fdiv z1.s, p0/m, z1.s, z2.s455; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]456; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]457; VBITS_GE_256-NEXT: ret458;459; VBITS_GE_512-LABEL: fdiv_v16f32:460; VBITS_GE_512: // %bb.0:461; VBITS_GE_512-NEXT: ptrue p0.s, vl16462; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]463; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]464; VBITS_GE_512-NEXT: fdiv z0.s, p0/m, z0.s, z1.s465; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]466; VBITS_GE_512-NEXT: ret467 %op1 = load <16 x float>, ptr %a468 %op2 = load <16 x float>, ptr %b469 %res = fdiv <16 x float> %op1, %op2470 store <16 x float> %res, ptr %a471 ret void472}473 474define void @fdiv_v32f32(ptr %a, ptr %b) vscale_range(8,0) #0 {475; CHECK-LABEL: fdiv_v32f32:476; CHECK: // %bb.0:477; CHECK-NEXT: ptrue p0.s, vl32478; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]479; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]480; CHECK-NEXT: fdiv z0.s, p0/m, z0.s, z1.s481; CHECK-NEXT: st1w { z0.s }, p0, [x0]482; CHECK-NEXT: ret483 %op1 = load <32 x float>, ptr %a484 %op2 = load <32 x float>, ptr %b485 %res = fdiv <32 x float> %op1, %op2486 store <32 x float> %res, ptr %a487 ret void488}489 490define void @fdiv_v64f32(ptr %a, ptr %b) vscale_range(16,0) #0 {491; CHECK-LABEL: fdiv_v64f32:492; CHECK: // %bb.0:493; CHECK-NEXT: ptrue p0.s, vl64494; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]495; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]496; CHECK-NEXT: fdiv z0.s, p0/m, z0.s, z1.s497; CHECK-NEXT: st1w { z0.s }, p0, [x0]498; CHECK-NEXT: ret499 %op1 = load <64 x float>, ptr %a500 %op2 = load <64 x float>, ptr %b501 %res = fdiv <64 x float> %op1, %op2502 store <64 x float> %res, ptr %a503 ret void504}505 506; Don't use SVE for 64-bit vectors.507define <1 x double> @fdiv_v1f64(<1 x double> %op1, <1 x double> %op2) vscale_range(2,0) #0 {508; CHECK-LABEL: fdiv_v1f64:509; CHECK: // %bb.0:510; CHECK-NEXT: fdiv d0, d0, d1511; CHECK-NEXT: ret512 %res = fdiv <1 x double> %op1, %op2513 ret <1 x double> %res514}515 516; Don't use SVE for 128-bit vectors.517define <2 x double> @fdiv_v2f64(<2 x double> %op1, <2 x double> %op2) vscale_range(2,0) #0 {518; CHECK-LABEL: fdiv_v2f64:519; CHECK: // %bb.0:520; CHECK-NEXT: fdiv v0.2d, v0.2d, v1.2d521; CHECK-NEXT: ret522 %res = fdiv <2 x double> %op1, %op2523 ret <2 x double> %res524}525 526define void @fdiv_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 {527; CHECK-LABEL: fdiv_v4f64:528; CHECK: // %bb.0:529; CHECK-NEXT: ptrue p0.d, vl4530; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]531; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]532; CHECK-NEXT: fdiv z0.d, p0/m, z0.d, z1.d533; CHECK-NEXT: st1d { z0.d }, p0, [x0]534; CHECK-NEXT: ret535 %op1 = load <4 x double>, ptr %a536 %op2 = load <4 x double>, ptr %b537 %res = fdiv <4 x double> %op1, %op2538 store <4 x double> %res, ptr %a539 ret void540}541 542define void @fdiv_v8f64(ptr %a, ptr %b) #0 {543; VBITS_GE_256-LABEL: fdiv_v8f64:544; VBITS_GE_256: // %bb.0:545; VBITS_GE_256-NEXT: ptrue p0.d, vl4546; VBITS_GE_256-NEXT: mov x8, #4 // =0x4547; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]548; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]549; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1]550; VBITS_GE_256-NEXT: fdiv z0.d, p0/m, z0.d, z1.d551; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]552; VBITS_GE_256-NEXT: fdiv z1.d, p0/m, z1.d, z2.d553; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]554; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]555; VBITS_GE_256-NEXT: ret556;557; VBITS_GE_512-LABEL: fdiv_v8f64:558; VBITS_GE_512: // %bb.0:559; VBITS_GE_512-NEXT: ptrue p0.d, vl8560; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]561; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]562; VBITS_GE_512-NEXT: fdiv z0.d, p0/m, z0.d, z1.d563; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]564; VBITS_GE_512-NEXT: ret565 %op1 = load <8 x double>, ptr %a566 %op2 = load <8 x double>, ptr %b567 %res = fdiv <8 x double> %op1, %op2568 store <8 x double> %res, ptr %a569 ret void570}571 572define void @fdiv_v16f64(ptr %a, ptr %b) vscale_range(8,0) #0 {573; CHECK-LABEL: fdiv_v16f64:574; CHECK: // %bb.0:575; CHECK-NEXT: ptrue p0.d, vl16576; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]577; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]578; CHECK-NEXT: fdiv z0.d, p0/m, z0.d, z1.d579; CHECK-NEXT: st1d { z0.d }, p0, [x0]580; CHECK-NEXT: ret581 %op1 = load <16 x double>, ptr %a582 %op2 = load <16 x double>, ptr %b583 %res = fdiv <16 x double> %op1, %op2584 store <16 x double> %res, ptr %a585 ret void586}587 588define void @fdiv_v32f64(ptr %a, ptr %b) vscale_range(16,0) #0 {589; CHECK-LABEL: fdiv_v32f64:590; CHECK: // %bb.0:591; CHECK-NEXT: ptrue p0.d, vl32592; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]593; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]594; CHECK-NEXT: fdiv z0.d, p0/m, z0.d, z1.d595; CHECK-NEXT: st1d { z0.d }, p0, [x0]596; CHECK-NEXT: ret597 %op1 = load <32 x double>, ptr %a598 %op2 = load <32 x double>, ptr %b599 %res = fdiv <32 x double> %op1, %op2600 store <32 x double> %res, ptr %a601 ret void602}603 604;605; FMA606;607 608; Don't use SVE for 64-bit vectors.609define <4 x half> @fma_v4f16(<4 x half> %op1, <4 x half> %op2, <4 x half> %op3) vscale_range(2,0) #0 {610; CHECK-LABEL: fma_v4f16:611; CHECK: // %bb.0:612; CHECK-NEXT: fmla v2.4h, v1.4h, v0.4h613; CHECK-NEXT: fmov d0, d2614; CHECK-NEXT: ret615 %res = call <4 x half> @llvm.fma.v4f16(<4 x half> %op1, <4 x half> %op2, <4 x half> %op3)616 ret <4 x half> %res617}618 619; Don't use SVE for 128-bit vectors.620define <8 x half> @fma_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3) vscale_range(2,0) #0 {621; CHECK-LABEL: fma_v8f16:622; CHECK: // %bb.0:623; CHECK-NEXT: fmla v2.8h, v1.8h, v0.8h624; CHECK-NEXT: mov v0.16b, v2.16b625; CHECK-NEXT: ret626 %res = call <8 x half> @llvm.fma.v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3)627 ret <8 x half> %res628}629 630define void @fma_v16f16(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 {631; CHECK-LABEL: fma_v16f16:632; CHECK: // %bb.0:633; CHECK-NEXT: ptrue p0.h, vl16634; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]635; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]636; CHECK-NEXT: ld1h { z2.h }, p0/z, [x2]637; CHECK-NEXT: fmad z0.h, p0/m, z1.h, z2.h638; CHECK-NEXT: st1h { z0.h }, p0, [x0]639; CHECK-NEXT: ret640 %op1 = load <16 x half>, ptr %a641 %op2 = load <16 x half>, ptr %b642 %op3 = load <16 x half>, ptr %c643 %res = call <16 x half> @llvm.fma.v16f16(<16 x half> %op1, <16 x half> %op2, <16 x half> %op3)644 store <16 x half> %res, ptr %a645 ret void646}647 648define void @fma_v32f16(ptr %a, ptr %b, ptr %c) #0 {649; VBITS_GE_256-LABEL: fma_v32f16:650; VBITS_GE_256: // %bb.0:651; VBITS_GE_256-NEXT: ptrue p0.h, vl16652; VBITS_GE_256-NEXT: mov x8, #16 // =0x10653; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]654; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]655; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x2, x8, lsl #1]656; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x0]657; VBITS_GE_256-NEXT: ld1h { z4.h }, p0/z, [x1]658; VBITS_GE_256-NEXT: ld1h { z5.h }, p0/z, [x2]659; VBITS_GE_256-NEXT: fmad z0.h, p0/m, z1.h, z2.h660; VBITS_GE_256-NEXT: fmad z3.h, p0/m, z4.h, z5.h661; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]662; VBITS_GE_256-NEXT: st1h { z3.h }, p0, [x0]663; VBITS_GE_256-NEXT: ret664;665; VBITS_GE_512-LABEL: fma_v32f16:666; VBITS_GE_512: // %bb.0:667; VBITS_GE_512-NEXT: ptrue p0.h, vl32668; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]669; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]670; VBITS_GE_512-NEXT: ld1h { z2.h }, p0/z, [x2]671; VBITS_GE_512-NEXT: fmad z0.h, p0/m, z1.h, z2.h672; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]673; VBITS_GE_512-NEXT: ret674 %op1 = load <32 x half>, ptr %a675 %op2 = load <32 x half>, ptr %b676 %op3 = load <32 x half>, ptr %c677 %res = call <32 x half> @llvm.fma.v32f16(<32 x half> %op1, <32 x half> %op2, <32 x half> %op3)678 store <32 x half> %res, ptr %a679 ret void680}681 682define void @fma_v64f16(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 {683; CHECK-LABEL: fma_v64f16:684; CHECK: // %bb.0:685; CHECK-NEXT: ptrue p0.h, vl64686; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]687; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]688; CHECK-NEXT: ld1h { z2.h }, p0/z, [x2]689; CHECK-NEXT: fmad z0.h, p0/m, z1.h, z2.h690; CHECK-NEXT: st1h { z0.h }, p0, [x0]691; CHECK-NEXT: ret692 %op1 = load <64 x half>, ptr %a693 %op2 = load <64 x half>, ptr %b694 %op3 = load <64 x half>, ptr %c695 %res = call <64 x half> @llvm.fma.v64f16(<64 x half> %op1, <64 x half> %op2, <64 x half> %op3)696 store <64 x half> %res, ptr %a697 ret void698}699 700define void @fma_v128f16(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 {701; CHECK-LABEL: fma_v128f16:702; CHECK: // %bb.0:703; CHECK-NEXT: ptrue p0.h, vl128704; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]705; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]706; CHECK-NEXT: ld1h { z2.h }, p0/z, [x2]707; CHECK-NEXT: fmad z0.h, p0/m, z1.h, z2.h708; CHECK-NEXT: st1h { z0.h }, p0, [x0]709; CHECK-NEXT: ret710 %op1 = load <128 x half>, ptr %a711 %op2 = load <128 x half>, ptr %b712 %op3 = load <128 x half>, ptr %c713 %res = call <128 x half> @llvm.fma.v128f16(<128 x half> %op1, <128 x half> %op2, <128 x half> %op3)714 store <128 x half> %res, ptr %a715 ret void716}717 718; Don't use SVE for 64-bit vectors.719define <2 x float> @fma_v2f32(<2 x float> %op1, <2 x float> %op2, <2 x float> %op3) vscale_range(2,0) #0 {720; CHECK-LABEL: fma_v2f32:721; CHECK: // %bb.0:722; CHECK-NEXT: fmla v2.2s, v1.2s, v0.2s723; CHECK-NEXT: fmov d0, d2724; CHECK-NEXT: ret725 %res = call <2 x float> @llvm.fma.v2f32(<2 x float> %op1, <2 x float> %op2, <2 x float> %op3)726 ret <2 x float> %res727}728 729; Don't use SVE for 128-bit vectors.730define <4 x float> @fma_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %op3) vscale_range(2,0) #0 {731; CHECK-LABEL: fma_v4f32:732; CHECK: // %bb.0:733; CHECK-NEXT: fmla v2.4s, v1.4s, v0.4s734; CHECK-NEXT: mov v0.16b, v2.16b735; CHECK-NEXT: ret736 %res = call <4 x float> @llvm.fma.v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %op3)737 ret <4 x float> %res738}739 740define void @fma_v8f32(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 {741; CHECK-LABEL: fma_v8f32:742; CHECK: // %bb.0:743; CHECK-NEXT: ptrue p0.s, vl8744; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]745; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]746; CHECK-NEXT: ld1w { z2.s }, p0/z, [x2]747; CHECK-NEXT: fmad z0.s, p0/m, z1.s, z2.s748; CHECK-NEXT: st1w { z0.s }, p0, [x0]749; CHECK-NEXT: ret750 %op1 = load <8 x float>, ptr %a751 %op2 = load <8 x float>, ptr %b752 %op3 = load <8 x float>, ptr %c753 %res = call <8 x float> @llvm.fma.v8f32(<8 x float> %op1, <8 x float> %op2, <8 x float> %op3)754 store <8 x float> %res, ptr %a755 ret void756}757 758define void @fma_v16f32(ptr %a, ptr %b, ptr %c) #0 {759; VBITS_GE_256-LABEL: fma_v16f32:760; VBITS_GE_256: // %bb.0:761; VBITS_GE_256-NEXT: ptrue p0.s, vl8762; VBITS_GE_256-NEXT: mov x8, #8 // =0x8763; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]764; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]765; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x2, x8, lsl #2]766; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x0]767; VBITS_GE_256-NEXT: ld1w { z4.s }, p0/z, [x1]768; VBITS_GE_256-NEXT: ld1w { z5.s }, p0/z, [x2]769; VBITS_GE_256-NEXT: fmad z0.s, p0/m, z1.s, z2.s770; VBITS_GE_256-NEXT: fmad z3.s, p0/m, z4.s, z5.s771; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]772; VBITS_GE_256-NEXT: st1w { z3.s }, p0, [x0]773; VBITS_GE_256-NEXT: ret774;775; VBITS_GE_512-LABEL: fma_v16f32:776; VBITS_GE_512: // %bb.0:777; VBITS_GE_512-NEXT: ptrue p0.s, vl16778; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]779; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]780; VBITS_GE_512-NEXT: ld1w { z2.s }, p0/z, [x2]781; VBITS_GE_512-NEXT: fmad z0.s, p0/m, z1.s, z2.s782; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]783; VBITS_GE_512-NEXT: ret784 %op1 = load <16 x float>, ptr %a785 %op2 = load <16 x float>, ptr %b786 %op3 = load <16 x float>, ptr %c787 %res = call <16 x float> @llvm.fma.v16f32(<16 x float> %op1, <16 x float> %op2, <16 x float> %op3)788 store <16 x float> %res, ptr %a789 ret void790}791 792define void @fma_v32f32(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 {793; CHECK-LABEL: fma_v32f32:794; CHECK: // %bb.0:795; CHECK-NEXT: ptrue p0.s, vl32796; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]797; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]798; CHECK-NEXT: ld1w { z2.s }, p0/z, [x2]799; CHECK-NEXT: fmad z0.s, p0/m, z1.s, z2.s800; CHECK-NEXT: st1w { z0.s }, p0, [x0]801; CHECK-NEXT: ret802 %op1 = load <32 x float>, ptr %a803 %op2 = load <32 x float>, ptr %b804 %op3 = load <32 x float>, ptr %c805 %res = call <32 x float> @llvm.fma.v32f32(<32 x float> %op1, <32 x float> %op2, <32 x float> %op3)806 store <32 x float> %res, ptr %a807 ret void808}809 810define void @fma_v64f32(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 {811; CHECK-LABEL: fma_v64f32:812; CHECK: // %bb.0:813; CHECK-NEXT: ptrue p0.s, vl64814; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]815; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]816; CHECK-NEXT: ld1w { z2.s }, p0/z, [x2]817; CHECK-NEXT: fmad z0.s, p0/m, z1.s, z2.s818; CHECK-NEXT: st1w { z0.s }, p0, [x0]819; CHECK-NEXT: ret820 %op1 = load <64 x float>, ptr %a821 %op2 = load <64 x float>, ptr %b822 %op3 = load <64 x float>, ptr %c823 %res = call <64 x float> @llvm.fma.v64f32(<64 x float> %op1, <64 x float> %op2, <64 x float> %op3)824 store <64 x float> %res, ptr %a825 ret void826}827 828; Don't use SVE for 64-bit vectors.829define <1 x double> @fma_v1f64(<1 x double> %op1, <1 x double> %op2, <1 x double> %op3) vscale_range(2,0) #0 {830; CHECK-LABEL: fma_v1f64:831; CHECK: // %bb.0:832; CHECK-NEXT: fmadd d0, d0, d1, d2833; CHECK-NEXT: ret834 %res = call <1 x double> @llvm.fma.v1f64(<1 x double> %op1, <1 x double> %op2, <1 x double> %op3)835 ret <1 x double> %res836}837 838; Don't use SVE for 128-bit vectors.839define <2 x double> @fma_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double> %op3) vscale_range(2,0) #0 {840; CHECK-LABEL: fma_v2f64:841; CHECK: // %bb.0:842; CHECK-NEXT: fmla v2.2d, v1.2d, v0.2d843; CHECK-NEXT: mov v0.16b, v2.16b844; CHECK-NEXT: ret845 %res = call <2 x double> @llvm.fma.v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double> %op3)846 ret <2 x double> %res847}848 849define void @fma_v4f64(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 {850; CHECK-LABEL: fma_v4f64:851; CHECK: // %bb.0:852; CHECK-NEXT: ptrue p0.d, vl4853; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]854; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]855; CHECK-NEXT: ld1d { z2.d }, p0/z, [x2]856; CHECK-NEXT: fmad z0.d, p0/m, z1.d, z2.d857; CHECK-NEXT: st1d { z0.d }, p0, [x0]858; CHECK-NEXT: ret859 %op1 = load <4 x double>, ptr %a860 %op2 = load <4 x double>, ptr %b861 %op3 = load <4 x double>, ptr %c862 %res = call <4 x double> @llvm.fma.v4f64(<4 x double> %op1, <4 x double> %op2, <4 x double> %op3)863 store <4 x double> %res, ptr %a864 ret void865}866 867define void @fma_v8f64(ptr %a, ptr %b, ptr %c) #0 {868; VBITS_GE_256-LABEL: fma_v8f64:869; VBITS_GE_256: // %bb.0:870; VBITS_GE_256-NEXT: ptrue p0.d, vl4871; VBITS_GE_256-NEXT: mov x8, #4 // =0x4872; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]873; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]874; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x2, x8, lsl #3]875; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x0]876; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1]877; VBITS_GE_256-NEXT: ld1d { z5.d }, p0/z, [x2]878; VBITS_GE_256-NEXT: fmad z0.d, p0/m, z1.d, z2.d879; VBITS_GE_256-NEXT: fmad z3.d, p0/m, z4.d, z5.d880; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]881; VBITS_GE_256-NEXT: st1d { z3.d }, p0, [x0]882; VBITS_GE_256-NEXT: ret883;884; VBITS_GE_512-LABEL: fma_v8f64:885; VBITS_GE_512: // %bb.0:886; VBITS_GE_512-NEXT: ptrue p0.d, vl8887; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]888; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]889; VBITS_GE_512-NEXT: ld1d { z2.d }, p0/z, [x2]890; VBITS_GE_512-NEXT: fmad z0.d, p0/m, z1.d, z2.d891; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]892; VBITS_GE_512-NEXT: ret893 %op1 = load <8 x double>, ptr %a894 %op2 = load <8 x double>, ptr %b895 %op3 = load <8 x double>, ptr %c896 %res = call <8 x double> @llvm.fma.v8f64(<8 x double> %op1, <8 x double> %op2, <8 x double> %op3)897 store <8 x double> %res, ptr %a898 ret void899}900 901define void @fma_v16f64(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 {902; CHECK-LABEL: fma_v16f64:903; CHECK: // %bb.0:904; CHECK-NEXT: ptrue p0.d, vl16905; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]906; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]907; CHECK-NEXT: ld1d { z2.d }, p0/z, [x2]908; CHECK-NEXT: fmad z0.d, p0/m, z1.d, z2.d909; CHECK-NEXT: st1d { z0.d }, p0, [x0]910; CHECK-NEXT: ret911 %op1 = load <16 x double>, ptr %a912 %op2 = load <16 x double>, ptr %b913 %op3 = load <16 x double>, ptr %c914 %res = call <16 x double> @llvm.fma.v16f64(<16 x double> %op1, <16 x double> %op2, <16 x double> %op3)915 store <16 x double> %res, ptr %a916 ret void917}918 919define void @fma_v32f64(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 {920; CHECK-LABEL: fma_v32f64:921; CHECK: // %bb.0:922; CHECK-NEXT: ptrue p0.d, vl32923; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]924; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]925; CHECK-NEXT: ld1d { z2.d }, p0/z, [x2]926; CHECK-NEXT: fmad z0.d, p0/m, z1.d, z2.d927; CHECK-NEXT: st1d { z0.d }, p0, [x0]928; CHECK-NEXT: ret929 %op1 = load <32 x double>, ptr %a930 %op2 = load <32 x double>, ptr %b931 %op3 = load <32 x double>, ptr %c932 %res = call <32 x double> @llvm.fma.v32f64(<32 x double> %op1, <32 x double> %op2, <32 x double> %op3)933 store <32 x double> %res, ptr %a934 ret void935}936 937;938; FMUL939;940 941; Don't use SVE for 64-bit vectors.942define <4 x half> @fmul_v4f16(<4 x half> %op1, <4 x half> %op2) vscale_range(2,0) #0 {943; CHECK-LABEL: fmul_v4f16:944; CHECK: // %bb.0:945; CHECK-NEXT: fmul v0.4h, v0.4h, v1.4h946; CHECK-NEXT: ret947 %res = fmul <4 x half> %op1, %op2948 ret <4 x half> %res949}950 951; Don't use SVE for 128-bit vectors.952define <8 x half> @fmul_v8f16(<8 x half> %op1, <8 x half> %op2) vscale_range(2,0) #0 {953; CHECK-LABEL: fmul_v8f16:954; CHECK: // %bb.0:955; CHECK-NEXT: fmul v0.8h, v0.8h, v1.8h956; CHECK-NEXT: ret957 %res = fmul <8 x half> %op1, %op2958 ret <8 x half> %res959}960 961define void @fmul_v16f16(ptr %a, ptr %b) vscale_range(2,0) #0 {962; CHECK-LABEL: fmul_v16f16:963; CHECK: // %bb.0:964; CHECK-NEXT: ptrue p0.h, vl16965; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]966; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]967; CHECK-NEXT: fmul z0.h, p0/m, z0.h, z1.h968; CHECK-NEXT: st1h { z0.h }, p0, [x0]969; CHECK-NEXT: ret970 %op1 = load <16 x half>, ptr %a971 %op2 = load <16 x half>, ptr %b972 %res = fmul <16 x half> %op1, %op2973 store <16 x half> %res, ptr %a974 ret void975}976 977define void @fmul_v32f16(ptr %a, ptr %b) #0 {978; VBITS_GE_256-LABEL: fmul_v32f16:979; VBITS_GE_256: // %bb.0:980; VBITS_GE_256-NEXT: ptrue p0.h, vl16981; VBITS_GE_256-NEXT: mov x8, #16 // =0x10982; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]983; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]984; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0]985; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1]986; VBITS_GE_256-NEXT: fmul z0.h, p0/m, z0.h, z1.h987; VBITS_GE_256-NEXT: fmul z2.h, p0/m, z2.h, z3.h988; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]989; VBITS_GE_256-NEXT: st1h { z2.h }, p0, [x0]990; VBITS_GE_256-NEXT: ret991;992; VBITS_GE_512-LABEL: fmul_v32f16:993; VBITS_GE_512: // %bb.0:994; VBITS_GE_512-NEXT: ptrue p0.h, vl32995; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]996; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]997; VBITS_GE_512-NEXT: fmul z0.h, p0/m, z0.h, z1.h998; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]999; VBITS_GE_512-NEXT: ret1000 %op1 = load <32 x half>, ptr %a1001 %op2 = load <32 x half>, ptr %b1002 %res = fmul <32 x half> %op1, %op21003 store <32 x half> %res, ptr %a1004 ret void1005}1006 1007define void @fmul_v64f16(ptr %a, ptr %b) vscale_range(8,0) #0 {1008; CHECK-LABEL: fmul_v64f16:1009; CHECK: // %bb.0:1010; CHECK-NEXT: ptrue p0.h, vl641011; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1012; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]1013; CHECK-NEXT: fmul z0.h, p0/m, z0.h, z1.h1014; CHECK-NEXT: st1h { z0.h }, p0, [x0]1015; CHECK-NEXT: ret1016 %op1 = load <64 x half>, ptr %a1017 %op2 = load <64 x half>, ptr %b1018 %res = fmul <64 x half> %op1, %op21019 store <64 x half> %res, ptr %a1020 ret void1021}1022 1023define void @fmul_v128f16(ptr %a, ptr %b) vscale_range(16,0) #0 {1024; CHECK-LABEL: fmul_v128f16:1025; CHECK: // %bb.0:1026; CHECK-NEXT: ptrue p0.h, vl1281027; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1028; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]1029; CHECK-NEXT: fmul z0.h, p0/m, z0.h, z1.h1030; CHECK-NEXT: st1h { z0.h }, p0, [x0]1031; CHECK-NEXT: ret1032 %op1 = load <128 x half>, ptr %a1033 %op2 = load <128 x half>, ptr %b1034 %res = fmul <128 x half> %op1, %op21035 store <128 x half> %res, ptr %a1036 ret void1037}1038 1039; Don't use SVE for 64-bit vectors.1040define <2 x float> @fmul_v2f32(<2 x float> %op1, <2 x float> %op2) vscale_range(2,0) #0 {1041; CHECK-LABEL: fmul_v2f32:1042; CHECK: // %bb.0:1043; CHECK-NEXT: fmul v0.2s, v0.2s, v1.2s1044; CHECK-NEXT: ret1045 %res = fmul <2 x float> %op1, %op21046 ret <2 x float> %res1047}1048 1049; Don't use SVE for 128-bit vectors.1050define <4 x float> @fmul_v4f32(<4 x float> %op1, <4 x float> %op2) vscale_range(2,0) #0 {1051; CHECK-LABEL: fmul_v4f32:1052; CHECK: // %bb.0:1053; CHECK-NEXT: fmul v0.4s, v0.4s, v1.4s1054; CHECK-NEXT: ret1055 %res = fmul <4 x float> %op1, %op21056 ret <4 x float> %res1057}1058 1059define void @fmul_v8f32(ptr %a, ptr %b) vscale_range(2,0) #0 {1060; CHECK-LABEL: fmul_v8f32:1061; CHECK: // %bb.0:1062; CHECK-NEXT: ptrue p0.s, vl81063; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1064; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1065; CHECK-NEXT: fmul z0.s, p0/m, z0.s, z1.s1066; CHECK-NEXT: st1w { z0.s }, p0, [x0]1067; CHECK-NEXT: ret1068 %op1 = load <8 x float>, ptr %a1069 %op2 = load <8 x float>, ptr %b1070 %res = fmul <8 x float> %op1, %op21071 store <8 x float> %res, ptr %a1072 ret void1073}1074 1075define void @fmul_v16f32(ptr %a, ptr %b) #0 {1076; VBITS_GE_256-LABEL: fmul_v16f32:1077; VBITS_GE_256: // %bb.0:1078; VBITS_GE_256-NEXT: ptrue p0.s, vl81079; VBITS_GE_256-NEXT: mov x8, #8 // =0x81080; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]1081; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]1082; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]1083; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]1084; VBITS_GE_256-NEXT: fmul z0.s, p0/m, z0.s, z1.s1085; VBITS_GE_256-NEXT: fmul z2.s, p0/m, z2.s, z3.s1086; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]1087; VBITS_GE_256-NEXT: st1w { z2.s }, p0, [x0]1088; VBITS_GE_256-NEXT: ret1089;1090; VBITS_GE_512-LABEL: fmul_v16f32:1091; VBITS_GE_512: // %bb.0:1092; VBITS_GE_512-NEXT: ptrue p0.s, vl161093; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]1094; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]1095; VBITS_GE_512-NEXT: fmul z0.s, p0/m, z0.s, z1.s1096; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]1097; VBITS_GE_512-NEXT: ret1098 %op1 = load <16 x float>, ptr %a1099 %op2 = load <16 x float>, ptr %b1100 %res = fmul <16 x float> %op1, %op21101 store <16 x float> %res, ptr %a1102 ret void1103}1104 1105define void @fmul_v32f32(ptr %a, ptr %b) vscale_range(8,0) #0 {1106; CHECK-LABEL: fmul_v32f32:1107; CHECK: // %bb.0:1108; CHECK-NEXT: ptrue p0.s, vl321109; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1110; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1111; CHECK-NEXT: fmul z0.s, p0/m, z0.s, z1.s1112; CHECK-NEXT: st1w { z0.s }, p0, [x0]1113; CHECK-NEXT: ret1114 %op1 = load <32 x float>, ptr %a1115 %op2 = load <32 x float>, ptr %b1116 %res = fmul <32 x float> %op1, %op21117 store <32 x float> %res, ptr %a1118 ret void1119}1120 1121define void @fmul_v64f32(ptr %a, ptr %b) vscale_range(16,0) #0 {1122; CHECK-LABEL: fmul_v64f32:1123; CHECK: // %bb.0:1124; CHECK-NEXT: ptrue p0.s, vl641125; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1126; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1127; CHECK-NEXT: fmul z0.s, p0/m, z0.s, z1.s1128; CHECK-NEXT: st1w { z0.s }, p0, [x0]1129; CHECK-NEXT: ret1130 %op1 = load <64 x float>, ptr %a1131 %op2 = load <64 x float>, ptr %b1132 %res = fmul <64 x float> %op1, %op21133 store <64 x float> %res, ptr %a1134 ret void1135}1136 1137; Don't use SVE for 64-bit vectors.1138define <1 x double> @fmul_v1f64(<1 x double> %op1, <1 x double> %op2) vscale_range(2,0) #0 {1139; CHECK-LABEL: fmul_v1f64:1140; CHECK: // %bb.0:1141; CHECK-NEXT: fmul d0, d0, d11142; CHECK-NEXT: ret1143 %res = fmul <1 x double> %op1, %op21144 ret <1 x double> %res1145}1146 1147; Don't use SVE for 128-bit vectors.1148define <2 x double> @fmul_v2f64(<2 x double> %op1, <2 x double> %op2) vscale_range(2,0) #0 {1149; CHECK-LABEL: fmul_v2f64:1150; CHECK: // %bb.0:1151; CHECK-NEXT: fmul v0.2d, v0.2d, v1.2d1152; CHECK-NEXT: ret1153 %res = fmul <2 x double> %op1, %op21154 ret <2 x double> %res1155}1156 1157define void @fmul_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 {1158; CHECK-LABEL: fmul_v4f64:1159; CHECK: // %bb.0:1160; CHECK-NEXT: ptrue p0.d, vl41161; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1162; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1163; CHECK-NEXT: fmul z0.d, p0/m, z0.d, z1.d1164; CHECK-NEXT: st1d { z0.d }, p0, [x0]1165; CHECK-NEXT: ret1166 %op1 = load <4 x double>, ptr %a1167 %op2 = load <4 x double>, ptr %b1168 %res = fmul <4 x double> %op1, %op21169 store <4 x double> %res, ptr %a1170 ret void1171}1172 1173define void @fmul_v8f64(ptr %a, ptr %b) #0 {1174; VBITS_GE_256-LABEL: fmul_v8f64:1175; VBITS_GE_256: // %bb.0:1176; VBITS_GE_256-NEXT: ptrue p0.d, vl41177; VBITS_GE_256-NEXT: mov x8, #4 // =0x41178; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]1179; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]1180; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]1181; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]1182; VBITS_GE_256-NEXT: fmul z0.d, p0/m, z0.d, z1.d1183; VBITS_GE_256-NEXT: fmul z2.d, p0/m, z2.d, z3.d1184; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]1185; VBITS_GE_256-NEXT: st1d { z2.d }, p0, [x0]1186; VBITS_GE_256-NEXT: ret1187;1188; VBITS_GE_512-LABEL: fmul_v8f64:1189; VBITS_GE_512: // %bb.0:1190; VBITS_GE_512-NEXT: ptrue p0.d, vl81191; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]1192; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]1193; VBITS_GE_512-NEXT: fmul z0.d, p0/m, z0.d, z1.d1194; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]1195; VBITS_GE_512-NEXT: ret1196 %op1 = load <8 x double>, ptr %a1197 %op2 = load <8 x double>, ptr %b1198 %res = fmul <8 x double> %op1, %op21199 store <8 x double> %res, ptr %a1200 ret void1201}1202 1203define void @fmul_v16f64(ptr %a, ptr %b) vscale_range(8,0) #0 {1204; CHECK-LABEL: fmul_v16f64:1205; CHECK: // %bb.0:1206; CHECK-NEXT: ptrue p0.d, vl161207; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1208; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1209; CHECK-NEXT: fmul z0.d, p0/m, z0.d, z1.d1210; CHECK-NEXT: st1d { z0.d }, p0, [x0]1211; CHECK-NEXT: ret1212 %op1 = load <16 x double>, ptr %a1213 %op2 = load <16 x double>, ptr %b1214 %res = fmul <16 x double> %op1, %op21215 store <16 x double> %res, ptr %a1216 ret void1217}1218 1219define void @fmul_v32f64(ptr %a, ptr %b) vscale_range(16,0) #0 {1220; CHECK-LABEL: fmul_v32f64:1221; CHECK: // %bb.0:1222; CHECK-NEXT: ptrue p0.d, vl321223; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1224; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1225; CHECK-NEXT: fmul z0.d, p0/m, z0.d, z1.d1226; CHECK-NEXT: st1d { z0.d }, p0, [x0]1227; CHECK-NEXT: ret1228 %op1 = load <32 x double>, ptr %a1229 %op2 = load <32 x double>, ptr %b1230 %res = fmul <32 x double> %op1, %op21231 store <32 x double> %res, ptr %a1232 ret void1233}1234 1235;1236; FNEG1237;1238 1239; Don't use SVE for 64-bit vectors.1240define <4 x half> @fneg_v4f16(<4 x half> %op) vscale_range(2,0) #0 {1241; CHECK-LABEL: fneg_v4f16:1242; CHECK: // %bb.0:1243; CHECK-NEXT: fneg v0.4h, v0.4h1244; CHECK-NEXT: ret1245 %res = fneg <4 x half> %op1246 ret <4 x half> %res1247}1248 1249; Don't use SVE for 128-bit vectors.1250define <8 x half> @fneg_v8f16(<8 x half> %op) vscale_range(2,0) #0 {1251; CHECK-LABEL: fneg_v8f16:1252; CHECK: // %bb.0:1253; CHECK-NEXT: fneg v0.8h, v0.8h1254; CHECK-NEXT: ret1255 %res = fneg <8 x half> %op1256 ret <8 x half> %res1257}1258 1259define void @fneg_v16f16(ptr %a, ptr %b) vscale_range(2,0) #0 {1260; CHECK-LABEL: fneg_v16f16:1261; CHECK: // %bb.0:1262; CHECK-NEXT: ptrue p0.h, vl161263; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1264; CHECK-NEXT: fneg z0.h, p0/m, z0.h1265; CHECK-NEXT: st1h { z0.h }, p0, [x0]1266; CHECK-NEXT: ret1267 %op = load <16 x half>, ptr %a1268 %res = fneg <16 x half> %op1269 store <16 x half> %res, ptr %a1270 ret void1271}1272 1273define void @fneg_v32f16(ptr %a) #0 {1274; VBITS_GE_256-LABEL: fneg_v32f16:1275; VBITS_GE_256: // %bb.0:1276; VBITS_GE_256-NEXT: ptrue p0.h, vl161277; VBITS_GE_256-NEXT: mov x8, #16 // =0x101278; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]1279; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]1280; VBITS_GE_256-NEXT: fneg z0.h, p0/m, z0.h1281; VBITS_GE_256-NEXT: fneg z1.h, p0/m, z1.h1282; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]1283; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]1284; VBITS_GE_256-NEXT: ret1285;1286; VBITS_GE_512-LABEL: fneg_v32f16:1287; VBITS_GE_512: // %bb.0:1288; VBITS_GE_512-NEXT: ptrue p0.h, vl321289; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]1290; VBITS_GE_512-NEXT: fneg z0.h, p0/m, z0.h1291; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]1292; VBITS_GE_512-NEXT: ret1293 %op = load <32 x half>, ptr %a1294 %res = fneg <32 x half> %op1295 store <32 x half> %res, ptr %a1296 ret void1297}1298 1299define void @fneg_v64f16(ptr %a) vscale_range(8,0) #0 {1300; CHECK-LABEL: fneg_v64f16:1301; CHECK: // %bb.0:1302; CHECK-NEXT: ptrue p0.h, vl641303; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1304; CHECK-NEXT: fneg z0.h, p0/m, z0.h1305; CHECK-NEXT: st1h { z0.h }, p0, [x0]1306; CHECK-NEXT: ret1307 %op = load <64 x half>, ptr %a1308 %res = fneg <64 x half> %op1309 store <64 x half> %res, ptr %a1310 ret void1311}1312 1313define void @fneg_v128f16(ptr %a) vscale_range(16,0) #0 {1314; CHECK-LABEL: fneg_v128f16:1315; CHECK: // %bb.0:1316; CHECK-NEXT: ptrue p0.h, vl1281317; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1318; CHECK-NEXT: fneg z0.h, p0/m, z0.h1319; CHECK-NEXT: st1h { z0.h }, p0, [x0]1320; CHECK-NEXT: ret1321 %op = load <128 x half>, ptr %a1322 %res = fneg <128 x half> %op1323 store <128 x half> %res, ptr %a1324 ret void1325}1326 1327; Don't use SVE for 64-bit vectors.1328define <2 x float> @fneg_v2f32(<2 x float> %op) vscale_range(2,0) #0 {1329; CHECK-LABEL: fneg_v2f32:1330; CHECK: // %bb.0:1331; CHECK-NEXT: fneg v0.2s, v0.2s1332; CHECK-NEXT: ret1333 %res = fneg <2 x float> %op1334 ret <2 x float> %res1335}1336 1337; Don't use SVE for 128-bit vectors.1338define <4 x float> @fneg_v4f32(<4 x float> %op) vscale_range(2,0) #0 {1339; CHECK-LABEL: fneg_v4f32:1340; CHECK: // %bb.0:1341; CHECK-NEXT: fneg v0.4s, v0.4s1342; CHECK-NEXT: ret1343 %res = fneg <4 x float> %op1344 ret <4 x float> %res1345}1346 1347define void @fneg_v8f32(ptr %a) vscale_range(2,0) #0 {1348; CHECK-LABEL: fneg_v8f32:1349; CHECK: // %bb.0:1350; CHECK-NEXT: ptrue p0.s, vl81351; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1352; CHECK-NEXT: fneg z0.s, p0/m, z0.s1353; CHECK-NEXT: st1w { z0.s }, p0, [x0]1354; CHECK-NEXT: ret1355 %op = load <8 x float>, ptr %a1356 %res = fneg <8 x float> %op1357 store <8 x float> %res, ptr %a1358 ret void1359}1360 1361define void @fneg_v16f32(ptr %a) #0 {1362; VBITS_GE_256-LABEL: fneg_v16f32:1363; VBITS_GE_256: // %bb.0:1364; VBITS_GE_256-NEXT: ptrue p0.s, vl81365; VBITS_GE_256-NEXT: mov x8, #8 // =0x81366; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]1367; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]1368; VBITS_GE_256-NEXT: fneg z0.s, p0/m, z0.s1369; VBITS_GE_256-NEXT: fneg z1.s, p0/m, z1.s1370; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]1371; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]1372; VBITS_GE_256-NEXT: ret1373;1374; VBITS_GE_512-LABEL: fneg_v16f32:1375; VBITS_GE_512: // %bb.0:1376; VBITS_GE_512-NEXT: ptrue p0.s, vl161377; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]1378; VBITS_GE_512-NEXT: fneg z0.s, p0/m, z0.s1379; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]1380; VBITS_GE_512-NEXT: ret1381 %op = load <16 x float>, ptr %a1382 %res = fneg <16 x float> %op1383 store <16 x float> %res, ptr %a1384 ret void1385}1386 1387define void @fneg_v32f32(ptr %a) vscale_range(8,0) #0 {1388; CHECK-LABEL: fneg_v32f32:1389; CHECK: // %bb.0:1390; CHECK-NEXT: ptrue p0.s, vl321391; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1392; CHECK-NEXT: fneg z0.s, p0/m, z0.s1393; CHECK-NEXT: st1w { z0.s }, p0, [x0]1394; CHECK-NEXT: ret1395 %op = load <32 x float>, ptr %a1396 %res = fneg <32 x float> %op1397 store <32 x float> %res, ptr %a1398 ret void1399}1400 1401define void @fneg_v64f32(ptr %a) vscale_range(16,0) #0 {1402; CHECK-LABEL: fneg_v64f32:1403; CHECK: // %bb.0:1404; CHECK-NEXT: ptrue p0.s, vl641405; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1406; CHECK-NEXT: fneg z0.s, p0/m, z0.s1407; CHECK-NEXT: st1w { z0.s }, p0, [x0]1408; CHECK-NEXT: ret1409 %op = load <64 x float>, ptr %a1410 %res = fneg <64 x float> %op1411 store <64 x float> %res, ptr %a1412 ret void1413}1414 1415; Don't use SVE for 64-bit vectors.1416define <1 x double> @fneg_v1f64(<1 x double> %op) vscale_range(2,0) #0 {1417; CHECK-LABEL: fneg_v1f64:1418; CHECK: // %bb.0:1419; CHECK-NEXT: fneg d0, d01420; CHECK-NEXT: ret1421 %res = fneg <1 x double> %op1422 ret <1 x double> %res1423}1424 1425; Don't use SVE for 128-bit vectors.1426define <2 x double> @fneg_v2f64(<2 x double> %op) vscale_range(2,0) #0 {1427; CHECK-LABEL: fneg_v2f64:1428; CHECK: // %bb.0:1429; CHECK-NEXT: fneg v0.2d, v0.2d1430; CHECK-NEXT: ret1431 %res = fneg <2 x double> %op1432 ret <2 x double> %res1433}1434 1435define void @fneg_v4f64(ptr %a) vscale_range(2,0) #0 {1436; CHECK-LABEL: fneg_v4f64:1437; CHECK: // %bb.0:1438; CHECK-NEXT: ptrue p0.d, vl41439; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1440; CHECK-NEXT: fneg z0.d, p0/m, z0.d1441; CHECK-NEXT: st1d { z0.d }, p0, [x0]1442; CHECK-NEXT: ret1443 %op = load <4 x double>, ptr %a1444 %res = fneg <4 x double> %op1445 store <4 x double> %res, ptr %a1446 ret void1447}1448 1449define void @fneg_v8f64(ptr %a) #0 {1450; VBITS_GE_256-LABEL: fneg_v8f64:1451; VBITS_GE_256: // %bb.0:1452; VBITS_GE_256-NEXT: ptrue p0.d, vl41453; VBITS_GE_256-NEXT: mov x8, #4 // =0x41454; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]1455; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]1456; VBITS_GE_256-NEXT: fneg z0.d, p0/m, z0.d1457; VBITS_GE_256-NEXT: fneg z1.d, p0/m, z1.d1458; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]1459; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]1460; VBITS_GE_256-NEXT: ret1461;1462; VBITS_GE_512-LABEL: fneg_v8f64:1463; VBITS_GE_512: // %bb.0:1464; VBITS_GE_512-NEXT: ptrue p0.d, vl81465; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]1466; VBITS_GE_512-NEXT: fneg z0.d, p0/m, z0.d1467; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]1468; VBITS_GE_512-NEXT: ret1469 %op = load <8 x double>, ptr %a1470 %res = fneg <8 x double> %op1471 store <8 x double> %res, ptr %a1472 ret void1473}1474 1475define void @fneg_v16f64(ptr %a) vscale_range(8,0) #0 {1476; CHECK-LABEL: fneg_v16f64:1477; CHECK: // %bb.0:1478; CHECK-NEXT: ptrue p0.d, vl161479; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1480; CHECK-NEXT: fneg z0.d, p0/m, z0.d1481; CHECK-NEXT: st1d { z0.d }, p0, [x0]1482; CHECK-NEXT: ret1483 %op = load <16 x double>, ptr %a1484 %res = fneg <16 x double> %op1485 store <16 x double> %res, ptr %a1486 ret void1487}1488 1489define void @fneg_v32f64(ptr %a) vscale_range(16,0) #0 {1490; CHECK-LABEL: fneg_v32f64:1491; CHECK: // %bb.0:1492; CHECK-NEXT: ptrue p0.d, vl321493; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1494; CHECK-NEXT: fneg z0.d, p0/m, z0.d1495; CHECK-NEXT: st1d { z0.d }, p0, [x0]1496; CHECK-NEXT: ret1497 %op = load <32 x double>, ptr %a1498 %res = fneg <32 x double> %op1499 store <32 x double> %res, ptr %a1500 ret void1501}1502 1503;1504; FSQRT1505;1506 1507; Don't use SVE for 64-bit vectors.1508define <4 x half> @fsqrt_v4f16(<4 x half> %op) vscale_range(2,0) #0 {1509; CHECK-LABEL: fsqrt_v4f16:1510; CHECK: // %bb.0:1511; CHECK-NEXT: fsqrt v0.4h, v0.4h1512; CHECK-NEXT: ret1513 %res = call <4 x half> @llvm.sqrt.v4f16(<4 x half> %op)1514 ret <4 x half> %res1515}1516 1517; Don't use SVE for 128-bit vectors.1518define <8 x half> @fsqrt_v8f16(<8 x half> %op) vscale_range(2,0) #0 {1519; CHECK-LABEL: fsqrt_v8f16:1520; CHECK: // %bb.0:1521; CHECK-NEXT: fsqrt v0.8h, v0.8h1522; CHECK-NEXT: ret1523 %res = call <8 x half> @llvm.sqrt.v8f16(<8 x half> %op)1524 ret <8 x half> %res1525}1526 1527define void @fsqrt_v16f16(ptr %a, ptr %b) vscale_range(2,0) #0 {1528; CHECK-LABEL: fsqrt_v16f16:1529; CHECK: // %bb.0:1530; CHECK-NEXT: ptrue p0.h, vl161531; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1532; CHECK-NEXT: fsqrt z0.h, p0/m, z0.h1533; CHECK-NEXT: st1h { z0.h }, p0, [x0]1534; CHECK-NEXT: ret1535 %op = load <16 x half>, ptr %a1536 %res = call <16 x half> @llvm.sqrt.v16f16(<16 x half> %op)1537 store <16 x half> %res, ptr %a1538 ret void1539}1540 1541define void @fsqrt_v32f16(ptr %a) #0 {1542; VBITS_GE_256-LABEL: fsqrt_v32f16:1543; VBITS_GE_256: // %bb.0:1544; VBITS_GE_256-NEXT: ptrue p0.h, vl161545; VBITS_GE_256-NEXT: mov x8, #16 // =0x101546; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]1547; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]1548; VBITS_GE_256-NEXT: fsqrt z0.h, p0/m, z0.h1549; VBITS_GE_256-NEXT: fsqrt z1.h, p0/m, z1.h1550; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]1551; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]1552; VBITS_GE_256-NEXT: ret1553;1554; VBITS_GE_512-LABEL: fsqrt_v32f16:1555; VBITS_GE_512: // %bb.0:1556; VBITS_GE_512-NEXT: ptrue p0.h, vl321557; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]1558; VBITS_GE_512-NEXT: fsqrt z0.h, p0/m, z0.h1559; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]1560; VBITS_GE_512-NEXT: ret1561 %op = load <32 x half>, ptr %a1562 %res = call <32 x half> @llvm.sqrt.v32f16(<32 x half> %op)1563 store <32 x half> %res, ptr %a1564 ret void1565}1566 1567define void @fsqrt_v64f16(ptr %a) vscale_range(8,0) #0 {1568; CHECK-LABEL: fsqrt_v64f16:1569; CHECK: // %bb.0:1570; CHECK-NEXT: ptrue p0.h, vl641571; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1572; CHECK-NEXT: fsqrt z0.h, p0/m, z0.h1573; CHECK-NEXT: st1h { z0.h }, p0, [x0]1574; CHECK-NEXT: ret1575 %op = load <64 x half>, ptr %a1576 %res = call <64 x half> @llvm.sqrt.v64f16(<64 x half> %op)1577 store <64 x half> %res, ptr %a1578 ret void1579}1580 1581define void @fsqrt_v128f16(ptr %a) vscale_range(16,0) #0 {1582; CHECK-LABEL: fsqrt_v128f16:1583; CHECK: // %bb.0:1584; CHECK-NEXT: ptrue p0.h, vl1281585; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1586; CHECK-NEXT: fsqrt z0.h, p0/m, z0.h1587; CHECK-NEXT: st1h { z0.h }, p0, [x0]1588; CHECK-NEXT: ret1589 %op = load <128 x half>, ptr %a1590 %res = call <128 x half> @llvm.sqrt.v128f16(<128 x half> %op)1591 store <128 x half> %res, ptr %a1592 ret void1593}1594 1595; Don't use SVE for 64-bit vectors.1596define <2 x float> @fsqrt_v2f32(<2 x float> %op) vscale_range(2,0) #0 {1597; CHECK-LABEL: fsqrt_v2f32:1598; CHECK: // %bb.0:1599; CHECK-NEXT: fsqrt v0.2s, v0.2s1600; CHECK-NEXT: ret1601 %res = call <2 x float> @llvm.sqrt.v2f32(<2 x float> %op)1602 ret <2 x float> %res1603}1604 1605; Don't use SVE for 128-bit vectors.1606define <4 x float> @fsqrt_v4f32(<4 x float> %op) vscale_range(2,0) #0 {1607; CHECK-LABEL: fsqrt_v4f32:1608; CHECK: // %bb.0:1609; CHECK-NEXT: fsqrt v0.4s, v0.4s1610; CHECK-NEXT: ret1611 %res = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %op)1612 ret <4 x float> %res1613}1614 1615define void @fsqrt_v8f32(ptr %a) vscale_range(2,0) #0 {1616; CHECK-LABEL: fsqrt_v8f32:1617; CHECK: // %bb.0:1618; CHECK-NEXT: ptrue p0.s, vl81619; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1620; CHECK-NEXT: fsqrt z0.s, p0/m, z0.s1621; CHECK-NEXT: st1w { z0.s }, p0, [x0]1622; CHECK-NEXT: ret1623 %op = load <8 x float>, ptr %a1624 %res = call <8 x float> @llvm.sqrt.v8f32(<8 x float> %op)1625 store <8 x float> %res, ptr %a1626 ret void1627}1628 1629define void @fsqrt_v16f32(ptr %a) #0 {1630; VBITS_GE_256-LABEL: fsqrt_v16f32:1631; VBITS_GE_256: // %bb.0:1632; VBITS_GE_256-NEXT: ptrue p0.s, vl81633; VBITS_GE_256-NEXT: mov x8, #8 // =0x81634; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]1635; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]1636; VBITS_GE_256-NEXT: fsqrt z0.s, p0/m, z0.s1637; VBITS_GE_256-NEXT: fsqrt z1.s, p0/m, z1.s1638; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]1639; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]1640; VBITS_GE_256-NEXT: ret1641;1642; VBITS_GE_512-LABEL: fsqrt_v16f32:1643; VBITS_GE_512: // %bb.0:1644; VBITS_GE_512-NEXT: ptrue p0.s, vl161645; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]1646; VBITS_GE_512-NEXT: fsqrt z0.s, p0/m, z0.s1647; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]1648; VBITS_GE_512-NEXT: ret1649 %op = load <16 x float>, ptr %a1650 %res = call <16 x float> @llvm.sqrt.v16f32(<16 x float> %op)1651 store <16 x float> %res, ptr %a1652 ret void1653}1654 1655define void @fsqrt_v32f32(ptr %a) vscale_range(8,0) #0 {1656; CHECK-LABEL: fsqrt_v32f32:1657; CHECK: // %bb.0:1658; CHECK-NEXT: ptrue p0.s, vl321659; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1660; CHECK-NEXT: fsqrt z0.s, p0/m, z0.s1661; CHECK-NEXT: st1w { z0.s }, p0, [x0]1662; CHECK-NEXT: ret1663 %op = load <32 x float>, ptr %a1664 %res = call <32 x float> @llvm.sqrt.v32f32(<32 x float> %op)1665 store <32 x float> %res, ptr %a1666 ret void1667}1668 1669define void @fsqrt_v64f32(ptr %a) vscale_range(16,0) #0 {1670; CHECK-LABEL: fsqrt_v64f32:1671; CHECK: // %bb.0:1672; CHECK-NEXT: ptrue p0.s, vl641673; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1674; CHECK-NEXT: fsqrt z0.s, p0/m, z0.s1675; CHECK-NEXT: st1w { z0.s }, p0, [x0]1676; CHECK-NEXT: ret1677 %op = load <64 x float>, ptr %a1678 %res = call <64 x float> @llvm.sqrt.v64f32(<64 x float> %op)1679 store <64 x float> %res, ptr %a1680 ret void1681}1682 1683; Don't use SVE for 64-bit vectors.1684define <1 x double> @fsqrt_v1f64(<1 x double> %op) vscale_range(2,0) #0 {1685; CHECK-LABEL: fsqrt_v1f64:1686; CHECK: // %bb.0:1687; CHECK-NEXT: fsqrt d0, d01688; CHECK-NEXT: ret1689 %res = call <1 x double> @llvm.sqrt.v1f64(<1 x double> %op)1690 ret <1 x double> %res1691}1692 1693; Don't use SVE for 128-bit vectors.1694define <2 x double> @fsqrt_v2f64(<2 x double> %op) vscale_range(2,0) #0 {1695; CHECK-LABEL: fsqrt_v2f64:1696; CHECK: // %bb.0:1697; CHECK-NEXT: fsqrt v0.2d, v0.2d1698; CHECK-NEXT: ret1699 %res = call <2 x double> @llvm.sqrt.v2f64(<2 x double> %op)1700 ret <2 x double> %res1701}1702 1703define void @fsqrt_v4f64(ptr %a) vscale_range(2,0) #0 {1704; CHECK-LABEL: fsqrt_v4f64:1705; CHECK: // %bb.0:1706; CHECK-NEXT: ptrue p0.d, vl41707; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1708; CHECK-NEXT: fsqrt z0.d, p0/m, z0.d1709; CHECK-NEXT: st1d { z0.d }, p0, [x0]1710; CHECK-NEXT: ret1711 %op = load <4 x double>, ptr %a1712 %res = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %op)1713 store <4 x double> %res, ptr %a1714 ret void1715}1716 1717define void @fsqrt_v8f64(ptr %a) #0 {1718; VBITS_GE_256-LABEL: fsqrt_v8f64:1719; VBITS_GE_256: // %bb.0:1720; VBITS_GE_256-NEXT: ptrue p0.d, vl41721; VBITS_GE_256-NEXT: mov x8, #4 // =0x41722; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]1723; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]1724; VBITS_GE_256-NEXT: fsqrt z0.d, p0/m, z0.d1725; VBITS_GE_256-NEXT: fsqrt z1.d, p0/m, z1.d1726; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]1727; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]1728; VBITS_GE_256-NEXT: ret1729;1730; VBITS_GE_512-LABEL: fsqrt_v8f64:1731; VBITS_GE_512: // %bb.0:1732; VBITS_GE_512-NEXT: ptrue p0.d, vl81733; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]1734; VBITS_GE_512-NEXT: fsqrt z0.d, p0/m, z0.d1735; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]1736; VBITS_GE_512-NEXT: ret1737 %op = load <8 x double>, ptr %a1738 %res = call <8 x double> @llvm.sqrt.v8f64(<8 x double> %op)1739 store <8 x double> %res, ptr %a1740 ret void1741}1742 1743define void @fsqrt_v16f64(ptr %a) vscale_range(8,0) #0 {1744; CHECK-LABEL: fsqrt_v16f64:1745; CHECK: // %bb.0:1746; CHECK-NEXT: ptrue p0.d, vl161747; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1748; CHECK-NEXT: fsqrt z0.d, p0/m, z0.d1749; CHECK-NEXT: st1d { z0.d }, p0, [x0]1750; CHECK-NEXT: ret1751 %op = load <16 x double>, ptr %a1752 %res = call <16 x double> @llvm.sqrt.v16f64(<16 x double> %op)1753 store <16 x double> %res, ptr %a1754 ret void1755}1756 1757define void @fsqrt_v32f64(ptr %a) vscale_range(16,0) #0 {1758; CHECK-LABEL: fsqrt_v32f64:1759; CHECK: // %bb.0:1760; CHECK-NEXT: ptrue p0.d, vl321761; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1762; CHECK-NEXT: fsqrt z0.d, p0/m, z0.d1763; CHECK-NEXT: st1d { z0.d }, p0, [x0]1764; CHECK-NEXT: ret1765 %op = load <32 x double>, ptr %a1766 %res = call <32 x double> @llvm.sqrt.v32f64(<32 x double> %op)1767 store <32 x double> %res, ptr %a1768 ret void1769}1770 1771;1772; FSUB1773;1774 1775; Don't use SVE for 64-bit vectors.1776define <4 x half> @fsub_v4f16(<4 x half> %op1, <4 x half> %op2) vscale_range(2,0) #0 {1777; CHECK-LABEL: fsub_v4f16:1778; CHECK: // %bb.0:1779; CHECK-NEXT: fsub v0.4h, v0.4h, v1.4h1780; CHECK-NEXT: ret1781 %res = fsub <4 x half> %op1, %op21782 ret <4 x half> %res1783}1784 1785; Don't use SVE for 128-bit vectors.1786define <8 x half> @fsub_v8f16(<8 x half> %op1, <8 x half> %op2) vscale_range(2,0) #0 {1787; CHECK-LABEL: fsub_v8f16:1788; CHECK: // %bb.0:1789; CHECK-NEXT: fsub v0.8h, v0.8h, v1.8h1790; CHECK-NEXT: ret1791 %res = fsub <8 x half> %op1, %op21792 ret <8 x half> %res1793}1794 1795define void @fsub_v16f16(ptr %a, ptr %b) vscale_range(2,0) #0 {1796; CHECK-LABEL: fsub_v16f16:1797; CHECK: // %bb.0:1798; CHECK-NEXT: ptrue p0.h, vl161799; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1800; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]1801; CHECK-NEXT: fsub z0.h, p0/m, z0.h, z1.h1802; CHECK-NEXT: st1h { z0.h }, p0, [x0]1803; CHECK-NEXT: ret1804 %op1 = load <16 x half>, ptr %a1805 %op2 = load <16 x half>, ptr %b1806 %res = fsub <16 x half> %op1, %op21807 store <16 x half> %res, ptr %a1808 ret void1809}1810 1811define void @fsub_v32f16(ptr %a, ptr %b) #0 {1812; VBITS_GE_256-LABEL: fsub_v32f16:1813; VBITS_GE_256: // %bb.0:1814; VBITS_GE_256-NEXT: ptrue p0.h, vl161815; VBITS_GE_256-NEXT: mov x8, #16 // =0x101816; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]1817; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]1818; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0]1819; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1]1820; VBITS_GE_256-NEXT: fsub z0.h, p0/m, z0.h, z1.h1821; VBITS_GE_256-NEXT: fsub z2.h, p0/m, z2.h, z3.h1822; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]1823; VBITS_GE_256-NEXT: st1h { z2.h }, p0, [x0]1824; VBITS_GE_256-NEXT: ret1825;1826; VBITS_GE_512-LABEL: fsub_v32f16:1827; VBITS_GE_512: // %bb.0:1828; VBITS_GE_512-NEXT: ptrue p0.h, vl321829; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]1830; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]1831; VBITS_GE_512-NEXT: fsub z0.h, p0/m, z0.h, z1.h1832; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]1833; VBITS_GE_512-NEXT: ret1834 %op1 = load <32 x half>, ptr %a1835 %op2 = load <32 x half>, ptr %b1836 %res = fsub <32 x half> %op1, %op21837 store <32 x half> %res, ptr %a1838 ret void1839}1840 1841define void @fsub_v64f16(ptr %a, ptr %b) vscale_range(8,0) #0 {1842; CHECK-LABEL: fsub_v64f16:1843; CHECK: // %bb.0:1844; CHECK-NEXT: ptrue p0.h, vl641845; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1846; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]1847; CHECK-NEXT: fsub z0.h, p0/m, z0.h, z1.h1848; CHECK-NEXT: st1h { z0.h }, p0, [x0]1849; CHECK-NEXT: ret1850 %op1 = load <64 x half>, ptr %a1851 %op2 = load <64 x half>, ptr %b1852 %res = fsub <64 x half> %op1, %op21853 store <64 x half> %res, ptr %a1854 ret void1855}1856 1857define void @fsub_v128f16(ptr %a, ptr %b) vscale_range(16,0) #0 {1858; CHECK-LABEL: fsub_v128f16:1859; CHECK: // %bb.0:1860; CHECK-NEXT: ptrue p0.h, vl1281861; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1862; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1]1863; CHECK-NEXT: fsub z0.h, p0/m, z0.h, z1.h1864; CHECK-NEXT: st1h { z0.h }, p0, [x0]1865; CHECK-NEXT: ret1866 %op1 = load <128 x half>, ptr %a1867 %op2 = load <128 x half>, ptr %b1868 %res = fsub <128 x half> %op1, %op21869 store <128 x half> %res, ptr %a1870 ret void1871}1872 1873; Don't use SVE for 64-bit vectors.1874define <2 x float> @fsub_v2f32(<2 x float> %op1, <2 x float> %op2) vscale_range(2,0) #0 {1875; CHECK-LABEL: fsub_v2f32:1876; CHECK: // %bb.0:1877; CHECK-NEXT: fsub v0.2s, v0.2s, v1.2s1878; CHECK-NEXT: ret1879 %res = fsub <2 x float> %op1, %op21880 ret <2 x float> %res1881}1882 1883; Don't use SVE for 128-bit vectors.1884define <4 x float> @fsub_v4f32(<4 x float> %op1, <4 x float> %op2) vscale_range(2,0) #0 {1885; CHECK-LABEL: fsub_v4f32:1886; CHECK: // %bb.0:1887; CHECK-NEXT: fsub v0.4s, v0.4s, v1.4s1888; CHECK-NEXT: ret1889 %res = fsub <4 x float> %op1, %op21890 ret <4 x float> %res1891}1892 1893define void @fsub_v8f32(ptr %a, ptr %b) vscale_range(2,0) #0 {1894; CHECK-LABEL: fsub_v8f32:1895; CHECK: // %bb.0:1896; CHECK-NEXT: ptrue p0.s, vl81897; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1898; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1899; CHECK-NEXT: fsub z0.s, p0/m, z0.s, z1.s1900; CHECK-NEXT: st1w { z0.s }, p0, [x0]1901; CHECK-NEXT: ret1902 %op1 = load <8 x float>, ptr %a1903 %op2 = load <8 x float>, ptr %b1904 %res = fsub <8 x float> %op1, %op21905 store <8 x float> %res, ptr %a1906 ret void1907}1908 1909define void @fsub_v16f32(ptr %a, ptr %b) #0 {1910; VBITS_GE_256-LABEL: fsub_v16f32:1911; VBITS_GE_256: // %bb.0:1912; VBITS_GE_256-NEXT: ptrue p0.s, vl81913; VBITS_GE_256-NEXT: mov x8, #8 // =0x81914; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]1915; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]1916; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]1917; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]1918; VBITS_GE_256-NEXT: fsub z0.s, p0/m, z0.s, z1.s1919; VBITS_GE_256-NEXT: fsub z2.s, p0/m, z2.s, z3.s1920; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]1921; VBITS_GE_256-NEXT: st1w { z2.s }, p0, [x0]1922; VBITS_GE_256-NEXT: ret1923;1924; VBITS_GE_512-LABEL: fsub_v16f32:1925; VBITS_GE_512: // %bb.0:1926; VBITS_GE_512-NEXT: ptrue p0.s, vl161927; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]1928; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]1929; VBITS_GE_512-NEXT: fsub z0.s, p0/m, z0.s, z1.s1930; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]1931; VBITS_GE_512-NEXT: ret1932 %op1 = load <16 x float>, ptr %a1933 %op2 = load <16 x float>, ptr %b1934 %res = fsub <16 x float> %op1, %op21935 store <16 x float> %res, ptr %a1936 ret void1937}1938 1939define void @fsub_v32f32(ptr %a, ptr %b) vscale_range(8,0) #0 {1940; CHECK-LABEL: fsub_v32f32:1941; CHECK: // %bb.0:1942; CHECK-NEXT: ptrue p0.s, vl321943; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1944; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1945; CHECK-NEXT: fsub z0.s, p0/m, z0.s, z1.s1946; CHECK-NEXT: st1w { z0.s }, p0, [x0]1947; CHECK-NEXT: ret1948 %op1 = load <32 x float>, ptr %a1949 %op2 = load <32 x float>, ptr %b1950 %res = fsub <32 x float> %op1, %op21951 store <32 x float> %res, ptr %a1952 ret void1953}1954 1955define void @fsub_v64f32(ptr %a, ptr %b) vscale_range(16,0) #0 {1956; CHECK-LABEL: fsub_v64f32:1957; CHECK: // %bb.0:1958; CHECK-NEXT: ptrue p0.s, vl641959; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1960; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]1961; CHECK-NEXT: fsub z0.s, p0/m, z0.s, z1.s1962; CHECK-NEXT: st1w { z0.s }, p0, [x0]1963; CHECK-NEXT: ret1964 %op1 = load <64 x float>, ptr %a1965 %op2 = load <64 x float>, ptr %b1966 %res = fsub <64 x float> %op1, %op21967 store <64 x float> %res, ptr %a1968 ret void1969}1970 1971; Don't use SVE for 64-bit vectors.1972define <1 x double> @fsub_v1f64(<1 x double> %op1, <1 x double> %op2) vscale_range(2,0) #0 {1973; CHECK-LABEL: fsub_v1f64:1974; CHECK: // %bb.0:1975; CHECK-NEXT: fsub d0, d0, d11976; CHECK-NEXT: ret1977 %res = fsub <1 x double> %op1, %op21978 ret <1 x double> %res1979}1980 1981; Don't use SVE for 128-bit vectors.1982define <2 x double> @fsub_v2f64(<2 x double> %op1, <2 x double> %op2) vscale_range(2,0) #0 {1983; CHECK-LABEL: fsub_v2f64:1984; CHECK: // %bb.0:1985; CHECK-NEXT: fsub v0.2d, v0.2d, v1.2d1986; CHECK-NEXT: ret1987 %res = fsub <2 x double> %op1, %op21988 ret <2 x double> %res1989}1990 1991define void @fsub_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 {1992; CHECK-LABEL: fsub_v4f64:1993; CHECK: // %bb.0:1994; CHECK-NEXT: ptrue p0.d, vl41995; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1996; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]1997; CHECK-NEXT: fsub z0.d, p0/m, z0.d, z1.d1998; CHECK-NEXT: st1d { z0.d }, p0, [x0]1999; CHECK-NEXT: ret2000 %op1 = load <4 x double>, ptr %a2001 %op2 = load <4 x double>, ptr %b2002 %res = fsub <4 x double> %op1, %op22003 store <4 x double> %res, ptr %a2004 ret void2005}2006 2007define void @fsub_v8f64(ptr %a, ptr %b) #0 {2008; VBITS_GE_256-LABEL: fsub_v8f64:2009; VBITS_GE_256: // %bb.0:2010; VBITS_GE_256-NEXT: ptrue p0.d, vl42011; VBITS_GE_256-NEXT: mov x8, #4 // =0x42012; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]2013; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]2014; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]2015; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]2016; VBITS_GE_256-NEXT: fsub z0.d, p0/m, z0.d, z1.d2017; VBITS_GE_256-NEXT: fsub z2.d, p0/m, z2.d, z3.d2018; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]2019; VBITS_GE_256-NEXT: st1d { z2.d }, p0, [x0]2020; VBITS_GE_256-NEXT: ret2021;2022; VBITS_GE_512-LABEL: fsub_v8f64:2023; VBITS_GE_512: // %bb.0:2024; VBITS_GE_512-NEXT: ptrue p0.d, vl82025; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]2026; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]2027; VBITS_GE_512-NEXT: fsub z0.d, p0/m, z0.d, z1.d2028; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]2029; VBITS_GE_512-NEXT: ret2030 %op1 = load <8 x double>, ptr %a2031 %op2 = load <8 x double>, ptr %b2032 %res = fsub <8 x double> %op1, %op22033 store <8 x double> %res, ptr %a2034 ret void2035}2036 2037define void @fsub_v16f64(ptr %a, ptr %b) vscale_range(8,0) #0 {2038; CHECK-LABEL: fsub_v16f64:2039; CHECK: // %bb.0:2040; CHECK-NEXT: ptrue p0.d, vl162041; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]2042; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]2043; CHECK-NEXT: fsub z0.d, p0/m, z0.d, z1.d2044; CHECK-NEXT: st1d { z0.d }, p0, [x0]2045; CHECK-NEXT: ret2046 %op1 = load <16 x double>, ptr %a2047 %op2 = load <16 x double>, ptr %b2048 %res = fsub <16 x double> %op1, %op22049 store <16 x double> %res, ptr %a2050 ret void2051}2052 2053define void @fsub_v32f64(ptr %a, ptr %b) vscale_range(16,0) #0 {2054; CHECK-LABEL: fsub_v32f64:2055; CHECK: // %bb.0:2056; CHECK-NEXT: ptrue p0.d, vl322057; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]2058; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1]2059; CHECK-NEXT: fsub z0.d, p0/m, z0.d, z1.d2060; CHECK-NEXT: st1d { z0.d }, p0, [x0]2061; CHECK-NEXT: ret2062 %op1 = load <32 x double>, ptr %a2063 %op2 = load <32 x double>, ptr %b2064 %res = fsub <32 x double> %op1, %op22065 store <32 x double> %res, ptr %a2066 ret void2067}2068 2069;2070; FABS2071;2072 2073; Don't use SVE for 64-bit vectors.2074define <4 x half> @fabs_v4f16(<4 x half> %op) vscale_range(2,0) #0 {2075; CHECK-LABEL: fabs_v4f16:2076; CHECK: // %bb.0:2077; CHECK-NEXT: fabs v0.4h, v0.4h2078; CHECK-NEXT: ret2079 %res = call <4 x half> @llvm.fabs.v4f16(<4 x half> %op)2080 ret <4 x half> %res2081}2082 2083; Don't use SVE for 128-bit vectors.2084define <8 x half> @fabs_v8f16(<8 x half> %op) vscale_range(2,0) #0 {2085; CHECK-LABEL: fabs_v8f16:2086; CHECK: // %bb.0:2087; CHECK-NEXT: fabs v0.8h, v0.8h2088; CHECK-NEXT: ret2089 %res = call <8 x half> @llvm.fabs.v8f16(<8 x half> %op)2090 ret <8 x half> %res2091}2092 2093define void @fabs_v16f16(ptr %a) vscale_range(2,0) #0 {2094; CHECK-LABEL: fabs_v16f16:2095; CHECK: // %bb.0:2096; CHECK-NEXT: ptrue p0.h, vl162097; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]2098; CHECK-NEXT: fabs z0.h, p0/m, z0.h2099; CHECK-NEXT: st1h { z0.h }, p0, [x0]2100; CHECK-NEXT: ret2101 %op = load <16 x half>, ptr %a2102 %res = call <16 x half> @llvm.fabs.v16f16(<16 x half> %op)2103 store <16 x half> %res, ptr %a2104 ret void2105}2106 2107define void @fabs_v32f16(ptr %a) #0 {2108; VBITS_GE_256-LABEL: fabs_v32f16:2109; VBITS_GE_256: // %bb.0:2110; VBITS_GE_256-NEXT: ptrue p0.h, vl162111; VBITS_GE_256-NEXT: mov x8, #16 // =0x102112; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]2113; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0]2114; VBITS_GE_256-NEXT: fabs z0.h, p0/m, z0.h2115; VBITS_GE_256-NEXT: fabs z1.h, p0/m, z1.h2116; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]2117; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0]2118; VBITS_GE_256-NEXT: ret2119;2120; VBITS_GE_512-LABEL: fabs_v32f16:2121; VBITS_GE_512: // %bb.0:2122; VBITS_GE_512-NEXT: ptrue p0.h, vl322123; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]2124; VBITS_GE_512-NEXT: fabs z0.h, p0/m, z0.h2125; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0]2126; VBITS_GE_512-NEXT: ret2127 %op = load <32 x half>, ptr %a2128 %res = call <32 x half> @llvm.fabs.v32f16(<32 x half> %op)2129 store <32 x half> %res, ptr %a2130 ret void2131}2132 2133define void @fabs_v64f16(ptr %a) vscale_range(8,0) #0 {2134; CHECK-LABEL: fabs_v64f16:2135; CHECK: // %bb.0:2136; CHECK-NEXT: ptrue p0.h, vl642137; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]2138; CHECK-NEXT: fabs z0.h, p0/m, z0.h2139; CHECK-NEXT: st1h { z0.h }, p0, [x0]2140; CHECK-NEXT: ret2141 %op = load <64 x half>, ptr %a2142 %res = call <64 x half> @llvm.fabs.v64f16(<64 x half> %op)2143 store <64 x half> %res, ptr %a2144 ret void2145}2146 2147define void @fabs_v128f16(ptr %a) vscale_range(16,0) #0 {2148; CHECK-LABEL: fabs_v128f16:2149; CHECK: // %bb.0:2150; CHECK-NEXT: ptrue p0.h, vl1282151; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]2152; CHECK-NEXT: fabs z0.h, p0/m, z0.h2153; CHECK-NEXT: st1h { z0.h }, p0, [x0]2154; CHECK-NEXT: ret2155 %op = load <128 x half>, ptr %a2156 %res = call <128 x half> @llvm.fabs.v128f16(<128 x half> %op)2157 store <128 x half> %res, ptr %a2158 ret void2159}2160 2161; Don't use SVE for 64-bit vectors.2162define <2 x float> @fabs_v2f32(<2 x float> %op) vscale_range(2,0) #0 {2163; CHECK-LABEL: fabs_v2f32:2164; CHECK: // %bb.0:2165; CHECK-NEXT: fabs v0.2s, v0.2s2166; CHECK-NEXT: ret2167 %res = call <2 x float> @llvm.fabs.v2f32(<2 x float> %op)2168 ret <2 x float> %res2169}2170 2171; Don't use SVE for 128-bit vectors.2172define <4 x float> @fabs_v4f32(<4 x float> %op) vscale_range(2,0) #0 {2173; CHECK-LABEL: fabs_v4f32:2174; CHECK: // %bb.0:2175; CHECK-NEXT: fabs v0.4s, v0.4s2176; CHECK-NEXT: ret2177 %res = call <4 x float> @llvm.fabs.v4f32(<4 x float> %op)2178 ret <4 x float> %res2179}2180 2181define void @fabs_v8f32(ptr %a) vscale_range(2,0) #0 {2182; CHECK-LABEL: fabs_v8f32:2183; CHECK: // %bb.0:2184; CHECK-NEXT: ptrue p0.s, vl82185; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]2186; CHECK-NEXT: fabs z0.s, p0/m, z0.s2187; CHECK-NEXT: st1w { z0.s }, p0, [x0]2188; CHECK-NEXT: ret2189 %op = load <8 x float>, ptr %a2190 %res = call <8 x float> @llvm.fabs.v8f32(<8 x float> %op)2191 store <8 x float> %res, ptr %a2192 ret void2193}2194 2195define void @fabs_v16f32(ptr %a) #0 {2196; VBITS_GE_256-LABEL: fabs_v16f32:2197; VBITS_GE_256: // %bb.0:2198; VBITS_GE_256-NEXT: ptrue p0.s, vl82199; VBITS_GE_256-NEXT: mov x8, #8 // =0x82200; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]2201; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]2202; VBITS_GE_256-NEXT: fabs z0.s, p0/m, z0.s2203; VBITS_GE_256-NEXT: fabs z1.s, p0/m, z1.s2204; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]2205; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]2206; VBITS_GE_256-NEXT: ret2207;2208; VBITS_GE_512-LABEL: fabs_v16f32:2209; VBITS_GE_512: // %bb.0:2210; VBITS_GE_512-NEXT: ptrue p0.s, vl162211; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]2212; VBITS_GE_512-NEXT: fabs z0.s, p0/m, z0.s2213; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]2214; VBITS_GE_512-NEXT: ret2215 %op = load <16 x float>, ptr %a2216 %res = call <16 x float> @llvm.fabs.v16f32(<16 x float> %op)2217 store <16 x float> %res, ptr %a2218 ret void2219}2220 2221define void @fabs_v32f32(ptr %a) vscale_range(8,0) #0 {2222; CHECK-LABEL: fabs_v32f32:2223; CHECK: // %bb.0:2224; CHECK-NEXT: ptrue p0.s, vl322225; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]2226; CHECK-NEXT: fabs z0.s, p0/m, z0.s2227; CHECK-NEXT: st1w { z0.s }, p0, [x0]2228; CHECK-NEXT: ret2229 %op = load <32 x float>, ptr %a2230 %res = call <32 x float> @llvm.fabs.v32f32(<32 x float> %op)2231 store <32 x float> %res, ptr %a2232 ret void2233}2234 2235define void @fabs_v64f32(ptr %a) vscale_range(16,0) #0 {2236; CHECK-LABEL: fabs_v64f32:2237; CHECK: // %bb.0:2238; CHECK-NEXT: ptrue p0.s, vl642239; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]2240; CHECK-NEXT: fabs z0.s, p0/m, z0.s2241; CHECK-NEXT: st1w { z0.s }, p0, [x0]2242; CHECK-NEXT: ret2243 %op = load <64 x float>, ptr %a2244 %res = call <64 x float> @llvm.fabs.v64f32(<64 x float> %op)2245 store <64 x float> %res, ptr %a2246 ret void2247}2248 2249; Don't use SVE for 64-bit vectors.2250define <1 x double> @fabs_v1f64(<1 x double> %op) vscale_range(2,0) #0 {2251; CHECK-LABEL: fabs_v1f64:2252; CHECK: // %bb.0:2253; CHECK-NEXT: fabs d0, d02254; CHECK-NEXT: ret2255 %res = call <1 x double> @llvm.fabs.v1f64(<1 x double> %op)2256 ret <1 x double> %res2257}2258 2259; Don't use SVE for 128-bit vectors.2260define <2 x double> @fabs_v2f64(<2 x double> %op) vscale_range(2,0) #0 {2261; CHECK-LABEL: fabs_v2f64:2262; CHECK: // %bb.0:2263; CHECK-NEXT: fabs v0.2d, v0.2d2264; CHECK-NEXT: ret2265 %res = call <2 x double> @llvm.fabs.v2f64(<2 x double> %op)2266 ret <2 x double> %res2267}2268 2269define void @fabs_v4f64(ptr %a) vscale_range(2,0) #0 {2270; CHECK-LABEL: fabs_v4f64:2271; CHECK: // %bb.0:2272; CHECK-NEXT: ptrue p0.d, vl42273; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]2274; CHECK-NEXT: fabs z0.d, p0/m, z0.d2275; CHECK-NEXT: st1d { z0.d }, p0, [x0]2276; CHECK-NEXT: ret2277 %op = load <4 x double>, ptr %a2278 %res = call <4 x double> @llvm.fabs.v4f64(<4 x double> %op)2279 store <4 x double> %res, ptr %a2280 ret void2281}2282 2283define void @fabs_v8f64(ptr %a) #0 {2284; VBITS_GE_256-LABEL: fabs_v8f64:2285; VBITS_GE_256: // %bb.0:2286; VBITS_GE_256-NEXT: ptrue p0.d, vl42287; VBITS_GE_256-NEXT: mov x8, #4 // =0x42288; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]2289; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]2290; VBITS_GE_256-NEXT: fabs z0.d, p0/m, z0.d2291; VBITS_GE_256-NEXT: fabs z1.d, p0/m, z1.d2292; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]2293; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]2294; VBITS_GE_256-NEXT: ret2295;2296; VBITS_GE_512-LABEL: fabs_v8f64:2297; VBITS_GE_512: // %bb.0:2298; VBITS_GE_512-NEXT: ptrue p0.d, vl82299; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]2300; VBITS_GE_512-NEXT: fabs z0.d, p0/m, z0.d2301; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]2302; VBITS_GE_512-NEXT: ret2303 %op = load <8 x double>, ptr %a2304 %res = call <8 x double> @llvm.fabs.v8f64(<8 x double> %op)2305 store <8 x double> %res, ptr %a2306 ret void2307}2308 2309define void @fabs_v16f64(ptr %a) vscale_range(8,0) #0 {2310; CHECK-LABEL: fabs_v16f64:2311; CHECK: // %bb.0:2312; CHECK-NEXT: ptrue p0.d, vl162313; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]2314; CHECK-NEXT: fabs z0.d, p0/m, z0.d2315; CHECK-NEXT: st1d { z0.d }, p0, [x0]2316; CHECK-NEXT: ret2317 %op = load <16 x double>, ptr %a2318 %res = call <16 x double> @llvm.fabs.v16f64(<16 x double> %op)2319 store <16 x double> %res, ptr %a2320 ret void2321}2322 2323define void @fabs_v32f64(ptr %a) vscale_range(16,0) #0 {2324; CHECK-LABEL: fabs_v32f64:2325; CHECK: // %bb.0:2326; CHECK-NEXT: ptrue p0.d, vl322327; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]2328; CHECK-NEXT: fabs z0.d, p0/m, z0.d2329; CHECK-NEXT: st1d { z0.d }, p0, [x0]2330; CHECK-NEXT: ret2331 %op = load <32 x double>, ptr %a2332 %res = call <32 x double> @llvm.fabs.v32f64(<32 x double> %op)2333 store <32 x double> %res, ptr %a2334 ret void2335}2336 2337attributes #0 = { "target-features"="+sve" }2338 2339declare <4 x half> @llvm.fma.v4f16(<4 x half>, <4 x half>, <4 x half>)2340declare <8 x half> @llvm.fma.v8f16(<8 x half>, <8 x half>, <8 x half>)2341declare <16 x half> @llvm.fma.v16f16(<16 x half>, <16 x half>, <16 x half>)2342declare <32 x half> @llvm.fma.v32f16(<32 x half>, <32 x half>, <32 x half>)2343declare <64 x half> @llvm.fma.v64f16(<64 x half>, <64 x half>, <64 x half>)2344declare <128 x half> @llvm.fma.v128f16(<128 x half>, <128 x half>, <128 x half>)2345declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)2346declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)2347declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>)2348declare <16 x float> @llvm.fma.v16f32(<16 x float>, <16 x float>, <16 x float>)2349declare <32 x float> @llvm.fma.v32f32(<32 x float>, <32 x float>, <32 x float>)2350declare <64 x float> @llvm.fma.v64f32(<64 x float>, <64 x float>, <64 x float>)2351declare <1 x double> @llvm.fma.v1f64(<1 x double>, <1 x double>, <1 x double>)2352declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)2353declare <4 x double> @llvm.fma.v4f64(<4 x double>, <4 x double>, <4 x double>)2354declare <8 x double> @llvm.fma.v8f64(<8 x double>, <8 x double>, <8 x double>)2355declare <16 x double> @llvm.fma.v16f64(<16 x double>, <16 x double>, <16 x double>)2356declare <32 x double> @llvm.fma.v32f64(<32 x double>, <32 x double>, <32 x double>)2357 2358declare <4 x half> @llvm.sqrt.v4f16(<4 x half>)2359declare <8 x half> @llvm.sqrt.v8f16(<8 x half>)2360declare <16 x half> @llvm.sqrt.v16f16(<16 x half>)2361declare <32 x half> @llvm.sqrt.v32f16(<32 x half>)2362declare <64 x half> @llvm.sqrt.v64f16(<64 x half>)2363declare <128 x half> @llvm.sqrt.v128f16(<128 x half>)2364declare <2 x float> @llvm.sqrt.v2f32(<2 x float>)2365declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)2366declare <8 x float> @llvm.sqrt.v8f32(<8 x float>)2367declare <16 x float> @llvm.sqrt.v16f32(<16 x float>)2368declare <32 x float> @llvm.sqrt.v32f32(<32 x float>)2369declare <64 x float> @llvm.sqrt.v64f32(<64 x float>)2370declare <1 x double> @llvm.sqrt.v1f64(<1 x double>)2371declare <2 x double> @llvm.sqrt.v2f64(<2 x double>)2372declare <4 x double> @llvm.sqrt.v4f64(<4 x double>)2373declare <8 x double> @llvm.sqrt.v8f64(<8 x double>)2374declare <16 x double> @llvm.sqrt.v16f64(<16 x double>)2375declare <32 x double> @llvm.sqrt.v32f64(<32 x double>)2376 2377declare <4 x half> @llvm.fabs.v4f16(<4 x half>)2378declare <8 x half> @llvm.fabs.v8f16(<8 x half>)2379declare <16 x half> @llvm.fabs.v16f16(<16 x half>)2380declare <32 x half> @llvm.fabs.v32f16(<32 x half>)2381declare <64 x half> @llvm.fabs.v64f16(<64 x half>)2382declare <128 x half> @llvm.fabs.v128f16(<128 x half>)2383declare <2 x float> @llvm.fabs.v2f32(<2 x float>)2384declare <4 x float> @llvm.fabs.v4f32(<4 x float>)2385declare <8 x float> @llvm.fabs.v8f32(<8 x float>)2386declare <16 x float> @llvm.fabs.v16f32(<16 x float>)2387declare <32 x float> @llvm.fabs.v32f32(<32 x float>)2388declare <64 x float> @llvm.fabs.v64f32(<64 x float>)2389declare <1 x double> @llvm.fabs.v1f64(<1 x double>)2390declare <2 x double> @llvm.fabs.v2f64(<2 x double>)2391declare <4 x double> @llvm.fabs.v4f64(<4 x double>)2392declare <8 x double> @llvm.fabs.v8f64(<8 x double>)2393declare <16 x double> @llvm.fabs.v16f64(<16 x double>)2394declare <32 x double> @llvm.fabs.v32f64(<32 x double>)2395