; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mattr=+sve2 < %s | FileCheck %s --check-prefixes=CHECK,SVE2
; RUN: llc -mattr=+sve2p1 < %s | FileCheck %s --check-prefixes=CHECK,SVE2P1

target triple = "aarch64-linux-gnu"

define void @fdot_v4f32(ptr %accptr, ptr %aptr, ptr %bptr) {
; SVE2-LABEL: fdot_v4f32:
; SVE2:       // %bb.0: // %entry
; SVE2-NEXT:    ldr q0, [x1]
; SVE2-NEXT:    ldr q1, [x2]
; SVE2-NEXT:    fcvtl v2.4s, v0.4h
; SVE2-NEXT:    fcvtl v3.4s, v1.4h
; SVE2-NEXT:    fcvtl2 v0.4s, v0.8h
; SVE2-NEXT:    fcvtl2 v1.4s, v1.8h
; SVE2-NEXT:    fmul v2.4s, v2.4s, v3.4s
; SVE2-NEXT:    ldr q3, [x0]
; SVE2-NEXT:    fmul v0.4s, v0.4s, v1.4s
; SVE2-NEXT:    fadd v1.4s, v3.4s, v2.4s
; SVE2-NEXT:    fadd v0.4s, v1.4s, v0.4s
; SVE2-NEXT:    str q0, [x0]
; SVE2-NEXT:    ret
;
; SVE2P1-LABEL: fdot_v4f32:
; SVE2P1:       // %bb.0: // %entry
; SVE2P1-NEXT:    ldr q0, [x0]
; SVE2P1-NEXT:    ldr q1, [x1]
; SVE2P1-NEXT:    ldr q2, [x2]
; SVE2P1-NEXT:    fdot z0.s, z1.h, z2.h
; SVE2P1-NEXT:    str q0, [x0]
; SVE2P1-NEXT:    ret
entry:
  %acc = load <4 x float>, ptr %accptr
  %a = load <8 x half>, ptr %aptr
  %b = load <8 x half>, ptr %bptr
  %a.wide = fpext <8 x half> %a to <8 x float>
  %b.wide = fpext <8 x half> %b to <8 x float>
  %mult = fmul <8 x float> %a.wide, %b.wide
  %partial.reduce = call <4 x float> @llvm.vector.partial.reduce.fadd(<4 x float> %acc, <8 x float> %mult)
  store <4 x float> %partial.reduce, ptr %accptr
  ret void
}

define void @fdot_wide_v8f32(ptr %accptr, ptr %aptr, ptr %bptr) vscale_range(2,0) {
; SVE2-LABEL: fdot_wide_v8f32:
; SVE2:       // %bb.0: // %entry
; SVE2-NEXT:    ptrue p0.s, vl8
; SVE2-NEXT:    mov x8, #8 // =0x8
; SVE2-NEXT:    ld1h { z0.s }, p0/z, [x1]
; SVE2-NEXT:    ld1h { z1.s }, p0/z, [x2]
; SVE2-NEXT:    ld1h { z2.s }, p0/z, [x1, x8, lsl #1]
; SVE2-NEXT:    ld1h { z3.s }, p0/z, [x2, x8, lsl #1]
; SVE2-NEXT:    fcvt z0.s, p0/m, z0.h
; SVE2-NEXT:    fcvt z1.s, p0/m, z1.h
; SVE2-NEXT:    fcvt z2.s, p0/m, z2.h
; SVE2-NEXT:    fcvt z3.s, p0/m, z3.h
; SVE2-NEXT:    fmul z0.s, p0/m, z0.s, z1.s
; SVE2-NEXT:    ld1w { z1.s }, p0/z, [x0]
; SVE2-NEXT:    fmul z2.s, p0/m, z2.s, z3.s
; SVE2-NEXT:    fadd z0.s, p0/m, z0.s, z1.s
; SVE2-NEXT:    fadd z0.s, p0/m, z0.s, z2.s
; SVE2-NEXT:    st1w { z0.s }, p0, [x0]
; SVE2-NEXT:    ret
;
; SVE2P1-LABEL: fdot_wide_v8f32:
; SVE2P1:       // %bb.0: // %entry
; SVE2P1-NEXT:    ptrue p0.s, vl8
; SVE2P1-NEXT:    ptrue p1.h, vl16
; SVE2P1-NEXT:    ld1w { z0.s }, p0/z, [x0]
; SVE2P1-NEXT:    ld1h { z1.h }, p1/z, [x1]
; SVE2P1-NEXT:    ld1h { z2.h }, p1/z, [x2]
; SVE2P1-NEXT:    fdot z0.s, z1.h, z2.h
; SVE2P1-NEXT:    st1w { z0.s }, p0, [x0]
; SVE2P1-NEXT:    ret
entry:
  %acc = load <8 x float>, ptr %accptr
  %a = load <16 x half>, ptr %aptr
  %b = load <16 x half>, ptr %bptr
  %a.wide = fpext <16 x half> %a to <16 x float>
  %b.wide = fpext <16 x half> %b to <16 x float>
  %mult = fmul <16 x float> %a.wide, %b.wide
  %partial.reduce = call <8 x float> @llvm.vector.partial.reduce.fadd(<8 x float> %acc, <16 x float> %mult)
  store <8 x float> %partial.reduce, ptr %accptr
  ret void
}

define void @fdot_wide_v16f32(ptr %accptr, ptr %aptr, ptr %bptr) vscale_range(4,0) {
; SVE2-LABEL: fdot_wide_v16f32:
; SVE2:       // %bb.0: // %entry
; SVE2-NEXT:    ptrue p0.s, vl16
; SVE2-NEXT:    mov x8, #16 // =0x10
; SVE2-NEXT:    ld1h { z0.s }, p0/z, [x1]
; SVE2-NEXT:    ld1h { z1.s }, p0/z, [x2]
; SVE2-NEXT:    ld1h { z2.s }, p0/z, [x1, x8, lsl #1]
; SVE2-NEXT:    ld1h { z3.s }, p0/z, [x2, x8, lsl #1]
; SVE2-NEXT:    fcvt z0.s, p0/m, z0.h
; SVE2-NEXT:    fcvt z1.s, p0/m, z1.h
; SVE2-NEXT:    fcvt z2.s, p0/m, z2.h
; SVE2-NEXT:    fcvt z3.s, p0/m, z3.h
; SVE2-NEXT:    fmul z0.s, p0/m, z0.s, z1.s
; SVE2-NEXT:    ld1w { z1.s }, p0/z, [x0]
; SVE2-NEXT:    fmul z2.s, p0/m, z2.s, z3.s
; SVE2-NEXT:    fadd z0.s, p0/m, z0.s, z1.s
; SVE2-NEXT:    fadd z0.s, p0/m, z0.s, z2.s
; SVE2-NEXT:    st1w { z0.s }, p0, [x0]
; SVE2-NEXT:    ret
;
; SVE2P1-LABEL: fdot_wide_v16f32:
; SVE2P1:       // %bb.0: // %entry
; SVE2P1-NEXT:    ptrue p0.s, vl16
; SVE2P1-NEXT:    ptrue p1.h, vl32
; SVE2P1-NEXT:    ld1w { z0.s }, p0/z, [x0]
; SVE2P1-NEXT:    ld1h { z1.h }, p1/z, [x1]
; SVE2P1-NEXT:    ld1h { z2.h }, p1/z, [x2]
; SVE2P1-NEXT:    fdot z0.s, z1.h, z2.h
; SVE2P1-NEXT:    st1w { z0.s }, p0, [x0]
; SVE2P1-NEXT:    ret
entry:
  %acc = load <16 x float>, ptr %accptr
  %a = load <32 x half>, ptr %aptr
  %b = load <32 x half>, ptr %bptr
  %a.wide = fpext <32 x half> %a to <32 x float>
  %b.wide = fpext <32 x half> %b to <32 x float>
  %mult = fmul <32 x float> %a.wide, %b.wide
  %partial.reduce = call <16 x float> @llvm.vector.partial.reduce.fadd(<16 x float> %acc, <32 x float> %mult)
  store <16 x float> %partial.reduce, ptr %accptr
  ret void
}

define void @fdot_wide_v32f32(ptr %accptr, ptr %aptr, ptr %bptr) vscale_range(8,0) {
; SVE2-LABEL: fdot_wide_v32f32:
; SVE2:       // %bb.0: // %entry
; SVE2-NEXT:    ptrue p0.s, vl32
; SVE2-NEXT:    mov x8, #32 // =0x20
; SVE2-NEXT:    ld1h { z0.s }, p0/z, [x1]
; SVE2-NEXT:    ld1h { z1.s }, p0/z, [x2]
; SVE2-NEXT:    ld1h { z2.s }, p0/z, [x1, x8, lsl #1]
; SVE2-NEXT:    ld1h { z3.s }, p0/z, [x2, x8, lsl #1]
; SVE2-NEXT:    fcvt z0.s, p0/m, z0.h
; SVE2-NEXT:    fcvt z1.s, p0/m, z1.h
; SVE2-NEXT:    fcvt z2.s, p0/m, z2.h
; SVE2-NEXT:    fcvt z3.s, p0/m, z3.h
; SVE2-NEXT:    fmul z0.s, p0/m, z0.s, z1.s
; SVE2-NEXT:    ld1w { z1.s }, p0/z, [x0]
; SVE2-NEXT:    fmul z2.s, p0/m, z2.s, z3.s
; SVE2-NEXT:    fadd z0.s, p0/m, z0.s, z1.s
; SVE2-NEXT:    fadd z0.s, p0/m, z0.s, z2.s
; SVE2-NEXT:    st1w { z0.s }, p0, [x0]
; SVE2-NEXT:    ret
;
; SVE2P1-LABEL: fdot_wide_v32f32:
; SVE2P1:       // %bb.0: // %entry
; SVE2P1-NEXT:    ptrue p0.s, vl32
; SVE2P1-NEXT:    ptrue p1.h, vl64
; SVE2P1-NEXT:    ld1w { z0.s }, p0/z, [x0]
; SVE2P1-NEXT:    ld1h { z1.h }, p1/z, [x1]
; SVE2P1-NEXT:    ld1h { z2.h }, p1/z, [x2]
; SVE2P1-NEXT:    fdot z0.s, z1.h, z2.h
; SVE2P1-NEXT:    st1w { z0.s }, p0, [x0]
; SVE2P1-NEXT:    ret
entry:
  %acc = load <32 x float>, ptr %accptr
  %a = load <64 x half>, ptr %aptr
  %b = load <64 x half>, ptr %bptr
  %a.wide = fpext <64 x half> %a to <64 x float>
  %b.wide = fpext <64 x half> %b to <64 x float>
  %mult = fmul <64 x float> %a.wide, %b.wide
  %partial.reduce = call <32 x float> @llvm.vector.partial.reduce.fadd(<32 x float> %acc, <64 x float> %mult)
  store <32 x float> %partial.reduce, ptr %accptr
  ret void
}

define void @fdot_wide_v64f32(ptr %accptr, ptr %aptr, ptr %bptr) vscale_range(16,0) {
; SVE2-LABEL: fdot_wide_v64f32:
; SVE2:       // %bb.0: // %entry
; SVE2-NEXT:    ptrue p0.s, vl64
; SVE2-NEXT:    mov x8, #64 // =0x40
; SVE2-NEXT:    ld1h { z0.s }, p0/z, [x1]
; SVE2-NEXT:    ld1h { z1.s }, p0/z, [x2]
; SVE2-NEXT:    ld1h { z2.s }, p0/z, [x1, x8, lsl #1]
; SVE2-NEXT:    ld1h { z3.s }, p0/z, [x2, x8, lsl #1]
; SVE2-NEXT:    fcvt z0.s, p0/m, z0.h
; SVE2-NEXT:    fcvt z1.s, p0/m, z1.h
; SVE2-NEXT:    fcvt z2.s, p0/m, z2.h
; SVE2-NEXT:    fcvt z3.s, p0/m, z3.h
; SVE2-NEXT:    fmul z0.s, p0/m, z0.s, z1.s
; SVE2-NEXT:    ld1w { z1.s }, p0/z, [x0]
; SVE2-NEXT:    fmul z2.s, p0/m, z2.s, z3.s
; SVE2-NEXT:    fadd z0.s, p0/m, z0.s, z1.s
; SVE2-NEXT:    fadd z0.s, p0/m, z0.s, z2.s
; SVE2-NEXT:    st1w { z0.s }, p0, [x0]
; SVE2-NEXT:    ret
;
; SVE2P1-LABEL: fdot_wide_v64f32:
; SVE2P1:       // %bb.0: // %entry
; SVE2P1-NEXT:    ptrue p0.s, vl64
; SVE2P1-NEXT:    ptrue p1.h, vl128
; SVE2P1-NEXT:    ld1w { z0.s }, p0/z, [x0]
; SVE2P1-NEXT:    ld1h { z1.h }, p1/z, [x1]
; SVE2P1-NEXT:    ld1h { z2.h }, p1/z, [x2]
; SVE2P1-NEXT:    fdot z0.s, z1.h, z2.h
; SVE2P1-NEXT:    st1w { z0.s }, p0, [x0]
; SVE2P1-NEXT:    ret
entry:
  %acc = load <64 x float>, ptr %accptr
  %a = load <128 x half>, ptr %aptr
  %b = load <128 x half>, ptr %bptr
  %a.wide = fpext <128 x half> %a to <128 x float>
  %b.wide = fpext <128 x half> %b to <128 x float>
  %mult = fmul <128 x float> %a.wide, %b.wide
  %partial.reduce = call <64 x float> @llvm.vector.partial.reduce.fadd(<64 x float> %acc, <128 x float> %mult)
  store <64 x float> %partial.reduce, ptr %accptr
  ret void
}

define <4 x float> @fixed_fdot_wide(<4 x float> %acc, <8 x half> %a, <8 x half> %b) {
; SVE2-LABEL: fixed_fdot_wide:
; SVE2:       // %bb.0: // %entry
; SVE2-NEXT:    fcvtl v3.4s, v1.4h
; SVE2-NEXT:    fcvtl v4.4s, v2.4h
; SVE2-NEXT:    fcvtl2 v1.4s, v1.8h
; SVE2-NEXT:    fcvtl2 v2.4s, v2.8h
; SVE2-NEXT:    fmul v3.4s, v3.4s, v4.4s
; SVE2-NEXT:    fmul v1.4s, v1.4s, v2.4s
; SVE2-NEXT:    fadd v0.4s, v0.4s, v3.4s
; SVE2-NEXT:    fadd v0.4s, v0.4s, v1.4s
; SVE2-NEXT:    ret
;
; SVE2P1-LABEL: fixed_fdot_wide:
; SVE2P1:       // %bb.0: // %entry
; SVE2P1-NEXT:    // kill: def $q0 killed $q0 def $z0
; SVE2P1-NEXT:    // kill: def $q2 killed $q2 def $z2
; SVE2P1-NEXT:    // kill: def $q1 killed $q1 def $z1
; SVE2P1-NEXT:    fdot z0.s, z1.h, z2.h
; SVE2P1-NEXT:    // kill: def $q0 killed $q0 killed $z0
; SVE2P1-NEXT:    ret
entry:
  %a.wide = fpext <8 x half> %a to <8 x float>
  %b.wide = fpext <8 x half> %b to <8 x float>
  %mult = fmul <8 x float> %a.wide, %b.wide
  %partial.reduce = call <4 x float> @llvm.vector.partial.reduce.fadd(<4 x float> %acc, <8 x float> %mult)
  ret <4 x float> %partial.reduce
}

define <8 x half> @partial_reduce_half(<8 x half> %acc, <16 x half> %a) {
; CHECK-LABEL: partial_reduce_half:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    fadd v0.8h, v0.8h, v1.8h
; CHECK-NEXT:    fadd v0.8h, v0.8h, v2.8h
; CHECK-NEXT:    ret
entry:
  %partial.reduce = call <8 x half> @llvm.vector.partial.reduce.fadd(<8 x half> %acc, <16 x half> %a)
  ret <8 x half> %partial.reduce
}

define <4 x float> @partial_reduce_float(<4 x float> %acc, <8 x float> %a) {
; CHECK-LABEL: partial_reduce_float:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    fadd v0.4s, v0.4s, v1.4s
; CHECK-NEXT:    fadd v0.4s, v0.4s, v2.4s
; CHECK-NEXT:    ret
entry:
  %partial.reduce = call <4 x float> @llvm.vector.partial.reduce.fadd(<4 x float> %acc, <8 x float> %a)
  ret <4 x float> %partial.reduce
}

define <2 x double> @partial_reduce_double(<2 x double> %acc, <4 x double> %a) {
; CHECK-LABEL: partial_reduce_double:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    fadd v0.2d, v0.2d, v1.2d
; CHECK-NEXT:    fadd v0.2d, v0.2d, v2.2d
; CHECK-NEXT:    ret
entry:
  %partial.reduce = call <2 x double> @llvm.vector.partial.reduce.fadd(<2 x double> %acc, <4 x double> %a)
  ret <2 x double> %partial.reduce
}