561 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+sme-i16i64 -mattr=+sme-f64f64 -force-streaming -enable-subreg-liveness -verify-machineinstrs < %s | FileCheck %s3 4;5; ADD Multi-Single x26;7 8define void @multi_vector_add_write_single_za_vg1x2_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zm) {9; CHECK-LABEL: multi_vector_add_write_single_za_vg1x2_i32:10; CHECK: // %bb.0:11; CHECK-NEXT: mov w8, w012; CHECK-NEXT: add za.s[w8, 0, vgx2], { z0.s, z1.s }, z2.s13; CHECK-NEXT: add za.s[w8, 7, vgx2], { z0.s, z1.s }, z2.s14; CHECK-NEXT: ret15 call void @llvm.aarch64.sme.add.write.single.za.vg1x2.nxv4i32(i32 %slice,16 <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,17 <vscale x 4 x i32> %zm)18 %slice.7 = add i32 %slice, 719 call void @llvm.aarch64.sme.add.write.single.za.vg1x2.nxv4i32(i32 %slice.7,20 <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,21 <vscale x 4 x i32> %zm)22 ret void23}24 25define void @multi_vector_add_write_single_za_vg1x2_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zm) {26; CHECK-LABEL: multi_vector_add_write_single_za_vg1x2_i64:27; CHECK: // %bb.0:28; CHECK-NEXT: mov w8, w029; CHECK-NEXT: add za.d[w8, 0, vgx2], { z0.d, z1.d }, z2.d30; CHECK-NEXT: add za.d[w8, 7, vgx2], { z0.d, z1.d }, z2.d31; CHECK-NEXT: ret32 call void @llvm.aarch64.sme.add.write.single.za.vg1x2.nxv2i64(i32 %slice,33 <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,34 <vscale x 2 x i64> %zm)35 %slice.7 = add i32 %slice, 736 call void @llvm.aarch64.sme.add.write.single.za.vg1x2.nxv2i64(i32 %slice.7,37 <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,38 <vscale x 2 x i64> %zm)39 ret void40}41 42;43; ADD Multi-Single x444;45 46define void @multi_vector_add_write_single_za_vg1x4_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,47; CHECK-LABEL: multi_vector_add_write_single_za_vg1x4_i32:48; CHECK: // %bb.0:49; CHECK-NEXT: mov w8, w050; CHECK-NEXT: add za.s[w8, 0, vgx4], { z0.s - z3.s }, z4.s51; CHECK-NEXT: add za.s[w8, 7, vgx4], { z0.s - z3.s }, z4.s52; CHECK-NEXT: ret53 <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3,54 <vscale x 4 x i32> %zm) {55 call void @llvm.aarch64.sme.add.write.single.za.vg1x4.nxv4i32(i32 %slice,56 <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,57 <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3,58 <vscale x 4 x i32> %zm)59 %slice.7 = add i32 %slice, 760 call void @llvm.aarch64.sme.add.write.single.za.vg1x4.nxv4i32(i32 %slice.7,61 <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,62 <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3,63 <vscale x 4 x i32> %zm)64 ret void65}66 67define void @multi_vector_add_write_single_za_vg1x4_i64(i32 %slice,68; CHECK-LABEL: multi_vector_add_write_single_za_vg1x4_i64:69; CHECK: // %bb.0:70; CHECK-NEXT: mov w8, w071; CHECK-NEXT: add za.d[w8, 0, vgx4], { z0.d - z3.d }, z4.d72; CHECK-NEXT: add za.d[w8, 7, vgx4], { z0.d - z3.d }, z4.d73; CHECK-NEXT: ret74 <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,75 <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3,76 <vscale x 2 x i64> %zm) {77 call void @llvm.aarch64.sme.add.write.single.za.vg1x4.nxv2i64(i32 %slice,78 <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,79 <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3,80 <vscale x 2 x i64> %zm)81 %slice.7 = add i32 %slice, 782 call void @llvm.aarch64.sme.add.write.single.za.vg1x4.nxv2i64(i32 %slice.7,83 <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,84 <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3,85 <vscale x 2 x i64> %zm)86 ret void87}88 89;90; ADD Multi-Multi x291;92 93define void @multi_vector_add_write_za_vg1x2_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,94; CHECK-LABEL: multi_vector_add_write_za_vg1x2_i32:95; CHECK: // %bb.0:96; CHECK-NEXT: mov w8, w097; CHECK-NEXT: add za.s[w8, 0, vgx2], { z0.s, z1.s }, { z2.s, z3.s }98; CHECK-NEXT: add za.s[w8, 7, vgx2], { z0.s, z1.s }, { z2.s, z3.s }99; CHECK-NEXT: ret100 <vscale x 4 x i32> %zm1, <vscale x 4 x i32> %zm2) {101 call void @llvm.aarch64.sme.add.write.za.vg1x2.nxv4i32(i32 %slice,102 <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,103 <vscale x 4 x i32> %zm1, <vscale x 4 x i32> %zm2)104 %slice.7 = add i32 %slice, 7105 call void @llvm.aarch64.sme.add.write.za.vg1x2.nxv4i32(i32 %slice.7,106 <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,107 <vscale x 4 x i32> %zm1, <vscale x 4 x i32> %zm2)108 ret void109}110 111 112define void @multi_vector_add_write_za_vg1x2_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,113; CHECK-LABEL: multi_vector_add_write_za_vg1x2_i64:114; CHECK: // %bb.0:115; CHECK-NEXT: mov w8, w0116; CHECK-NEXT: add za.d[w8, 0, vgx2], { z0.d, z1.d }, { z2.d, z3.d }117; CHECK-NEXT: add za.d[w8, 7, vgx2], { z0.d, z1.d }, { z2.d, z3.d }118; CHECK-NEXT: ret119 <vscale x 2 x i64> %zm1, <vscale x 2 x i64> %zm2) {120 call void @llvm.aarch64.sme.add.write.za.vg1x2.nxv2i64(i32 %slice,121 <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,122 <vscale x 2 x i64> %zm1, <vscale x 2 x i64> %zm2)123 %slice.7 = add i32 %slice, 7124 call void @llvm.aarch64.sme.add.write.za.vg1x2.nxv2i64(i32 %slice.7,125 <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,126 <vscale x 2 x i64> %zm1, <vscale x 2 x i64> %zm2)127 ret void128}129 130 131;132; ADD Multi-Multi x4133;134 135define void @multi_vector_add_write_za_vg1x4_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,136; CHECK-LABEL: multi_vector_add_write_za_vg1x4_i32:137; CHECK: // %bb.0:138; CHECK-NEXT: mov w8, w0139; CHECK-NEXT: add za.s[w8, 0, vgx4], { z0.s - z3.s }, { z4.s - z7.s }140; CHECK-NEXT: add za.s[w8, 7, vgx4], { z0.s - z3.s }, { z4.s - z7.s }141; CHECK-NEXT: ret142 <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3,143 <vscale x 4 x i32> %zm0, <vscale x 4 x i32> %zm1,144 <vscale x 4 x i32> %zm2, <vscale x 4 x i32> %zm3) {145 call void @llvm.aarch64.sme.add.write.za.vg1x4.nxv4i32(i32 %slice,146 <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,147 <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3,148 <vscale x 4 x i32> %zm0, <vscale x 4 x i32> %zm1,149 <vscale x 4 x i32> %zm2, <vscale x 4 x i32> %zm3)150 %slice.7 = add i32 %slice, 7151 call void @llvm.aarch64.sme.add.write.za.vg1x4.nxv4i32(i32 %slice.7,152 <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,153 <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3,154 <vscale x 4 x i32> %zm0, <vscale x 4 x i32> %zm1,155 <vscale x 4 x i32> %zm2, <vscale x 4 x i32> %zm3)156 ret void157}158 159define void @multi_vector_add_write_za_vg1x4_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,160; CHECK-LABEL: multi_vector_add_write_za_vg1x4_i64:161; CHECK: // %bb.0:162; CHECK-NEXT: mov w8, w0163; CHECK-NEXT: add za.d[w8, 0, vgx4], { z0.d - z3.d }, { z4.d - z7.d }164; CHECK-NEXT: add za.d[w8, 7, vgx4], { z0.d - z3.d }, { z4.d - z7.d }165; CHECK-NEXT: ret166 <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3,167 <vscale x 2 x i64> %zm0, <vscale x 2 x i64> %zm1,168 <vscale x 2 x i64> %zm2, <vscale x 2 x i64> %zm3) {169 call void @llvm.aarch64.sme.add.write.za.vg1x4.nxv2i64(i32 %slice,170 <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,171 <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3,172 <vscale x 2 x i64> %zm0, <vscale x 2 x i64> %zm1,173 <vscale x 2 x i64> %zm2, <vscale x 2 x i64> %zm3)174 %slice.7 = add i32 %slice, 7175 call void @llvm.aarch64.sme.add.write.za.vg1x4.nxv2i64(i32 %slice.7,176 <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,177 <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3,178 <vscale x 2 x i64> %zm0, <vscale x 2 x i64> %zm1,179 <vscale x 2 x i64> %zm2, <vscale x 2 x i64> %zm3)180 ret void181}182 183;184; ADD and accumulate into ZA185;186; x2187define void @multi_vector_add_za_vg1x2_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1) {188; CHECK-LABEL: multi_vector_add_za_vg1x2_i32:189; CHECK: // %bb.0:190; CHECK-NEXT: mov w8, w0191; CHECK-NEXT: add za.s[w8, 0, vgx2], { z0.s, z1.s }192; CHECK-NEXT: add za.s[w8, 7, vgx2], { z0.s, z1.s }193; CHECK-NEXT: ret194 call void @llvm.aarch64.sme.add.za32.vg1x2.nxv4i32(i32 %slice,<vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1)195 %slice.7 = add i32 %slice, 7196 call void @llvm.aarch64.sme.add.za32.vg1x2.nxv4i32(i32 %slice.7, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1)197 ret void198}199 200define void @multi_vector_add_za_vg1x2_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1) {201; CHECK-LABEL: multi_vector_add_za_vg1x2_i64:202; CHECK: // %bb.0:203; CHECK-NEXT: mov w8, w0204; CHECK-NEXT: add za.d[w8, 0, vgx2], { z0.d, z1.d }205; CHECK-NEXT: add za.d[w8, 7, vgx2], { z0.d, z1.d }206; CHECK-NEXT: ret207 call void @llvm.aarch64.sme.add.za64.vg1x2.nxv2i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1)208 %slice.7 = add i32 %slice, 7209 call void @llvm.aarch64.sme.add.za64.vg1x2.nxv2i64(i32 %slice.7, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1)210 ret void211}212 213define void @multi_vector_add_za_vg1x2_f32(i32 %slice, <vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1) {214; CHECK-LABEL: multi_vector_add_za_vg1x2_f32:215; CHECK: // %bb.0:216; CHECK-NEXT: mov w8, w0217; CHECK-NEXT: fadd za.s[w8, 0, vgx2], { z0.s, z1.s }218; CHECK-NEXT: fadd za.s[w8, 7, vgx2], { z0.s, z1.s }219; CHECK-NEXT: ret220 call void @llvm.aarch64.sme.add.za32.vg1x2.nxv4f32(i32 %slice,221 <vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1)222 %slice.7 = add i32 %slice, 7223 call void @llvm.aarch64.sme.add.za32.vg1x2.nxv4f32(i32 %slice.7,224 <vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1)225 ret void226}227 228define void @multi_vector_add_za_vg1x2_f64(i32 %slice, <vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1) {229; CHECK-LABEL: multi_vector_add_za_vg1x2_f64:230; CHECK: // %bb.0:231; CHECK-NEXT: mov w8, w0232; CHECK-NEXT: fadd za.d[w8, 0, vgx2], { z0.d, z1.d }233; CHECK-NEXT: fadd za.d[w8, 7, vgx2], { z0.d, z1.d }234; CHECK-NEXT: ret235 call void @llvm.aarch64.sme.add.za64.vg1x2.nxv2f64(i32 %slice,236 <vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1)237 %slice.7 = add i32 %slice, 7238 call void @llvm.aarch64.sme.add.za64.vg1x2.nxv2f64(i32 %slice.7,239 <vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1)240 ret void241}242 243define void @multi_vector_add_za_vg1x2_f64_tuple(i64 %stride, ptr %ptr) {244; CHECK-LABEL: multi_vector_add_za_vg1x2_f64_tuple:245; CHECK: // %bb.0: // %entry246; CHECK-NEXT: ptrue pn8.b247; CHECK-NEXT: add x9, x1, x0248; CHECK-NEXT: mov w8, wzr249; CHECK-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x1]250; CHECK-NEXT: ld1d { z17.d, z25.d }, pn8/z, [x9]251; CHECK-NEXT: fadd za.d[w8, 0, vgx2], { z16.d, z17.d }252; CHECK-NEXT: fadd za.d[w8, 0, vgx2], { z24.d, z25.d }253; CHECK-NEXT: ret254entry:255 %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()256 %1 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount") %0, ptr %ptr)257 %2 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %1, 0258 %3 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %1, 1259 %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride260 %4 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount") %0, ptr %arrayidx2)261 %5 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %4, 0262 %6 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %4, 1263 call void @llvm.aarch64.sme.add.za64.vg1x2.nxv2f64(i32 0, <vscale x 2 x double> %2, <vscale x 2 x double> %5)264 call void @llvm.aarch64.sme.add.za64.vg1x2.nxv2f64(i32 0, <vscale x 2 x double> %3, <vscale x 2 x double> %6)265 ret void266}267 268; x4269 270define void @multi_vector_add_za_vg1x4_i32(i32 %slice, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3) {271; CHECK-LABEL: multi_vector_add_za_vg1x4_i32:272; CHECK: // %bb.0:273; CHECK-NEXT: mov w8, w0274; CHECK-NEXT: add za.s[w8, 0, vgx4], { z0.s - z3.s }275; CHECK-NEXT: add za.s[w8, 7, vgx4], { z0.s - z3.s }276; CHECK-NEXT: ret277 call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4i32(i32 %slice,278 <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,279 <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3)280 %slice.7 = add i32 %slice, 7281 call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4i32(i32 %slice.7,282 <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1,283 <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3)284 ret void285}286 287define void @multi_vector_add_za_vg1x4_i64(i32 %slice, <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1, <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3) {288; CHECK-LABEL: multi_vector_add_za_vg1x4_i64:289; CHECK: // %bb.0:290; CHECK-NEXT: mov w8, w0291; CHECK-NEXT: add za.d[w8, 0, vgx4], { z0.d - z3.d }292; CHECK-NEXT: add za.d[w8, 7, vgx4], { z0.d - z3.d }293; CHECK-NEXT: ret294 call void @llvm.aarch64.sme.add.za64.vg1x4.nxv2i64(i32 %slice,295 <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,296 <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3)297 %slice.7 = add i32 %slice, 7298 call void @llvm.aarch64.sme.add.za64.vg1x4.nxv2i64(i32 %slice.7,299 <vscale x 2 x i64> %zn0, <vscale x 2 x i64> %zn1,300 <vscale x 2 x i64> %zn2, <vscale x 2 x i64> %zn3)301 ret void302}303 304define void @multi_vector_add_za_vg1x4_f32(i32 %slice, <vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3) {305; CHECK-LABEL: multi_vector_add_za_vg1x4_f32:306; CHECK: // %bb.0:307; CHECK-NEXT: mov w8, w0308; CHECK-NEXT: fadd za.s[w8, 0, vgx4], { z0.s - z3.s }309; CHECK-NEXT: fadd za.s[w8, 7, vgx4], { z0.s - z3.s }310; CHECK-NEXT: ret311 call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 %slice,312 <vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1,313 <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3)314 %slice.7 = add i32 %slice, 7315 call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 %slice.7,316 <vscale x 4 x float> %zn0, <vscale x 4 x float> %zn1,317 <vscale x 4 x float> %zn2, <vscale x 4 x float> %zn3)318 ret void319}320 321define void @multi_vector_add_za_vg1x4_f32_tuple(i64 %stride, ptr %ptr) {322; CHECK-LABEL: multi_vector_add_za_vg1x4_f32_tuple:323; CHECK: // %bb.0: // %entry324; CHECK-NEXT: lsl x9, x0, #1325; CHECK-NEXT: add x10, x1, x0326; CHECK-NEXT: ptrue pn8.b327; CHECK-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x1]328; CHECK-NEXT: ld1w { z17.s, z21.s, z25.s, z29.s }, pn8/z, [x10]329; CHECK-NEXT: mov w8, wzr330; CHECK-NEXT: add x11, x1, x9331; CHECK-NEXT: add x9, x10, x9332; CHECK-NEXT: ld1w { z18.s, z22.s, z26.s, z30.s }, pn8/z, [x11]333; CHECK-NEXT: ld1w { z19.s, z23.s, z27.s, z31.s }, pn8/z, [x9]334; CHECK-NEXT: fadd za.s[w8, 0, vgx4], { z16.s - z19.s }335; CHECK-NEXT: fadd za.s[w8, 0, vgx4], { z20.s - z23.s }336; CHECK-NEXT: fadd za.s[w8, 0, vgx4], { z24.s - z27.s }337; CHECK-NEXT: fadd za.s[w8, 0, vgx4], { z28.s - z31.s }338; CHECK-NEXT: ret339entry:340 %0 = tail call target("aarch64.svcount") @llvm.aarch64.sve.ptrue.c8()341 %1 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %ptr)342 %2 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 0343 %3 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 1344 %4 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 2345 %5 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, 3346 %arrayidx2 = getelementptr inbounds i8, ptr %ptr, i64 %stride347 %6 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx2)348 %7 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 0349 %8 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 1350 %9 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 2351 %10 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %6, 3352 %mul3 = shl i64 %stride, 1353 %arrayidx4 = getelementptr inbounds i8, ptr %ptr, i64 %mul3354 %11 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx4)355 %12 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 0356 %13 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 1357 %14 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 2358 %15 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %11, 3359 %mul5 = mul i64 %stride, 3360 %arrayidx6 = getelementptr inbounds i8, ptr %ptr, i64 %mul5361 %16 = tail call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %0, ptr %arrayidx6)362 %17 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 0363 %18 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 1364 %19 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 2365 %20 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %16, 3366 call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 0, <vscale x 4 x float> %2, <vscale x 4 x float> %7, <vscale x 4 x float> %12, <vscale x 4 x float> %17)367 call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 0, <vscale x 4 x float> %3, <vscale x 4 x float> %8, <vscale x 4 x float> %13, <vscale x 4 x float> %18)368 call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 0, <vscale x 4 x float> %4, <vscale x 4 x float> %9, <vscale x 4 x float> %14, <vscale x 4 x float> %19)369 call void @llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32 0, <vscale x 4 x float> %5, <vscale x 4 x float> %10, <vscale x 4 x float> %15, <vscale x 4 x float> %20)370 ret void371}372 373define void @multi_vector_add_za_vg1x4_f64(i32 %slice, <vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1, <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3) {374; CHECK-LABEL: multi_vector_add_za_vg1x4_f64:375; CHECK: // %bb.0:376; CHECK-NEXT: mov w8, w0377; CHECK-NEXT: fadd za.d[w8, 0, vgx4], { z0.d - z3.d }378; CHECK-NEXT: fadd za.d[w8, 7, vgx4], { z0.d - z3.d }379; CHECK-NEXT: ret380 call void @llvm.aarch64.sme.add.za64.vg1x4.nxv2f64(i32 %slice,381 <vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1,382 <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3)383 %slice.7 = add i32 %slice, 7384 call void @llvm.aarch64.sme.add.za64.vg1x4.nxv2f64(i32 %slice.7,385 <vscale x 2 x double> %zn0, <vscale x 2 x double> %zn1,386 <vscale x 2 x double> %zn2, <vscale x 2 x double> %zn3)387 ret void388}389 390;391; ADD Vectors Multi-Single x2392;393 394define { <vscale x 16 x i8>, <vscale x 16 x i8> } @multi_vec_add_single_x2_s8(<vscale x 16 x i8> %unused, <vscale x 16 x i8> %zdn1, <vscale x 16 x i8> %zdn2, <vscale x 16 x i8> %zm) {395; CHECK-LABEL: multi_vec_add_single_x2_s8:396; CHECK: // %bb.0:397; CHECK-NEXT: mov z5.d, z2.d398; CHECK-NEXT: mov z4.d, z1.d399; CHECK-NEXT: add { z4.b, z5.b }, { z4.b, z5.b }, z3.b400; CHECK-NEXT: mov z0.d, z4.d401; CHECK-NEXT: mov z1.d, z5.d402; CHECK-NEXT: ret403 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> }404 @llvm.aarch64.sve.add.single.x2.nxv16i8(<vscale x 16 x i8> %zdn1, <vscale x 16 x i8> %zdn2,405 <vscale x 16 x i8> %zm)406 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res407}408 409define { <vscale x 8 x i16>, <vscale x 8 x i16> } @multi_vec_add_single_x2_s16(<vscale x 8 x i16> %unused, <vscale x 8 x i16> %zdn1, <vscale x 8 x i16> %zdn2, <vscale x 8 x i16> %zm) {410; CHECK-LABEL: multi_vec_add_single_x2_s16:411; CHECK: // %bb.0:412; CHECK-NEXT: mov z5.d, z2.d413; CHECK-NEXT: mov z4.d, z1.d414; CHECK-NEXT: add { z4.h, z5.h }, { z4.h, z5.h }, z3.h415; CHECK-NEXT: mov z0.d, z4.d416; CHECK-NEXT: mov z1.d, z5.d417; CHECK-NEXT: ret418 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16> }419 @llvm.aarch64.sve.add.single.x2.nxv8i16(<vscale x 8 x i16> %zdn1, <vscale x 8 x i16> %zdn2,420 <vscale x 8 x i16> %zm)421 ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %res422}423 424define { <vscale x 4 x i32>, <vscale x 4 x i32> } @multi_vec_add_single_x2_s32(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zdn1, <vscale x 4 x i32> %zdn2, <vscale x 4 x i32> %zm) {425; CHECK-LABEL: multi_vec_add_single_x2_s32:426; CHECK: // %bb.0:427; CHECK-NEXT: mov z5.d, z2.d428; CHECK-NEXT: mov z4.d, z1.d429; CHECK-NEXT: add { z4.s, z5.s }, { z4.s, z5.s }, z3.s430; CHECK-NEXT: mov z0.d, z4.d431; CHECK-NEXT: mov z1.d, z5.d432; CHECK-NEXT: ret433 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> }434 @llvm.aarch64.sve.add.single.x2.nxv4i32(<vscale x 4 x i32> %zdn1, <vscale x 4 x i32> %zdn2,435 <vscale x 4 x i32> %zm)436 ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res437}438 439define { <vscale x 2 x i64>, <vscale x 2 x i64> } @multi_vec_add_single_x2_s64(<vscale x 2 x i64> %unused, <vscale x 2 x i64> %zdn1, <vscale x 2 x i64> %zdn2, <vscale x 2 x i64> %zm) {440; CHECK-LABEL: multi_vec_add_single_x2_s64:441; CHECK: // %bb.0:442; CHECK-NEXT: mov z5.d, z2.d443; CHECK-NEXT: mov z4.d, z1.d444; CHECK-NEXT: add { z4.d, z5.d }, { z4.d, z5.d }, z3.d445; CHECK-NEXT: mov z0.d, z4.d446; CHECK-NEXT: mov z1.d, z5.d447; CHECK-NEXT: ret448 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> }449 @llvm.aarch64.sve.add.single.x2.nxv2i64(<vscale x 2 x i64> %zdn1, <vscale x 2 x i64> %zdn2,450 <vscale x 2 x i64> %zm)451 ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res452}453 454;455; ADD Vectors Multi-Single x4456;457 458define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @multi_vec_add_single_x4_s8(<vscale x 16 x i8> %unused, <vscale x 16 x i8> %zdn1, <vscale x 16 x i8> %zdn2, <vscale x 16 x i8> %zdn3, <vscale x 16 x i8> %zdn4, <vscale x 16 x i8>%zm) {459; CHECK-LABEL: multi_vec_add_single_x4_s8:460; CHECK: // %bb.0:461; CHECK-NEXT: mov z27.d, z4.d462; CHECK-NEXT: mov z26.d, z3.d463; CHECK-NEXT: mov z25.d, z2.d464; CHECK-NEXT: mov z24.d, z1.d465; CHECK-NEXT: add { z24.b - z27.b }, { z24.b - z27.b }, z5.b466; CHECK-NEXT: mov z0.d, z24.d467; CHECK-NEXT: mov z1.d, z25.d468; CHECK-NEXT: mov z2.d, z26.d469; CHECK-NEXT: mov z3.d, z27.d470; CHECK-NEXT: ret471 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> }472 @llvm.aarch64.sve.add.single.x4.nxv16i8(<vscale x 16 x i8> %zdn1, <vscale x 16 x i8> %zdn2,473 <vscale x 16 x i8> %zdn3, <vscale x 16 x i8> %zdn4,474 <vscale x 16 x i8> %zm)475 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res476}477 478define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @multi_vec_add_x4_single_s16(<vscale x 8 x i16> %unused, <vscale x 8 x i16> %zdn1, <vscale x 8 x i16> %zdn2, <vscale x 8 x i16> %zdn3, <vscale x 8 x i16> %zdn4, <vscale x 8 x i16> %zm) {479; CHECK-LABEL: multi_vec_add_x4_single_s16:480; CHECK: // %bb.0:481; CHECK-NEXT: mov z27.d, z4.d482; CHECK-NEXT: mov z26.d, z3.d483; CHECK-NEXT: mov z25.d, z2.d484; CHECK-NEXT: mov z24.d, z1.d485; CHECK-NEXT: add { z24.h - z27.h }, { z24.h - z27.h }, z5.h486; CHECK-NEXT: mov z0.d, z24.d487; CHECK-NEXT: mov z1.d, z25.d488; CHECK-NEXT: mov z2.d, z26.d489; CHECK-NEXT: mov z3.d, z27.d490; CHECK-NEXT: ret491 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> }492 @llvm.aarch64.sve.add.single.x4.nxv8i16(<vscale x 8 x i16> %zdn1, <vscale x 8 x i16> %zdn2,493 <vscale x 8 x i16> %zdn3, <vscale x 8 x i16> %zdn4,494 <vscale x 8 x i16> %zm)495 ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res496}497 498define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @multi_vec_add_x4_single_s32(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zdn1, <vscale x 4 x i32> %zdn2, <vscale x 4 x i32> %zdn3, <vscale x 4 x i32> %zdn4, <vscale x 4 x i32> %zm) {499; CHECK-LABEL: multi_vec_add_x4_single_s32:500; CHECK: // %bb.0:501; CHECK-NEXT: mov z27.d, z4.d502; CHECK-NEXT: mov z26.d, z3.d503; CHECK-NEXT: mov z25.d, z2.d504; CHECK-NEXT: mov z24.d, z1.d505; CHECK-NEXT: add { z24.s - z27.s }, { z24.s - z27.s }, z5.s506; CHECK-NEXT: mov z0.d, z24.d507; CHECK-NEXT: mov z1.d, z25.d508; CHECK-NEXT: mov z2.d, z26.d509; CHECK-NEXT: mov z3.d, z27.d510; CHECK-NEXT: ret511 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> }512 @llvm.aarch64.sve.add.single.x4.nxv4i32(<vscale x 4 x i32> %zdn1, <vscale x 4 x i32> %zdn2,513 <vscale x 4 x i32> %zdn3, <vscale x 4 x i32> %zdn4,514 <vscale x 4 x i32> %zm)515 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res516}517 518define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @multi_vec_add_x4_single_s64(<vscale x 2 x i64> %unused, <vscale x 2 x i64> %zdn1, <vscale x 2 x i64> %zdn2, <vscale x 2 x i64> %zdn3, <vscale x 2 x i64> %zdn4, <vscale x 2 x i64> %zm) {519; CHECK-LABEL: multi_vec_add_x4_single_s64:520; CHECK: // %bb.0:521; CHECK-NEXT: mov z27.d, z4.d522; CHECK-NEXT: mov z26.d, z3.d523; CHECK-NEXT: mov z25.d, z2.d524; CHECK-NEXT: mov z24.d, z1.d525; CHECK-NEXT: add { z24.d - z27.d }, { z24.d - z27.d }, z5.d526; CHECK-NEXT: mov z0.d, z24.d527; CHECK-NEXT: mov z1.d, z25.d528; CHECK-NEXT: mov z2.d, z26.d529; CHECK-NEXT: mov z3.d, z27.d530; CHECK-NEXT: ret531 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> }532 @llvm.aarch64.sve.add.single.x4.nxv2i64(<vscale x 2 x i64> %zdn1, <vscale x 2 x i64> %zdn2,533 <vscale x 2 x i64> %zdn3, <vscale x 2 x i64> %zdn4,534 <vscale x 2 x i64> %zm)535 ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res536}537declare void@llvm.aarch64.sme.add.write.single.za.vg1x2.nxv4i32(i32, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)538declare void@llvm.aarch64.sme.add.write.single.za.vg1x2.nxv2i64(i32, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)539declare void@llvm.aarch64.sme.add.write.single.za.vg1x4.nxv4i32(i32, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)540declare void@llvm.aarch64.sme.add.write.single.za.vg1x4.nxv2i64(i32, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)541declare void@llvm.aarch64.sme.add.write.za.vg1x2.nxv4i32(i32, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)542declare void@llvm.aarch64.sme.add.write.za.vg1x2.nxv2i64(i32, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)543declare void@llvm.aarch64.sme.add.write.za.vg1x4.nxv4i32(i32, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)544declare void@llvm.aarch64.sme.add.write.za.vg1x4.nxv2i64(i32, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)545declare void@llvm.aarch64.sme.add.za32.vg1x2.nxv4i32(i32, <vscale x 4 x i32>,<vscale x 4 x i32>)546declare void@llvm.aarch64.sme.add.za64.vg1x2.nxv2i64(i32, <vscale x 2 x i64>,<vscale x 2 x i64>)547declare void@llvm.aarch64.sme.add.za32.vg1x4.nxv4i32(i32, <vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>,<vscale x 4 x i32>)548declare void@llvm.aarch64.sme.add.za64.vg1x4.nxv2i64(i32, <vscale x 2 x i64>,<vscale x 2 x i64>,<vscale x 2 x i64>, <vscale x 2 x i64>)549declare void@llvm.aarch64.sme.add.za32.vg1x2.nxv4f32(i32, <vscale x 4 x float>, <vscale x 4 x float>)550declare void@llvm.aarch64.sme.add.za64.vg1x2.nxv2f64(i32, <vscale x 2 x double>, <vscale x 2 x double>)551declare void@llvm.aarch64.sme.add.za32.vg1x4.nxv4f32(i32, <vscale x 4 x float>, <vscale x 4 x float>,<vscale x 4 x float>, <vscale x 4 x float>)552declare void@llvm.aarch64.sme.add.za64.vg1x4.nxv2f64(i32, <vscale x 2 x double>, <vscale x 2 x double>,<vscale x 2 x double>, <vscale x 2 x double>)553declare { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.add.single.x2.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)554declare { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.add.single.x2.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)555declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.add.single.x2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)556declare { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.add.single.x2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)557declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.add.single.x4.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)558declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.add.single.x4.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)559declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.add.single.x4.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)560declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.add.single.x4.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)561