801 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s3; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s4; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s5; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s6 7;;LD2Q8 9define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld2q_si_i8_off16(<vscale x 16 x i1> %pg, ptr %addr ) {10; CHECK-LABEL: ld2q_si_i8_off16:11; CHECK: // %bb.0:12; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, #-16, mul vl]13; CHECK-NEXT: ret14 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -1615 %base_ptr = bitcast ptr %base to ptr16 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2q.sret.nxv16i8(<vscale x 16 x i1> %pg, ptr %base_ptr);17 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res18}19 20define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld2q_si_i8_off14(<vscale x 16 x i1> %pg, ptr %addr ) {21; CHECK-LABEL: ld2q_si_i8_off14:22; CHECK: // %bb.0:23; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, #14, mul vl]24; CHECK-NEXT: ret25 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 1426 %base_ptr = bitcast ptr %base to ptr27 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2q.sret.nxv16i8(<vscale x 16 x i1> %pg, ptr %base_ptr);28 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res29}30 31define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld2q_ss_i8(<vscale x 16 x i1> %pg, ptr %addr, i64 %a) {32; CHECK-LABEL: ld2q_ss_i8:33; CHECK: // %bb.0:34; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, x1, lsl #4]35; CHECK-NEXT: ret36 %addr2 = getelementptr i128, ptr %addr, i64 %a37 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2q.sret.nxv16i8(<vscale x 16 x i1> %pg, ptr %addr2);38 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res39}40 41define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld2q_i8(<vscale x 16 x i1> %pg, ptr %addr) {42; CHECK-LABEL: ld2q_i8:43; CHECK: // %bb.0:44; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0]45; CHECK-NEXT: ret46 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2q.sret.nxv16i8(<vscale x 16 x i1> %pg, ptr %addr);47 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res48}49 50define { <vscale x 8 x i16>, <vscale x 8 x i16> } @ld2q_si_i16(<vscale x 8 x i1> %pg, ptr %addr ) {51; CHECK-LABEL: ld2q_si_i16:52; CHECK: // %bb.0:53; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, #-16, mul vl]54; CHECK-NEXT: ret55 %base = getelementptr <vscale x 8 x i16>, ptr %addr, i64 -1656 %base_ptr = bitcast ptr %base to ptr57 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2q.sret.nxv8i16(<vscale x 8 x i1> %pg, ptr %base_ptr);58 ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %res59}60 61define { <vscale x 8 x i16>, <vscale x 8 x i16> } @ld2q_ss_i16(<vscale x 8 x i1> %pg, ptr %addr, i64 %a) {62; CHECK-LABEL: ld2q_ss_i16:63; CHECK: // %bb.0:64; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, x1, lsl #4]65; CHECK-NEXT: ret66 %addr2 = getelementptr i128, ptr %addr, i64 %a67 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2q.sret.nxv8i16(<vscale x 8 x i1> %pg, ptr %addr2);68 ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %res69}70 71define { <vscale x 8 x i16>, <vscale x 8 x i16> } @ld2q_i16(<vscale x 8 x i1> %pg, ptr %addr) {72; CHECK-LABEL: ld2q_i16:73; CHECK: // %bb.0:74; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0]75; CHECK-NEXT: ret76 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2q.sret.nxv8i16(<vscale x 8 x i1> %pg, ptr %addr);77 ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %res78}79 80define { <vscale x 4 x i32>, <vscale x 4 x i32> } @ld2q_si_i32(<vscale x 4 x i1> %pg, ptr %addr ) {81; CHECK-LABEL: ld2q_si_i32:82; CHECK: // %bb.0:83; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, #-16, mul vl]84; CHECK-NEXT: ret85 %base = getelementptr <vscale x 4 x i32>, ptr %addr, i64 -1686 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2q.sret.nxv4i32(<vscale x 4 x i1> %pg, ptr %base);87 ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res88}89 90define { <vscale x 4 x i32>, <vscale x 4 x i32> } @ld2q_ss_i32(<vscale x 4 x i1> %pg, ptr %addr, i64 %a) {91; CHECK-LABEL: ld2q_ss_i32:92; CHECK: // %bb.0:93; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, x1, lsl #4]94; CHECK-NEXT: ret95 %addr2 = getelementptr i128, ptr %addr, i64 %a96 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2q.sret.nxv4i32(<vscale x 4 x i1> %pg, ptr %addr2);97 ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res98}99 100define { <vscale x 4 x i32>, <vscale x 4 x i32> } @ld2q_i32(<vscale x 4 x i1> %pg, ptr %addr) {101; CHECK-LABEL: ld2q_i32:102; CHECK: // %bb.0:103; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0]104; CHECK-NEXT: ret105 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2q.sret.nxv4i32(<vscale x 4 x i1> %pg, ptr %addr);106 ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res107}108 109define { <vscale x 2 x i64>, <vscale x 2 x i64> } @ld2q_si_i64(<vscale x 2 x i1> %pg, ptr %addr ) {110; CHECK-LABEL: ld2q_si_i64:111; CHECK: // %bb.0:112; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, #-16, mul vl]113; CHECK-NEXT: ret114 %base = getelementptr <vscale x 2 x i64>, ptr %addr, i64 -16115 %base_ptr = bitcast ptr %base to ptr116 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2q.sret.nxv2i64(<vscale x 2 x i1> %pg, ptr %base_ptr);117 ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res118}119 120define { <vscale x 2 x i64>, <vscale x 2 x i64> } @ld2q_ss_i64(<vscale x 2 x i1> %pg, ptr %addr, i64 %a) {121; CHECK-LABEL: ld2q_ss_i64:122; CHECK: // %bb.0:123; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, x1, lsl #4]124; CHECK-NEXT: ret125 %addr2 = getelementptr i128, ptr %addr, i64 %a126 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2q.sret.nxv2i64(<vscale x 2 x i1> %pg, ptr %addr2);127 ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res128}129 130define { <vscale x 2 x i64>, <vscale x 2 x i64> } @ld2q_i64(<vscale x 2 x i1> %pg, ptr %addr) {131; CHECK-LABEL: ld2q_i64:132; CHECK: // %bb.0:133; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0]134; CHECK-NEXT: ret135 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2q.sret.nxv2i64(<vscale x 2 x i1> %pg, ptr %addr);136 ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res137}138 139define { <vscale x 8 x half>, <vscale x 8 x half> } @ld2q_si_f16(<vscale x 8 x i1> %pg, ptr %addr ) {140; CHECK-LABEL: ld2q_si_f16:141; CHECK: // %bb.0:142; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, #-16, mul vl]143; CHECK-NEXT: ret144 %base = getelementptr <vscale x 8 x half>, ptr %addr, i64 -16145 %base_ptr = bitcast ptr %base to ptr146 %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld2q.sret.nxv8f16(<vscale x 8 x i1> %pg, ptr %base_ptr);147 ret { <vscale x 8 x half>, <vscale x 8 x half> } %res148}149 150define { <vscale x 8 x half>, <vscale x 8 x half> } @ld2q_ss_f16(<vscale x 8 x i1> %pg, ptr %addr, i64 %a) {151; CHECK-LABEL: ld2q_ss_f16:152; CHECK: // %bb.0:153; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, x1, lsl #4]154; CHECK-NEXT: ret155 %addr2 = getelementptr i128, ptr %addr, i64 %a156 %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld2q.sret.nxv8f16(<vscale x 8 x i1> %pg, ptr %addr2);157 ret { <vscale x 8 x half>, <vscale x 8 x half> } %res158}159 160define { <vscale x 8 x half>, <vscale x 8 x half> } @ld2q_f16(<vscale x 8 x i1> %pg, ptr %addr) {161; CHECK-LABEL: ld2q_f16:162; CHECK: // %bb.0:163; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0]164; CHECK-NEXT: ret165 %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld2q.sret.nxv8f16(<vscale x 8 x i1> %pg, ptr %addr);166 ret { <vscale x 8 x half>, <vscale x 8 x half> } %res167}168 169define { <vscale x 4 x float>, <vscale x 4 x float> } @ld2q_si_f32(<vscale x 4 x i1> %pg, ptr %addr ) {170; CHECK-LABEL: ld2q_si_f32:171; CHECK: // %bb.0:172; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, #-16, mul vl]173; CHECK-NEXT: ret174 %base = getelementptr <vscale x 4 x float>, ptr %addr, i64 -16175 %base_ptr = bitcast ptr %base to ptr176 %res = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld2q.sret.nxv4f32(<vscale x 4 x i1> %pg, ptr %base_ptr);177 ret { <vscale x 4 x float>, <vscale x 4 x float> } %res178}179 180define { <vscale x 4 x float>, <vscale x 4 x float> } @ld2q_ss_f32(<vscale x 4 x i1> %pg, ptr %addr, i64 %a) {181; CHECK-LABEL: ld2q_ss_f32:182; CHECK: // %bb.0:183; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, x1, lsl #4]184; CHECK-NEXT: ret185 %addr2 = getelementptr i128, ptr %addr, i64 %a186 %res = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld2q.sret.nxv4f32(<vscale x 4 x i1> %pg, ptr %addr2);187 ret { <vscale x 4 x float>, <vscale x 4 x float> } %res188}189 190define { <vscale x 4 x float>, <vscale x 4 x float> } @ld2q_f32(<vscale x 4 x i1> %pg, ptr %addr) {191; CHECK-LABEL: ld2q_f32:192; CHECK: // %bb.0:193; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0]194; CHECK-NEXT: ret195 %res = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld2q.sret.nxv4f32(<vscale x 4 x i1> %pg, ptr %addr);196 ret { <vscale x 4 x float>, <vscale x 4 x float> } %res197}198 199define { <vscale x 2 x double>, <vscale x 2 x double> } @ld2q_si_f64(<vscale x 2 x i1> %pg, ptr %addr ) {200; CHECK-LABEL: ld2q_si_f64:201; CHECK: // %bb.0:202; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, #-16, mul vl]203; CHECK-NEXT: ret204 %base = getelementptr <vscale x 2 x double>, ptr %addr, i64 -16205 %base_ptr = bitcast ptr %base to ptr206 %res = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2q.sret.nxv2f64(<vscale x 2 x i1> %pg, ptr %base_ptr);207 ret { <vscale x 2 x double>, <vscale x 2 x double> } %res208}209 210define { <vscale x 2 x double>, <vscale x 2 x double> } @ld2q_ss_f64(<vscale x 2 x i1> %pg, ptr %addr, i64 %a) {211; CHECK-LABEL: ld2q_ss_f64:212; CHECK: // %bb.0:213; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, x1, lsl #4]214; CHECK-NEXT: ret215 %addr2 = getelementptr i128, ptr %addr, i64 %a216 %res = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2q.sret.nxv2f64(<vscale x 2 x i1> %pg, ptr %addr2);217 ret { <vscale x 2 x double>, <vscale x 2 x double> } %res218}219 220define { <vscale x 2 x double>, <vscale x 2 x double> } @ld2q_f64(<vscale x 2 x i1> %pg, ptr %addr) {221; CHECK-LABEL: ld2q_f64:222; CHECK: // %bb.0:223; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0]224; CHECK-NEXT: ret225 %res = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2q.sret.nxv2f64(<vscale x 2 x i1> %pg, ptr %addr);226 ret { <vscale x 2 x double>, <vscale x 2 x double> } %res227}228 229define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld2q_si_bf16(<vscale x 8 x i1> %pg, ptr %addr ) {230; CHECK-LABEL: ld2q_si_bf16:231; CHECK: // %bb.0:232; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, #-16, mul vl]233; CHECK-NEXT: ret234 %base = getelementptr <vscale x 8 x bfloat>, ptr %addr, i64 -16235 %base_ptr = bitcast ptr %base to ptr236 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld2q.sret.nxv8bf16(<vscale x 8 x i1> %pg, ptr %base_ptr);237 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res238}239 240define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld2q_ss_bf16(<vscale x 8 x i1> %pg, ptr %addr, i64 %a) {241; CHECK-LABEL: ld2q_ss_bf16:242; CHECK: // %bb.0:243; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0, x1, lsl #4]244; CHECK-NEXT: ret245 %addr2 = getelementptr i128, ptr %addr, i64 %a246 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld2q.sret.nxv8bf16(<vscale x 8 x i1> %pg, ptr %addr2);247 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res248}249 250define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld2q_bf16(<vscale x 8 x i1> %pg, ptr %addr) {251; CHECK-LABEL: ld2q_bf16:252; CHECK: // %bb.0:253; CHECK-NEXT: ld2q { z0.q, z1.q }, p0/z, [x0]254; CHECK-NEXT: ret255 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld2q.sret.nxv8bf16(<vscale x 8 x i1> %pg, ptr %addr);256 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res257}258 259;; LD3Q260define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld3q_si_i8_off24(<vscale x 16 x i1> %pg, ptr %addr ) {261; CHECK-LABEL: ld3q_si_i8_off24:262; CHECK: // %bb.0:263; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, #-24, mul vl]264; CHECK-NEXT: ret265 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -24266 %base_ptr = bitcast ptr %base to ptr267 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3q.sret.nxv16i8(<vscale x 16 x i1> %pg, ptr %base_ptr);268 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res269}270 271define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld3q_si_i8_off21(<vscale x 16 x i1> %pg, ptr %addr ) {272; CHECK-LABEL: ld3q_si_i8_off21:273; CHECK: // %bb.0:274; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, #21, mul vl]275; CHECK-NEXT: ret276 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 21277 %base_ptr = bitcast ptr %base to ptr278 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3q.sret.nxv16i8(<vscale x 16 x i1> %pg, ptr %base_ptr);279 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res280}281 282define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld3q_ss_i8(<vscale x 16 x i1> %pg, ptr %addr, i64 %a) {283; CHECK-LABEL: ld3q_ss_i8:284; CHECK: // %bb.0:285; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, x1, lsl #4]286; CHECK-NEXT: ret287 %addr2 = getelementptr i128, ptr %addr, i64 %a288 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3q.sret.nxv16i8(<vscale x 16 x i1> %pg, ptr %addr2);289 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res290}291 292define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @ld3q_i8(<vscale x 16 x i1> %pg, ptr %addr) {293; CHECK-LABEL: ld3q_i8:294; CHECK: // %bb.0:295; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0]296; CHECK-NEXT: ret297 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sve.ld3q.sret.nxv16i8(<vscale x 16 x i1> %pg, ptr %addr);298 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} %res299}300 301define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @ld3q_si_i16(<vscale x 8 x i1> %pg, ptr %addr ) {302; CHECK-LABEL: ld3q_si_i16:303; CHECK: // %bb.0:304; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, #-24, mul vl]305; CHECK-NEXT: ret306 %base = getelementptr <vscale x 8 x i16>, ptr %addr, i64 -24307 %base_ptr = bitcast ptr %base to ptr308 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld3q.sret.nxv8i16(<vscale x 8 x i1> %pg, ptr %base_ptr);309 ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res310}311 312define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @ld3q_ss_i16(<vscale x 8 x i1> %pg, ptr %addr, i64 %a) {313; CHECK-LABEL: ld3q_ss_i16:314; CHECK: // %bb.0:315; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, x1, lsl #4]316; CHECK-NEXT: ret317 %addr2 = getelementptr i128, ptr %addr, i64 %a318 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld3q.sret.nxv8i16(<vscale x 8 x i1> %pg, ptr %addr2);319 ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res320}321 322define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @ld3q_i16(<vscale x 8 x i1> %pg, ptr %addr) {323; CHECK-LABEL: ld3q_i16:324; CHECK: // %bb.0:325; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0]326; CHECK-NEXT: ret327 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld3q.sret.nxv8i16(<vscale x 8 x i1> %pg, ptr %addr);328 ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res329}330 331define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @ld3q_si_i32(<vscale x 4 x i1> %pg, ptr %addr ) {332; CHECK-LABEL: ld3q_si_i32:333; CHECK: // %bb.0:334; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, #-24, mul vl]335; CHECK-NEXT: ret336 %base = getelementptr <vscale x 4 x i32>, ptr %addr, i64 -24337 %base_ptr = bitcast ptr %base to ptr338 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld3q.sret.nxv4i32(<vscale x 4 x i1> %pg, ptr %base_ptr);339 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res340}341 342define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @ld3q_ss_i32(<vscale x 4 x i1> %pg, ptr %addr, i64 %a) {343; CHECK-LABEL: ld3q_ss_i32:344; CHECK: // %bb.0:345; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, x1, lsl #4]346; CHECK-NEXT: ret347 %addr2 = getelementptr i128, ptr %addr, i64 %a348 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld3q.sret.nxv4i32(<vscale x 4 x i1> %pg, ptr %addr2);349 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res350}351 352define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @ld3q_i32(<vscale x 4 x i1> %pg, ptr %addr) {353; CHECK-LABEL: ld3q_i32:354; CHECK: // %bb.0:355; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0]356; CHECK-NEXT: ret357 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld3q.sret.nxv4i32(<vscale x 4 x i1> %pg, ptr %addr);358 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res359}360 361define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @ld3q_si_i64(<vscale x 2 x i1> %pg, ptr %addr ) {362; CHECK-LABEL: ld3q_si_i64:363; CHECK: // %bb.0:364; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, #-24, mul vl]365; CHECK-NEXT: ret366 %addr2 = getelementptr <vscale x 4 x i32>, ptr %addr, i64 -24367 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld3q.sret.nxv2i64(<vscale x 2 x i1> %pg, ptr %addr2);368 ret {<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res369}370 371define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @ld3q_ss_i64(<vscale x 2 x i1> %pg, ptr %addr, i64 %a) {372; CHECK-LABEL: ld3q_ss_i64:373; CHECK: // %bb.0:374; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, x1, lsl #4]375; CHECK-NEXT: ret376 %addr2 = getelementptr i128, ptr %addr, i64 %a377 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld3q.sret.nxv2i64(<vscale x 2 x i1> %pg, ptr %addr2);378 ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res379}380 381define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @ld3q_i64(<vscale x 2 x i1> %pg, ptr %addr) {382; CHECK-LABEL: ld3q_i64:383; CHECK: // %bb.0:384; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0]385; CHECK-NEXT: ret386 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld3q.sret.nxv2i64(<vscale x 2 x i1> %pg, ptr %addr);387 ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res388}389 390define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @ld3q_si_f16(<vscale x 8 x i1> %pg, ptr %addr ) {391; CHECK-LABEL: ld3q_si_f16:392; CHECK: // %bb.0:393; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, #-24, mul vl]394; CHECK-NEXT: ret395 %base = getelementptr <vscale x 8 x half>, ptr %addr, i64 -24396 %base_ptr = bitcast ptr %base to ptr397 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld3q.sret.nxv8f16(<vscale x 8 x i1> %pg, ptr %base_ptr);398 ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res399}400 401define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @ld3q_ss_f16(<vscale x 8 x i1> %pg, ptr %addr, i64 %a) {402; CHECK-LABEL: ld3q_ss_f16:403; CHECK: // %bb.0:404; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, x1, lsl #4]405; CHECK-NEXT: ret406 %addr2 = getelementptr i128, ptr %addr, i64 %a407 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld3q.sret.nxv8f16(<vscale x 8 x i1> %pg, ptr %addr2);408 ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res409}410 411define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @ld3q_f16(<vscale x 8 x i1> %pg, ptr %addr) {412; CHECK-LABEL: ld3q_f16:413; CHECK: // %bb.0:414; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0]415; CHECK-NEXT: ret416 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld3q.sret.nxv8f16(<vscale x 8 x i1> %pg, ptr %addr);417 ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res418}419 420define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @ld3q_si_f32(<vscale x 4 x i1> %pg, ptr %addr ) {421; CHECK-LABEL: ld3q_si_f32:422; CHECK: // %bb.0:423; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, #-24, mul vl]424; CHECK-NEXT: ret425 %base = getelementptr <vscale x 4 x float>, ptr %addr, i64 -24426 %base_ptr = bitcast ptr %base to ptr427 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld3q.sret.nxv4f32(<vscale x 4 x i1> %pg, ptr %base_ptr);428 ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res429}430 431define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @ld3q_ss_f32(<vscale x 4 x i1> %pg, ptr %addr, i64 %a) {432; CHECK-LABEL: ld3q_ss_f32:433; CHECK: // %bb.0:434; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, x1, lsl #4]435; CHECK-NEXT: ret436 %addr2 = getelementptr i128, ptr %addr, i64 %a437 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld3q.sret.nxv4f32(<vscale x 4 x i1> %pg, ptr %addr2);438 ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res439}440 441define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @ld3q_f32(<vscale x 4 x i1> %pg, ptr %addr) {442; CHECK-LABEL: ld3q_f32:443; CHECK: // %bb.0:444; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0]445; CHECK-NEXT: ret446 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld3q.sret.nxv4f32(<vscale x 4 x i1> %pg, ptr %addr);447 ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res448}449 450define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @ld3q_si_f64(<vscale x 2 x i1> %pg, ptr %addr ) {451; CHECK-LABEL: ld3q_si_f64:452; CHECK: // %bb.0:453; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, #-24, mul vl]454; CHECK-NEXT: ret455 %base = getelementptr <vscale x 2 x double>, ptr %addr, i64 -24456 %base_ptr = bitcast ptr %base to ptr457 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld3q.sret.nxv2f64(<vscale x 2 x i1> %pg, ptr %base_ptr);458 ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res459}460 461define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @ld3q_ss_f64(<vscale x 2 x i1> %pg, ptr %addr, i64 %a) {462; CHECK-LABEL: ld3q_ss_f64:463; CHECK: // %bb.0:464; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, x1, lsl #4]465; CHECK-NEXT: ret466 %addr2 = getelementptr i128, ptr %addr, i64 %a467 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld3q.sret.nxv2f64(<vscale x 2 x i1> %pg, ptr %addr2);468 ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res469}470 471define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @ld3q_f64(<vscale x 2 x i1> %pg, ptr %addr) {472; CHECK-LABEL: ld3q_f64:473; CHECK: // %bb.0:474; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0]475; CHECK-NEXT: ret476 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld3q.sret.nxv2f64(<vscale x 2 x i1> %pg, ptr %addr);477 ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res478}479 480define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld3q_si_bf16(<vscale x 8 x i1> %pg, ptr %addr ) {481; CHECK-LABEL: ld3q_si_bf16:482; CHECK: // %bb.0:483; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, #-24, mul vl]484; CHECK-NEXT: ret485 %base = getelementptr <vscale x 8 x bfloat>, ptr %addr, i64 -24486 %base_ptr = bitcast ptr %base to ptr487 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld3q.sret.nxv8bf16(<vscale x 8 x i1> %pg, ptr %base_ptr);488 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res489}490 491define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld3q_ss_bf16(<vscale x 8 x i1> %pg, ptr %addr, i64 %a) {492; CHECK-LABEL: ld3q_ss_bf16:493; CHECK: // %bb.0:494; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0, x1, lsl #4]495; CHECK-NEXT: ret496 %addr2 = getelementptr i128, ptr %addr, i64 %a497 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld3q.sret.nxv8bf16(<vscale x 8 x i1> %pg, ptr %addr2);498 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res499}500 501define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld3q_bf16(<vscale x 8 x i1> %pg, ptr %addr) {502; CHECK-LABEL: ld3q_bf16:503; CHECK: // %bb.0:504; CHECK-NEXT: ld3q { z0.q - z2.q }, p0/z, [x0]505; CHECK-NEXT: ret506 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld3q.sret.nxv8bf16(<vscale x 8 x i1> %pg, ptr %addr);507 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res508}509 510;; LD4Q511define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld4q_si_i8_off32(<vscale x 16 x i1> %pg, ptr %addr ) {512; CHECK-LABEL: ld4q_si_i8_off32:513; CHECK: // %bb.0:514; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, #-32, mul vl]515; CHECK-NEXT: ret516 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -32517 %base_ptr = bitcast ptr %base to ptr518 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4q.sret.nxv16i8(<vscale x 16 x i1> %pg, ptr %base_ptr);519 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res520}521 522define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld4q_si_i8_off28(<vscale x 16 x i1> %pg, ptr %addr ) {523; CHECK-LABEL: ld4q_si_i8_off28:524; CHECK: // %bb.0:525; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, #28, mul vl]526; CHECK-NEXT: ret527 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 28528 %base_ptr = bitcast ptr %base to ptr529 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4q.sret.nxv16i8(<vscale x 16 x i1> %pg, ptr %base_ptr);530 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res531}532 533define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld4q_ss_i8(<vscale x 16 x i1> %pg, ptr %addr, i64 %a) {534; CHECK-LABEL: ld4q_ss_i8:535; CHECK: // %bb.0:536; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, x1, lsl #4]537; CHECK-NEXT: ret538 %addr2 = getelementptr i128, ptr %addr, i64 %a539 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4q.sret.nxv16i8(<vscale x 16 x i1> %pg, ptr %addr2);540 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res541}542 543define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @ld4q_i8(<vscale x 16 x i1> %pg, ptr %addr) {544; CHECK-LABEL: ld4q_i8:545; CHECK: // %bb.0:546; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0]547; CHECK-NEXT: ret548 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.aarch64.sve.ld4q.sret.nxv16i8(<vscale x 16 x i1> %pg, ptr %addr);549 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>} %res550}551 552define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @ld4q_si_i16(<vscale x 8 x i1> %pg, ptr %addr ) {553; CHECK-LABEL: ld4q_si_i16:554; CHECK: // %bb.0:555; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, #-32, mul vl]556; CHECK-NEXT: ret557 %base = getelementptr <vscale x 8 x i16>, ptr %addr, i64 -32558 %base_ptr = bitcast ptr %base to ptr559 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld4q.sret.nxv8i16(<vscale x 8 x i1> %pg, ptr %base_ptr);560 ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res561}562 563define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @ld4q_ss_i16(<vscale x 8 x i1> %pg, ptr %addr, i64 %a) {564; CHECK-LABEL: ld4q_ss_i16:565; CHECK: // %bb.0:566; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, x1, lsl #4]567; CHECK-NEXT: ret568 %addr2 = getelementptr i128, ptr %addr, i64 %a569 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld4q.sret.nxv8i16(<vscale x 8 x i1> %pg, ptr %addr2);570 ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res571}572 573define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @ld4q_i16(<vscale x 8 x i1> %pg, ptr %addr) {574; CHECK-LABEL: ld4q_i16:575; CHECK: // %bb.0:576; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0]577; CHECK-NEXT: ret578 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld4q.sret.nxv8i16(<vscale x 8 x i1> %pg, ptr %addr);579 ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res580}581 582define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @ld4q_si_i32(<vscale x 4 x i1> %pg, ptr %addr ) {583; CHECK-LABEL: ld4q_si_i32:584; CHECK: // %bb.0:585; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, #-32, mul vl]586; CHECK-NEXT: ret587 %base = getelementptr <vscale x 4 x i32>, ptr %addr, i64 -32588 %base_ptr = bitcast ptr %base to ptr589 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4q.sret.nxv4i32(<vscale x 4 x i1> %pg, ptr %base_ptr);590 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res591}592 593define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @ld4q_ss_i32(<vscale x 4 x i1> %pg, ptr %addr, i64 %a) {594; CHECK-LABEL: ld4q_ss_i32:595; CHECK: // %bb.0:596; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, x1, lsl #4]597; CHECK-NEXT: ret598 %addr2 = getelementptr i128, ptr %addr, i64 %a599 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4q.sret.nxv4i32(<vscale x 4 x i1> %pg, ptr %addr2);600 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res601}602 603define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @ld4q_i32(<vscale x 4 x i1> %pg, ptr %addr) {604; CHECK-LABEL: ld4q_i32:605; CHECK: // %bb.0:606; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0]607; CHECK-NEXT: ret608 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4q.sret.nxv4i32(<vscale x 4 x i1> %pg, ptr %addr);609 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res610}611 612define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @ld4q_si_i64(<vscale x 2 x i1> %pg, ptr %addr ) {613; CHECK-LABEL: ld4q_si_i64:614; CHECK: // %bb.0:615; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, #-32, mul vl]616; CHECK-NEXT: ret617 %base = getelementptr <vscale x 2 x i64>, ptr %addr, i64 -32618 %base_ptr = bitcast ptr %base to ptr619 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4q.sret.nxv2i64(<vscale x 2 x i1> %pg, ptr %base_ptr);620 ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res621}622 623define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @ld4q_ss_i64(<vscale x 2 x i1> %pg, ptr %addr, i64 %a) {624; CHECK-LABEL: ld4q_ss_i64:625; CHECK: // %bb.0:626; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, x1, lsl #4]627; CHECK-NEXT: ret628 %addr2 = getelementptr i128, ptr %addr, i64 %a629 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4q.sret.nxv2i64(<vscale x 2 x i1> %pg, ptr %addr2);630 ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res631}632 633define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @ld4q_i64(<vscale x 2 x i1> %pg, ptr %addr) {634; CHECK-LABEL: ld4q_i64:635; CHECK: // %bb.0:636; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0]637; CHECK-NEXT: ret638 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4q.sret.nxv2i64(<vscale x 2 x i1> %pg, ptr %addr);639 ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res640}641 642define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @ld4q_si_f16(<vscale x 8 x i1> %pg, ptr %addr ) {643; CHECK-LABEL: ld4q_si_f16:644; CHECK: // %bb.0:645; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, #-32, mul vl]646; CHECK-NEXT: ret647 %base = getelementptr <vscale x 8 x half>, ptr %addr, i64 -32648 %base_ptr = bitcast ptr %base to ptr649 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld4q.sret.nxv8f16(<vscale x 8 x i1> %pg, ptr %base_ptr);650 ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res651}652 653define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @ld4q_ss_f16(<vscale x 8 x i1> %pg, ptr %addr, i64 %a) {654; CHECK-LABEL: ld4q_ss_f16:655; CHECK: // %bb.0:656; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, x1, lsl #4]657; CHECK-NEXT: ret658 %addr2 = getelementptr i128, ptr %addr, i64 %a659 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld4q.sret.nxv8f16(<vscale x 8 x i1> %pg, ptr %addr2);660 ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res661}662 663define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @ld4q_f16(<vscale x 8 x i1> %pg, ptr %addr) {664; CHECK-LABEL: ld4q_f16:665; CHECK: // %bb.0:666; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0]667; CHECK-NEXT: ret668 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld4q.sret.nxv8f16(<vscale x 8 x i1> %pg, ptr %addr);669 ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res670}671 672define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @ld4q_si_f32(<vscale x 4 x i1> %pg, ptr %addr ) {673; CHECK-LABEL: ld4q_si_f32:674; CHECK: // %bb.0:675; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, #-32, mul vl]676; CHECK-NEXT: ret677 %base = getelementptr <vscale x 4 x float>, ptr %addr, i64 -32678 %base_ptr = bitcast ptr %base to ptr679 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld4q.sret.nxv4f32(<vscale x 4 x i1> %pg, ptr %base_ptr);680 ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res681}682 683define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @ld4q_ss_f32(<vscale x 4 x i1> %pg, ptr %addr, i64 %a) {684; CHECK-LABEL: ld4q_ss_f32:685; CHECK: // %bb.0:686; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, x1, lsl #4]687; CHECK-NEXT: ret688 %addr2 = getelementptr i128, ptr %addr, i64 %a689 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld4q.sret.nxv4f32(<vscale x 4 x i1> %pg, ptr %addr2);690 ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res691}692 693define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @ld4q_f32(<vscale x 4 x i1> %pg, ptr %addr) {694; CHECK-LABEL: ld4q_f32:695; CHECK: // %bb.0:696; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0]697; CHECK-NEXT: ret698 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld4q.sret.nxv4f32(<vscale x 4 x i1> %pg, ptr %addr);699 ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res700}701 702define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @ld4q_si_f64(<vscale x 2 x i1> %pg, ptr %addr ) {703; CHECK-LABEL: ld4q_si_f64:704; CHECK: // %bb.0:705; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, #-32, mul vl]706; CHECK-NEXT: ret707 %base = getelementptr <vscale x 2 x double>, ptr %addr, i64 -32708 %base_ptr = bitcast ptr %base to ptr709 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld4q.sret.nxv2f64(<vscale x 2 x i1> %pg, ptr %base_ptr);710 ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res711}712 713define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @ld4q_ss_f64(<vscale x 2 x i1> %pg, ptr %addr, i64 %a) {714; CHECK-LABEL: ld4q_ss_f64:715; CHECK: // %bb.0:716; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, x1, lsl #4]717; CHECK-NEXT: ret718 %addr2 = getelementptr i128, ptr %addr, i64 %a719 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld4q.sret.nxv2f64(<vscale x 2 x i1> %pg, ptr %addr2);720 ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res721}722 723define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @ld4q_f64(<vscale x 2 x i1> %pg, ptr %addr) {724; CHECK-LABEL: ld4q_f64:725; CHECK: // %bb.0:726; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0]727; CHECK-NEXT: ret728 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld4q.sret.nxv2f64(<vscale x 2 x i1> %pg, ptr %addr);729 ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res730}731 732define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld4q_si_bf16(<vscale x 8 x i1> %pg, ptr %addr ) {733; CHECK-LABEL: ld4q_si_bf16:734; CHECK: // %bb.0:735; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, #-32, mul vl]736; CHECK-NEXT: ret737 %base = getelementptr <vscale x 8 x bfloat>, ptr %addr, i64 -32738 %base_ptr = bitcast ptr %base to ptr739 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld4q.sret.nxv8bf16(<vscale x 8 x i1> %pg, ptr %base_ptr);740 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res741}742 743define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld4q_ss_bf16(<vscale x 8 x i1> %pg, ptr %addr, i64 %a) {744; CHECK-LABEL: ld4q_ss_bf16:745; CHECK: // %bb.0:746; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0, x1, lsl #4]747; CHECK-NEXT: ret748 %addr2 = getelementptr i128, ptr %addr, i64 %a749 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld4q.sret.nxv8bf16(<vscale x 8 x i1> %pg, ptr %addr2);750 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res751}752 753define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld4q_bf16(<vscale x 8 x i1> %pg, ptr %addr) {754; CHECK-LABEL: ld4q_bf16:755; CHECK: // %bb.0:756; CHECK-NEXT: ld4q { z0.q - z3.q }, p0/z, [x0]757; CHECK-NEXT: ret758 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld4q.sret.nxv8bf16(<vscale x 8 x i1> %pg, ptr %addr);759 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res760}761 762 763declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2i64(target("aarch64.svcount"), ptr)764declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4i32(target("aarch64.svcount"), ptr)765declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8i16(target("aarch64.svcount"), ptr)766declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount"), ptr)767declare { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2f64(target("aarch64.svcount"), ptr)768declare { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4f32(target("aarch64.svcount"), ptr)769declare { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8f16(target("aarch64.svcount"), ptr)770declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8bf16(target("aarch64.svcount"), ptr)771 772declare { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2q.sret.nxv16i8(<vscale x 16 x i1>, ptr)773declare { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2q.sret.nxv8i16(<vscale x 8 x i1>, ptr)774declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2q.sret.nxv4i32(<vscale x 4 x i1>, ptr)775declare { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2q.sret.nxv2i64(<vscale x 2 x i1>, ptr)776 777declare { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld2q.sret.nxv8f16(<vscale x 8 x i1>, ptr)778declare { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld2q.sret.nxv4f32(<vscale x 4 x i1>, ptr)779declare { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2q.sret.nxv2f64(<vscale x 2 x i1>, ptr)780declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld2q.sret.nxv8bf16(<vscale x 8 x i1>, ptr)781 782declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3q.sret.nxv16i8(<vscale x 16 x i1>, ptr)783declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld3q.sret.nxv8i16(<vscale x 8 x i1>, ptr)784declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld3q.sret.nxv4i32(<vscale x 4 x i1>, ptr)785declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld3q.sret.nxv2i64(<vscale x 2 x i1>, ptr)786 787declare { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld3q.sret.nxv8f16(<vscale x 8 x i1>, ptr)788declare { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld3q.sret.nxv4f32(<vscale x 4 x i1>, ptr)789declare { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld3q.sret.nxv2f64(<vscale x 2 x i1>, ptr)790declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld3q.sret.nxv8bf16(<vscale x 8 x i1>, ptr)791 792declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4q.sret.nxv16i8(<vscale x 16 x i1>, ptr)793declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld4q.sret.nxv8i16(<vscale x 8 x i1>, ptr)794declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4q.sret.nxv4i32(<vscale x 4 x i1>, ptr)795declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4q.sret.nxv2i64(<vscale x 2 x i1>, ptr)796 797declare { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld4q.sret.nxv8f16(<vscale x 8 x i1>, ptr)798declare { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld4q.sret.nxv4f32(<vscale x 4 x i1>, ptr)799declare { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld4q.sret.nxv2f64(<vscale x 2 x i1>, ptr)800declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld4q.sret.nxv8bf16(<vscale x 8 x i1>, ptr)801