1281 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s3; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming < %s | FileCheck %s4; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s5 6; == Normal Multi-Vector Consecutive Loads ==7 8define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld1_x2_i8(target("aarch64.svcount") %pn, ptr %ptr) nounwind {9; CHECK-LABEL: ld1_x2_i8:10; CHECK: // %bb.0:11; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill12; CHECK-NEXT: addvl sp, sp, #-113; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill14; CHECK-NEXT: mov p8.b, p0.b15; CHECK-NEXT: ld1b { z0.b, z1.b }, pn8/z, [x0]16; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload17; CHECK-NEXT: addvl sp, sp, #118; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload19; CHECK-NEXT: ret20 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %pn, ptr %ptr);21 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res22}23 24define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld1_x2_i8_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {25; CHECK-LABEL: ld1_x2_i8_scalar:26; CHECK: // %bb.0:27; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill28; CHECK-NEXT: addvl sp, sp, #-129; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill30; CHECK-NEXT: mov p8.b, p0.b31; CHECK-NEXT: ld1b { z0.b, z1.b }, pn8/z, [x0, x1]32; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload33; CHECK-NEXT: addvl sp, sp, #134; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload35; CHECK-NEXT: ret36 %base = getelementptr i8, ptr %ptr, i64 %index37 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %pn, ptr %base);38 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res39}40 41define { <vscale x 8 x i16>, <vscale x 8 x i16> } @ld1_x2_i16(target("aarch64.svcount") %pn, ptr %ptr) nounwind {42; CHECK-LABEL: ld1_x2_i16:43; CHECK: // %bb.0:44; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill45; CHECK-NEXT: addvl sp, sp, #-146; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill47; CHECK-NEXT: mov p8.b, p0.b48; CHECK-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0]49; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload50; CHECK-NEXT: addvl sp, sp, #151; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload52; CHECK-NEXT: ret53 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") %pn, ptr %ptr);54 ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %res55}56 57define { <vscale x 8 x i16>, <vscale x 8 x i16> } @ld1_x2_i16_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {58; CHECK-LABEL: ld1_x2_i16_scalar:59; CHECK: // %bb.0:60; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill61; CHECK-NEXT: addvl sp, sp, #-162; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill63; CHECK-NEXT: mov p8.b, p0.b64; CHECK-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0, x1, lsl #1]65; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload66; CHECK-NEXT: addvl sp, sp, #167; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload68; CHECK-NEXT: ret69 %base = getelementptr i16, ptr %ptr, i64 %index70 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount") %pn, ptr %base);71 ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %res72}73 74define { <vscale x 4 x i32>, <vscale x 4 x i32> } @ld1_x2_i32(target("aarch64.svcount") %pn, ptr %ptr) nounwind {75; CHECK-LABEL: ld1_x2_i32:76; CHECK: // %bb.0:77; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill78; CHECK-NEXT: addvl sp, sp, #-179; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill80; CHECK-NEXT: mov p8.b, p0.b81; CHECK-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0]82; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload83; CHECK-NEXT: addvl sp, sp, #184; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload85; CHECK-NEXT: ret86 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount") %pn, ptr %ptr);87 ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res88}89 90define { <vscale x 4 x i32>, <vscale x 4 x i32> } @ld1_x2_i32_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {91; CHECK-LABEL: ld1_x2_i32_scalar:92; CHECK: // %bb.0:93; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill94; CHECK-NEXT: addvl sp, sp, #-195; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill96; CHECK-NEXT: mov p8.b, p0.b97; CHECK-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0, x1, lsl #2]98; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload99; CHECK-NEXT: addvl sp, sp, #1100; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload101; CHECK-NEXT: ret102 %base = getelementptr i32, ptr %ptr, i64 %index103 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount") %pn, ptr %base);104 ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res105}106 107define { <vscale x 2 x i64>, <vscale x 2 x i64> } @ld1_x2_i64(target("aarch64.svcount") %pn, ptr %ptr) nounwind {108; CHECK-LABEL: ld1_x2_i64:109; CHECK: // %bb.0:110; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill111; CHECK-NEXT: addvl sp, sp, #-1112; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill113; CHECK-NEXT: mov p8.b, p0.b114; CHECK-NEXT: ld1d { z0.d, z1.d }, pn8/z, [x0]115; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload116; CHECK-NEXT: addvl sp, sp, #1117; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload118; CHECK-NEXT: ret119 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x2.nxv2i64(target("aarch64.svcount") %pn, ptr %ptr);120 ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res121}122 123define { <vscale x 2 x i64>, <vscale x 2 x i64> } @ld1_x2_i64_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {124; CHECK-LABEL: ld1_x2_i64_scalar:125; CHECK: // %bb.0:126; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill127; CHECK-NEXT: addvl sp, sp, #-1128; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill129; CHECK-NEXT: mov p8.b, p0.b130; CHECK-NEXT: ld1d { z0.d, z1.d }, pn8/z, [x0, x1, lsl #3]131; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload132; CHECK-NEXT: addvl sp, sp, #1133; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload134; CHECK-NEXT: ret135 %base = getelementptr i64, ptr %ptr, i64 %index136 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x2.nxv2i64(target("aarch64.svcount") %pn, ptr %base);137 ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res138}139 140define { <vscale x 8 x half>, <vscale x 8 x half> } @ld1_x2_f16(target("aarch64.svcount") %pn, ptr %ptr) nounwind {141; CHECK-LABEL: ld1_x2_f16:142; CHECK: // %bb.0:143; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill144; CHECK-NEXT: addvl sp, sp, #-1145; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill146; CHECK-NEXT: mov p8.b, p0.b147; CHECK-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0]148; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload149; CHECK-NEXT: addvl sp, sp, #1150; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload151; CHECK-NEXT: ret152 %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x2.nxv8f16(target("aarch64.svcount") %pn, ptr %ptr);153 ret { <vscale x 8 x half>, <vscale x 8 x half> } %res154}155 156define { <vscale x 8 x half>, <vscale x 8 x half> } @ld1_x2_f16_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {157; CHECK-LABEL: ld1_x2_f16_scalar:158; CHECK: // %bb.0:159; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill160; CHECK-NEXT: addvl sp, sp, #-1161; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill162; CHECK-NEXT: mov p8.b, p0.b163; CHECK-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0, x1, lsl #1]164; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload165; CHECK-NEXT: addvl sp, sp, #1166; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload167; CHECK-NEXT: ret168 %base = getelementptr half, ptr %ptr, i64 %index169 %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x2.nxv8f16(target("aarch64.svcount") %pn, ptr %base);170 ret { <vscale x 8 x half>, <vscale x 8 x half> } %res171}172 173define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld1_x2_bf16(target("aarch64.svcount") %pn, ptr %ptr) nounwind {174; CHECK-LABEL: ld1_x2_bf16:175; CHECK: // %bb.0:176; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill177; CHECK-NEXT: addvl sp, sp, #-1178; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill179; CHECK-NEXT: mov p8.b, p0.b180; CHECK-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0]181; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload182; CHECK-NEXT: addvl sp, sp, #1183; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload184; CHECK-NEXT: ret185 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld1.pn.x2.nxv8bf16(target("aarch64.svcount") %pn, ptr %ptr);186 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res187}188 189define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld1_x2_bf16_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {190; CHECK-LABEL: ld1_x2_bf16_scalar:191; CHECK: // %bb.0:192; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill193; CHECK-NEXT: addvl sp, sp, #-1194; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill195; CHECK-NEXT: mov p8.b, p0.b196; CHECK-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0, x1, lsl #1]197; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload198; CHECK-NEXT: addvl sp, sp, #1199; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload200; CHECK-NEXT: ret201 %base = getelementptr bfloat, ptr %ptr, i64 %index202 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld1.pn.x2.nxv8bf16(target("aarch64.svcount") %pn, ptr %base);203 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res204}205 206define { <vscale x 4 x float>, <vscale x 4 x float> } @ld1_x2_f32(target("aarch64.svcount") %pn, ptr %ptr) nounwind {207; CHECK-LABEL: ld1_x2_f32:208; CHECK: // %bb.0:209; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill210; CHECK-NEXT: addvl sp, sp, #-1211; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill212; CHECK-NEXT: mov p8.b, p0.b213; CHECK-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0]214; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload215; CHECK-NEXT: addvl sp, sp, #1216; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload217; CHECK-NEXT: ret218 %res = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") %pn, ptr %ptr);219 ret { <vscale x 4 x float>, <vscale x 4 x float> } %res220}221 222define { <vscale x 4 x float>, <vscale x 4 x float> } @ld1_x2_f32_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {223; CHECK-LABEL: ld1_x2_f32_scalar:224; CHECK: // %bb.0:225; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill226; CHECK-NEXT: addvl sp, sp, #-1227; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill228; CHECK-NEXT: mov p8.b, p0.b229; CHECK-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0, x1, lsl #2]230; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload231; CHECK-NEXT: addvl sp, sp, #1232; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload233; CHECK-NEXT: ret234 %base = getelementptr float, ptr %ptr, i64 %index235 %res = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount") %pn, ptr %base);236 ret { <vscale x 4 x float>, <vscale x 4 x float> } %res237}238 239define { <vscale x 2 x double>, <vscale x 2 x double> } @ld1_x2_f64(target("aarch64.svcount") %pn, ptr %ptr) nounwind {240; CHECK-LABEL: ld1_x2_f64:241; CHECK: // %bb.0:242; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill243; CHECK-NEXT: addvl sp, sp, #-1244; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill245; CHECK-NEXT: mov p8.b, p0.b246; CHECK-NEXT: ld1d { z0.d, z1.d }, pn8/z, [x0]247; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload248; CHECK-NEXT: addvl sp, sp, #1249; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload250; CHECK-NEXT: ret251 %res = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount") %pn, ptr %ptr);252 ret { <vscale x 2 x double>, <vscale x 2 x double> } %res253}254 255define { <vscale x 2 x double>, <vscale x 2 x double> } @ld1_x2_f64_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {256; CHECK-LABEL: ld1_x2_f64_scalar:257; CHECK: // %bb.0:258; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill259; CHECK-NEXT: addvl sp, sp, #-1260; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill261; CHECK-NEXT: mov p8.b, p0.b262; CHECK-NEXT: ld1d { z0.d, z1.d }, pn8/z, [x0, x1, lsl #3]263; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload264; CHECK-NEXT: addvl sp, sp, #1265; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload266; CHECK-NEXT: ret267 %base = getelementptr double, ptr %ptr, i64 %index268 %res = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount") %pn, ptr %base);269 ret { <vscale x 2 x double>, <vscale x 2 x double> } %res270}271 272; Test to ensure we load into the correct registers for the instruction273define <vscale x 16 x i8> @ld1_x2_i8_z0_taken(target("aarch64.svcount") %pn, ptr %ptr, <vscale x 16 x i8> %val) {274; CHECK-LABEL: ld1_x2_i8_z0_taken:275; CHECK: // %bb.0:276; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill277; CHECK-NEXT: addvl sp, sp, #-1278; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill279; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG280; CHECK-NEXT: .cfi_offset w29, -16281; CHECK-NEXT: mov p8.b, p0.b282; CHECK-NEXT: ld1b { z2.b, z3.b }, pn8/z, [x0]283; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload284; CHECK-NEXT: add z0.b, z0.b, z2.b285; CHECK-NEXT: addvl sp, sp, #1286; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload287; CHECK-NEXT: ret288 %ld1 = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %pn, ptr %ptr);289 %ld1_0 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %ld1, 0290 %res = add <vscale x 16 x i8> %val, %ld1_0291 ret <vscale x 16 x i8> %res292}293 294; Test to ensure we load into the correct registers for the instruction295define <vscale x 16 x i8> @ld1_x2_i8_z0_taken_scalar(target("aarch64.svcount") %pn, ptr %ptr, <vscale x 16 x i8> %val, i64 %index) {296; CHECK-LABEL: ld1_x2_i8_z0_taken_scalar:297; CHECK: // %bb.0:298; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill299; CHECK-NEXT: addvl sp, sp, #-1300; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill301; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG302; CHECK-NEXT: .cfi_offset w29, -16303; CHECK-NEXT: mov p8.b, p0.b304; CHECK-NEXT: ld1b { z2.b, z3.b }, pn8/z, [x0, x1]305; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload306; CHECK-NEXT: add z0.b, z0.b, z2.b307; CHECK-NEXT: addvl sp, sp, #1308; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload309; CHECK-NEXT: ret310 %base = getelementptr i8, ptr %ptr, i64 %index311 %ld1 = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount") %pn, ptr %base);312 %ld1_0 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %ld1, 0313 %res = add <vscale x 16 x i8> %val, %ld1_0314 ret <vscale x 16 x i8> %res315}316 317define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld1_x4_i8(target("aarch64.svcount") %pn, ptr %ptr) nounwind {318; CHECK-LABEL: ld1_x4_i8:319; CHECK: // %bb.0:320; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill321; CHECK-NEXT: addvl sp, sp, #-1322; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill323; CHECK-NEXT: mov p8.b, p0.b324; CHECK-NEXT: ld1b { z0.b - z3.b }, pn8/z, [x0]325; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload326; CHECK-NEXT: addvl sp, sp, #1327; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload328; CHECK-NEXT: ret329 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %pn, ptr %ptr);330 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res331}332 333define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld1_x4_i8_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {334; CHECK-LABEL: ld1_x4_i8_scalar:335; CHECK: // %bb.0:336; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill337; CHECK-NEXT: addvl sp, sp, #-1338; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill339; CHECK-NEXT: mov p8.b, p0.b340; CHECK-NEXT: ld1b { z0.b - z3.b }, pn8/z, [x0, x1]341; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload342; CHECK-NEXT: addvl sp, sp, #1343; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload344; CHECK-NEXT: ret345 %base = getelementptr i8, ptr %ptr, i64 %index346 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount") %pn, ptr %base);347 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res348}349 350define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @ld1_x4_i16(target("aarch64.svcount") %pn, ptr %ptr) nounwind {351; CHECK-LABEL: ld1_x4_i16:352; CHECK: // %bb.0:353; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill354; CHECK-NEXT: addvl sp, sp, #-1355; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill356; CHECK-NEXT: mov p8.b, p0.b357; CHECK-NEXT: ld1h { z0.h - z3.h }, pn8/z, [x0]358; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload359; CHECK-NEXT: addvl sp, sp, #1360; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload361; CHECK-NEXT: ret362 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %pn, ptr %ptr);363 ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res364}365 366define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @ld1_x4_i16_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {367; CHECK-LABEL: ld1_x4_i16_scalar:368; CHECK: // %bb.0:369; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill370; CHECK-NEXT: addvl sp, sp, #-1371; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill372; CHECK-NEXT: mov p8.b, p0.b373; CHECK-NEXT: ld1h { z0.h - z3.h }, pn8/z, [x0, x1, lsl #1]374; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload375; CHECK-NEXT: addvl sp, sp, #1376; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload377; CHECK-NEXT: ret378 %base = getelementptr i16, ptr %ptr, i64 %index379 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %pn, ptr %base);380 ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res381}382 383define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @ld1_x4_i32(target("aarch64.svcount") %pn, ptr %ptr) nounwind {384; CHECK-LABEL: ld1_x4_i32:385; CHECK: // %bb.0:386; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill387; CHECK-NEXT: addvl sp, sp, #-1388; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill389; CHECK-NEXT: mov p8.b, p0.b390; CHECK-NEXT: ld1w { z0.s - z3.s }, pn8/z, [x0]391; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload392; CHECK-NEXT: addvl sp, sp, #1393; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload394; CHECK-NEXT: ret395 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") %pn, ptr %ptr);396 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res397}398 399define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @ld1_x4_i32_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {400; CHECK-LABEL: ld1_x4_i32_scalar:401; CHECK: // %bb.0:402; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill403; CHECK-NEXT: addvl sp, sp, #-1404; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill405; CHECK-NEXT: mov p8.b, p0.b406; CHECK-NEXT: ld1w { z0.s - z3.s }, pn8/z, [x0, x1, lsl #2]407; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload408; CHECK-NEXT: addvl sp, sp, #1409; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload410; CHECK-NEXT: ret411 %base = getelementptr i32, ptr %ptr, i64 %index412 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount") %pn, ptr %base);413 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res414}415 416define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @ld1_x4_i64(target("aarch64.svcount") %pn, ptr %ptr) nounwind {417; CHECK-LABEL: ld1_x4_i64:418; CHECK: // %bb.0:419; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill420; CHECK-NEXT: addvl sp, sp, #-1421; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill422; CHECK-NEXT: mov p8.b, p0.b423; CHECK-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x0]424; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload425; CHECK-NEXT: addvl sp, sp, #1426; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload427; CHECK-NEXT: ret428 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount") %pn, ptr %ptr);429 ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res430}431 432define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @ld1_x4_i64_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {433; CHECK-LABEL: ld1_x4_i64_scalar:434; CHECK: // %bb.0:435; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill436; CHECK-NEXT: addvl sp, sp, #-1437; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill438; CHECK-NEXT: mov p8.b, p0.b439; CHECK-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x0, x1, lsl #3]440; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload441; CHECK-NEXT: addvl sp, sp, #1442; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload443; CHECK-NEXT: ret444 %base = getelementptr i64, ptr %ptr, i64 %index445 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount") %pn, ptr %base);446 ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res447}448 449define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @ld1_x4_f16(target("aarch64.svcount") %pn, ptr %ptr) nounwind {450; CHECK-LABEL: ld1_x4_f16:451; CHECK: // %bb.0:452; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill453; CHECK-NEXT: addvl sp, sp, #-1454; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill455; CHECK-NEXT: mov p8.b, p0.b456; CHECK-NEXT: ld1h { z0.h - z3.h }, pn8/z, [x0]457; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload458; CHECK-NEXT: addvl sp, sp, #1459; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload460; CHECK-NEXT: ret461 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x4.nxv8f16(target("aarch64.svcount") %pn, ptr %ptr);462 ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res463}464 465define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @ld1_x4_f16_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {466; CHECK-LABEL: ld1_x4_f16_scalar:467; CHECK: // %bb.0:468; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill469; CHECK-NEXT: addvl sp, sp, #-1470; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill471; CHECK-NEXT: mov p8.b, p0.b472; CHECK-NEXT: ld1h { z0.h - z3.h }, pn8/z, [x0, x1, lsl #1]473; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload474; CHECK-NEXT: addvl sp, sp, #1475; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload476; CHECK-NEXT: ret477 %base = getelementptr half, ptr %ptr, i64 %index478 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x4.nxv8f16(target("aarch64.svcount") %pn, ptr %base);479 ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res480}481 482define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld1_x4_bf16(target("aarch64.svcount") %pn, ptr %ptr) nounwind {483; CHECK-LABEL: ld1_x4_bf16:484; CHECK: // %bb.0:485; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill486; CHECK-NEXT: addvl sp, sp, #-1487; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill488; CHECK-NEXT: mov p8.b, p0.b489; CHECK-NEXT: ld1h { z0.h - z3.h }, pn8/z, [x0]490; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload491; CHECK-NEXT: addvl sp, sp, #1492; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload493; CHECK-NEXT: ret494 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld1.pn.x4.nxv8bf16(target("aarch64.svcount") %pn, ptr %ptr);495 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res496}497 498define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld1_x4_bf16_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {499; CHECK-LABEL: ld1_x4_bf16_scalar:500; CHECK: // %bb.0:501; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill502; CHECK-NEXT: addvl sp, sp, #-1503; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill504; CHECK-NEXT: mov p8.b, p0.b505; CHECK-NEXT: ld1h { z0.h - z3.h }, pn8/z, [x0, x1, lsl #1]506; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload507; CHECK-NEXT: addvl sp, sp, #1508; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload509; CHECK-NEXT: ret510 %base = getelementptr bfloat, ptr %ptr, i64 %index511 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld1.pn.x4.nxv8bf16(target("aarch64.svcount") %pn, ptr %base);512 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res513}514 515define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @ld1_x4_f32(target("aarch64.svcount") %pn, ptr %ptr) nounwind {516; CHECK-LABEL: ld1_x4_f32:517; CHECK: // %bb.0:518; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill519; CHECK-NEXT: addvl sp, sp, #-1520; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill521; CHECK-NEXT: mov p8.b, p0.b522; CHECK-NEXT: ld1w { z0.s - z3.s }, pn8/z, [x0]523; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload524; CHECK-NEXT: addvl sp, sp, #1525; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload526; CHECK-NEXT: ret527 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %pn, ptr %ptr);528 ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res529}530 531define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @ld1_x4_f32_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {532; CHECK-LABEL: ld1_x4_f32_scalar:533; CHECK: // %bb.0:534; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill535; CHECK-NEXT: addvl sp, sp, #-1536; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill537; CHECK-NEXT: mov p8.b, p0.b538; CHECK-NEXT: ld1w { z0.s - z3.s }, pn8/z, [x0, x1, lsl #2]539; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload540; CHECK-NEXT: addvl sp, sp, #1541; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload542; CHECK-NEXT: ret543 %base = getelementptr float, ptr %ptr, i64 %index544 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount") %pn, ptr %base);545 ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res546}547 548define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @ld1_x4_f64(target("aarch64.svcount") %pn, ptr %ptr) nounwind {549; CHECK-LABEL: ld1_x4_f64:550; CHECK: // %bb.0:551; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill552; CHECK-NEXT: addvl sp, sp, #-1553; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill554; CHECK-NEXT: mov p8.b, p0.b555; CHECK-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x0]556; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload557; CHECK-NEXT: addvl sp, sp, #1558; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload559; CHECK-NEXT: ret560 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %pn, ptr %ptr);561 ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res562}563 564define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @ld1_x4_f64_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {565; CHECK-LABEL: ld1_x4_f64_scalar:566; CHECK: // %bb.0:567; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill568; CHECK-NEXT: addvl sp, sp, #-1569; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill570; CHECK-NEXT: mov p8.b, p0.b571; CHECK-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x0, x1, lsl #3]572; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload573; CHECK-NEXT: addvl sp, sp, #1574; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload575; CHECK-NEXT: ret576 %base = getelementptr double, ptr %ptr, i64 %index577 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount") %pn, ptr %base);578 ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res579}580 581; Test to ensure we load into the correct registers for the instruction582define <vscale x 8 x i16> @ld1_x4_i16_z0_taken(target("aarch64.svcount") %pn, ptr %ptr, <vscale x 8 x i16> %val) {583; CHECK-LABEL: ld1_x4_i16_z0_taken:584; CHECK: // %bb.0:585; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill586; CHECK-NEXT: addvl sp, sp, #-1587; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill588; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG589; CHECK-NEXT: .cfi_offset w29, -16590; CHECK-NEXT: mov p8.b, p0.b591; CHECK-NEXT: ld1h { z4.h - z7.h }, pn8/z, [x0]592; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload593; CHECK-NEXT: add z0.h, z0.h, z4.h594; CHECK-NEXT: addvl sp, sp, #1595; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload596; CHECK-NEXT: ret597 %ld1 = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %pn, ptr %ptr);598 %ld1_0 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %ld1, 0599 %res = add <vscale x 8 x i16> %val, %ld1_0600 ret <vscale x 8 x i16> %res601}602 603; Test to ensure we load into the correct registers for the instruction604define <vscale x 8 x i16> @ld1_x4_i16_z0_taken_scalar(target("aarch64.svcount") %pn, ptr %ptr, <vscale x 8 x i16> %val, i64 %index) {605; CHECK-LABEL: ld1_x4_i16_z0_taken_scalar:606; CHECK: // %bb.0:607; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill608; CHECK-NEXT: addvl sp, sp, #-1609; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill610; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG611; CHECK-NEXT: .cfi_offset w29, -16612; CHECK-NEXT: mov p8.b, p0.b613; CHECK-NEXT: ld1h { z4.h - z7.h }, pn8/z, [x0, x1, lsl #1]614; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload615; CHECK-NEXT: add z0.h, z0.h, z4.h616; CHECK-NEXT: addvl sp, sp, #1617; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload618; CHECK-NEXT: ret619 %base = getelementptr i16, ptr %ptr, i64 %index620 %ld1 = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount") %pn, ptr %base);621 %ld1_0 = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %ld1, 0622 %res = add <vscale x 8 x i16> %val, %ld1_0623 ret <vscale x 8 x i16> %res624}625 626; == Non-temporal Multi-Vector Consecutive Loads ==627 628define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ldnt1_x2_i8(target("aarch64.svcount") %pn, ptr %ptr) nounwind {629; CHECK-LABEL: ldnt1_x2_i8:630; CHECK: // %bb.0:631; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill632; CHECK-NEXT: addvl sp, sp, #-1633; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill634; CHECK-NEXT: mov p8.b, p0.b635; CHECK-NEXT: ldnt1b { z0.b, z1.b }, pn8/z, [x0]636; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload637; CHECK-NEXT: addvl sp, sp, #1638; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload639; CHECK-NEXT: ret640 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount") %pn, ptr %ptr);641 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res642}643 644define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ldnt1_x2_i8_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {645; CHECK-LABEL: ldnt1_x2_i8_scalar:646; CHECK: // %bb.0:647; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill648; CHECK-NEXT: addvl sp, sp, #-1649; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill650; CHECK-NEXT: mov p8.b, p0.b651; CHECK-NEXT: ldnt1b { z0.b, z1.b }, pn8/z, [x0, x1]652; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload653; CHECK-NEXT: addvl sp, sp, #1654; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload655; CHECK-NEXT: ret656 %base = getelementptr i8, ptr %ptr, i64 %index657 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount") %pn, ptr %base);658 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res659}660 661define { <vscale x 8 x i16>, <vscale x 8 x i16> } @ldnt1_x2_i16(target("aarch64.svcount") %pn, ptr %ptr) nounwind {662; CHECK-LABEL: ldnt1_x2_i16:663; CHECK: // %bb.0:664; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill665; CHECK-NEXT: addvl sp, sp, #-1666; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill667; CHECK-NEXT: mov p8.b, p0.b668; CHECK-NEXT: ldnt1h { z0.h, z1.h }, pn8/z, [x0]669; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload670; CHECK-NEXT: addvl sp, sp, #1671; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload672; CHECK-NEXT: ret673 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8i16(target("aarch64.svcount") %pn, ptr %ptr);674 ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %res675}676 677define { <vscale x 8 x i16>, <vscale x 8 x i16> } @ldnt1_x2_i16_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {678; CHECK-LABEL: ldnt1_x2_i16_scalar:679; CHECK: // %bb.0:680; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill681; CHECK-NEXT: addvl sp, sp, #-1682; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill683; CHECK-NEXT: mov p8.b, p0.b684; CHECK-NEXT: ldnt1h { z0.h, z1.h }, pn8/z, [x0, x1, lsl #1]685; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload686; CHECK-NEXT: addvl sp, sp, #1687; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload688; CHECK-NEXT: ret689 %base = getelementptr i16, ptr %ptr, i64 %index690 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8i16(target("aarch64.svcount") %pn, ptr %base);691 ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %res692}693 694define { <vscale x 4 x i32>, <vscale x 4 x i32> } @ldnt1_x2_i32(target("aarch64.svcount") %pn, ptr %ptr) nounwind {695; CHECK-LABEL: ldnt1_x2_i32:696; CHECK: // %bb.0:697; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill698; CHECK-NEXT: addvl sp, sp, #-1699; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill700; CHECK-NEXT: mov p8.b, p0.b701; CHECK-NEXT: ldnt1w { z0.s, z1.s }, pn8/z, [x0]702; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload703; CHECK-NEXT: addvl sp, sp, #1704; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload705; CHECK-NEXT: ret706 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4i32(target("aarch64.svcount") %pn, ptr %ptr);707 ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res708}709 710define { <vscale x 4 x i32>, <vscale x 4 x i32> } @ldnt1_x2_i32_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {711; CHECK-LABEL: ldnt1_x2_i32_scalar:712; CHECK: // %bb.0:713; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill714; CHECK-NEXT: addvl sp, sp, #-1715; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill716; CHECK-NEXT: mov p8.b, p0.b717; CHECK-NEXT: ldnt1w { z0.s, z1.s }, pn8/z, [x0, x1, lsl #2]718; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload719; CHECK-NEXT: addvl sp, sp, #1720; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload721; CHECK-NEXT: ret722 %base = getelementptr i32, ptr %ptr, i64 %index723 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4i32(target("aarch64.svcount") %pn, ptr %base);724 ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res725}726 727define { <vscale x 2 x i64>, <vscale x 2 x i64> } @ldnt1_x2_i64(target("aarch64.svcount") %pn, ptr %ptr) nounwind {728; CHECK-LABEL: ldnt1_x2_i64:729; CHECK: // %bb.0:730; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill731; CHECK-NEXT: addvl sp, sp, #-1732; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill733; CHECK-NEXT: mov p8.b, p0.b734; CHECK-NEXT: ldnt1d { z0.d, z1.d }, pn8/z, [x0]735; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload736; CHECK-NEXT: addvl sp, sp, #1737; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload738; CHECK-NEXT: ret739 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2i64(target("aarch64.svcount") %pn, ptr %ptr);740 ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res741}742 743define { <vscale x 2 x i64>, <vscale x 2 x i64> } @ldnt1_x2_i64_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {744; CHECK-LABEL: ldnt1_x2_i64_scalar:745; CHECK: // %bb.0:746; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill747; CHECK-NEXT: addvl sp, sp, #-1748; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill749; CHECK-NEXT: mov p8.b, p0.b750; CHECK-NEXT: ldnt1d { z0.d, z1.d }, pn8/z, [x0, x1, lsl #3]751; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload752; CHECK-NEXT: addvl sp, sp, #1753; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload754; CHECK-NEXT: ret755 %base = getelementptr i64, ptr %ptr, i64 %index756 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2i64(target("aarch64.svcount") %pn, ptr %base);757 ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res758}759 760define { <vscale x 8 x half>, <vscale x 8 x half> } @ldnt1_x2_f16(target("aarch64.svcount") %pn, ptr %ptr) nounwind {761; CHECK-LABEL: ldnt1_x2_f16:762; CHECK: // %bb.0:763; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill764; CHECK-NEXT: addvl sp, sp, #-1765; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill766; CHECK-NEXT: mov p8.b, p0.b767; CHECK-NEXT: ldnt1h { z0.h, z1.h }, pn8/z, [x0]768; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload769; CHECK-NEXT: addvl sp, sp, #1770; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload771; CHECK-NEXT: ret772 %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8f16(target("aarch64.svcount") %pn, ptr %ptr);773 ret { <vscale x 8 x half>, <vscale x 8 x half> } %res774}775 776define { <vscale x 8 x half>, <vscale x 8 x half> } @ldnt1_x2_f16_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {777; CHECK-LABEL: ldnt1_x2_f16_scalar:778; CHECK: // %bb.0:779; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill780; CHECK-NEXT: addvl sp, sp, #-1781; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill782; CHECK-NEXT: mov p8.b, p0.b783; CHECK-NEXT: ldnt1h { z0.h, z1.h }, pn8/z, [x0, x1, lsl #1]784; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload785; CHECK-NEXT: addvl sp, sp, #1786; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload787; CHECK-NEXT: ret788 %base = getelementptr i16, ptr %ptr, i64 %index789 %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8f16(target("aarch64.svcount") %pn, ptr %base);790 ret { <vscale x 8 x half>, <vscale x 8 x half> } %res791}792 793define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ldnt1_x2_bf16(target("aarch64.svcount") %pn, ptr %ptr) nounwind {794; CHECK-LABEL: ldnt1_x2_bf16:795; CHECK: // %bb.0:796; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill797; CHECK-NEXT: addvl sp, sp, #-1798; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill799; CHECK-NEXT: mov p8.b, p0.b800; CHECK-NEXT: ldnt1h { z0.h, z1.h }, pn8/z, [x0]801; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload802; CHECK-NEXT: addvl sp, sp, #1803; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload804; CHECK-NEXT: ret805 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8bf16(target("aarch64.svcount") %pn, ptr %ptr);806 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res807}808 809define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ldnt1_x2_bf16_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {810; CHECK-LABEL: ldnt1_x2_bf16_scalar:811; CHECK: // %bb.0:812; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill813; CHECK-NEXT: addvl sp, sp, #-1814; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill815; CHECK-NEXT: mov p8.b, p0.b816; CHECK-NEXT: ldnt1h { z0.h, z1.h }, pn8/z, [x0, x1, lsl #1]817; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload818; CHECK-NEXT: addvl sp, sp, #1819; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload820; CHECK-NEXT: ret821 %base = getelementptr bfloat, ptr %ptr, i64 %index822 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8bf16(target("aarch64.svcount") %pn, ptr %base);823 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res824}825 826define { <vscale x 4 x float>, <vscale x 4 x float> } @ldnt1_x2_f32(target("aarch64.svcount") %pn, ptr %ptr) nounwind {827; CHECK-LABEL: ldnt1_x2_f32:828; CHECK: // %bb.0:829; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill830; CHECK-NEXT: addvl sp, sp, #-1831; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill832; CHECK-NEXT: mov p8.b, p0.b833; CHECK-NEXT: ldnt1w { z0.s, z1.s }, pn8/z, [x0]834; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload835; CHECK-NEXT: addvl sp, sp, #1836; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload837; CHECK-NEXT: ret838 %res = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4f32(target("aarch64.svcount") %pn, ptr %ptr);839 ret { <vscale x 4 x float>, <vscale x 4 x float> } %res840}841 842define { <vscale x 4 x float>, <vscale x 4 x float> } @ldnt1_x2_f32_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {843; CHECK-LABEL: ldnt1_x2_f32_scalar:844; CHECK: // %bb.0:845; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill846; CHECK-NEXT: addvl sp, sp, #-1847; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill848; CHECK-NEXT: mov p8.b, p0.b849; CHECK-NEXT: ldnt1w { z0.s, z1.s }, pn8/z, [x0, x1, lsl #2]850; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload851; CHECK-NEXT: addvl sp, sp, #1852; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload853; CHECK-NEXT: ret854 %base = getelementptr float, ptr %ptr, i64 %index855 %res = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4f32(target("aarch64.svcount") %pn, ptr %base);856 ret { <vscale x 4 x float>, <vscale x 4 x float> } %res857}858 859define { <vscale x 2 x double>, <vscale x 2 x double> } @ldnt1_x2_f64(target("aarch64.svcount") %pn, ptr %ptr) nounwind {860; CHECK-LABEL: ldnt1_x2_f64:861; CHECK: // %bb.0:862; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill863; CHECK-NEXT: addvl sp, sp, #-1864; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill865; CHECK-NEXT: mov p8.b, p0.b866; CHECK-NEXT: ldnt1d { z0.d, z1.d }, pn8/z, [x0]867; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload868; CHECK-NEXT: addvl sp, sp, #1869; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload870; CHECK-NEXT: ret871 %res = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2f64(target("aarch64.svcount") %pn, ptr %ptr);872 ret { <vscale x 2 x double>, <vscale x 2 x double> } %res873}874 875define { <vscale x 2 x double>, <vscale x 2 x double> } @ldnt1_x2_f64_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {876; CHECK-LABEL: ldnt1_x2_f64_scalar:877; CHECK: // %bb.0:878; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill879; CHECK-NEXT: addvl sp, sp, #-1880; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill881; CHECK-NEXT: mov p8.b, p0.b882; CHECK-NEXT: ldnt1d { z0.d, z1.d }, pn8/z, [x0, x1, lsl #3]883; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload884; CHECK-NEXT: addvl sp, sp, #1885; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload886; CHECK-NEXT: ret887 %base = getelementptr double, ptr %ptr, i64 %index888 %res = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2f64(target("aarch64.svcount") %pn, ptr %base);889 ret { <vscale x 2 x double>, <vscale x 2 x double> } %res890}891 892; Test to ensure we load into the correct registers for the instruction893define <vscale x 4 x i32> @ldnt1_x2_i32_z0_taken(target("aarch64.svcount") %pn, ptr %ptr, <vscale x 4 x i32> %val) {894; CHECK-LABEL: ldnt1_x2_i32_z0_taken:895; CHECK: // %bb.0:896; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill897; CHECK-NEXT: addvl sp, sp, #-1898; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill899; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG900; CHECK-NEXT: .cfi_offset w29, -16901; CHECK-NEXT: mov p8.b, p0.b902; CHECK-NEXT: ldnt1w { z2.s, z3.s }, pn8/z, [x0]903; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload904; CHECK-NEXT: add z0.s, z0.s, z2.s905; CHECK-NEXT: addvl sp, sp, #1906; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload907; CHECK-NEXT: ret908 %ld1 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4i32(target("aarch64.svcount") %pn, ptr %ptr);909 %ld1_0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %ld1, 0910 %res = add <vscale x 4 x i32> %val, %ld1_0911 ret <vscale x 4 x i32> %res912}913 914; Test to ensure we load into the correct registers for the instruction915define <vscale x 4 x i32> @ldnt1_x2_i32_z0_taken_scalar(target("aarch64.svcount") %pn, ptr %ptr, <vscale x 4 x i32> %val, i64 %index) {916; CHECK-LABEL: ldnt1_x2_i32_z0_taken_scalar:917; CHECK: // %bb.0:918; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill919; CHECK-NEXT: addvl sp, sp, #-1920; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill921; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG922; CHECK-NEXT: .cfi_offset w29, -16923; CHECK-NEXT: mov p8.b, p0.b924; CHECK-NEXT: ldnt1w { z2.s, z3.s }, pn8/z, [x0, x1, lsl #2]925; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload926; CHECK-NEXT: add z0.s, z0.s, z2.s927; CHECK-NEXT: addvl sp, sp, #1928; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload929; CHECK-NEXT: ret930 %base = getelementptr i32, ptr %ptr, i64 %index931 %ld1 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4i32(target("aarch64.svcount") %pn, ptr %base);932 %ld1_0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %ld1, 0933 %res = add <vscale x 4 x i32> %val, %ld1_0934 ret <vscale x 4 x i32> %res935}936 937define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ldnt1_x4_i8(target("aarch64.svcount") %pn, ptr %ptr) nounwind {938; CHECK-LABEL: ldnt1_x4_i8:939; CHECK: // %bb.0:940; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill941; CHECK-NEXT: addvl sp, sp, #-1942; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill943; CHECK-NEXT: mov p8.b, p0.b944; CHECK-NEXT: ldnt1b { z0.b - z3.b }, pn8/z, [x0]945; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload946; CHECK-NEXT: addvl sp, sp, #1947; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload948; CHECK-NEXT: ret949 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount") %pn, ptr %ptr);950 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res951}952 953define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ldnt1_x4_i8_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {954; CHECK-LABEL: ldnt1_x4_i8_scalar:955; CHECK: // %bb.0:956; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill957; CHECK-NEXT: addvl sp, sp, #-1958; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill959; CHECK-NEXT: mov p8.b, p0.b960; CHECK-NEXT: ldnt1b { z0.b - z3.b }, pn8/z, [x0, x1]961; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload962; CHECK-NEXT: addvl sp, sp, #1963; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload964; CHECK-NEXT: ret965 %base = getelementptr i8, ptr %ptr, i64 %index966 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount") %pn, ptr %base);967 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res968}969 970define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @ldnt1_x4_i16(target("aarch64.svcount") %pn, ptr %ptr) nounwind {971; CHECK-LABEL: ldnt1_x4_i16:972; CHECK: // %bb.0:973; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill974; CHECK-NEXT: addvl sp, sp, #-1975; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill976; CHECK-NEXT: mov p8.b, p0.b977; CHECK-NEXT: ldnt1h { z0.h - z3.h }, pn8/z, [x0]978; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload979; CHECK-NEXT: addvl sp, sp, #1980; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload981; CHECK-NEXT: ret982 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8i16(target("aarch64.svcount") %pn, ptr %ptr);983 ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res984}985 986define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @ldnt1_x4_i16_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {987; CHECK-LABEL: ldnt1_x4_i16_scalar:988; CHECK: // %bb.0:989; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill990; CHECK-NEXT: addvl sp, sp, #-1991; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill992; CHECK-NEXT: mov p8.b, p0.b993; CHECK-NEXT: ldnt1h { z0.h - z3.h }, pn8/z, [x0, x1, lsl #1]994; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload995; CHECK-NEXT: addvl sp, sp, #1996; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload997; CHECK-NEXT: ret998 %base = getelementptr i16, ptr %ptr, i64 %index999 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8i16(target("aarch64.svcount") %pn, ptr %base);1000 ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res1001}1002 1003define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @ldnt1_x4_i32(target("aarch64.svcount") %pn, ptr %ptr) nounwind {1004; CHECK-LABEL: ldnt1_x4_i32:1005; CHECK: // %bb.0:1006; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1007; CHECK-NEXT: addvl sp, sp, #-11008; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill1009; CHECK-NEXT: mov p8.b, p0.b1010; CHECK-NEXT: ldnt1w { z0.s - z3.s }, pn8/z, [x0]1011; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload1012; CHECK-NEXT: addvl sp, sp, #11013; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1014; CHECK-NEXT: ret1015 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4i32(target("aarch64.svcount") %pn, ptr %ptr);1016 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res1017}1018 1019define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @ldnt1_x4_i32_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {1020; CHECK-LABEL: ldnt1_x4_i32_scalar:1021; CHECK: // %bb.0:1022; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1023; CHECK-NEXT: addvl sp, sp, #-11024; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill1025; CHECK-NEXT: mov p8.b, p0.b1026; CHECK-NEXT: ldnt1w { z0.s - z3.s }, pn8/z, [x0, x1, lsl #2]1027; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload1028; CHECK-NEXT: addvl sp, sp, #11029; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1030; CHECK-NEXT: ret1031 %base = getelementptr i32, ptr %ptr, i64 %index1032 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4i32(target("aarch64.svcount") %pn, ptr %base);1033 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res1034}1035 1036define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @ldnt1_x4_i64(target("aarch64.svcount") %pn, ptr %ptr) nounwind {1037; CHECK-LABEL: ldnt1_x4_i64:1038; CHECK: // %bb.0:1039; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1040; CHECK-NEXT: addvl sp, sp, #-11041; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill1042; CHECK-NEXT: mov p8.b, p0.b1043; CHECK-NEXT: ldnt1d { z0.d - z3.d }, pn8/z, [x0]1044; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload1045; CHECK-NEXT: addvl sp, sp, #11046; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1047; CHECK-NEXT: ret1048 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2i64(target("aarch64.svcount") %pn, ptr %ptr);1049 ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res1050}1051 1052define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @ldnt1_x4_i64_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {1053; CHECK-LABEL: ldnt1_x4_i64_scalar:1054; CHECK: // %bb.0:1055; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1056; CHECK-NEXT: addvl sp, sp, #-11057; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill1058; CHECK-NEXT: mov p8.b, p0.b1059; CHECK-NEXT: ldnt1d { z0.d - z3.d }, pn8/z, [x0, x1, lsl #3]1060; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload1061; CHECK-NEXT: addvl sp, sp, #11062; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1063; CHECK-NEXT: ret1064 %base = getelementptr i64, ptr %ptr, i64 %index1065 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2i64(target("aarch64.svcount") %pn, ptr %base);1066 ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res1067}1068 1069define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @ldnt1_x4_f16(target("aarch64.svcount") %pn, ptr %ptr) nounwind {1070; CHECK-LABEL: ldnt1_x4_f16:1071; CHECK: // %bb.0:1072; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1073; CHECK-NEXT: addvl sp, sp, #-11074; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill1075; CHECK-NEXT: mov p8.b, p0.b1076; CHECK-NEXT: ldnt1h { z0.h - z3.h }, pn8/z, [x0]1077; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload1078; CHECK-NEXT: addvl sp, sp, #11079; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1080; CHECK-NEXT: ret1081 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8f16(target("aarch64.svcount") %pn, ptr %ptr);1082 ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res1083}1084 1085define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @ldnt1_x4_f16_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {1086; CHECK-LABEL: ldnt1_x4_f16_scalar:1087; CHECK: // %bb.0:1088; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1089; CHECK-NEXT: addvl sp, sp, #-11090; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill1091; CHECK-NEXT: mov p8.b, p0.b1092; CHECK-NEXT: ldnt1h { z0.h - z3.h }, pn8/z, [x0, x1, lsl #1]1093; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload1094; CHECK-NEXT: addvl sp, sp, #11095; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1096; CHECK-NEXT: ret1097 %base = getelementptr half, ptr %ptr, i64 %index1098 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8f16(target("aarch64.svcount") %pn, ptr %base);1099 ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res1100}1101 1102define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ldnt1_x4_bf16(target("aarch64.svcount") %pn, ptr %ptr) nounwind {1103; CHECK-LABEL: ldnt1_x4_bf16:1104; CHECK: // %bb.0:1105; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1106; CHECK-NEXT: addvl sp, sp, #-11107; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill1108; CHECK-NEXT: mov p8.b, p0.b1109; CHECK-NEXT: ldnt1h { z0.h - z3.h }, pn8/z, [x0]1110; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload1111; CHECK-NEXT: addvl sp, sp, #11112; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1113; CHECK-NEXT: ret1114 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8bf16(target("aarch64.svcount") %pn, ptr %ptr);1115 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res1116}1117 1118define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ldnt1_x4_bf16_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {1119; CHECK-LABEL: ldnt1_x4_bf16_scalar:1120; CHECK: // %bb.0:1121; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1122; CHECK-NEXT: addvl sp, sp, #-11123; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill1124; CHECK-NEXT: mov p8.b, p0.b1125; CHECK-NEXT: ldnt1h { z0.h - z3.h }, pn8/z, [x0, x1, lsl #1]1126; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload1127; CHECK-NEXT: addvl sp, sp, #11128; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1129; CHECK-NEXT: ret1130 %base = getelementptr bfloat, ptr %ptr, i64 %index1131 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8bf16(target("aarch64.svcount") %pn, ptr %base);1132 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res1133}1134 1135define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @ldnt1_x4_f32(target("aarch64.svcount") %pn, ptr %ptr) nounwind {1136; CHECK-LABEL: ldnt1_x4_f32:1137; CHECK: // %bb.0:1138; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1139; CHECK-NEXT: addvl sp, sp, #-11140; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill1141; CHECK-NEXT: mov p8.b, p0.b1142; CHECK-NEXT: ldnt1w { z0.s - z3.s }, pn8/z, [x0]1143; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload1144; CHECK-NEXT: addvl sp, sp, #11145; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1146; CHECK-NEXT: ret1147 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4f32(target("aarch64.svcount") %pn, ptr %ptr);1148 ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res1149}1150 1151define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @ldnt1_x4_f32_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {1152; CHECK-LABEL: ldnt1_x4_f32_scalar:1153; CHECK: // %bb.0:1154; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1155; CHECK-NEXT: addvl sp, sp, #-11156; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill1157; CHECK-NEXT: mov p8.b, p0.b1158; CHECK-NEXT: ldnt1w { z0.s - z3.s }, pn8/z, [x0, x1, lsl #2]1159; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload1160; CHECK-NEXT: addvl sp, sp, #11161; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1162; CHECK-NEXT: ret1163 %base = getelementptr float, ptr %ptr, i64 %index1164 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4f32(target("aarch64.svcount") %pn, ptr %base);1165 ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res1166}1167 1168define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @ldnt1_x4_f64(target("aarch64.svcount") %pn, ptr %ptr) nounwind {1169; CHECK-LABEL: ldnt1_x4_f64:1170; CHECK: // %bb.0:1171; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1172; CHECK-NEXT: addvl sp, sp, #-11173; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill1174; CHECK-NEXT: mov p8.b, p0.b1175; CHECK-NEXT: ldnt1d { z0.d - z3.d }, pn8/z, [x0]1176; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload1177; CHECK-NEXT: addvl sp, sp, #11178; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1179; CHECK-NEXT: ret1180 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2f64(target("aarch64.svcount") %pn, ptr %ptr);1181 ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res1182}1183 1184define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @ldnt1_x4_f64_scalar(target("aarch64.svcount") %pn, ptr %ptr, i64 %index) nounwind {1185; CHECK-LABEL: ldnt1_x4_f64_scalar:1186; CHECK: // %bb.0:1187; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1188; CHECK-NEXT: addvl sp, sp, #-11189; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill1190; CHECK-NEXT: mov p8.b, p0.b1191; CHECK-NEXT: ldnt1d { z0.d - z3.d }, pn8/z, [x0, x1, lsl #3]1192; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload1193; CHECK-NEXT: addvl sp, sp, #11194; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1195; CHECK-NEXT: ret1196 %base = getelementptr double, ptr %ptr, i64 %index1197 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2f64(target("aarch64.svcount") %pn, ptr %base);1198 ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res1199}1200 1201; Test to ensure we load into the correct registers for the instruction1202define <vscale x 2 x i64> @ldnt1_x4_i64_z0_taken(target("aarch64.svcount") %pn, ptr %ptr, <vscale x 2 x i64> %val) {1203; CHECK-LABEL: ldnt1_x4_i64_z0_taken:1204; CHECK: // %bb.0:1205; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1206; CHECK-NEXT: addvl sp, sp, #-11207; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill1208; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG1209; CHECK-NEXT: .cfi_offset w29, -161210; CHECK-NEXT: mov p8.b, p0.b1211; CHECK-NEXT: ldnt1d { z4.d - z7.d }, pn8/z, [x0]1212; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload1213; CHECK-NEXT: add z0.d, z0.d, z4.d1214; CHECK-NEXT: addvl sp, sp, #11215; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1216; CHECK-NEXT: ret1217 %ld1 = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2i64(target("aarch64.svcount") %pn, ptr %ptr);1218 %ld1_0 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %ld1, 01219 %res = add <vscale x 2 x i64> %val, %ld1_01220 ret <vscale x 2 x i64> %res1221}1222 1223; Test to ensure we load into the correct registers for the instruction1224define <vscale x 2 x i64> @ldnt1_x4_i64_z0_taken_scalar(target("aarch64.svcount") %pn, ptr %ptr, <vscale x 2 x i64> %val, i64 %index) {1225; CHECK-LABEL: ldnt1_x4_i64_z0_taken_scalar:1226; CHECK: // %bb.0:1227; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1228; CHECK-NEXT: addvl sp, sp, #-11229; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill1230; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG1231; CHECK-NEXT: .cfi_offset w29, -161232; CHECK-NEXT: mov p8.b, p0.b1233; CHECK-NEXT: ldnt1d { z4.d - z7.d }, pn8/z, [x0, x1, lsl #3]1234; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload1235; CHECK-NEXT: add z0.d, z0.d, z4.d1236; CHECK-NEXT: addvl sp, sp, #11237; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1238; CHECK-NEXT: ret1239 %base = getelementptr i64, ptr %ptr, i64 %index1240 %ld1 = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2i64(target("aarch64.svcount") %pn, ptr %base);1241 %ld1_0 = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %ld1, 01242 %res = add <vscale x 2 x i64> %val, %ld1_01243 ret <vscale x 2 x i64> %res1244}1245 1246declare { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x2.nxv2i64(target("aarch64.svcount"), ptr)1247declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x2.nxv4i32(target("aarch64.svcount"), ptr)1248declare { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x2.nxv8i16(target("aarch64.svcount"), ptr)1249declare { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x2.nxv16i8(target("aarch64.svcount"), ptr)1250declare { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x2.nxv2f64(target("aarch64.svcount"), ptr)1251declare { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x2.nxv4f32(target("aarch64.svcount"), ptr)1252declare { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x2.nxv8f16(target("aarch64.svcount"), ptr)1253declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld1.pn.x2.nxv8bf16(target("aarch64.svcount"), ptr)1254 1255declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld1.pn.x4.nxv2i64(target("aarch64.svcount"), ptr)1256declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld1.pn.x4.nxv4i32(target("aarch64.svcount"), ptr)1257declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld1.pn.x4.nxv8i16(target("aarch64.svcount"), ptr)1258declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld1.pn.x4.nxv16i8(target("aarch64.svcount"), ptr)1259declare { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld1.pn.x4.nxv2f64(target("aarch64.svcount"), ptr)1260declare { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld1.pn.x4.nxv4f32(target("aarch64.svcount"), ptr)1261declare { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld1.pn.x4.nxv8f16(target("aarch64.svcount"), ptr)1262declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld1.pn.x4.nxv8bf16(target("aarch64.svcount"), ptr)1263 1264declare { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2i64(target("aarch64.svcount"), ptr)1265declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4i32(target("aarch64.svcount"), ptr)1266declare { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8i16(target("aarch64.svcount"), ptr)1267declare { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv16i8(target("aarch64.svcount"), ptr)1268declare { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv2f64(target("aarch64.svcount"), ptr)1269declare { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv4f32(target("aarch64.svcount"), ptr)1270declare { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8f16(target("aarch64.svcount"), ptr)1271declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ldnt1.pn.x2.nxv8bf16(target("aarch64.svcount"), ptr)1272 1273declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2i64(target("aarch64.svcount"), ptr)1274declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4i32(target("aarch64.svcount"), ptr)1275declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8i16(target("aarch64.svcount"), ptr)1276declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv16i8(target("aarch64.svcount"), ptr)1277declare { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv2f64(target("aarch64.svcount"), ptr)1278declare { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv4f32(target("aarch64.svcount"), ptr)1279declare { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8f16(target("aarch64.svcount"), ptr)1280declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ldnt1.pn.x4.nxv8bf16(target("aarch64.svcount"), ptr)1281