1533 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,CHECK-LD1R3; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+no-sve-fp-ld1r < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NO-LD1R4;5; Check that ldr1* instruction is generated to splat scalar during load,6; rather than mov from scalar to vector register (which would require the vector unit).7;8; one-off: ld1r_stack checks that ldr1b works with stack objects.9;10; Test axes:11; types = [i8, i16, i32, i64, half, float, double]12; methods = [direct load, gep upper bound - 1, gep out of range x {neg,pos}, sext..., zext..., unpacked_floats...]13;14 15@g8 = external global i816 17; One-off test for splatted value coming from stack load.18define <vscale x 16 x i8> @ld1r_stack() {19; CHECK-LABEL: ld1r_stack:20; CHECK: // %bb.0:21; CHECK-NEXT: sub sp, sp, #1622; CHECK-NEXT: .cfi_def_cfa_offset 1623; CHECK-NEXT: adrp x8, :got:g824; CHECK-NEXT: ptrue p0.b25; CHECK-NEXT: ldr x8, [x8, :got_lo12:g8]26; CHECK-NEXT: ldrb w8, [x8]27; CHECK-NEXT: strb w8, [sp, #12]28; CHECK-NEXT: ld1rb { z0.b }, p0/z, [sp, #14]29; CHECK-NEXT: add sp, sp, #1630; CHECK-NEXT: ret31 %valp = alloca i832 %valp2 = load volatile i8, ptr @g833 store volatile i8 %valp2, ptr %valp34 %valp3 = getelementptr i8, ptr %valp, i32 235 %val = load i8, ptr %valp336 %1 = insertelement <vscale x 16 x i8> poison, i8 %val, i32 037 %2 = shufflevector <vscale x 16 x i8> %1, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer38 ret <vscale x 16 x i8> %239}40 41define <vscale x 16 x i8> @ld1rb(ptr %valp) {42; CHECK-LABEL: ld1rb:43; CHECK: // %bb.0:44; CHECK-NEXT: ptrue p0.b45; CHECK-NEXT: ld1rb { z0.b }, p0/z, [x0]46; CHECK-NEXT: ret47 %val = load i8, ptr %valp48 %ins = insertelement <vscale x 16 x i8> poison, i8 %val, i32 049 %shf = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer50 ret <vscale x 16 x i8> %shf51}52 53define <vscale x 16 x i8> @ld1rb_gep(ptr %valp) {54; CHECK-LABEL: ld1rb_gep:55; CHECK: // %bb.0:56; CHECK-NEXT: ptrue p0.b57; CHECK-NEXT: ld1rb { z0.b }, p0/z, [x0, #63]58; CHECK-NEXT: ret59 %valp2 = getelementptr i8, ptr %valp, i32 6360 %val = load i8, ptr %valp261 %ins = insertelement <vscale x 16 x i8> poison, i8 %val, i32 062 %shf = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer63 ret <vscale x 16 x i8> %shf64}65 66define <vscale x 16 x i8> @ld1rb_gep_out_of_range_up(ptr %valp) {67; CHECK-LABEL: ld1rb_gep_out_of_range_up:68; CHECK: // %bb.0:69; CHECK-NEXT: ptrue p0.b70; CHECK-NEXT: add x8, x0, #6471; CHECK-NEXT: ld1rb { z0.b }, p0/z, [x8]72; CHECK-NEXT: ret73 %valp2 = getelementptr i8, ptr %valp, i32 6474 %val = load i8, ptr %valp275 %ins = insertelement <vscale x 16 x i8> poison, i8 %val, i32 076 %shf = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer77 ret <vscale x 16 x i8> %shf78}79 80define <vscale x 16 x i8> @ld1rb_gep_out_of_range_down(ptr %valp) {81; CHECK-LABEL: ld1rb_gep_out_of_range_down:82; CHECK: // %bb.0:83; CHECK-NEXT: ptrue p0.b84; CHECK-NEXT: sub x8, x0, #185; CHECK-NEXT: ld1rb { z0.b }, p0/z, [x8]86; CHECK-NEXT: ret87 %valp2 = getelementptr i8, ptr %valp, i32 -188 %val = load i8, ptr %valp289 %ins = insertelement <vscale x 16 x i8> poison, i8 %val, i32 090 %shf = shufflevector <vscale x 16 x i8> %ins, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer91 ret <vscale x 16 x i8> %shf92}93 94define <vscale x 8 x i16> @ld1rb_i8_i16_zext(ptr %valp) {95; CHECK-LABEL: ld1rb_i8_i16_zext:96; CHECK: // %bb.0:97; CHECK-NEXT: ptrue p0.h98; CHECK-NEXT: ld1rb { z0.h }, p0/z, [x0]99; CHECK-NEXT: ret100 %val = load i8, ptr %valp101 %ext = zext i8 %val to i16102 %ins = insertelement <vscale x 8 x i16> poison, i16 %ext, i32 0103 %shf = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer104 ret <vscale x 8 x i16> %shf105}106 107define <vscale x 8 x i16> @ld1rb_i8_i16_sext(ptr %valp) {108; CHECK-LABEL: ld1rb_i8_i16_sext:109; CHECK: // %bb.0:110; CHECK-NEXT: ptrue p0.h111; CHECK-NEXT: ld1rsb { z0.h }, p0/z, [x0]112; CHECK-NEXT: ret113 %val = load i8, ptr %valp114 %ext = sext i8 %val to i16115 %ins = insertelement <vscale x 8 x i16> poison, i16 %ext, i32 0116 %shf = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer117 ret <vscale x 8 x i16> %shf118}119 120define <vscale x 4 x i32> @ld1rb_i8_i32_zext(ptr %valp) {121; CHECK-LABEL: ld1rb_i8_i32_zext:122; CHECK: // %bb.0:123; CHECK-NEXT: ptrue p0.s124; CHECK-NEXT: ld1rb { z0.s }, p0/z, [x0]125; CHECK-NEXT: ret126 %val = load i8, ptr %valp127 %ext = zext i8 %val to i32128 %ins = insertelement <vscale x 4 x i32> poison, i32 %ext, i32 0129 %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer130 ret <vscale x 4 x i32> %shf131}132 133define <vscale x 4 x i32> @ld1rb_i8_i32_sext(ptr %valp) {134; CHECK-LABEL: ld1rb_i8_i32_sext:135; CHECK: // %bb.0:136; CHECK-NEXT: ptrue p0.s137; CHECK-NEXT: ld1rsb { z0.s }, p0/z, [x0]138; CHECK-NEXT: ret139 %val = load i8, ptr %valp140 %ext = sext i8 %val to i32141 %ins = insertelement <vscale x 4 x i32> poison, i32 %ext, i32 0142 %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer143 ret <vscale x 4 x i32> %shf144}145 146define <vscale x 2 x i64> @ld1rb_i8_i64_zext(ptr %valp) {147; CHECK-LABEL: ld1rb_i8_i64_zext:148; CHECK: // %bb.0:149; CHECK-NEXT: ptrue p0.d150; CHECK-NEXT: ld1rb { z0.d }, p0/z, [x0]151; CHECK-NEXT: ret152 %val = load i8, ptr %valp153 %ext = zext i8 %val to i64154 %ins = insertelement <vscale x 2 x i64> poison, i64 %ext, i32 0155 %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer156 ret <vscale x 2 x i64> %shf157}158 159define <vscale x 2 x i64> @ld1rb_i8_i64_sext(ptr %valp) {160; CHECK-LABEL: ld1rb_i8_i64_sext:161; CHECK: // %bb.0:162; CHECK-NEXT: ptrue p0.d163; CHECK-NEXT: ld1rsb { z0.d }, p0/z, [x0]164; CHECK-NEXT: ret165 %val = load i8, ptr %valp166 %ext = sext i8 %val to i64167 %ins = insertelement <vscale x 2 x i64> poison, i64 %ext, i32 0168 %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer169 ret <vscale x 2 x i64> %shf170}171 172define <vscale x 8 x i16> @ld1rh(ptr %valp) {173; CHECK-LABEL: ld1rh:174; CHECK: // %bb.0:175; CHECK-NEXT: ptrue p0.h176; CHECK-NEXT: ld1rh { z0.h }, p0/z, [x0]177; CHECK-NEXT: ret178 %val = load i16, ptr %valp179 %ins = insertelement <vscale x 8 x i16> poison, i16 %val, i32 0180 %shf = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer181 ret <vscale x 8 x i16> %shf182}183 184define <vscale x 8 x i16> @ld1rh_gep(ptr %valp) {185; CHECK-LABEL: ld1rh_gep:186; CHECK: // %bb.0:187; CHECK-NEXT: ptrue p0.h188; CHECK-NEXT: ld1rh { z0.h }, p0/z, [x0, #126]189; CHECK-NEXT: ret190 %valp2 = getelementptr i16, ptr %valp, i32 63191 %val = load i16, ptr %valp2192 %ins = insertelement <vscale x 8 x i16> poison, i16 %val, i32 0193 %shf = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer194 ret <vscale x 8 x i16> %shf195}196 197define <vscale x 8 x i16> @ld1rh_gep_out_of_range_up(ptr %valp) {198; CHECK-LABEL: ld1rh_gep_out_of_range_up:199; CHECK: // %bb.0:200; CHECK-NEXT: ptrue p0.h201; CHECK-NEXT: add x8, x0, #128202; CHECK-NEXT: ld1rh { z0.h }, p0/z, [x8]203; CHECK-NEXT: ret204 %valp2 = getelementptr i16, ptr %valp, i32 64205 %val = load i16, ptr %valp2206 %ins = insertelement <vscale x 8 x i16> poison, i16 %val, i32 0207 %shf = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer208 ret <vscale x 8 x i16> %shf209}210 211define <vscale x 8 x i16> @ld1rh_gep_out_of_range_down(ptr %valp) {212; CHECK-LABEL: ld1rh_gep_out_of_range_down:213; CHECK: // %bb.0:214; CHECK-NEXT: ptrue p0.h215; CHECK-NEXT: sub x8, x0, #2216; CHECK-NEXT: ld1rh { z0.h }, p0/z, [x8]217; CHECK-NEXT: ret218 %valp2 = getelementptr i16, ptr %valp, i32 -1219 %val = load i16, ptr %valp2220 %ins = insertelement <vscale x 8 x i16> poison, i16 %val, i32 0221 %shf = shufflevector <vscale x 8 x i16> %ins, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer222 ret <vscale x 8 x i16> %shf223}224 225define <vscale x 4 x i32> @ld1rh_i16_i32_zext(ptr %valp) {226; CHECK-LABEL: ld1rh_i16_i32_zext:227; CHECK: // %bb.0:228; CHECK-NEXT: ptrue p0.s229; CHECK-NEXT: ld1rh { z0.s }, p0/z, [x0]230; CHECK-NEXT: ret231 %val = load i16, ptr %valp232 %ext = zext i16 %val to i32233 %ins = insertelement <vscale x 4 x i32> poison, i32 %ext, i32 0234 %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer235 ret <vscale x 4 x i32> %shf236}237 238define <vscale x 4 x i32> @ld1rh_i16_i32_sext(ptr %valp) {239; CHECK-LABEL: ld1rh_i16_i32_sext:240; CHECK: // %bb.0:241; CHECK-NEXT: ptrue p0.s242; CHECK-NEXT: ld1rsh { z0.s }, p0/z, [x0]243; CHECK-NEXT: ret244 %val = load i16, ptr %valp245 %ext = sext i16 %val to i32246 %ins = insertelement <vscale x 4 x i32> poison, i32 %ext, i32 0247 %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer248 ret <vscale x 4 x i32> %shf249}250 251define <vscale x 2 x i64> @ld1rh_i16_i64_zext(ptr %valp) {252; CHECK-LABEL: ld1rh_i16_i64_zext:253; CHECK: // %bb.0:254; CHECK-NEXT: ptrue p0.d255; CHECK-NEXT: ld1rh { z0.d }, p0/z, [x0]256; CHECK-NEXT: ret257 %val = load i16, ptr %valp258 %ext = zext i16 %val to i64259 %ins = insertelement <vscale x 2 x i64> poison, i64 %ext, i32 0260 %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer261 ret <vscale x 2 x i64> %shf262}263 264define <vscale x 2 x i64> @ld1rh_i16_i64_sext(ptr %valp) {265; CHECK-LABEL: ld1rh_i16_i64_sext:266; CHECK: // %bb.0:267; CHECK-NEXT: ptrue p0.d268; CHECK-NEXT: ld1rsh { z0.d }, p0/z, [x0]269; CHECK-NEXT: ret270 %val = load i16, ptr %valp271 %ext = sext i16 %val to i64272 %ins = insertelement <vscale x 2 x i64> poison, i64 %ext, i32 0273 %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer274 ret <vscale x 2 x i64> %shf275}276 277define <vscale x 4 x i32> @ld1rw(ptr %valp) {278; CHECK-LABEL: ld1rw:279; CHECK: // %bb.0:280; CHECK-NEXT: ptrue p0.s281; CHECK-NEXT: ld1rw { z0.s }, p0/z, [x0]282; CHECK-NEXT: ret283 %val = load i32, ptr %valp284 %ins = insertelement <vscale x 4 x i32> poison, i32 %val, i32 0285 %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer286 ret <vscale x 4 x i32> %shf287}288 289define <vscale x 4 x i32> @ld1rw_gep(ptr %valp) {290; CHECK-LABEL: ld1rw_gep:291; CHECK: // %bb.0:292; CHECK-NEXT: ptrue p0.s293; CHECK-NEXT: ld1rw { z0.s }, p0/z, [x0, #252]294; CHECK-NEXT: ret295 %valp2 = getelementptr i32, ptr %valp, i32 63296 %val = load i32, ptr %valp2297 %ins = insertelement <vscale x 4 x i32> poison, i32 %val, i32 0298 %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer299 ret <vscale x 4 x i32> %shf300}301 302define <vscale x 4 x i32> @ld1rw_gep_out_of_range_up(ptr %valp) {303; CHECK-LABEL: ld1rw_gep_out_of_range_up:304; CHECK: // %bb.0:305; CHECK-NEXT: ptrue p0.s306; CHECK-NEXT: add x8, x0, #256307; CHECK-NEXT: ld1rw { z0.s }, p0/z, [x8]308; CHECK-NEXT: ret309 %valp2 = getelementptr i32, ptr %valp, i32 64310 %val = load i32, ptr %valp2311 %ins = insertelement <vscale x 4 x i32> poison, i32 %val, i32 0312 %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer313 ret <vscale x 4 x i32> %shf314}315 316define <vscale x 4 x i32> @ld1rw_gep_out_of_range_down(ptr %valp) {317; CHECK-LABEL: ld1rw_gep_out_of_range_down:318; CHECK: // %bb.0:319; CHECK-NEXT: ptrue p0.s320; CHECK-NEXT: sub x8, x0, #4321; CHECK-NEXT: ld1rw { z0.s }, p0/z, [x8]322; CHECK-NEXT: ret323 %valp2 = getelementptr i32, ptr %valp, i32 -1324 %val = load i32, ptr %valp2325 %ins = insertelement <vscale x 4 x i32> poison, i32 %val, i32 0326 %shf = shufflevector <vscale x 4 x i32> %ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer327 ret <vscale x 4 x i32> %shf328}329 330define <vscale x 2 x i64> @ld1rw_i32_i64_zext(ptr %valp) {331; CHECK-LABEL: ld1rw_i32_i64_zext:332; CHECK: // %bb.0:333; CHECK-NEXT: ptrue p0.d334; CHECK-NEXT: ld1rw { z0.d }, p0/z, [x0]335; CHECK-NEXT: ret336 %val = load i32, ptr %valp337 %ext = zext i32 %val to i64338 %ins = insertelement <vscale x 2 x i64> poison, i64 %ext, i32 0339 %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer340 ret <vscale x 2 x i64> %shf341}342 343define <vscale x 2 x i64> @ld1rw_i32_i64_sext(ptr %valp) {344; CHECK-LABEL: ld1rw_i32_i64_sext:345; CHECK: // %bb.0:346; CHECK-NEXT: ptrue p0.d347; CHECK-NEXT: ld1rsw { z0.d }, p0/z, [x0]348; CHECK-NEXT: ret349 %val = load i32, ptr %valp350 %ext = sext i32 %val to i64351 %ins = insertelement <vscale x 2 x i64> poison, i64 %ext, i32 0352 %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer353 ret <vscale x 2 x i64> %shf354}355 356define <vscale x 2 x i64> @ld1rd(ptr %valp) {357; CHECK-LABEL: ld1rd:358; CHECK: // %bb.0:359; CHECK-NEXT: ptrue p0.d360; CHECK-NEXT: ld1rd { z0.d }, p0/z, [x0]361; CHECK-NEXT: ret362 %val = load i64, ptr %valp363 %ins = insertelement <vscale x 2 x i64> poison, i64 %val, i32 0364 %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer365 ret <vscale x 2 x i64> %shf366}367 368define <vscale x 2 x i64> @ld1rd_gep(ptr %valp) {369; CHECK-LABEL: ld1rd_gep:370; CHECK: // %bb.0:371; CHECK-NEXT: ptrue p0.d372; CHECK-NEXT: ld1rd { z0.d }, p0/z, [x0, #504]373; CHECK-NEXT: ret374 %valp2 = getelementptr i64, ptr %valp, i32 63375 %val = load i64, ptr %valp2376 %ins = insertelement <vscale x 2 x i64> poison, i64 %val, i32 0377 %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer378 ret <vscale x 2 x i64> %shf379}380 381define <vscale x 2 x i64> @ld1rd_gep_out_of_range_up(ptr %valp) {382; CHECK-LABEL: ld1rd_gep_out_of_range_up:383; CHECK: // %bb.0:384; CHECK-NEXT: ptrue p0.d385; CHECK-NEXT: add x8, x0, #512386; CHECK-NEXT: ld1rd { z0.d }, p0/z, [x8]387; CHECK-NEXT: ret388 %valp2 = getelementptr i64, ptr %valp, i32 64389 %val = load i64, ptr %valp2390 %ins = insertelement <vscale x 2 x i64> poison, i64 %val, i32 0391 %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer392 ret <vscale x 2 x i64> %shf393}394 395define <vscale x 2 x i64> @ld1rd_gep_out_of_range_down(ptr %valp) {396; CHECK-LABEL: ld1rd_gep_out_of_range_down:397; CHECK: // %bb.0:398; CHECK-NEXT: ptrue p0.d399; CHECK-NEXT: sub x8, x0, #8400; CHECK-NEXT: ld1rd { z0.d }, p0/z, [x8]401; CHECK-NEXT: ret402 %valp2 = getelementptr i64, ptr %valp, i32 -1403 %val = load i64, ptr %valp2404 %ins = insertelement <vscale x 2 x i64> poison, i64 %val, i32 0405 %shf = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer406 ret <vscale x 2 x i64> %shf407}408 409define <vscale x 8 x half> @ld1rh_half(ptr %valp) {410; CHECK-LD1R-LABEL: ld1rh_half:411; CHECK-LD1R: // %bb.0:412; CHECK-LD1R-NEXT: ptrue p0.h413; CHECK-LD1R-NEXT: ld1rh { z0.h }, p0/z, [x0]414; CHECK-LD1R-NEXT: ret415;416; CHECK-NO-LD1R-LABEL: ld1rh_half:417; CHECK-NO-LD1R: // %bb.0:418; CHECK-NO-LD1R-NEXT: ldr h0, [x0]419; CHECK-NO-LD1R-NEXT: mov z0.h, h0420; CHECK-NO-LD1R-NEXT: ret421 %val = load half, ptr %valp422 %ins = insertelement <vscale x 8 x half> poison, half %val, i32 0423 %shf = shufflevector <vscale x 8 x half> %ins, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer424 ret <vscale x 8 x half> %shf425}426 427define <vscale x 8 x half> @ld1rh_half_neoverse(ptr %valp) #1 {428; CHECK-LABEL: ld1rh_half_neoverse:429; CHECK: // %bb.0:430; CHECK-NEXT: ldr h0, [x0]431; CHECK-NEXT: mov z0.h, h0432; CHECK-NEXT: ret433 %val = load half, ptr %valp434 %ins = insertelement <vscale x 8 x half> poison, half %val, i32 0435 %shf = shufflevector <vscale x 8 x half> %ins, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer436 ret <vscale x 8 x half> %shf437}438 439define <vscale x 8 x half> @ld1rh_half_gep(ptr %valp) {440; CHECK-LD1R-LABEL: ld1rh_half_gep:441; CHECK-LD1R: // %bb.0:442; CHECK-LD1R-NEXT: ptrue p0.h443; CHECK-LD1R-NEXT: ld1rh { z0.h }, p0/z, [x0, #126]444; CHECK-LD1R-NEXT: ret445;446; CHECK-NO-LD1R-LABEL: ld1rh_half_gep:447; CHECK-NO-LD1R: // %bb.0:448; CHECK-NO-LD1R-NEXT: ldr h0, [x0, #126]449; CHECK-NO-LD1R-NEXT: mov z0.h, h0450; CHECK-NO-LD1R-NEXT: ret451 %valp2 = getelementptr half, ptr %valp, i32 63452 %val = load half, ptr %valp2453 %ins = insertelement <vscale x 8 x half> poison, half %val, i32 0454 %shf = shufflevector <vscale x 8 x half> %ins, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer455 ret <vscale x 8 x half> %shf456}457 458define <vscale x 8 x half> @ld1rh_half_gep_out_of_range_up(ptr %valp) {459; CHECK-LD1R-LABEL: ld1rh_half_gep_out_of_range_up:460; CHECK-LD1R: // %bb.0:461; CHECK-LD1R-NEXT: ptrue p0.h462; CHECK-LD1R-NEXT: add x8, x0, #128463; CHECK-LD1R-NEXT: ld1rh { z0.h }, p0/z, [x8]464; CHECK-LD1R-NEXT: ret465;466; CHECK-NO-LD1R-LABEL: ld1rh_half_gep_out_of_range_up:467; CHECK-NO-LD1R: // %bb.0:468; CHECK-NO-LD1R-NEXT: ldr h0, [x0, #128]469; CHECK-NO-LD1R-NEXT: mov z0.h, h0470; CHECK-NO-LD1R-NEXT: ret471 %valp2 = getelementptr half, ptr %valp, i32 64472 %val = load half, ptr %valp2473 %ins = insertelement <vscale x 8 x half> poison, half %val, i32 0474 %shf = shufflevector <vscale x 8 x half> %ins, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer475 ret <vscale x 8 x half> %shf476}477 478define <vscale x 8 x half> @ld1rh_half_gep_out_of_range_down(ptr %valp) {479; CHECK-LD1R-LABEL: ld1rh_half_gep_out_of_range_down:480; CHECK-LD1R: // %bb.0:481; CHECK-LD1R-NEXT: ptrue p0.h482; CHECK-LD1R-NEXT: sub x8, x0, #2483; CHECK-LD1R-NEXT: ld1rh { z0.h }, p0/z, [x8]484; CHECK-LD1R-NEXT: ret485;486; CHECK-NO-LD1R-LABEL: ld1rh_half_gep_out_of_range_down:487; CHECK-NO-LD1R: // %bb.0:488; CHECK-NO-LD1R-NEXT: ldur h0, [x0, #-2]489; CHECK-NO-LD1R-NEXT: mov z0.h, h0490; CHECK-NO-LD1R-NEXT: ret491 %valp2 = getelementptr half, ptr %valp, i32 -1492 %val = load half, ptr %valp2493 %ins = insertelement <vscale x 8 x half> poison, half %val, i32 0494 %shf = shufflevector <vscale x 8 x half> %ins, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer495 ret <vscale x 8 x half> %shf496}497 498define <vscale x 4 x half> @ld1rh_half_unpacked4(ptr %valp) {499; CHECK-LD1R-LABEL: ld1rh_half_unpacked4:500; CHECK-LD1R: // %bb.0:501; CHECK-LD1R-NEXT: ptrue p0.s502; CHECK-LD1R-NEXT: ld1rh { z0.s }, p0/z, [x0]503; CHECK-LD1R-NEXT: ret504;505; CHECK-NO-LD1R-LABEL: ld1rh_half_unpacked4:506; CHECK-NO-LD1R: // %bb.0:507; CHECK-NO-LD1R-NEXT: ldr h0, [x0]508; CHECK-NO-LD1R-NEXT: mov z0.h, h0509; CHECK-NO-LD1R-NEXT: ret510 %val = load half, ptr %valp511 %ins = insertelement <vscale x 4 x half> poison, half %val, i32 0512 %shf = shufflevector <vscale x 4 x half> %ins, <vscale x 4 x half> poison, <vscale x 4 x i32> zeroinitializer513 ret <vscale x 4 x half> %shf514}515 516define <vscale x 4 x half> @ld1rh_half_unpacked4_gep(ptr %valp) {517; CHECK-LD1R-LABEL: ld1rh_half_unpacked4_gep:518; CHECK-LD1R: // %bb.0:519; CHECK-LD1R-NEXT: ptrue p0.s520; CHECK-LD1R-NEXT: ld1rh { z0.s }, p0/z, [x0, #126]521; CHECK-LD1R-NEXT: ret522;523; CHECK-NO-LD1R-LABEL: ld1rh_half_unpacked4_gep:524; CHECK-NO-LD1R: // %bb.0:525; CHECK-NO-LD1R-NEXT: ldr h0, [x0, #126]526; CHECK-NO-LD1R-NEXT: mov z0.h, h0527; CHECK-NO-LD1R-NEXT: ret528 %valp2 = getelementptr half, ptr %valp, i32 63529 %val = load half, ptr %valp2530 %ins = insertelement <vscale x 4 x half> poison, half %val, i32 0531 %shf = shufflevector <vscale x 4 x half> %ins, <vscale x 4 x half> poison, <vscale x 4 x i32> zeroinitializer532 ret <vscale x 4 x half> %shf533}534 535define <vscale x 4 x half> @ld1rh_half_unpacked4_gep_out_of_range_up(ptr %valp) {536; CHECK-LD1R-LABEL: ld1rh_half_unpacked4_gep_out_of_range_up:537; CHECK-LD1R: // %bb.0:538; CHECK-LD1R-NEXT: ptrue p0.s539; CHECK-LD1R-NEXT: add x8, x0, #128540; CHECK-LD1R-NEXT: ld1rh { z0.s }, p0/z, [x8]541; CHECK-LD1R-NEXT: ret542;543; CHECK-NO-LD1R-LABEL: ld1rh_half_unpacked4_gep_out_of_range_up:544; CHECK-NO-LD1R: // %bb.0:545; CHECK-NO-LD1R-NEXT: ldr h0, [x0, #128]546; CHECK-NO-LD1R-NEXT: mov z0.h, h0547; CHECK-NO-LD1R-NEXT: ret548 %valp2 = getelementptr half, ptr %valp, i32 64549 %val = load half, ptr %valp2550 %ins = insertelement <vscale x 4 x half> poison, half %val, i32 0551 %shf = shufflevector <vscale x 4 x half> %ins, <vscale x 4 x half> poison, <vscale x 4 x i32> zeroinitializer552 ret <vscale x 4 x half> %shf553}554 555define <vscale x 4 x half> @ld1rh_half_unpacked4_gep_out_of_range_down(ptr %valp) {556; CHECK-LD1R-LABEL: ld1rh_half_unpacked4_gep_out_of_range_down:557; CHECK-LD1R: // %bb.0:558; CHECK-LD1R-NEXT: ptrue p0.s559; CHECK-LD1R-NEXT: sub x8, x0, #2560; CHECK-LD1R-NEXT: ld1rh { z0.s }, p0/z, [x8]561; CHECK-LD1R-NEXT: ret562;563; CHECK-NO-LD1R-LABEL: ld1rh_half_unpacked4_gep_out_of_range_down:564; CHECK-NO-LD1R: // %bb.0:565; CHECK-NO-LD1R-NEXT: ldur h0, [x0, #-2]566; CHECK-NO-LD1R-NEXT: mov z0.h, h0567; CHECK-NO-LD1R-NEXT: ret568 %valp2 = getelementptr half, ptr %valp, i32 -1569 %val = load half, ptr %valp2570 %ins = insertelement <vscale x 4 x half> poison, half %val, i32 0571 %shf = shufflevector <vscale x 4 x half> %ins, <vscale x 4 x half> poison, <vscale x 4 x i32> zeroinitializer572 ret <vscale x 4 x half> %shf573}574 575define <vscale x 2 x half> @ld1rh_half_unpacked2(ptr %valp) {576; CHECK-LD1R-LABEL: ld1rh_half_unpacked2:577; CHECK-LD1R: // %bb.0:578; CHECK-LD1R-NEXT: ptrue p0.d579; CHECK-LD1R-NEXT: ld1rh { z0.d }, p0/z, [x0]580; CHECK-LD1R-NEXT: ret581;582; CHECK-NO-LD1R-LABEL: ld1rh_half_unpacked2:583; CHECK-NO-LD1R: // %bb.0:584; CHECK-NO-LD1R-NEXT: ldr h0, [x0]585; CHECK-NO-LD1R-NEXT: mov z0.h, h0586; CHECK-NO-LD1R-NEXT: ret587 %val = load half, ptr %valp588 %ins = insertelement <vscale x 2 x half> poison, half %val, i32 0589 %shf = shufflevector <vscale x 2 x half> %ins, <vscale x 2 x half> poison, <vscale x 2 x i32> zeroinitializer590 ret <vscale x 2 x half> %shf591}592 593define <vscale x 2 x half> @ld1rh_half_unpacked2_gep(ptr %valp) {594; CHECK-LD1R-LABEL: ld1rh_half_unpacked2_gep:595; CHECK-LD1R: // %bb.0:596; CHECK-LD1R-NEXT: ptrue p0.d597; CHECK-LD1R-NEXT: ld1rh { z0.d }, p0/z, [x0, #126]598; CHECK-LD1R-NEXT: ret599;600; CHECK-NO-LD1R-LABEL: ld1rh_half_unpacked2_gep:601; CHECK-NO-LD1R: // %bb.0:602; CHECK-NO-LD1R-NEXT: ldr h0, [x0, #126]603; CHECK-NO-LD1R-NEXT: mov z0.h, h0604; CHECK-NO-LD1R-NEXT: ret605 %valp2 = getelementptr half, ptr %valp, i32 63606 %val = load half, ptr %valp2607 %ins = insertelement <vscale x 2 x half> poison, half %val, i32 0608 %shf = shufflevector <vscale x 2 x half> %ins, <vscale x 2 x half> poison, <vscale x 2 x i32> zeroinitializer609 ret <vscale x 2 x half> %shf610}611 612define <vscale x 2 x half> @ld1rh_half_unpacked2_gep_out_of_range_up(ptr %valp) {613; CHECK-LD1R-LABEL: ld1rh_half_unpacked2_gep_out_of_range_up:614; CHECK-LD1R: // %bb.0:615; CHECK-LD1R-NEXT: ptrue p0.d616; CHECK-LD1R-NEXT: add x8, x0, #128617; CHECK-LD1R-NEXT: ld1rh { z0.d }, p0/z, [x8]618; CHECK-LD1R-NEXT: ret619;620; CHECK-NO-LD1R-LABEL: ld1rh_half_unpacked2_gep_out_of_range_up:621; CHECK-NO-LD1R: // %bb.0:622; CHECK-NO-LD1R-NEXT: ldr h0, [x0, #128]623; CHECK-NO-LD1R-NEXT: mov z0.h, h0624; CHECK-NO-LD1R-NEXT: ret625 %valp2 = getelementptr half, ptr %valp, i32 64626 %val = load half, ptr %valp2627 %ins = insertelement <vscale x 2 x half> poison, half %val, i32 0628 %shf = shufflevector <vscale x 2 x half> %ins, <vscale x 2 x half> poison, <vscale x 2 x i32> zeroinitializer629 ret <vscale x 2 x half> %shf630}631 632define <vscale x 2 x half> @ld1rh_half_unpacked2_gep_out_of_range_down(ptr %valp) {633; CHECK-LD1R-LABEL: ld1rh_half_unpacked2_gep_out_of_range_down:634; CHECK-LD1R: // %bb.0:635; CHECK-LD1R-NEXT: ptrue p0.d636; CHECK-LD1R-NEXT: sub x8, x0, #2637; CHECK-LD1R-NEXT: ld1rh { z0.d }, p0/z, [x8]638; CHECK-LD1R-NEXT: ret639;640; CHECK-NO-LD1R-LABEL: ld1rh_half_unpacked2_gep_out_of_range_down:641; CHECK-NO-LD1R: // %bb.0:642; CHECK-NO-LD1R-NEXT: ldur h0, [x0, #-2]643; CHECK-NO-LD1R-NEXT: mov z0.h, h0644; CHECK-NO-LD1R-NEXT: ret645 %valp2 = getelementptr half, ptr %valp, i32 -1646 %val = load half, ptr %valp2647 %ins = insertelement <vscale x 2 x half> poison, half %val, i32 0648 %shf = shufflevector <vscale x 2 x half> %ins, <vscale x 2 x half> poison, <vscale x 2 x i32> zeroinitializer649 ret <vscale x 2 x half> %shf650}651 652define <vscale x 4 x float> @ld1rw_float(ptr %valp) {653; CHECK-LD1R-LABEL: ld1rw_float:654; CHECK-LD1R: // %bb.0:655; CHECK-LD1R-NEXT: ptrue p0.s656; CHECK-LD1R-NEXT: ld1rw { z0.s }, p0/z, [x0]657; CHECK-LD1R-NEXT: ret658;659; CHECK-NO-LD1R-LABEL: ld1rw_float:660; CHECK-NO-LD1R: // %bb.0:661; CHECK-NO-LD1R-NEXT: ldr s0, [x0]662; CHECK-NO-LD1R-NEXT: mov z0.s, s0663; CHECK-NO-LD1R-NEXT: ret664 %val = load float, ptr %valp665 %ins = insertelement <vscale x 4 x float> poison, float %val, i32 0666 %shf = shufflevector <vscale x 4 x float> %ins, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer667 ret <vscale x 4 x float> %shf668}669 670define <vscale x 4 x float> @ld1rw_float_gep(ptr %valp) {671; CHECK-LD1R-LABEL: ld1rw_float_gep:672; CHECK-LD1R: // %bb.0:673; CHECK-LD1R-NEXT: ptrue p0.s674; CHECK-LD1R-NEXT: ld1rw { z0.s }, p0/z, [x0, #252]675; CHECK-LD1R-NEXT: ret676;677; CHECK-NO-LD1R-LABEL: ld1rw_float_gep:678; CHECK-NO-LD1R: // %bb.0:679; CHECK-NO-LD1R-NEXT: ldr s0, [x0, #252]680; CHECK-NO-LD1R-NEXT: mov z0.s, s0681; CHECK-NO-LD1R-NEXT: ret682 %valp2 = getelementptr float, ptr %valp, i32 63683 %val = load float, ptr %valp2684 %ins = insertelement <vscale x 4 x float> poison, float %val, i32 0685 %shf = shufflevector <vscale x 4 x float> %ins, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer686 ret <vscale x 4 x float> %shf687}688 689define <vscale x 4 x float> @ld1rw_float_gep_out_of_range_up(ptr %valp) {690; CHECK-LD1R-LABEL: ld1rw_float_gep_out_of_range_up:691; CHECK-LD1R: // %bb.0:692; CHECK-LD1R-NEXT: ptrue p0.s693; CHECK-LD1R-NEXT: add x8, x0, #256694; CHECK-LD1R-NEXT: ld1rw { z0.s }, p0/z, [x8]695; CHECK-LD1R-NEXT: ret696;697; CHECK-NO-LD1R-LABEL: ld1rw_float_gep_out_of_range_up:698; CHECK-NO-LD1R: // %bb.0:699; CHECK-NO-LD1R-NEXT: ldr s0, [x0, #256]700; CHECK-NO-LD1R-NEXT: mov z0.s, s0701; CHECK-NO-LD1R-NEXT: ret702 %valp2 = getelementptr float, ptr %valp, i32 64703 %val = load float, ptr %valp2704 %ins = insertelement <vscale x 4 x float> poison, float %val, i32 0705 %shf = shufflevector <vscale x 4 x float> %ins, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer706 ret <vscale x 4 x float> %shf707}708 709define <vscale x 4 x float> @ld1rw_float_gep_out_of_range_down(ptr %valp) {710; CHECK-LD1R-LABEL: ld1rw_float_gep_out_of_range_down:711; CHECK-LD1R: // %bb.0:712; CHECK-LD1R-NEXT: ptrue p0.s713; CHECK-LD1R-NEXT: sub x8, x0, #4714; CHECK-LD1R-NEXT: ld1rw { z0.s }, p0/z, [x8]715; CHECK-LD1R-NEXT: ret716;717; CHECK-NO-LD1R-LABEL: ld1rw_float_gep_out_of_range_down:718; CHECK-NO-LD1R: // %bb.0:719; CHECK-NO-LD1R-NEXT: ldur s0, [x0, #-4]720; CHECK-NO-LD1R-NEXT: mov z0.s, s0721; CHECK-NO-LD1R-NEXT: ret722 %valp2 = getelementptr float, ptr %valp, i32 -1723 %val = load float, ptr %valp2724 %ins = insertelement <vscale x 4 x float> poison, float %val, i32 0725 %shf = shufflevector <vscale x 4 x float> %ins, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer726 ret <vscale x 4 x float> %shf727}728 729define <vscale x 2 x float> @ld1rw_float_unpacked2(ptr %valp) {730; CHECK-LD1R-LABEL: ld1rw_float_unpacked2:731; CHECK-LD1R: // %bb.0:732; CHECK-LD1R-NEXT: ptrue p0.d733; CHECK-LD1R-NEXT: ld1rw { z0.d }, p0/z, [x0]734; CHECK-LD1R-NEXT: ret735;736; CHECK-NO-LD1R-LABEL: ld1rw_float_unpacked2:737; CHECK-NO-LD1R: // %bb.0:738; CHECK-NO-LD1R-NEXT: ldr s0, [x0]739; CHECK-NO-LD1R-NEXT: mov z0.s, s0740; CHECK-NO-LD1R-NEXT: ret741 %val = load float, ptr %valp742 %ins = insertelement <vscale x 2 x float> poison, float %val, i32 0743 %shf = shufflevector <vscale x 2 x float> %ins, <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer744 ret <vscale x 2 x float> %shf745}746 747define <vscale x 2 x float> @ld1rw_float_unpacked2_gep(ptr %valp) {748; CHECK-LD1R-LABEL: ld1rw_float_unpacked2_gep:749; CHECK-LD1R: // %bb.0:750; CHECK-LD1R-NEXT: ptrue p0.d751; CHECK-LD1R-NEXT: ld1rw { z0.d }, p0/z, [x0, #252]752; CHECK-LD1R-NEXT: ret753;754; CHECK-NO-LD1R-LABEL: ld1rw_float_unpacked2_gep:755; CHECK-NO-LD1R: // %bb.0:756; CHECK-NO-LD1R-NEXT: ldr s0, [x0, #252]757; CHECK-NO-LD1R-NEXT: mov z0.s, s0758; CHECK-NO-LD1R-NEXT: ret759 %valp2 = getelementptr float, ptr %valp, i32 63760 %val = load float, ptr %valp2761 %ins = insertelement <vscale x 2 x float> poison, float %val, i32 0762 %shf = shufflevector <vscale x 2 x float> %ins, <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer763 ret <vscale x 2 x float> %shf764}765 766define <vscale x 2 x float> @ld1rw_float_unpacked2_gep_out_of_range_up(ptr %valp) {767; CHECK-LD1R-LABEL: ld1rw_float_unpacked2_gep_out_of_range_up:768; CHECK-LD1R: // %bb.0:769; CHECK-LD1R-NEXT: ptrue p0.d770; CHECK-LD1R-NEXT: add x8, x0, #256771; CHECK-LD1R-NEXT: ld1rw { z0.d }, p0/z, [x8]772; CHECK-LD1R-NEXT: ret773;774; CHECK-NO-LD1R-LABEL: ld1rw_float_unpacked2_gep_out_of_range_up:775; CHECK-NO-LD1R: // %bb.0:776; CHECK-NO-LD1R-NEXT: ldr s0, [x0, #256]777; CHECK-NO-LD1R-NEXT: mov z0.s, s0778; CHECK-NO-LD1R-NEXT: ret779 %valp2 = getelementptr float, ptr %valp, i32 64780 %val = load float, ptr %valp2781 %ins = insertelement <vscale x 2 x float> poison, float %val, i32 0782 %shf = shufflevector <vscale x 2 x float> %ins, <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer783 ret <vscale x 2 x float> %shf784}785 786define <vscale x 2 x float> @ld1rw_float_unpacked2_gep_out_of_range_down(ptr %valp) {787; CHECK-LD1R-LABEL: ld1rw_float_unpacked2_gep_out_of_range_down:788; CHECK-LD1R: // %bb.0:789; CHECK-LD1R-NEXT: ptrue p0.d790; CHECK-LD1R-NEXT: sub x8, x0, #4791; CHECK-LD1R-NEXT: ld1rw { z0.d }, p0/z, [x8]792; CHECK-LD1R-NEXT: ret793;794; CHECK-NO-LD1R-LABEL: ld1rw_float_unpacked2_gep_out_of_range_down:795; CHECK-NO-LD1R: // %bb.0:796; CHECK-NO-LD1R-NEXT: ldur s0, [x0, #-4]797; CHECK-NO-LD1R-NEXT: mov z0.s, s0798; CHECK-NO-LD1R-NEXT: ret799 %valp2 = getelementptr float, ptr %valp, i32 -1800 %val = load float, ptr %valp2801 %ins = insertelement <vscale x 2 x float> poison, float %val, i32 0802 %shf = shufflevector <vscale x 2 x float> %ins, <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer803 ret <vscale x 2 x float> %shf804}805 806define <vscale x 2 x double> @ld1rd_double(ptr %valp) {807; CHECK-LD1R-LABEL: ld1rd_double:808; CHECK-LD1R: // %bb.0:809; CHECK-LD1R-NEXT: ptrue p0.d810; CHECK-LD1R-NEXT: ld1rd { z0.d }, p0/z, [x0]811; CHECK-LD1R-NEXT: ret812;813; CHECK-NO-LD1R-LABEL: ld1rd_double:814; CHECK-NO-LD1R: // %bb.0:815; CHECK-NO-LD1R-NEXT: ldr d0, [x0]816; CHECK-NO-LD1R-NEXT: mov z0.d, d0817; CHECK-NO-LD1R-NEXT: ret818 %val = load double, ptr %valp819 %ins = insertelement <vscale x 2 x double> poison, double %val, i32 0820 %shf = shufflevector <vscale x 2 x double> %ins, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer821 ret <vscale x 2 x double> %shf822}823 824define <vscale x 2 x double> @ld1rd_double_gep(ptr %valp) {825; CHECK-LD1R-LABEL: ld1rd_double_gep:826; CHECK-LD1R: // %bb.0:827; CHECK-LD1R-NEXT: ptrue p0.d828; CHECK-LD1R-NEXT: ld1rd { z0.d }, p0/z, [x0, #504]829; CHECK-LD1R-NEXT: ret830;831; CHECK-NO-LD1R-LABEL: ld1rd_double_gep:832; CHECK-NO-LD1R: // %bb.0:833; CHECK-NO-LD1R-NEXT: ldr d0, [x0, #504]834; CHECK-NO-LD1R-NEXT: mov z0.d, d0835; CHECK-NO-LD1R-NEXT: ret836 %valp2 = getelementptr double, ptr %valp, i32 63837 %val = load double, ptr %valp2838 %ins = insertelement <vscale x 2 x double> poison, double %val, i32 0839 %shf = shufflevector <vscale x 2 x double> %ins, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer840 ret <vscale x 2 x double> %shf841}842 843define <vscale x 2 x double> @ld1rd_double_gep_out_of_range_up(ptr %valp) {844; CHECK-LD1R-LABEL: ld1rd_double_gep_out_of_range_up:845; CHECK-LD1R: // %bb.0:846; CHECK-LD1R-NEXT: ptrue p0.d847; CHECK-LD1R-NEXT: add x8, x0, #512848; CHECK-LD1R-NEXT: ld1rd { z0.d }, p0/z, [x8]849; CHECK-LD1R-NEXT: ret850;851; CHECK-NO-LD1R-LABEL: ld1rd_double_gep_out_of_range_up:852; CHECK-NO-LD1R: // %bb.0:853; CHECK-NO-LD1R-NEXT: ldr d0, [x0, #512]854; CHECK-NO-LD1R-NEXT: mov z0.d, d0855; CHECK-NO-LD1R-NEXT: ret856 %valp2 = getelementptr double, ptr %valp, i32 64857 %val = load double, ptr %valp2858 %ins = insertelement <vscale x 2 x double> poison, double %val, i32 0859 %shf = shufflevector <vscale x 2 x double> %ins, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer860 ret <vscale x 2 x double> %shf861}862 863define <vscale x 2 x double> @ld1rd_double_gep_out_of_range_down(ptr %valp) {864; CHECK-LD1R-LABEL: ld1rd_double_gep_out_of_range_down:865; CHECK-LD1R: // %bb.0:866; CHECK-LD1R-NEXT: ptrue p0.d867; CHECK-LD1R-NEXT: sub x8, x0, #8868; CHECK-LD1R-NEXT: ld1rd { z0.d }, p0/z, [x8]869; CHECK-LD1R-NEXT: ret870;871; CHECK-NO-LD1R-LABEL: ld1rd_double_gep_out_of_range_down:872; CHECK-NO-LD1R: // %bb.0:873; CHECK-NO-LD1R-NEXT: ldur d0, [x0, #-8]874; CHECK-NO-LD1R-NEXT: mov z0.d, d0875; CHECK-NO-LD1R-NEXT: ret876 %valp2 = getelementptr double, ptr %valp, i32 -1877 %val = load double, ptr %valp2878 %ins = insertelement <vscale x 2 x double> poison, double %val, i32 0879 %shf = shufflevector <vscale x 2 x double> %ins, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer880 ret <vscale x 2 x double> %shf881}882 883define <vscale x 2 x double> @dupq_ld1rqd_f64(ptr %a) {884; CHECK-LABEL: dupq_ld1rqd_f64:885; CHECK: // %bb.0:886; CHECK-NEXT: ptrue p0.d887; CHECK-NEXT: ld1rqd { z0.d }, p0/z, [x0]888; CHECK-NEXT: ret889 %1 = load <2 x double>, ptr %a890 %2 = tail call fast <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> poison, <2 x double> %1, i64 0)891 %3 = tail call fast <vscale x 2 x double> @llvm.aarch64.sve.dupq.lane.nxv2f64(<vscale x 2 x double> %2, i64 0)892 ret <vscale x 2 x double> %3893}894 895define <vscale x 4 x float> @dupq_ld1rqw_f32(ptr %a) {896; CHECK-LABEL: dupq_ld1rqw_f32:897; CHECK: // %bb.0:898; CHECK-NEXT: ptrue p0.s899; CHECK-NEXT: ld1rqw { z0.s }, p0/z, [x0]900; CHECK-NEXT: ret901 %1 = load <4 x float>, ptr %a902 %2 = tail call fast <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> poison, <4 x float> %1, i64 0)903 %3 = tail call fast <vscale x 4 x float> @llvm.aarch64.sve.dupq.lane.nxv4f32(<vscale x 4 x float> %2, i64 0)904 ret <vscale x 4 x float> %3905}906 907define <vscale x 8 x half> @dupq_ld1rqh_f16(ptr %a) {908; CHECK-LABEL: dupq_ld1rqh_f16:909; CHECK: // %bb.0:910; CHECK-NEXT: ptrue p0.h911; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0]912; CHECK-NEXT: ret913 %1 = load <8 x half>, ptr %a914 %2 = tail call fast <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v8f16(<vscale x 8 x half> poison, <8 x half> %1, i64 0)915 %3 = tail call fast <vscale x 8 x half> @llvm.aarch64.sve.dupq.lane.nxv8f16(<vscale x 8 x half> %2, i64 0)916 ret <vscale x 8 x half> %3917}918 919define <vscale x 8 x bfloat> @dupq_ld1rqh_bf16(ptr %a) #0 {920; CHECK-LABEL: dupq_ld1rqh_bf16:921; CHECK: // %bb.0:922; CHECK-NEXT: ptrue p0.h923; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0]924; CHECK-NEXT: ret925 %1 = load <8 x bfloat>, ptr %a926 %2 = tail call fast <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat> poison, <8 x bfloat> %1, i64 0)927 %3 = tail call fast <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat> %2, i64 0)928 ret <vscale x 8 x bfloat> %3929}930 931define <vscale x 2 x i64> @dupq_ld1rqd_i64(ptr %a) #0 {932; CHECK-LABEL: dupq_ld1rqd_i64:933; CHECK: // %bb.0:934; CHECK-NEXT: ptrue p0.d935; CHECK-NEXT: ld1rqd { z0.d }, p0/z, [x0]936; CHECK-NEXT: ret937 %1 = load <2 x i64>, ptr %a938 %2 = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> poison, <2 x i64> %1, i64 0)939 %3 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.dupq.lane.nxv2i64(<vscale x 2 x i64> %2, i64 0)940 ret <vscale x 2 x i64> %3941}942 943define <vscale x 4 x i32> @dupq_ld1rqw_i32(ptr %a) #0 {944; CHECK-LABEL: dupq_ld1rqw_i32:945; CHECK: // %bb.0:946; CHECK-NEXT: ptrue p0.s947; CHECK-NEXT: ld1rqw { z0.s }, p0/z, [x0]948; CHECK-NEXT: ret949 %1 = load <4 x i32>, ptr %a950 %2 = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> poison, <4 x i32> %1, i64 0)951 %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.dupq.lane.nxv4i32(<vscale x 4 x i32> %2, i64 0)952 ret <vscale x 4 x i32> %3953}954 955define <vscale x 8 x i16> @dupq_ld1rqw_i16(ptr %a) #0 {956; CHECK-LABEL: dupq_ld1rqw_i16:957; CHECK: // %bb.0:958; CHECK-NEXT: ptrue p0.h959; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0]960; CHECK-NEXT: ret961 %1 = load <8 x i16>, ptr %a962 %2 = tail call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v8i16(<vscale x 8 x i16> poison, <8 x i16> %1, i64 0)963 %3 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.dupq.lane.nxv8i16(<vscale x 8 x i16> %2, i64 0)964 ret <vscale x 8 x i16> %3965}966 967define <vscale x 16 x i8> @dupq_ld1rqw_i8(ptr %a) #0 {968; CHECK-LABEL: dupq_ld1rqw_i8:969; CHECK: // %bb.0:970; CHECK-NEXT: ptrue p0.b971; CHECK-NEXT: ld1rqb { z0.b }, p0/z, [x0]972; CHECK-NEXT: ret973 %1 = load <16 x i8>, ptr %a974 %2 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> poison, <16 x i8> %1, i64 0)975 %3 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.dupq.lane.nxv16i8(<vscale x 16 x i8> %2, i64 0)976 ret <vscale x 16 x i8> %3977}978 979;980;981; Tests for dup:982;983; Positive tests:984; * dup with passthru=poison or passthru=zero.985; * sign/zero extending.986; * unpacked types.987;988; Negative tests:989; * dup with passthru as a parameter.990;991;992 993define <vscale x 16 x i8> @dup_ld1rb_i8_passthruundef_nxv16i8(<vscale x 16 x i1> %pg, ptr %addr) {994; CHECK-LABEL: dup_ld1rb_i8_passthruundef_nxv16i8:995; CHECK: // %bb.0:996; CHECK-NEXT: ld1rb { z0.b }, p0/z, [x0]997; CHECK-NEXT: ret998 %ld = load i8, ptr %addr999 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.dup.nxv16i8(<vscale x 16 x i8> poison, <vscale x 16 x i1> %pg, i8 %ld)1000 ret <vscale x 16 x i8> %res1001}1002define <vscale x 8 x i16> @dup_ld1rh_i16_passthruundef_nxv8i16(<vscale x 8 x i1> %pg, ptr %addr) {1003; CHECK-LABEL: dup_ld1rh_i16_passthruundef_nxv8i16:1004; CHECK: // %bb.0:1005; CHECK-NEXT: ld1rh { z0.h }, p0/z, [x0]1006; CHECK-NEXT: ret1007 %ld = load i16, ptr %addr1008 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.dup.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, i16 %ld)1009 ret <vscale x 8 x i16> %res1010}1011define <vscale x 8 x i16> @dup_ld1rh_i8_passthruundef_nxv8i16_sext(<vscale x 8 x i1> %pg, ptr %addr) {1012; CHECK-LABEL: dup_ld1rh_i8_passthruundef_nxv8i16_sext:1013; CHECK: // %bb.0:1014; CHECK-NEXT: ld1rsb { z0.h }, p0/z, [x0]1015; CHECK-NEXT: ret1016 %ld = load i8, ptr %addr1017 %ext = sext i8 %ld to i161018 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.dup.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, i16 %ext)1019 ret <vscale x 8 x i16> %res1020}1021define <vscale x 8 x i16> @dup_ld1rh_i8_passthruundef_nxv8i16_zext(<vscale x 8 x i1> %pg, ptr %addr) {1022; CHECK-LABEL: dup_ld1rh_i8_passthruundef_nxv8i16_zext:1023; CHECK: // %bb.0:1024; CHECK-NEXT: ld1rb { z0.h }, p0/z, [x0]1025; CHECK-NEXT: ret1026 %ld = load i8, ptr %addr1027 %ext = zext i8 %ld to i161028 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.dup.nxv8i16(<vscale x 8 x i16> poison, <vscale x 8 x i1> %pg, i16 %ext)1029 ret <vscale x 8 x i16> %res1030}1031define <vscale x 4 x i32> @dup_ld1rs_i32_passthruundef_nxv4i32(<vscale x 4 x i1> %pg, ptr %addr) {1032; CHECK-LABEL: dup_ld1rs_i32_passthruundef_nxv4i32:1033; CHECK: // %bb.0:1034; CHECK-NEXT: ld1rw { z0.s }, p0/z, [x0]1035; CHECK-NEXT: ret1036 %ld = load i32, ptr %addr1037 %res = call <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, i32 %ld)1038 ret <vscale x 4 x i32> %res1039}1040define <vscale x 4 x i32> @dup_ld1rs_i8_passthruundef_nxv4i32_sext(<vscale x 4 x i1> %pg, ptr %addr) {1041; CHECK-LABEL: dup_ld1rs_i8_passthruundef_nxv4i32_sext:1042; CHECK: // %bb.0:1043; CHECK-NEXT: ld1rsb { z0.s }, p0/z, [x0]1044; CHECK-NEXT: ret1045 %ld = load i8, ptr %addr1046 %ext = sext i8 %ld to i321047 %res = call <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, i32 %ext)1048 ret <vscale x 4 x i32> %res1049}1050define <vscale x 4 x i32> @dup_ld1rs_i8_passthruundef_nxv4i32_zext(<vscale x 4 x i1> %pg, ptr %addr) {1051; CHECK-LABEL: dup_ld1rs_i8_passthruundef_nxv4i32_zext:1052; CHECK: // %bb.0:1053; CHECK-NEXT: ld1rb { z0.s }, p0/z, [x0]1054; CHECK-NEXT: ret1055 %ld = load i8, ptr %addr1056 %ext = zext i8 %ld to i321057 %res = call <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, i32 %ext)1058 ret <vscale x 4 x i32> %res1059}1060define <vscale x 4 x i32> @dup_ld1rs_i16_passthruundef_nxv4i32_sext(<vscale x 4 x i1> %pg, ptr %addr) {1061; CHECK-LABEL: dup_ld1rs_i16_passthruundef_nxv4i32_sext:1062; CHECK: // %bb.0:1063; CHECK-NEXT: ld1rsh { z0.s }, p0/z, [x0]1064; CHECK-NEXT: ret1065 %ld = load i16, ptr %addr1066 %ext = sext i16 %ld to i321067 %res = call <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, i32 %ext)1068 ret <vscale x 4 x i32> %res1069}1070define <vscale x 4 x i32> @dup_ld1rs_i16_passthruundef_nxv4i32_zext(<vscale x 4 x i1> %pg, ptr %addr) {1071; CHECK-LABEL: dup_ld1rs_i16_passthruundef_nxv4i32_zext:1072; CHECK: // %bb.0:1073; CHECK-NEXT: ld1rh { z0.s }, p0/z, [x0]1074; CHECK-NEXT: ret1075 %ld = load i16, ptr %addr1076 %ext = zext i16 %ld to i321077 %res = call <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, i32 %ext)1078 ret <vscale x 4 x i32> %res1079}1080define <vscale x 2 x i64> @dup_ld1rd_i64_passthruundef_nxv2i64(<vscale x 2 x i1> %pg, ptr %addr) {1081; CHECK-LABEL: dup_ld1rd_i64_passthruundef_nxv2i64:1082; CHECK: // %bb.0:1083; CHECK-NEXT: ld1rd { z0.d }, p0/z, [x0]1084; CHECK-NEXT: ret1085 %ld = load i64, ptr %addr1086 %res = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, i64 %ld)1087 ret <vscale x 2 x i64> %res1088}1089define <vscale x 2 x i64> @dup_ld1rs_i8_passthruundef_nxv2i64_sext(<vscale x 2 x i1> %pg, ptr %addr) {1090; CHECK-LABEL: dup_ld1rs_i8_passthruundef_nxv2i64_sext:1091; CHECK: // %bb.0:1092; CHECK-NEXT: ld1rsb { z0.d }, p0/z, [x0]1093; CHECK-NEXT: ret1094 %ld = load i8, ptr %addr1095 %ext = sext i8 %ld to i641096 %res = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, i64 %ext)1097 ret <vscale x 2 x i64> %res1098}1099define <vscale x 2 x i64> @dup_ld1rs_i8_passthruundef_nxv2i64_zext(<vscale x 2 x i1> %pg, ptr %addr) {1100; CHECK-LABEL: dup_ld1rs_i8_passthruundef_nxv2i64_zext:1101; CHECK: // %bb.0:1102; CHECK-NEXT: ld1rb { z0.d }, p0/z, [x0]1103; CHECK-NEXT: ret1104 %ld = load i8, ptr %addr1105 %ext = zext i8 %ld to i641106 %res = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, i64 %ext)1107 ret <vscale x 2 x i64> %res1108}1109define <vscale x 2 x i64> @dup_ld1rs_i16_passthruundef_nxv2i64_sext(<vscale x 2 x i1> %pg, ptr %addr) {1110; CHECK-LABEL: dup_ld1rs_i16_passthruundef_nxv2i64_sext:1111; CHECK: // %bb.0:1112; CHECK-NEXT: ld1rsh { z0.d }, p0/z, [x0]1113; CHECK-NEXT: ret1114 %ld = load i16, ptr %addr1115 %ext = sext i16 %ld to i641116 %res = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, i64 %ext)1117 ret <vscale x 2 x i64> %res1118}1119define <vscale x 2 x i64> @dup_ld1rs_i16_passthruundef_nxv2i64_zext(<vscale x 2 x i1> %pg, ptr %addr) {1120; CHECK-LABEL: dup_ld1rs_i16_passthruundef_nxv2i64_zext:1121; CHECK: // %bb.0:1122; CHECK-NEXT: ld1rh { z0.d }, p0/z, [x0]1123; CHECK-NEXT: ret1124 %ld = load i16, ptr %addr1125 %ext = zext i16 %ld to i641126 %res = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, i64 %ext)1127 ret <vscale x 2 x i64> %res1128}1129define <vscale x 2 x i64> @dup_ld1rs_i32_passthruundef_nxv2i64_sext(<vscale x 2 x i1> %pg, ptr %addr) {1130; CHECK-LABEL: dup_ld1rs_i32_passthruundef_nxv2i64_sext:1131; CHECK: // %bb.0:1132; CHECK-NEXT: ld1rsw { z0.d }, p0/z, [x0]1133; CHECK-NEXT: ret1134 %ld = load i32, ptr %addr1135 %ext = sext i32 %ld to i641136 %res = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, i64 %ext)1137 ret <vscale x 2 x i64> %res1138}1139define <vscale x 2 x i64> @dup_ld1rs_i32_passthruundef_nxv2i64_zext(<vscale x 2 x i1> %pg, ptr %addr) {1140; CHECK-LABEL: dup_ld1rs_i32_passthruundef_nxv2i64_zext:1141; CHECK: // %bb.0:1142; CHECK-NEXT: ld1rw { z0.d }, p0/z, [x0]1143; CHECK-NEXT: ret1144 %ld = load i32, ptr %addr1145 %ext = zext i32 %ld to i641146 %res = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, i64 %ext)1147 ret <vscale x 2 x i64> %res1148}1149define <vscale x 8 x half> @dup_ld1rh_half_passthruundef_nxv8f16(<vscale x 8 x i1> %pg, ptr %addr) {1150; CHECK-LD1R-LABEL: dup_ld1rh_half_passthruundef_nxv8f16:1151; CHECK-LD1R: // %bb.0:1152; CHECK-LD1R-NEXT: ld1rh { z0.h }, p0/z, [x0]1153; CHECK-LD1R-NEXT: ret1154;1155; CHECK-NO-LD1R-LABEL: dup_ld1rh_half_passthruundef_nxv8f16:1156; CHECK-NO-LD1R: // %bb.0:1157; CHECK-NO-LD1R-NEXT: ldr h0, [x0]1158; CHECK-NO-LD1R-NEXT: mov z0.h, p0/m, h01159; CHECK-NO-LD1R-NEXT: ret1160 %ld = load half, ptr %addr1161 %res = call <vscale x 8 x half> @llvm.aarch64.sve.dup.nxv8f16(<vscale x 8 x half> poison, <vscale x 8 x i1> %pg, half %ld)1162 ret <vscale x 8 x half> %res1163}1164define <vscale x 4 x float> @dup_ld1rs_float_passthruundef_nxv4f32(<vscale x 4 x i1> %pg, ptr %addr) {1165; CHECK-LD1R-LABEL: dup_ld1rs_float_passthruundef_nxv4f32:1166; CHECK-LD1R: // %bb.0:1167; CHECK-LD1R-NEXT: ld1rw { z0.s }, p0/z, [x0]1168; CHECK-LD1R-NEXT: ret1169;1170; CHECK-NO-LD1R-LABEL: dup_ld1rs_float_passthruundef_nxv4f32:1171; CHECK-NO-LD1R: // %bb.0:1172; CHECK-NO-LD1R-NEXT: ldr s0, [x0]1173; CHECK-NO-LD1R-NEXT: mov z0.s, p0/m, s01174; CHECK-NO-LD1R-NEXT: ret1175 %ld = load float, ptr %addr1176 %res = call <vscale x 4 x float> @llvm.aarch64.sve.dup.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x i1> %pg, float %ld)1177 ret <vscale x 4 x float> %res1178}1179define <vscale x 2 x double> @dup_ld1rd_double_passthruundef_nxv2f64(<vscale x 2 x i1> %pg, ptr %addr) {1180; CHECK-LD1R-LABEL: dup_ld1rd_double_passthruundef_nxv2f64:1181; CHECK-LD1R: // %bb.0:1182; CHECK-LD1R-NEXT: ld1rd { z0.d }, p0/z, [x0]1183; CHECK-LD1R-NEXT: ret1184;1185; CHECK-NO-LD1R-LABEL: dup_ld1rd_double_passthruundef_nxv2f64:1186; CHECK-NO-LD1R: // %bb.0:1187; CHECK-NO-LD1R-NEXT: ldr d0, [x0]1188; CHECK-NO-LD1R-NEXT: mov z0.d, p0/m, d01189; CHECK-NO-LD1R-NEXT: ret1190 %ld = load double, ptr %addr1191 %res = call <vscale x 2 x double> @llvm.aarch64.sve.dup.nxv2f64(<vscale x 2 x double> poison, <vscale x 2 x i1> %pg, double %ld)1192 ret <vscale x 2 x double> %res1193}1194define <vscale x 4 x half> @dup_ld1rh_half_passthruundef_nxv4f16(<vscale x 4 x i1> %pg, ptr %addr) {1195; CHECK-LD1R-LABEL: dup_ld1rh_half_passthruundef_nxv4f16:1196; CHECK-LD1R: // %bb.0:1197; CHECK-LD1R-NEXT: ld1rh { z0.s }, p0/z, [x0]1198; CHECK-LD1R-NEXT: ret1199;1200; CHECK-NO-LD1R-LABEL: dup_ld1rh_half_passthruundef_nxv4f16:1201; CHECK-NO-LD1R: // %bb.0:1202; CHECK-NO-LD1R-NEXT: ldr h0, [x0]1203; CHECK-NO-LD1R-NEXT: mov z0.h, p0/m, h01204; CHECK-NO-LD1R-NEXT: ret1205 %ld = load half, ptr %addr1206 %res = call <vscale x 4 x half> @llvm.aarch64.sve.dup.nxv4f16(<vscale x 4 x half> poison, <vscale x 4 x i1> %pg, half %ld)1207 ret <vscale x 4 x half> %res1208}1209define <vscale x 16 x i8> @dup_ld1rb_i8_passthruzero_nxv16i8(<vscale x 16 x i1> %pg, ptr %addr) {1210; CHECK-LABEL: dup_ld1rb_i8_passthruzero_nxv16i8:1211; CHECK: // %bb.0:1212; CHECK-NEXT: ld1rb { z0.b }, p0/z, [x0]1213; CHECK-NEXT: ret1214 %ld = load i8, ptr %addr1215 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.dup.nxv16i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i1> %pg, i8 %ld)1216 ret <vscale x 16 x i8> %res1217}1218define <vscale x 8 x i16> @dup_ld1rh_i16_passthruzero_nxv8i16(<vscale x 8 x i1> %pg, ptr %addr) {1219; CHECK-LABEL: dup_ld1rh_i16_passthruzero_nxv8i16:1220; CHECK: // %bb.0:1221; CHECK-NEXT: ld1rh { z0.h }, p0/z, [x0]1222; CHECK-NEXT: ret1223 %ld = load i16, ptr %addr1224 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.dup.nxv8i16(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x i1> %pg, i16 %ld)1225 ret <vscale x 8 x i16> %res1226}1227define <vscale x 4 x i32> @dup_ld1rs_i32_passthruzero_nxv4i32(<vscale x 4 x i1> %pg, ptr %addr) {1228; CHECK-LABEL: dup_ld1rs_i32_passthruzero_nxv4i32:1229; CHECK: // %bb.0:1230; CHECK-NEXT: ld1rw { z0.s }, p0/z, [x0]1231; CHECK-NEXT: ret1232 %ld = load i32, ptr %addr1233 %res = call <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x i1> %pg, i32 %ld)1234 ret <vscale x 4 x i32> %res1235}1236define <vscale x 2 x i64> @dup_ld1rd_i64_passthruzero_nxv2i64(<vscale x 2 x i1> %pg, ptr %addr) {1237; CHECK-LABEL: dup_ld1rd_i64_passthruzero_nxv2i64:1238; CHECK: // %bb.0:1239; CHECK-NEXT: ld1rd { z0.d }, p0/z, [x0]1240; CHECK-NEXT: ret1241 %ld = load i64, ptr %addr1242 %res = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, i64 %ld)1243 ret <vscale x 2 x i64> %res1244}1245define <vscale x 8 x half> @dup_ld1rh_half_passthruzero_nxv8f16(<vscale x 8 x i1> %pg, ptr %addr) {1246; CHECK-LD1R-LABEL: dup_ld1rh_half_passthruzero_nxv8f16:1247; CHECK-LD1R: // %bb.0:1248; CHECK-LD1R-NEXT: ld1rh { z0.h }, p0/z, [x0]1249; CHECK-LD1R-NEXT: ret1250;1251; CHECK-NO-LD1R-LABEL: dup_ld1rh_half_passthruzero_nxv8f16:1252; CHECK-NO-LD1R: // %bb.0:1253; CHECK-NO-LD1R-NEXT: movi v0.2d, #00000000000000001254; CHECK-NO-LD1R-NEXT: ldr h1, [x0]1255; CHECK-NO-LD1R-NEXT: mov z0.h, p0/m, h11256; CHECK-NO-LD1R-NEXT: ret1257 %ld = load half, ptr %addr1258 %res = call <vscale x 8 x half> @llvm.aarch64.sve.dup.nxv8f16(<vscale x 8 x half> zeroinitializer, <vscale x 8 x i1> %pg, half %ld)1259 ret <vscale x 8 x half> %res1260}1261define <vscale x 4 x float> @dup_ld1rs_float_passthruzero_nxv4f32(<vscale x 4 x i1> %pg, ptr %addr) {1262; CHECK-LD1R-LABEL: dup_ld1rs_float_passthruzero_nxv4f32:1263; CHECK-LD1R: // %bb.0:1264; CHECK-LD1R-NEXT: ld1rw { z0.s }, p0/z, [x0]1265; CHECK-LD1R-NEXT: ret1266;1267; CHECK-NO-LD1R-LABEL: dup_ld1rs_float_passthruzero_nxv4f32:1268; CHECK-NO-LD1R: // %bb.0:1269; CHECK-NO-LD1R-NEXT: movi v0.2d, #00000000000000001270; CHECK-NO-LD1R-NEXT: ldr s1, [x0]1271; CHECK-NO-LD1R-NEXT: mov z0.s, p0/m, s11272; CHECK-NO-LD1R-NEXT: ret1273 %ld = load float, ptr %addr1274 %res = call <vscale x 4 x float> @llvm.aarch64.sve.dup.nxv4f32(<vscale x 4 x float> zeroinitializer, <vscale x 4 x i1> %pg, float %ld)1275 ret <vscale x 4 x float> %res1276}1277define <vscale x 2 x double> @dup_ld1rd_double_passthruzero_nxv2f64(<vscale x 2 x i1> %pg, ptr %addr) {1278; CHECK-LD1R-LABEL: dup_ld1rd_double_passthruzero_nxv2f64:1279; CHECK-LD1R: // %bb.0:1280; CHECK-LD1R-NEXT: ld1rd { z0.d }, p0/z, [x0]1281; CHECK-LD1R-NEXT: ret1282;1283; CHECK-NO-LD1R-LABEL: dup_ld1rd_double_passthruzero_nxv2f64:1284; CHECK-NO-LD1R: // %bb.0:1285; CHECK-NO-LD1R-NEXT: movi v0.2d, #00000000000000001286; CHECK-NO-LD1R-NEXT: ldr d1, [x0]1287; CHECK-NO-LD1R-NEXT: mov z0.d, p0/m, d11288; CHECK-NO-LD1R-NEXT: ret1289 %ld = load double, ptr %addr1290 %res = call <vscale x 2 x double> @llvm.aarch64.sve.dup.nxv2f64(<vscale x 2 x double> zeroinitializer, <vscale x 2 x i1> %pg, double %ld)1291 ret <vscale x 2 x double> %res1292}1293define <vscale x 4 x half> @dup_ld1rh_half_passthruzero_nxv4f16(<vscale x 4 x i1> %pg, ptr %addr) {1294; CHECK-LD1R-LABEL: dup_ld1rh_half_passthruzero_nxv4f16:1295; CHECK-LD1R: // %bb.0:1296; CHECK-LD1R-NEXT: ld1rh { z0.s }, p0/z, [x0]1297; CHECK-LD1R-NEXT: ret1298;1299; CHECK-NO-LD1R-LABEL: dup_ld1rh_half_passthruzero_nxv4f16:1300; CHECK-NO-LD1R: // %bb.0:1301; CHECK-NO-LD1R-NEXT: movi v0.2d, #00000000000000001302; CHECK-NO-LD1R-NEXT: ldr h1, [x0]1303; CHECK-NO-LD1R-NEXT: mov z0.h, p0/m, h11304; CHECK-NO-LD1R-NEXT: ret1305 %ld = load half, ptr %addr1306 %res = call <vscale x 4 x half> @llvm.aarch64.sve.dup.nxv4f16(<vscale x 4 x half> zeroinitializer, <vscale x 4 x i1> %pg, half %ld)1307 ret <vscale x 4 x half> %res1308}1309define <vscale x 2 x half> @dup_ld1rh_half_passthruzero_nxv2f16(<vscale x 2 x i1> %pg, ptr %addr) {1310; CHECK-LD1R-LABEL: dup_ld1rh_half_passthruzero_nxv2f16:1311; CHECK-LD1R: // %bb.0:1312; CHECK-LD1R-NEXT: ld1rh { z0.d }, p0/z, [x0]1313; CHECK-LD1R-NEXT: ret1314;1315; CHECK-NO-LD1R-LABEL: dup_ld1rh_half_passthruzero_nxv2f16:1316; CHECK-NO-LD1R: // %bb.0:1317; CHECK-NO-LD1R-NEXT: movi v0.2d, #00000000000000001318; CHECK-NO-LD1R-NEXT: ldr h1, [x0]1319; CHECK-NO-LD1R-NEXT: mov z0.h, p0/m, h11320; CHECK-NO-LD1R-NEXT: ret1321 %ld = load half, ptr %addr1322 %res = call <vscale x 2 x half> @llvm.aarch64.sve.dup.nxv2f16(<vscale x 2 x half> zeroinitializer, <vscale x 2 x i1> %pg, half %ld)1323 ret <vscale x 2 x half> %res1324}1325define <vscale x 2 x float> @dup_ld1rs_float_passthruzero_nxv2f32(<vscale x 2 x i1> %pg, ptr %addr) {1326; CHECK-LD1R-LABEL: dup_ld1rs_float_passthruzero_nxv2f32:1327; CHECK-LD1R: // %bb.0:1328; CHECK-LD1R-NEXT: ld1rw { z0.d }, p0/z, [x0]1329; CHECK-LD1R-NEXT: ret1330;1331; CHECK-NO-LD1R-LABEL: dup_ld1rs_float_passthruzero_nxv2f32:1332; CHECK-NO-LD1R: // %bb.0:1333; CHECK-NO-LD1R-NEXT: movi v0.2d, #00000000000000001334; CHECK-NO-LD1R-NEXT: ldr s1, [x0]1335; CHECK-NO-LD1R-NEXT: mov z0.s, p0/m, s11336; CHECK-NO-LD1R-NEXT: ret1337 %ld = load float, ptr %addr1338 %res = call <vscale x 2 x float> @llvm.aarch64.sve.dup.nxv2f32(<vscale x 2 x float> zeroinitializer, <vscale x 2 x i1> %pg, float %ld)1339 ret <vscale x 2 x float> %res1340}1341define <vscale x 16 x i8> @negtest_dup_ld1rb_i8_passthru_nxv16i8(<vscale x 16 x i8> %pt, <vscale x 16 x i1> %pg, ptr %addr) {1342; CHECK-LABEL: negtest_dup_ld1rb_i8_passthru_nxv16i8:1343; CHECK: // %bb.0:1344; CHECK-NEXT: ldrb w8, [x0]1345; CHECK-NEXT: mov z0.b, p0/m, w81346; CHECK-NEXT: ret1347 %ld = load i8, ptr %addr1348 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.dup.nxv16i8(<vscale x 16 x i8> %pt, <vscale x 16 x i1> %pg, i8 %ld)1349 ret <vscale x 16 x i8> %res1350}1351define <vscale x 8 x i16> @negtest_dup_ld1rh_i16_passthru_nxv8i16(<vscale x 8 x i16> %pt, <vscale x 8 x i1> %pg, ptr %addr) {1352; CHECK-LABEL: negtest_dup_ld1rh_i16_passthru_nxv8i16:1353; CHECK: // %bb.0:1354; CHECK-NEXT: ldrh w8, [x0]1355; CHECK-NEXT: mov z0.h, p0/m, w81356; CHECK-NEXT: ret1357 %ld = load i16, ptr %addr1358 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.dup.nxv8i16(<vscale x 8 x i16> %pt, <vscale x 8 x i1> %pg, i16 %ld)1359 ret <vscale x 8 x i16> %res1360}1361define <vscale x 4 x i32> @negtest_dup_ld1rs_i32_passthru_nxv4i32(<vscale x 4 x i32> %pt, <vscale x 4 x i1> %pg, ptr %addr) {1362; CHECK-LABEL: negtest_dup_ld1rs_i32_passthru_nxv4i32:1363; CHECK: // %bb.0:1364; CHECK-NEXT: ldr w8, [x0]1365; CHECK-NEXT: mov z0.s, p0/m, w81366; CHECK-NEXT: ret1367 %ld = load i32, ptr %addr1368 %res = call <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32> %pt, <vscale x 4 x i1> %pg, i32 %ld)1369 ret <vscale x 4 x i32> %res1370}1371define <vscale x 2 x i64> @negtest_dup_ld1rd_i64_passthru_nxv2i64(<vscale x 2 x i64> %pt, <vscale x 2 x i1> %pg, ptr %addr) {1372; CHECK-LABEL: negtest_dup_ld1rd_i64_passthru_nxv2i64:1373; CHECK: // %bb.0:1374; CHECK-NEXT: ldr x8, [x0]1375; CHECK-NEXT: mov z0.d, p0/m, x81376; CHECK-NEXT: ret1377 %ld = load i64, ptr %addr1378 %res = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> %pt, <vscale x 2 x i1> %pg, i64 %ld)1379 ret <vscale x 2 x i64> %res1380}1381define <vscale x 8 x half> @negtest_dup_ld1rh_half_passthru_nxv8f16(<vscale x 8 x half> %pt, <vscale x 8 x i1> %pg, ptr %addr) {1382; CHECK-LABEL: negtest_dup_ld1rh_half_passthru_nxv8f16:1383; CHECK: // %bb.0:1384; CHECK-NEXT: ldr h1, [x0]1385; CHECK-NEXT: mov z0.h, p0/m, h11386; CHECK-NEXT: ret1387 %ld = load half, ptr %addr1388 %res = call <vscale x 8 x half> @llvm.aarch64.sve.dup.nxv8f16(<vscale x 8 x half> %pt, <vscale x 8 x i1> %pg, half %ld)1389 ret <vscale x 8 x half> %res1390}1391define <vscale x 4 x float> @negtest_dup_ld1rs_float_passthru_nxv4f32(<vscale x 4 x float> %pt, <vscale x 4 x i1> %pg, ptr %addr) {1392; CHECK-LABEL: negtest_dup_ld1rs_float_passthru_nxv4f32:1393; CHECK: // %bb.0:1394; CHECK-NEXT: ldr s1, [x0]1395; CHECK-NEXT: mov z0.s, p0/m, s11396; CHECK-NEXT: ret1397 %ld = load float, ptr %addr1398 %res = call <vscale x 4 x float> @llvm.aarch64.sve.dup.nxv4f32(<vscale x 4 x float> %pt, <vscale x 4 x i1> %pg, float %ld)1399 ret <vscale x 4 x float> %res1400}1401define <vscale x 2 x double> @negtest_dup_ld1rd_double_passthru_nxv2f64(<vscale x 2 x double> %pt, <vscale x 2 x i1> %pg, ptr %addr) {1402; CHECK-LABEL: negtest_dup_ld1rd_double_passthru_nxv2f64:1403; CHECK: // %bb.0:1404; CHECK-NEXT: ldr d1, [x0]1405; CHECK-NEXT: mov z0.d, p0/m, d11406; CHECK-NEXT: ret1407 %ld = load double, ptr %addr1408 %res = call <vscale x 2 x double> @llvm.aarch64.sve.dup.nxv2f64(<vscale x 2 x double> %pt, <vscale x 2 x i1> %pg, double %ld)1409 ret <vscale x 2 x double> %res1410}1411 1412 1413; Check that a load consumed by a scalable splat prefers a replicating load.1414define ptr @avoid_preindex_load(ptr %src, ptr %out) {1415; CHECK-LABEL: avoid_preindex_load:1416; CHECK: // %bb.0:1417; CHECK-NEXT: ptrue p0.d1418; CHECK-NEXT: ld1rsb { z0.d }, p0/z, [x0, #1]1419; CHECK-NEXT: add x0, x0, #11420; CHECK-NEXT: str z0, [x1]1421; CHECK-NEXT: ret1422 %ptr = getelementptr inbounds i8, ptr %src, i64 11423 %tmp = load i8, ptr %ptr, align 41424 %ext = sext i8 %tmp to i641425 %ins = insertelement <vscale x 2 x i64> poison, i64 %ext, i32 01426 %dup = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer1427 store <vscale x 2 x i64> %dup, ptr %out1428 ret ptr %ptr1429}1430 1431; Check that a load consumed by a scalable splat prefers a replicating1432; load over a pre-indexed load.1433define ptr @avoid_preindex_load_dup(ptr %src, <vscale x 2 x i1> %pg, ptr %out) {1434; CHECK-LABEL: avoid_preindex_load_dup:1435; CHECK: // %bb.0:1436; CHECK-NEXT: ld1rsb { z0.d }, p0/z, [x0, #1]1437; CHECK-NEXT: add x0, x0, #11438; CHECK-NEXT: str z0, [x1]1439; CHECK-NEXT: ret1440 %ptr = getelementptr inbounds i8, ptr %src, i64 11441 %tmp = load i8, ptr %ptr, align 41442 %ext = sext i8 %tmp to i641443 %dup = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> poison, <vscale x 2 x i1> %pg, i64 %ext)1444 store <vscale x 2 x i64> %dup, ptr %out1445 ret ptr %ptr1446}1447 1448; Same as avoid_preindex_load_dup, but with zero passthru.1449define ptr @avoid_preindex_load_dup_passthru_zero(ptr %src, <vscale x 2 x i1> %pg, ptr %out) {1450; CHECK-LABEL: avoid_preindex_load_dup_passthru_zero:1451; CHECK: // %bb.0:1452; CHECK-NEXT: ld1rsb { z0.d }, p0/z, [x0, #1]1453; CHECK-NEXT: add x0, x0, #11454; CHECK-NEXT: str z0, [x1]1455; CHECK-NEXT: ret1456 %ptr = getelementptr inbounds i8, ptr %src, i64 11457 %tmp = load i8, ptr %ptr, align 41458 %ext = sext i8 %tmp to i641459 %dup = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> %pg, i64 %ext)1460 store <vscale x 2 x i64> %dup, ptr %out1461 ret ptr %ptr1462}1463 1464; If a dup has a non-undef passthru, stick with the pre-indexed load.1465define ptr @preindex_load_dup_passthru(<vscale x 2 x i64> %passthru, ptr %src, <vscale x 2 x i1> %pg, ptr %out) {1466; CHECK-LABEL: preindex_load_dup_passthru:1467; CHECK: // %bb.0:1468; CHECK-NEXT: ldrsb x8, [x0, #1]!1469; CHECK-NEXT: mov z0.d, p0/m, x81470; CHECK-NEXT: str z0, [x1]1471; CHECK-NEXT: ret1472 %ptr = getelementptr inbounds i8, ptr %src, i64 11473 %tmp = load i8, ptr %ptr, align 41474 %ext = sext i8 %tmp to i641475 %dup = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64> %passthru, <vscale x 2 x i1> %pg, i64 %ext)1476 store <vscale x 2 x i64> %dup, ptr %out1477 ret ptr %ptr1478}1479 1480; Show that a second user of the load prevents the replicating load1481; check which would ordinarily inhibit indexed loads from firing.1482define ptr @preidx8sext64_instead_of_ld1r(ptr %src, ptr %out, ptr %dst) {1483; CHECK-LABEL: preidx8sext64_instead_of_ld1r:1484; CHECK: // %bb.0:1485; CHECK-NEXT: ldrsb x8, [x0, #1]!1486; CHECK-NEXT: mov z0.d, x81487; CHECK-NEXT: str z0, [x1]1488; CHECK-NEXT: str x8, [x2]1489; CHECK-NEXT: ret1490 %ptr = getelementptr inbounds i8, ptr %src, i64 11491 %tmp = load i8, ptr %ptr, align 41492 %ext = sext i8 %tmp to i641493 %ins = insertelement <vscale x 2 x i64> poison, i64 %ext, i32 01494 %dup = shufflevector <vscale x 2 x i64> %ins, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer1495 store <vscale x 2 x i64> %dup, ptr %out1496 store i64 %ext, ptr %dst1497 ret ptr %ptr1498}1499 1500 1501declare <vscale x 16 x i8> @llvm.aarch64.sve.dupq.lane.nxv16i8(<vscale x 16 x i8>, i64)1502declare <vscale x 8 x i16> @llvm.aarch64.sve.dupq.lane.nxv8i16(<vscale x 8 x i16>, i64)1503declare <vscale x 4 x i32> @llvm.aarch64.sve.dupq.lane.nxv4i32(<vscale x 4 x i32>, i64)1504declare <vscale x 2 x i64> @llvm.aarch64.sve.dupq.lane.nxv2i64(<vscale x 2 x i64>, i64)1505declare <vscale x 8 x half> @llvm.aarch64.sve.dupq.lane.nxv8f16(<vscale x 8 x half>, i64)1506declare <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat>, i64)1507declare <vscale x 4 x float> @llvm.aarch64.sve.dupq.lane.nxv4f32(<vscale x 4 x float>, i64)1508declare <vscale x 2 x double> @llvm.aarch64.sve.dupq.lane.nxv2f64(<vscale x 2 x double>, i64)1509 1510declare <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double>, <2 x double>, i64)1511declare <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float>, <4 x float>, i64)1512declare <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v8f16(<vscale x 8 x half>, <8 x half>, i64)1513declare <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64>, <2 x i64>, i64)1514declare <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32>, <4 x i32>, i64)1515declare <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v8i16(<vscale x 8 x i16>, <8 x i16>, i64)1516declare <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8>, <16 x i8>, i64)1517declare <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat>, <8 x bfloat>, i64)1518 1519declare <vscale x 16 x i8> @llvm.aarch64.sve.dup.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, i8)1520declare <vscale x 8 x i16> @llvm.aarch64.sve.dup.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, i16)1521declare <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32)1522declare <vscale x 2 x i64> @llvm.aarch64.sve.dup.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, i64)1523declare <vscale x 8 x half> @llvm.aarch64.sve.dup.nxv8f16(<vscale x 8 x half>, <vscale x 8 x i1>, half)1524declare <vscale x 4 x float> @llvm.aarch64.sve.dup.nxv4f32(<vscale x 4 x float>, <vscale x 4 x i1>, float)1525declare <vscale x 2 x double> @llvm.aarch64.sve.dup.nxv2f64(<vscale x 2 x double>, <vscale x 2 x i1>, double)1526declare <vscale x 4 x half> @llvm.aarch64.sve.dup.nxv4f16(<vscale x 4 x half>, <vscale x 4 x i1>, half)1527declare <vscale x 2 x half> @llvm.aarch64.sve.dup.nxv2f16(<vscale x 2 x half>, <vscale x 2 x i1>, half)1528declare <vscale x 2 x float> @llvm.aarch64.sve.dup.nxv2f32(<vscale x 2 x float>, <vscale x 2 x i1>, float)1529 1530 1531attributes #0 = { "target-features"="+sve,+bf16" }1532attributes #1 = { "target-cpu"="neoverse-v1" }1533