1283 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16,+bf16,+sve | FileCheck %s3 4define <8 x i8> @loadv8i8(ptr %p) {5; CHECK-LABEL: loadv8i8:6; CHECK: // %bb.0:7; CHECK-NEXT: ldr b0, [x0]8; CHECK-NEXT: ret9 %l = load i8, ptr %p10 %v = insertelement <8 x i8> zeroinitializer, i8 %l, i32 011 ret <8 x i8> %v12}13 14define <16 x i8> @loadv16i8(ptr %p) {15; CHECK-LABEL: loadv16i8:16; CHECK: // %bb.0:17; CHECK-NEXT: ldr b0, [x0]18; CHECK-NEXT: ret19 %l = load i8, ptr %p20 %v = insertelement <16 x i8> zeroinitializer, i8 %l, i32 021 ret <16 x i8> %v22}23 24define <4 x i16> @loadv4i16(ptr %p) {25; CHECK-LABEL: loadv4i16:26; CHECK: // %bb.0:27; CHECK-NEXT: ldr h0, [x0]28; CHECK-NEXT: ret29 %l = load i16, ptr %p30 %v = insertelement <4 x i16> zeroinitializer, i16 %l, i32 031 ret <4 x i16> %v32}33 34define <8 x i16> @loadv8i16(ptr %p) {35; CHECK-LABEL: loadv8i16:36; CHECK: // %bb.0:37; CHECK-NEXT: ldr h0, [x0]38; CHECK-NEXT: ret39 %l = load i16, ptr %p40 %v = insertelement <8 x i16> zeroinitializer, i16 %l, i32 041 ret <8 x i16> %v42}43 44define <2 x i32> @loadv2i32(ptr %p) {45; CHECK-LABEL: loadv2i32:46; CHECK: // %bb.0:47; CHECK-NEXT: ldr s0, [x0]48; CHECK-NEXT: ret49 %l = load i32, ptr %p50 %v = insertelement <2 x i32> zeroinitializer, i32 %l, i32 051 ret <2 x i32> %v52}53 54define <4 x i32> @loadv4i32(ptr %p) {55; CHECK-LABEL: loadv4i32:56; CHECK: // %bb.0:57; CHECK-NEXT: ldr s0, [x0]58; CHECK-NEXT: ret59 %l = load i32, ptr %p60 %v = insertelement <4 x i32> zeroinitializer, i32 %l, i32 061 ret <4 x i32> %v62}63 64define <2 x i64> @loadv2i64(ptr %p) {65; CHECK-LABEL: loadv2i64:66; CHECK: // %bb.0:67; CHECK-NEXT: ldr d0, [x0]68; CHECK-NEXT: ret69 %l = load i64, ptr %p70 %v = insertelement <2 x i64> zeroinitializer, i64 %l, i32 071 ret <2 x i64> %v72}73 74 75define <4 x half> @loadv4f16(ptr %p) {76; CHECK-LABEL: loadv4f16:77; CHECK: // %bb.0:78; CHECK-NEXT: ldr h0, [x0]79; CHECK-NEXT: ret80 %l = load half, ptr %p81 %v = insertelement <4 x half> zeroinitializer, half %l, i32 082 ret <4 x half> %v83}84 85define <8 x half> @loadv8f16(ptr %p) {86; CHECK-LABEL: loadv8f16:87; CHECK: // %bb.0:88; CHECK-NEXT: ldr h0, [x0]89; CHECK-NEXT: ret90 %l = load half, ptr %p91 %v = insertelement <8 x half> zeroinitializer, half %l, i32 092 ret <8 x half> %v93}94 95define <4 x bfloat> @loadv4bf16(ptr %p) {96; CHECK-LABEL: loadv4bf16:97; CHECK: // %bb.0:98; CHECK-NEXT: ldr h0, [x0]99; CHECK-NEXT: ret100 %l = load bfloat, ptr %p101 %v = insertelement <4 x bfloat> zeroinitializer, bfloat %l, i32 0102 ret <4 x bfloat> %v103}104 105define <8 x bfloat> @loadv8bf16(ptr %p) {106; CHECK-LABEL: loadv8bf16:107; CHECK: // %bb.0:108; CHECK-NEXT: ldr h0, [x0]109; CHECK-NEXT: ret110 %l = load bfloat, ptr %p111 %v = insertelement <8 x bfloat> zeroinitializer, bfloat %l, i32 0112 ret <8 x bfloat> %v113}114 115define <2 x float> @loadv2f32(ptr %p) {116; CHECK-LABEL: loadv2f32:117; CHECK: // %bb.0:118; CHECK-NEXT: ldr s0, [x0]119; CHECK-NEXT: ret120 %l = load float, ptr %p121 %v = insertelement <2 x float> zeroinitializer, float %l, i32 0122 ret <2 x float> %v123}124 125define <4 x float> @loadv4f32(ptr %p) {126; CHECK-LABEL: loadv4f32:127; CHECK: // %bb.0:128; CHECK-NEXT: ldr s0, [x0]129; CHECK-NEXT: ret130 %l = load float, ptr %p131 %v = insertelement <4 x float> zeroinitializer, float %l, i32 0132 ret <4 x float> %v133}134 135define <2 x double> @loadv2f64(ptr %p) {136; CHECK-LABEL: loadv2f64:137; CHECK: // %bb.0:138; CHECK-NEXT: ldr d0, [x0]139; CHECK-NEXT: ret140 %l = load double, ptr %p141 %v = insertelement <2 x double> zeroinitializer, double %l, i32 0142 ret <2 x double> %v143}144 145 146; Unscaled147 148define <8 x i8> @loadv8i8_offset(ptr %p) {149; CHECK-LABEL: loadv8i8_offset:150; CHECK: // %bb.0:151; CHECK-NEXT: ldr b0, [x0, #1]152; CHECK-NEXT: ret153 %g = getelementptr inbounds i8, ptr %p, i64 1154 %l = load i8, ptr %g155 %v = insertelement <8 x i8> zeroinitializer, i8 %l, i32 0156 ret <8 x i8> %v157}158 159define <16 x i8> @loadv16i8_offset(ptr %p) {160; CHECK-LABEL: loadv16i8_offset:161; CHECK: // %bb.0:162; CHECK-NEXT: ldr b0, [x0, #1]163; CHECK-NEXT: ret164 %g = getelementptr inbounds i8, ptr %p, i64 1165 %l = load i8, ptr %g166 %v = insertelement <16 x i8> zeroinitializer, i8 %l, i32 0167 ret <16 x i8> %v168}169 170define <4 x i16> @loadv4i16_offset(ptr %p) {171; CHECK-LABEL: loadv4i16_offset:172; CHECK: // %bb.0:173; CHECK-NEXT: ldur h0, [x0, #1]174; CHECK-NEXT: ret175 %g = getelementptr inbounds i8, ptr %p, i64 1176 %l = load i16, ptr %g177 %v = insertelement <4 x i16> zeroinitializer, i16 %l, i32 0178 ret <4 x i16> %v179}180 181define <8 x i16> @loadv8i16_offset(ptr %p) {182; CHECK-LABEL: loadv8i16_offset:183; CHECK: // %bb.0:184; CHECK-NEXT: ldur h0, [x0, #1]185; CHECK-NEXT: ret186 %g = getelementptr inbounds i8, ptr %p, i64 1187 %l = load i16, ptr %g188 %v = insertelement <8 x i16> zeroinitializer, i16 %l, i32 0189 ret <8 x i16> %v190}191 192define <2 x i32> @loadv2i32_offset(ptr %p) {193; CHECK-LABEL: loadv2i32_offset:194; CHECK: // %bb.0:195; CHECK-NEXT: ldur s0, [x0, #1]196; CHECK-NEXT: ret197 %g = getelementptr inbounds i8, ptr %p, i64 1198 %l = load i32, ptr %g199 %v = insertelement <2 x i32> zeroinitializer, i32 %l, i32 0200 ret <2 x i32> %v201}202 203define <4 x i32> @loadv4i32_offset(ptr %p) {204; CHECK-LABEL: loadv4i32_offset:205; CHECK: // %bb.0:206; CHECK-NEXT: ldur s0, [x0, #1]207; CHECK-NEXT: ret208 %g = getelementptr inbounds i8, ptr %p, i64 1209 %l = load i32, ptr %g210 %v = insertelement <4 x i32> zeroinitializer, i32 %l, i32 0211 ret <4 x i32> %v212}213 214define <2 x i64> @loadv2i64_offset(ptr %p) {215; CHECK-LABEL: loadv2i64_offset:216; CHECK: // %bb.0:217; CHECK-NEXT: ldur d0, [x0, #1]218; CHECK-NEXT: ret219 %g = getelementptr inbounds i8, ptr %p, i64 1220 %l = load i64, ptr %g221 %v = insertelement <2 x i64> zeroinitializer, i64 %l, i32 0222 ret <2 x i64> %v223}224 225 226define <4 x half> @loadv4f16_offset(ptr %p) {227; CHECK-LABEL: loadv4f16_offset:228; CHECK: // %bb.0:229; CHECK-NEXT: ldur h0, [x0, #1]230; CHECK-NEXT: ret231 %g = getelementptr inbounds i8, ptr %p, i64 1232 %l = load half, ptr %g233 %v = insertelement <4 x half> zeroinitializer, half %l, i32 0234 ret <4 x half> %v235}236 237define <8 x half> @loadv8f16_offset(ptr %p) {238; CHECK-LABEL: loadv8f16_offset:239; CHECK: // %bb.0:240; CHECK-NEXT: ldur h0, [x0, #1]241; CHECK-NEXT: ret242 %g = getelementptr inbounds i8, ptr %p, i64 1243 %l = load half, ptr %g244 %v = insertelement <8 x half> zeroinitializer, half %l, i32 0245 ret <8 x half> %v246}247 248define <4 x bfloat> @loadv4bf16_offset(ptr %p) {249; CHECK-LABEL: loadv4bf16_offset:250; CHECK: // %bb.0:251; CHECK-NEXT: ldur h0, [x0, #1]252; CHECK-NEXT: ret253 %g = getelementptr inbounds i8, ptr %p, i64 1254 %l = load bfloat, ptr %g255 %v = insertelement <4 x bfloat> zeroinitializer, bfloat %l, i32 0256 ret <4 x bfloat> %v257}258 259define <8 x bfloat> @loadv8bf16_offset(ptr %p) {260; CHECK-LABEL: loadv8bf16_offset:261; CHECK: // %bb.0:262; CHECK-NEXT: ldur h0, [x0, #1]263; CHECK-NEXT: ret264 %g = getelementptr inbounds i8, ptr %p, i64 1265 %l = load bfloat, ptr %g266 %v = insertelement <8 x bfloat> zeroinitializer, bfloat %l, i32 0267 ret <8 x bfloat> %v268}269 270define <2 x float> @loadv2f32_offset(ptr %p) {271; CHECK-LABEL: loadv2f32_offset:272; CHECK: // %bb.0:273; CHECK-NEXT: ldur s0, [x0, #1]274; CHECK-NEXT: ret275 %g = getelementptr inbounds i8, ptr %p, i64 1276 %l = load float, ptr %g277 %v = insertelement <2 x float> zeroinitializer, float %l, i32 0278 ret <2 x float> %v279}280 281define <4 x float> @loadv4f32_offset(ptr %p) {282; CHECK-LABEL: loadv4f32_offset:283; CHECK: // %bb.0:284; CHECK-NEXT: ldur s0, [x0, #1]285; CHECK-NEXT: ret286 %g = getelementptr inbounds i8, ptr %p, i64 1287 %l = load float, ptr %g288 %v = insertelement <4 x float> zeroinitializer, float %l, i32 0289 ret <4 x float> %v290}291 292define <2 x double> @loadv2f64_offset(ptr %p) {293; CHECK-LABEL: loadv2f64_offset:294; CHECK: // %bb.0:295; CHECK-NEXT: ldur d0, [x0, #1]296; CHECK-NEXT: ret297 %g = getelementptr inbounds i8, ptr %p, i64 1298 %l = load double, ptr %g299 %v = insertelement <2 x double> zeroinitializer, double %l, i32 0300 ret <2 x double> %v301}302 303 304define <8 x i8> @loadv8i8_noffset(ptr %p) {305; CHECK-LABEL: loadv8i8_noffset:306; CHECK: // %bb.0:307; CHECK-NEXT: ldur b0, [x0, #-1]308; CHECK-NEXT: ret309 %g = getelementptr inbounds i8, ptr %p, i64 -1310 %l = load i8, ptr %g311 %v = insertelement <8 x i8> zeroinitializer, i8 %l, i32 0312 ret <8 x i8> %v313}314 315define <16 x i8> @loadv16i8_noffset(ptr %p) {316; CHECK-LABEL: loadv16i8_noffset:317; CHECK: // %bb.0:318; CHECK-NEXT: ldur b0, [x0, #-1]319; CHECK-NEXT: ret320 %g = getelementptr inbounds i8, ptr %p, i64 -1321 %l = load i8, ptr %g322 %v = insertelement <16 x i8> zeroinitializer, i8 %l, i32 0323 ret <16 x i8> %v324}325 326define <4 x i16> @loadv4i16_noffset(ptr %p) {327; CHECK-LABEL: loadv4i16_noffset:328; CHECK: // %bb.0:329; CHECK-NEXT: ldur h0, [x0, #-1]330; CHECK-NEXT: ret331 %g = getelementptr inbounds i8, ptr %p, i64 -1332 %l = load i16, ptr %g333 %v = insertelement <4 x i16> zeroinitializer, i16 %l, i32 0334 ret <4 x i16> %v335}336 337define <8 x i16> @loadv8i16_noffset(ptr %p) {338; CHECK-LABEL: loadv8i16_noffset:339; CHECK: // %bb.0:340; CHECK-NEXT: ldur h0, [x0, #-1]341; CHECK-NEXT: ret342 %g = getelementptr inbounds i8, ptr %p, i64 -1343 %l = load i16, ptr %g344 %v = insertelement <8 x i16> zeroinitializer, i16 %l, i32 0345 ret <8 x i16> %v346}347 348define <2 x i32> @loadv2i32_noffset(ptr %p) {349; CHECK-LABEL: loadv2i32_noffset:350; CHECK: // %bb.0:351; CHECK-NEXT: ldur s0, [x0, #-1]352; CHECK-NEXT: ret353 %g = getelementptr inbounds i8, ptr %p, i64 -1354 %l = load i32, ptr %g355 %v = insertelement <2 x i32> zeroinitializer, i32 %l, i32 0356 ret <2 x i32> %v357}358 359define <4 x i32> @loadv4i32_noffset(ptr %p) {360; CHECK-LABEL: loadv4i32_noffset:361; CHECK: // %bb.0:362; CHECK-NEXT: ldur s0, [x0, #-1]363; CHECK-NEXT: ret364 %g = getelementptr inbounds i8, ptr %p, i64 -1365 %l = load i32, ptr %g366 %v = insertelement <4 x i32> zeroinitializer, i32 %l, i32 0367 ret <4 x i32> %v368}369 370define <2 x i64> @loadv2i64_noffset(ptr %p) {371; CHECK-LABEL: loadv2i64_noffset:372; CHECK: // %bb.0:373; CHECK-NEXT: ldur d0, [x0, #-1]374; CHECK-NEXT: ret375 %g = getelementptr inbounds i8, ptr %p, i64 -1376 %l = load i64, ptr %g377 %v = insertelement <2 x i64> zeroinitializer, i64 %l, i32 0378 ret <2 x i64> %v379}380 381define <4 x half> @loadv4f16_noffset(ptr %p) {382; CHECK-LABEL: loadv4f16_noffset:383; CHECK: // %bb.0:384; CHECK-NEXT: ldur h0, [x0, #-1]385; CHECK-NEXT: ret386 %g = getelementptr inbounds i8, ptr %p, i64 -1387 %l = load half, ptr %g388 %v = insertelement <4 x half> zeroinitializer, half %l, i32 0389 ret <4 x half> %v390}391 392define <8 x half> @loadv8f16_noffset(ptr %p) {393; CHECK-LABEL: loadv8f16_noffset:394; CHECK: // %bb.0:395; CHECK-NEXT: ldur h0, [x0, #-1]396; CHECK-NEXT: ret397 %g = getelementptr inbounds i8, ptr %p, i64 -1398 %l = load half, ptr %g399 %v = insertelement <8 x half> zeroinitializer, half %l, i32 0400 ret <8 x half> %v401}402 403define <4 x bfloat> @loadv4bf16_noffset(ptr %p) {404; CHECK-LABEL: loadv4bf16_noffset:405; CHECK: // %bb.0:406; CHECK-NEXT: ldur h0, [x0, #-1]407; CHECK-NEXT: ret408 %g = getelementptr inbounds i8, ptr %p, i64 -1409 %l = load bfloat, ptr %g410 %v = insertelement <4 x bfloat> zeroinitializer, bfloat %l, i32 0411 ret <4 x bfloat> %v412}413 414define <8 x bfloat> @loadv8bf16_noffset(ptr %p) {415; CHECK-LABEL: loadv8bf16_noffset:416; CHECK: // %bb.0:417; CHECK-NEXT: ldur h0, [x0, #-1]418; CHECK-NEXT: ret419 %g = getelementptr inbounds i8, ptr %p, i64 -1420 %l = load bfloat, ptr %g421 %v = insertelement <8 x bfloat> zeroinitializer, bfloat %l, i32 0422 ret <8 x bfloat> %v423}424 425define <2 x float> @loadv2f32_noffset(ptr %p) {426; CHECK-LABEL: loadv2f32_noffset:427; CHECK: // %bb.0:428; CHECK-NEXT: ldur s0, [x0, #-1]429; CHECK-NEXT: ret430 %g = getelementptr inbounds i8, ptr %p, i64 -1431 %l = load float, ptr %g432 %v = insertelement <2 x float> zeroinitializer, float %l, i32 0433 ret <2 x float> %v434}435 436define <4 x float> @loadv4f32_noffset(ptr %p) {437; CHECK-LABEL: loadv4f32_noffset:438; CHECK: // %bb.0:439; CHECK-NEXT: ldur s0, [x0, #-1]440; CHECK-NEXT: ret441 %g = getelementptr inbounds i8, ptr %p, i64 -1442 %l = load float, ptr %g443 %v = insertelement <4 x float> zeroinitializer, float %l, i32 0444 ret <4 x float> %v445}446 447define <2 x double> @loadv2f64_noffset(ptr %p) {448; CHECK-LABEL: loadv2f64_noffset:449; CHECK: // %bb.0:450; CHECK-NEXT: ldur d0, [x0, #-1]451; CHECK-NEXT: ret452 %g = getelementptr inbounds i8, ptr %p, i64 -1453 %l = load double, ptr %g454 %v = insertelement <2 x double> zeroinitializer, double %l, i32 0455 ret <2 x double> %v456}457 458 459; ROW addressing modes460 461define <8 x i8> @loadv8i8_roW(ptr %p, i32 %o) {462; CHECK-LABEL: loadv8i8_roW:463; CHECK: // %bb.0:464; CHECK-NEXT: ldr b0, [x0, w1, sxtw]465; CHECK-NEXT: ret466 %g = getelementptr inbounds i8, ptr %p, i32 %o467 %l = load i8, ptr %g468 %v = insertelement <8 x i8> zeroinitializer, i8 %l, i32 0469 ret <8 x i8> %v470}471 472define <16 x i8> @loadv16i8_roW(ptr %p, i32 %o) {473; CHECK-LABEL: loadv16i8_roW:474; CHECK: // %bb.0:475; CHECK-NEXT: ldr b0, [x0, w1, sxtw]476; CHECK-NEXT: ret477 %g = getelementptr inbounds i8, ptr %p, i32 %o478 %l = load i8, ptr %g479 %v = insertelement <16 x i8> zeroinitializer, i8 %l, i32 0480 ret <16 x i8> %v481}482 483define <4 x i16> @loadv4i16_roW(ptr %p, i32 %o) {484; CHECK-LABEL: loadv4i16_roW:485; CHECK: // %bb.0:486; CHECK-NEXT: ldr h0, [x0, w1, sxtw #1]487; CHECK-NEXT: ret488 %g = getelementptr inbounds i16, ptr %p, i32 %o489 %l = load i16, ptr %g490 %v = insertelement <4 x i16> zeroinitializer, i16 %l, i32 0491 ret <4 x i16> %v492}493 494define <8 x i16> @loadv8i16_roW(ptr %p, i32 %o) {495; CHECK-LABEL: loadv8i16_roW:496; CHECK: // %bb.0:497; CHECK-NEXT: ldr h0, [x0, w1, sxtw #1]498; CHECK-NEXT: ret499 %g = getelementptr inbounds i16, ptr %p, i32 %o500 %l = load i16, ptr %g501 %v = insertelement <8 x i16> zeroinitializer, i16 %l, i32 0502 ret <8 x i16> %v503}504 505define <2 x i32> @loadv2i32_roW(ptr %p, i32 %o) {506; CHECK-LABEL: loadv2i32_roW:507; CHECK: // %bb.0:508; CHECK-NEXT: ldr s0, [x0, w1, sxtw #2]509; CHECK-NEXT: ret510 %g = getelementptr inbounds i32, ptr %p, i32 %o511 %l = load i32, ptr %g512 %v = insertelement <2 x i32> zeroinitializer, i32 %l, i32 0513 ret <2 x i32> %v514}515 516define <4 x i32> @loadv4i32_roW(ptr %p, i32 %o) {517; CHECK-LABEL: loadv4i32_roW:518; CHECK: // %bb.0:519; CHECK-NEXT: ldr s0, [x0, w1, sxtw #2]520; CHECK-NEXT: ret521 %g = getelementptr inbounds i32, ptr %p, i32 %o522 %l = load i32, ptr %g523 %v = insertelement <4 x i32> zeroinitializer, i32 %l, i32 0524 ret <4 x i32> %v525}526 527define <2 x i64> @loadv2i64_roW(ptr %p, i32 %o) {528; CHECK-LABEL: loadv2i64_roW:529; CHECK: // %bb.0:530; CHECK-NEXT: ldr d0, [x0, w1, sxtw #3]531; CHECK-NEXT: ret532 %g = getelementptr inbounds i64, ptr %p, i32 %o533 %l = load i64, ptr %g534 %v = insertelement <2 x i64> zeroinitializer, i64 %l, i32 0535 ret <2 x i64> %v536}537 538define <4 x half> @loadv4f16_roW(ptr %p, i32 %o) {539; CHECK-LABEL: loadv4f16_roW:540; CHECK: // %bb.0:541; CHECK-NEXT: ldr h0, [x0, w1, sxtw #1]542; CHECK-NEXT: ret543 %g = getelementptr inbounds half, ptr %p, i32 %o544 %l = load half, ptr %g545 %v = insertelement <4 x half> zeroinitializer, half %l, i32 0546 ret <4 x half> %v547}548 549define <8 x half> @loadv8f16_roW(ptr %p, i32 %o) {550; CHECK-LABEL: loadv8f16_roW:551; CHECK: // %bb.0:552; CHECK-NEXT: ldr h0, [x0, w1, sxtw #1]553; CHECK-NEXT: ret554 %g = getelementptr inbounds half, ptr %p, i32 %o555 %l = load half, ptr %g556 %v = insertelement <8 x half> zeroinitializer, half %l, i32 0557 ret <8 x half> %v558}559 560define <4 x bfloat> @loadv4bf16_roW(ptr %p, i32 %o) {561; CHECK-LABEL: loadv4bf16_roW:562; CHECK: // %bb.0:563; CHECK-NEXT: ldr h0, [x0, w1, sxtw #1]564; CHECK-NEXT: ret565 %g = getelementptr inbounds bfloat, ptr %p, i32 %o566 %l = load bfloat, ptr %g567 %v = insertelement <4 x bfloat> zeroinitializer, bfloat %l, i32 0568 ret <4 x bfloat> %v569}570 571define <8 x bfloat> @loadv8bf16_roW(ptr %p, i32 %o) {572; CHECK-LABEL: loadv8bf16_roW:573; CHECK: // %bb.0:574; CHECK-NEXT: ldr h0, [x0, w1, sxtw #1]575; CHECK-NEXT: ret576 %g = getelementptr inbounds bfloat, ptr %p, i32 %o577 %l = load bfloat, ptr %g578 %v = insertelement <8 x bfloat> zeroinitializer, bfloat %l, i32 0579 ret <8 x bfloat> %v580}581 582define <2 x float> @loadv2f32_roW(ptr %p, i32 %o) {583; CHECK-LABEL: loadv2f32_roW:584; CHECK: // %bb.0:585; CHECK-NEXT: ldr s0, [x0, w1, sxtw #2]586; CHECK-NEXT: ret587 %g = getelementptr inbounds float, ptr %p, i32 %o588 %l = load float, ptr %g589 %v = insertelement <2 x float> zeroinitializer, float %l, i32 0590 ret <2 x float> %v591}592 593define <4 x float> @loadv4f32_roW(ptr %p, i32 %o) {594; CHECK-LABEL: loadv4f32_roW:595; CHECK: // %bb.0:596; CHECK-NEXT: ldr s0, [x0, w1, sxtw #2]597; CHECK-NEXT: ret598 %g = getelementptr inbounds float, ptr %p, i32 %o599 %l = load float, ptr %g600 %v = insertelement <4 x float> zeroinitializer, float %l, i32 0601 ret <4 x float> %v602}603 604define <2 x double> @loadv2f64_roW(ptr %p, i32 %o) {605; CHECK-LABEL: loadv2f64_roW:606; CHECK: // %bb.0:607; CHECK-NEXT: ldr d0, [x0, w1, sxtw #3]608; CHECK-NEXT: ret609 %g = getelementptr inbounds double, ptr %p, i32 %o610 %l = load double, ptr %g611 %v = insertelement <2 x double> zeroinitializer, double %l, i32 0612 ret <2 x double> %v613}614 615 616; roX617 618define <8 x i8> @loadv8i8_roX(ptr %p, i64 %o) {619; CHECK-LABEL: loadv8i8_roX:620; CHECK: // %bb.0:621; CHECK-NEXT: ldr b0, [x0, x1]622; CHECK-NEXT: ret623 %g = getelementptr inbounds i8, ptr %p, i64 %o624 %l = load i8, ptr %g625 %v = insertelement <8 x i8> zeroinitializer, i8 %l, i32 0626 ret <8 x i8> %v627}628 629define <16 x i8> @loadv16i8_roX(ptr %p, i64 %o) {630; CHECK-LABEL: loadv16i8_roX:631; CHECK: // %bb.0:632; CHECK-NEXT: ldr b0, [x0, x1]633; CHECK-NEXT: ret634 %g = getelementptr inbounds i8, ptr %p, i64 %o635 %l = load i8, ptr %g636 %v = insertelement <16 x i8> zeroinitializer, i8 %l, i32 0637 ret <16 x i8> %v638}639 640define <4 x i16> @loadv4i16_roX(ptr %p, i64 %o) {641; CHECK-LABEL: loadv4i16_roX:642; CHECK: // %bb.0:643; CHECK-NEXT: ldr h0, [x0, x1, lsl #1]644; CHECK-NEXT: ret645 %g = getelementptr inbounds i16, ptr %p, i64 %o646 %l = load i16, ptr %g647 %v = insertelement <4 x i16> zeroinitializer, i16 %l, i32 0648 ret <4 x i16> %v649}650 651define <8 x i16> @loadv8i16_roX(ptr %p, i64 %o) {652; CHECK-LABEL: loadv8i16_roX:653; CHECK: // %bb.0:654; CHECK-NEXT: ldr h0, [x0, x1, lsl #1]655; CHECK-NEXT: ret656 %g = getelementptr inbounds i16, ptr %p, i64 %o657 %l = load i16, ptr %g658 %v = insertelement <8 x i16> zeroinitializer, i16 %l, i32 0659 ret <8 x i16> %v660}661 662define <2 x i32> @loadv2i32_roX(ptr %p, i64 %o) {663; CHECK-LABEL: loadv2i32_roX:664; CHECK: // %bb.0:665; CHECK-NEXT: ldr s0, [x0, x1, lsl #2]666; CHECK-NEXT: ret667 %g = getelementptr inbounds i32, ptr %p, i64 %o668 %l = load i32, ptr %g669 %v = insertelement <2 x i32> zeroinitializer, i32 %l, i32 0670 ret <2 x i32> %v671}672 673define <4 x i32> @loadv4i32_roX(ptr %p, i64 %o) {674; CHECK-LABEL: loadv4i32_roX:675; CHECK: // %bb.0:676; CHECK-NEXT: ldr s0, [x0, x1, lsl #2]677; CHECK-NEXT: ret678 %g = getelementptr inbounds i32, ptr %p, i64 %o679 %l = load i32, ptr %g680 %v = insertelement <4 x i32> zeroinitializer, i32 %l, i32 0681 ret <4 x i32> %v682}683 684define <2 x i64> @loadv2i64_roX(ptr %p, i64 %o) {685; CHECK-LABEL: loadv2i64_roX:686; CHECK: // %bb.0:687; CHECK-NEXT: ldr d0, [x0, x1, lsl #3]688; CHECK-NEXT: ret689 %g = getelementptr inbounds i64, ptr %p, i64 %o690 %l = load i64, ptr %g691 %v = insertelement <2 x i64> zeroinitializer, i64 %l, i32 0692 ret <2 x i64> %v693}694 695define <4 x half> @loadv4f16_roX(ptr %p, i64 %o) {696; CHECK-LABEL: loadv4f16_roX:697; CHECK: // %bb.0:698; CHECK-NEXT: ldr h0, [x0, x1, lsl #1]699; CHECK-NEXT: ret700 %g = getelementptr inbounds half, ptr %p, i64 %o701 %l = load half, ptr %g702 %v = insertelement <4 x half> zeroinitializer, half %l, i32 0703 ret <4 x half> %v704}705 706define <8 x half> @loadv8f16_roX(ptr %p, i64 %o) {707; CHECK-LABEL: loadv8f16_roX:708; CHECK: // %bb.0:709; CHECK-NEXT: ldr h0, [x0, x1, lsl #1]710; CHECK-NEXT: ret711 %g = getelementptr inbounds half, ptr %p, i64 %o712 %l = load half, ptr %g713 %v = insertelement <8 x half> zeroinitializer, half %l, i32 0714 ret <8 x half> %v715}716 717define <4 x bfloat> @loadv4bf16_roX(ptr %p, i64 %o) {718; CHECK-LABEL: loadv4bf16_roX:719; CHECK: // %bb.0:720; CHECK-NEXT: ldr h0, [x0, x1, lsl #1]721; CHECK-NEXT: ret722 %g = getelementptr inbounds bfloat, ptr %p, i64 %o723 %l = load bfloat, ptr %g724 %v = insertelement <4 x bfloat> zeroinitializer, bfloat %l, i32 0725 ret <4 x bfloat> %v726}727 728define <8 x bfloat> @loadv8bf16_roX(ptr %p, i64 %o) {729; CHECK-LABEL: loadv8bf16_roX:730; CHECK: // %bb.0:731; CHECK-NEXT: ldr h0, [x0, x1, lsl #1]732; CHECK-NEXT: ret733 %g = getelementptr inbounds bfloat, ptr %p, i64 %o734 %l = load bfloat, ptr %g735 %v = insertelement <8 x bfloat> zeroinitializer, bfloat %l, i32 0736 ret <8 x bfloat> %v737}738 739define <2 x float> @loadv2f32_roX(ptr %p, i64 %o) {740; CHECK-LABEL: loadv2f32_roX:741; CHECK: // %bb.0:742; CHECK-NEXT: ldr s0, [x0, x1, lsl #2]743; CHECK-NEXT: ret744 %g = getelementptr inbounds float, ptr %p, i64 %o745 %l = load float, ptr %g746 %v = insertelement <2 x float> zeroinitializer, float %l, i32 0747 ret <2 x float> %v748}749 750define <4 x float> @loadv4f32_roX(ptr %p, i64 %o) {751; CHECK-LABEL: loadv4f32_roX:752; CHECK: // %bb.0:753; CHECK-NEXT: ldr s0, [x0, x1, lsl #2]754; CHECK-NEXT: ret755 %g = getelementptr inbounds float, ptr %p, i64 %o756 %l = load float, ptr %g757 %v = insertelement <4 x float> zeroinitializer, float %l, i32 0758 ret <4 x float> %v759}760 761define <2 x double> @loadv2f64_roX(ptr %p, i64 %o) {762; CHECK-LABEL: loadv2f64_roX:763; CHECK: // %bb.0:764; CHECK-NEXT: ldr d0, [x0, x1, lsl #3]765; CHECK-NEXT: ret766 %g = getelementptr inbounds double, ptr %p, i64 %o767 %l = load double, ptr %g768 %v = insertelement <2 x double> zeroinitializer, double %l, i32 0769 ret <2 x double> %v770}771 772 773define void @predictor_4x4_neon(ptr nocapture noundef writeonly %0, i64 noundef %1, ptr nocapture noundef readonly %2, ptr nocapture noundef readnone %3) {774; CHECK-LABEL: predictor_4x4_neon:775; CHECK: // %bb.0:776; CHECK-NEXT: movi v0.2d, #0000000000000000777; CHECK-NEXT: ldur w8, [x2, #2]778; CHECK-NEXT: ldr s1, [x2]779; CHECK-NEXT: ldur s2, [x2, #1]780; CHECK-NEXT: ushll v3.8h, v2.8b, #1781; CHECK-NEXT: mov v0.s[0], w8782; CHECK-NEXT: lsr w8, w8, #24783; CHECK-NEXT: uaddl v0.8h, v0.8b, v1.8b784; CHECK-NEXT: urhadd v1.8b, v1.8b, v2.8b785; CHECK-NEXT: add v0.8h, v0.8h, v3.8h786; CHECK-NEXT: dup v3.8b, w8787; CHECK-NEXT: str s1, [x0]788; CHECK-NEXT: lsl x8, x1, #1789; CHECK-NEXT: rshrn v0.8b, v0.8h, #2790; CHECK-NEXT: zip1 v2.2s, v1.2s, v3.2s791; CHECK-NEXT: str s0, [x0, x1]792; CHECK-NEXT: zip1 v3.2s, v0.2s, v3.2s793; CHECK-NEXT: ext v2.8b, v2.8b, v0.8b, #1794; CHECK-NEXT: ext v1.8b, v3.8b, v0.8b, #1795; CHECK-NEXT: str s2, [x0, x8]796; CHECK-NEXT: add x8, x8, x1797; CHECK-NEXT: str s1, [x0, x8]798; CHECK-NEXT: ret799 %5 = load i32, ptr %2, align 4800 %6 = insertelement <2 x i32> <i32 poison, i32 0>, i32 %5, i64 0801 %7 = bitcast <2 x i32> %6 to <8 x i8>802 %8 = getelementptr inbounds i8, ptr %2, i64 1803 %9 = load i32, ptr %8, align 4804 %10 = insertelement <2 x i32> <i32 poison, i32 0>, i32 %9, i64 0805 %11 = bitcast <2 x i32> %10 to <8 x i8>806 %12 = getelementptr inbounds i8, ptr %2, i64 2807 %13 = load i32, ptr %12, align 4808 %14 = insertelement <2 x i32> <i32 poison, i32 0>, i32 %13, i64 0809 %15 = bitcast <2 x i32> %14 to <8 x i8>810 %16 = lshr i32 %13, 24811 %17 = trunc i32 %16 to i8812 %18 = insertelement <8 x i8> undef, i8 %17, i64 0813 %19 = shufflevector <8 x i8> %18, <8 x i8> poison, <8 x i32> zeroinitializer814 %20 = tail call <8 x i8> @llvm.aarch64.neon.urhadd.v8i8(<8 x i8> %7, <8 x i8> %11)815 %21 = zext <8 x i8> %7 to <8 x i16>816 %22 = zext <8 x i8> %11 to <8 x i16>817 %23 = zext <8 x i8> %15 to <8 x i16>818 %24 = shl nuw nsw <8 x i16> %22, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>819 %25 = add nuw nsw <8 x i16> %23, %21820 %26 = add nuw nsw <8 x i16> %25, %24821 %27 = tail call <8 x i8> @llvm.aarch64.neon.rshrn.v8i8(<8 x i16> %26, i32 2)822 %28 = bitcast <8 x i8> %20 to <2 x i32>823 %29 = extractelement <2 x i32> %28, i64 0824 store i32 %29, ptr %0, align 4825 %30 = bitcast <8 x i8> %27 to <2 x i32>826 %31 = getelementptr inbounds i8, ptr %0, i64 %1827 %32 = extractelement <2 x i32> %30, i64 0828 store i32 %32, ptr %31, align 4829 %33 = bitcast <8 x i8> %19 to <2 x i32>830 %34 = shufflevector <2 x i32> %28, <2 x i32> %33, <2 x i32> <i32 0, i32 2>831 %35 = bitcast <2 x i32> %34 to <8 x i8>832 %36 = shufflevector <2 x i32> %30, <2 x i32> %33, <2 x i32> <i32 0, i32 2>833 %37 = bitcast <2 x i32> %36 to <8 x i8>834 %38 = shufflevector <8 x i8> %35, <8 x i8> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 undef, i32 undef, i32 undef, i32 undef>835 %39 = bitcast <8 x i8> %38 to <2 x i32>836 %40 = shl nsw i64 %1, 1837 %41 = getelementptr inbounds i8, ptr %0, i64 %40838 %42 = extractelement <2 x i32> %39, i64 0839 store i32 %42, ptr %41, align 4840 %43 = shufflevector <8 x i8> %37, <8 x i8> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 undef, i32 undef, i32 undef, i32 undef>841 %44 = bitcast <8 x i8> %43 to <2 x i32>842 %45 = mul nsw i64 %1, 3843 %46 = getelementptr inbounds i8, ptr %0, i64 %45844 %47 = extractelement <2 x i32> %44, i64 0845 store i32 %47, ptr %46, align 4846 ret void847}848 849define void @predictor_4x4_neon_new(ptr nocapture noundef writeonly %0, i64 noundef %1, ptr nocapture noundef readonly %2, ptr nocapture noundef readnone %3) {850; CHECK-LABEL: predictor_4x4_neon_new:851; CHECK: // %bb.0:852; CHECK-NEXT: ldr s0, [x2]853; CHECK-NEXT: ldur s1, [x2, #1]854; CHECK-NEXT: lsl x8, x1, #1855; CHECK-NEXT: ldur s2, [x2, #2]856; CHECK-NEXT: ldur s3, [x2, #3]857; CHECK-NEXT: uaddl v4.8h, v1.8b, v0.8b858; CHECK-NEXT: urhadd v0.8b, v0.8b, v1.8b859; CHECK-NEXT: add x9, x8, x1860; CHECK-NEXT: uaddl v5.8h, v2.8b, v1.8b861; CHECK-NEXT: uaddl v3.8h, v3.8b, v2.8b862; CHECK-NEXT: urhadd v1.8b, v1.8b, v2.8b863; CHECK-NEXT: str s0, [x0]864; CHECK-NEXT: add v4.8h, v4.8h, v5.8h865; CHECK-NEXT: add v3.8h, v3.8h, v5.8h866; CHECK-NEXT: rshrn v4.8b, v4.8h, #2867; CHECK-NEXT: rshrn v0.8b, v3.8h, #2868; CHECK-NEXT: str s4, [x0, x1]869; CHECK-NEXT: str s1, [x0, x8]870; CHECK-NEXT: str s0, [x0, x9]871; CHECK-NEXT: ret872 %5 = load i32, ptr %2, align 4873 %6 = insertelement <2 x i32> <i32 poison, i32 0>, i32 %5, i64 0874 %7 = bitcast <2 x i32> %6 to <8 x i8>875 %8 = getelementptr inbounds i8, ptr %2, i64 1876 %9 = load i32, ptr %8, align 4877 %10 = insertelement <2 x i32> <i32 poison, i32 0>, i32 %9, i64 0878 %11 = bitcast <2 x i32> %10 to <8 x i8>879 %12 = getelementptr inbounds i8, ptr %2, i64 2880 %13 = load i32, ptr %12, align 4881 %14 = insertelement <2 x i32> <i32 poison, i32 0>, i32 %13, i64 0882 %15 = bitcast <2 x i32> %14 to <8 x i8>883 %16 = getelementptr inbounds i8, ptr %2, i64 3884 %17 = load i32, ptr %16, align 4885 %18 = insertelement <2 x i32> <i32 poison, i32 0>, i32 %17, i64 0886 %19 = bitcast <2 x i32> %18 to <8 x i8>887 %20 = tail call <8 x i8> @llvm.aarch64.neon.urhadd.v8i8(<8 x i8> %7, <8 x i8> %11)888 %21 = tail call <8 x i8> @llvm.aarch64.neon.urhadd.v8i8(<8 x i8> %11, <8 x i8> %15)889 %22 = zext <8 x i8> %7 to <8 x i16>890 %23 = zext <8 x i8> %11 to <8 x i16>891 %24 = add nuw nsw <8 x i16> %23, %22892 %25 = zext <8 x i8> %15 to <8 x i16>893 %26 = add nuw nsw <8 x i16> %25, %23894 %27 = add nuw nsw <8 x i16> %24, %26895 %28 = tail call <8 x i8> @llvm.aarch64.neon.rshrn.v8i8(<8 x i16> %27, i32 2)896 %29 = zext <8 x i8> %19 to <8 x i16>897 %30 = add nuw nsw <8 x i16> %29, %25898 %31 = add nuw nsw <8 x i16> %30, %26899 %32 = tail call <8 x i8> @llvm.aarch64.neon.rshrn.v8i8(<8 x i16> %31, i32 2)900 %33 = bitcast <8 x i8> %20 to <2 x i32>901 %34 = extractelement <2 x i32> %33, i64 0902 store i32 %34, ptr %0, align 4903 %35 = bitcast <8 x i8> %28 to <2 x i32>904 %36 = getelementptr inbounds i8, ptr %0, i64 %1905 %37 = extractelement <2 x i32> %35, i64 0906 store i32 %37, ptr %36, align 4907 %38 = bitcast <8 x i8> %21 to <2 x i32>908 %39 = shl nsw i64 %1, 1909 %40 = getelementptr inbounds i8, ptr %0, i64 %39910 %41 = extractelement <2 x i32> %38, i64 0911 store i32 %41, ptr %40, align 4912 %42 = bitcast <8 x i8> %32 to <2 x i32>913 %43 = mul nsw i64 %1, 3914 %44 = getelementptr inbounds i8, ptr %0, i64 %43915 %45 = extractelement <2 x i32> %42, i64 0916 store i32 %45, ptr %44, align 4917 ret void918}919 920 921define <vscale x 8 x i8> @loadnxv8i8(ptr %p) {922; CHECK-LABEL: loadnxv8i8:923; CHECK: // %bb.0:924; CHECK-NEXT: movi v0.2d, #0000000000000000925; CHECK-NEXT: ldrb w8, [x0]926; CHECK-NEXT: ptrue p0.h, vl1927; CHECK-NEXT: mov z0.h, p0/m, w8928; CHECK-NEXT: ret929 %l = load i8, ptr %p930 %v = insertelement <vscale x 8 x i8> zeroinitializer, i8 %l, i32 0931 ret <vscale x 8 x i8> %v932}933 934define <vscale x 16 x i8> @loadnxv16i8(ptr %p) {935; CHECK-LABEL: loadnxv16i8:936; CHECK: // %bb.0:937; CHECK-NEXT: ldr b0, [x0]938; CHECK-NEXT: ret939 %l = load i8, ptr %p940 %v = insertelement <vscale x 16 x i8> zeroinitializer, i8 %l, i32 0941 ret <vscale x 16 x i8> %v942}943 944define <vscale x 4 x i16> @loadnxv4i16(ptr %p) {945; CHECK-LABEL: loadnxv4i16:946; CHECK: // %bb.0:947; CHECK-NEXT: movi v0.2d, #0000000000000000948; CHECK-NEXT: ldrh w8, [x0]949; CHECK-NEXT: ptrue p0.s, vl1950; CHECK-NEXT: mov z0.s, p0/m, w8951; CHECK-NEXT: ret952 %l = load i16, ptr %p953 %v = insertelement <vscale x 4 x i16> zeroinitializer, i16 %l, i32 0954 ret <vscale x 4 x i16> %v955}956 957define <vscale x 8 x i16> @loadnxv8i16(ptr %p) {958; CHECK-LABEL: loadnxv8i16:959; CHECK: // %bb.0:960; CHECK-NEXT: ldr h0, [x0]961; CHECK-NEXT: ret962 %l = load i16, ptr %p963 %v = insertelement <vscale x 8 x i16> zeroinitializer, i16 %l, i32 0964 ret <vscale x 8 x i16> %v965}966 967define <vscale x 2 x i32> @loadnxv2i32(ptr %p) {968; CHECK-LABEL: loadnxv2i32:969; CHECK: // %bb.0:970; CHECK-NEXT: movi v0.2d, #0000000000000000971; CHECK-NEXT: ldr w8, [x0]972; CHECK-NEXT: ptrue p0.d, vl1973; CHECK-NEXT: mov z0.d, p0/m, x8974; CHECK-NEXT: ret975 %l = load i32, ptr %p976 %v = insertelement <vscale x 2 x i32> zeroinitializer, i32 %l, i32 0977 ret <vscale x 2 x i32> %v978}979 980define <vscale x 4 x i32> @loadnxv4i32(ptr %p) {981; CHECK-LABEL: loadnxv4i32:982; CHECK: // %bb.0:983; CHECK-NEXT: ldr s0, [x0]984; CHECK-NEXT: ret985 %l = load i32, ptr %p986 %v = insertelement <vscale x 4 x i32> zeroinitializer, i32 %l, i32 0987 ret <vscale x 4 x i32> %v988}989 990define <vscale x 2 x i64> @loadnxv2i64(ptr %p) {991; CHECK-LABEL: loadnxv2i64:992; CHECK: // %bb.0:993; CHECK-NEXT: ldr d0, [x0]994; CHECK-NEXT: ret995 %l = load i64, ptr %p996 %v = insertelement <vscale x 2 x i64> zeroinitializer, i64 %l, i32 0997 ret <vscale x 2 x i64> %v998}999 1000 1001define <vscale x 4 x half> @loadnxv4f16(ptr %p) {1002; CHECK-LABEL: loadnxv4f16:1003; CHECK: // %bb.0:1004; CHECK-NEXT: mov w8, wzr1005; CHECK-NEXT: index z0.s, #0, #11006; CHECK-NEXT: ptrue p0.s1007; CHECK-NEXT: mov z1.s, w81008; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s1009; CHECK-NEXT: movi v0.2d, #00000000000000001010; CHECK-NEXT: ldr h1, [x0]1011; CHECK-NEXT: mov z0.h, p0/m, h11012; CHECK-NEXT: ret1013 %l = load half, ptr %p1014 %v = insertelement <vscale x 4 x half> zeroinitializer, half %l, i32 01015 ret <vscale x 4 x half> %v1016}1017 1018define <vscale x 8 x half> @loadnxv8f16(ptr %p) {1019; CHECK-LABEL: loadnxv8f16:1020; CHECK: // %bb.0:1021; CHECK-NEXT: ldr h0, [x0]1022; CHECK-NEXT: ret1023 %l = load half, ptr %p1024 %v = insertelement <vscale x 8 x half> zeroinitializer, half %l, i32 01025 ret <vscale x 8 x half> %v1026}1027 1028define <vscale x 4 x bfloat> @loadnxv4bf16(ptr %p) {1029; CHECK-LABEL: loadnxv4bf16:1030; CHECK: // %bb.0:1031; CHECK-NEXT: mov w8, wzr1032; CHECK-NEXT: index z0.s, #0, #11033; CHECK-NEXT: ptrue p0.s1034; CHECK-NEXT: mov z1.s, w81035; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s1036; CHECK-NEXT: movi v0.2d, #00000000000000001037; CHECK-NEXT: ldr h1, [x0]1038; CHECK-NEXT: mov z0.h, p0/m, h11039; CHECK-NEXT: ret1040 %l = load bfloat, ptr %p1041 %v = insertelement <vscale x 4 x bfloat> zeroinitializer, bfloat %l, i32 01042 ret <vscale x 4 x bfloat> %v1043}1044 1045define <vscale x 8 x bfloat> @loadnxv8bf16(ptr %p) {1046; CHECK-LABEL: loadnxv8bf16:1047; CHECK: // %bb.0:1048; CHECK-NEXT: ldr h0, [x0]1049; CHECK-NEXT: ret1050 %l = load bfloat, ptr %p1051 %v = insertelement <vscale x 8 x bfloat> zeroinitializer, bfloat %l, i32 01052 ret <vscale x 8 x bfloat> %v1053}1054 1055define <vscale x 2 x float> @loadnxv2f32(ptr %p) {1056; CHECK-LABEL: loadnxv2f32:1057; CHECK: // %bb.0:1058; CHECK-NEXT: mov x8, xzr1059; CHECK-NEXT: index z0.d, #0, #11060; CHECK-NEXT: ptrue p0.d1061; CHECK-NEXT: mov z1.d, x81062; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d1063; CHECK-NEXT: movi v0.2d, #00000000000000001064; CHECK-NEXT: ldr s1, [x0]1065; CHECK-NEXT: mov z0.s, p0/m, s11066; CHECK-NEXT: ret1067 %l = load float, ptr %p1068 %v = insertelement <vscale x 2 x float> zeroinitializer, float %l, i32 01069 ret <vscale x 2 x float> %v1070}1071 1072define <vscale x 4 x float> @loadnxv4f32(ptr %p) {1073; CHECK-LABEL: loadnxv4f32:1074; CHECK: // %bb.0:1075; CHECK-NEXT: ldr s0, [x0]1076; CHECK-NEXT: ret1077 %l = load float, ptr %p1078 %v = insertelement <vscale x 4 x float> zeroinitializer, float %l, i32 01079 ret <vscale x 4 x float> %v1080}1081 1082define <vscale x 2 x double> @loadnxv2f64(ptr %p) {1083; CHECK-LABEL: loadnxv2f64:1084; CHECK: // %bb.0:1085; CHECK-NEXT: ldr d0, [x0]1086; CHECK-NEXT: ret1087 %l = load double, ptr %p1088 %v = insertelement <vscale x 2 x double> zeroinitializer, double %l, i32 01089 ret <vscale x 2 x double> %v1090}1091 1092 1093; Unscaled1094 1095define <vscale x 8 x i8> @loadnxv8i8_offset(ptr %p) {1096; CHECK-LABEL: loadnxv8i8_offset:1097; CHECK: // %bb.0:1098; CHECK-NEXT: movi v0.2d, #00000000000000001099; CHECK-NEXT: ldrb w8, [x0, #1]1100; CHECK-NEXT: ptrue p0.h, vl11101; CHECK-NEXT: mov z0.h, p0/m, w81102; CHECK-NEXT: ret1103 %g = getelementptr inbounds i8, ptr %p, i64 11104 %l = load i8, ptr %g1105 %v = insertelement <vscale x 8 x i8> zeroinitializer, i8 %l, i32 01106 ret <vscale x 8 x i8> %v1107}1108 1109define <vscale x 16 x i8> @loadnxv16i8_offset(ptr %p) {1110; CHECK-LABEL: loadnxv16i8_offset:1111; CHECK: // %bb.0:1112; CHECK-NEXT: ldr b0, [x0, #1]1113; CHECK-NEXT: ret1114 %g = getelementptr inbounds i8, ptr %p, i64 11115 %l = load i8, ptr %g1116 %v = insertelement <vscale x 16 x i8> zeroinitializer, i8 %l, i32 01117 ret <vscale x 16 x i8> %v1118}1119 1120define <vscale x 4 x i16> @loadnxv4i16_offset(ptr %p) {1121; CHECK-LABEL: loadnxv4i16_offset:1122; CHECK: // %bb.0:1123; CHECK-NEXT: movi v0.2d, #00000000000000001124; CHECK-NEXT: ldurh w8, [x0, #1]1125; CHECK-NEXT: ptrue p0.s, vl11126; CHECK-NEXT: mov z0.s, p0/m, w81127; CHECK-NEXT: ret1128 %g = getelementptr inbounds i8, ptr %p, i64 11129 %l = load i16, ptr %g1130 %v = insertelement <vscale x 4 x i16> zeroinitializer, i16 %l, i32 01131 ret <vscale x 4 x i16> %v1132}1133 1134define <vscale x 8 x i16> @loadnxv8i16_offset(ptr %p) {1135; CHECK-LABEL: loadnxv8i16_offset:1136; CHECK: // %bb.0:1137; CHECK-NEXT: ldur h0, [x0, #1]1138; CHECK-NEXT: ret1139 %g = getelementptr inbounds i8, ptr %p, i64 11140 %l = load i16, ptr %g1141 %v = insertelement <vscale x 8 x i16> zeroinitializer, i16 %l, i32 01142 ret <vscale x 8 x i16> %v1143}1144 1145define <vscale x 2 x i32> @loadnxv2i32_offset(ptr %p) {1146; CHECK-LABEL: loadnxv2i32_offset:1147; CHECK: // %bb.0:1148; CHECK-NEXT: movi v0.2d, #00000000000000001149; CHECK-NEXT: ldur w8, [x0, #1]1150; CHECK-NEXT: ptrue p0.d, vl11151; CHECK-NEXT: mov z0.d, p0/m, x81152; CHECK-NEXT: ret1153 %g = getelementptr inbounds i8, ptr %p, i64 11154 %l = load i32, ptr %g1155 %v = insertelement <vscale x 2 x i32> zeroinitializer, i32 %l, i32 01156 ret <vscale x 2 x i32> %v1157}1158 1159define <vscale x 4 x i32> @loadnxv4i32_offset(ptr %p) {1160; CHECK-LABEL: loadnxv4i32_offset:1161; CHECK: // %bb.0:1162; CHECK-NEXT: ldur s0, [x0, #1]1163; CHECK-NEXT: ret1164 %g = getelementptr inbounds i8, ptr %p, i64 11165 %l = load i32, ptr %g1166 %v = insertelement <vscale x 4 x i32> zeroinitializer, i32 %l, i32 01167 ret <vscale x 4 x i32> %v1168}1169 1170define <vscale x 2 x i64> @loadnxv2i64_offset(ptr %p) {1171; CHECK-LABEL: loadnxv2i64_offset:1172; CHECK: // %bb.0:1173; CHECK-NEXT: ldur d0, [x0, #1]1174; CHECK-NEXT: ret1175 %g = getelementptr inbounds i8, ptr %p, i64 11176 %l = load i64, ptr %g1177 %v = insertelement <vscale x 2 x i64> zeroinitializer, i64 %l, i32 01178 ret <vscale x 2 x i64> %v1179}1180 1181 1182define <vscale x 4 x half> @loadnxv4f16_offset(ptr %p) {1183; CHECK-LABEL: loadnxv4f16_offset:1184; CHECK: // %bb.0:1185; CHECK-NEXT: mov w8, wzr1186; CHECK-NEXT: index z0.s, #0, #11187; CHECK-NEXT: ptrue p0.s1188; CHECK-NEXT: mov z1.s, w81189; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s1190; CHECK-NEXT: movi v0.2d, #00000000000000001191; CHECK-NEXT: ldur h1, [x0, #1]1192; CHECK-NEXT: mov z0.h, p0/m, h11193; CHECK-NEXT: ret1194 %g = getelementptr inbounds i8, ptr %p, i64 11195 %l = load half, ptr %g1196 %v = insertelement <vscale x 4 x half> zeroinitializer, half %l, i32 01197 ret <vscale x 4 x half> %v1198}1199 1200define <vscale x 8 x half> @loadnxv8f16_offset(ptr %p) {1201; CHECK-LABEL: loadnxv8f16_offset:1202; CHECK: // %bb.0:1203; CHECK-NEXT: ldur h0, [x0, #1]1204; CHECK-NEXT: ret1205 %g = getelementptr inbounds i8, ptr %p, i64 11206 %l = load half, ptr %g1207 %v = insertelement <vscale x 8 x half> zeroinitializer, half %l, i32 01208 ret <vscale x 8 x half> %v1209}1210 1211define <vscale x 4 x bfloat> @loadnxv4bf16_offset(ptr %p) {1212; CHECK-LABEL: loadnxv4bf16_offset:1213; CHECK: // %bb.0:1214; CHECK-NEXT: mov w8, wzr1215; CHECK-NEXT: index z0.s, #0, #11216; CHECK-NEXT: ptrue p0.s1217; CHECK-NEXT: mov z1.s, w81218; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s1219; CHECK-NEXT: movi v0.2d, #00000000000000001220; CHECK-NEXT: ldur h1, [x0, #1]1221; CHECK-NEXT: mov z0.h, p0/m, h11222; CHECK-NEXT: ret1223 %g = getelementptr inbounds i8, ptr %p, i64 11224 %l = load bfloat, ptr %g1225 %v = insertelement <vscale x 4 x bfloat> zeroinitializer, bfloat %l, i32 01226 ret <vscale x 4 x bfloat> %v1227}1228 1229define <vscale x 8 x bfloat> @loadnxv8bf16_offset(ptr %p) {1230; CHECK-LABEL: loadnxv8bf16_offset:1231; CHECK: // %bb.0:1232; CHECK-NEXT: ldur h0, [x0, #1]1233; CHECK-NEXT: ret1234 %g = getelementptr inbounds i8, ptr %p, i64 11235 %l = load bfloat, ptr %g1236 %v = insertelement <vscale x 8 x bfloat> zeroinitializer, bfloat %l, i32 01237 ret <vscale x 8 x bfloat> %v1238}1239 1240define <vscale x 2 x float> @loadnxv2f32_offset(ptr %p) {1241; CHECK-LABEL: loadnxv2f32_offset:1242; CHECK: // %bb.0:1243; CHECK-NEXT: mov x8, xzr1244; CHECK-NEXT: index z0.d, #0, #11245; CHECK-NEXT: ptrue p0.d1246; CHECK-NEXT: mov z1.d, x81247; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d1248; CHECK-NEXT: movi v0.2d, #00000000000000001249; CHECK-NEXT: ldur s1, [x0, #1]1250; CHECK-NEXT: mov z0.s, p0/m, s11251; CHECK-NEXT: ret1252 %g = getelementptr inbounds i8, ptr %p, i64 11253 %l = load float, ptr %g1254 %v = insertelement <vscale x 2 x float> zeroinitializer, float %l, i32 01255 ret <vscale x 2 x float> %v1256}1257 1258define <vscale x 4 x float> @loadnxv4f32_offset(ptr %p) {1259; CHECK-LABEL: loadnxv4f32_offset:1260; CHECK: // %bb.0:1261; CHECK-NEXT: ldur s0, [x0, #1]1262; CHECK-NEXT: ret1263 %g = getelementptr inbounds i8, ptr %p, i64 11264 %l = load float, ptr %g1265 %v = insertelement <vscale x 4 x float> zeroinitializer, float %l, i32 01266 ret <vscale x 4 x float> %v1267}1268 1269define <vscale x 2 x double> @loadnxv2f64_offset(ptr %p) {1270; CHECK-LABEL: loadnxv2f64_offset:1271; CHECK: // %bb.0:1272; CHECK-NEXT: ldur d0, [x0, #1]1273; CHECK-NEXT: ret1274 %g = getelementptr inbounds i8, ptr %p, i64 11275 %l = load double, ptr %g1276 %v = insertelement <vscale x 2 x double> zeroinitializer, double %l, i32 01277 ret <vscale x 2 x double> %v1278}1279 1280 1281declare <8 x i8> @llvm.aarch64.neon.rshrn.v8i8(<8 x i16>, i32) #11282declare <8 x i8> @llvm.aarch64.neon.urhadd.v8i8(<8 x i8>, <8 x i8>) #11283