2855 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 62; RUN: llc -mattr=+sve < %s | FileCheck %s3 4target triple = "aarch64-unknown-linux-gnu"5 6define void @sve_load_store_nxv1i8(ptr %a, ptr %b) {7; CHECK-LABEL: sve_load_store_nxv1i8:8; CHECK: // %bb.0:9; CHECK-NEXT: rdvl x8, #110; CHECK-NEXT: lsr x8, x8, #411; CHECK-NEXT: whilelo p0.b, xzr, x812; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]13; CHECK-NEXT: st1b { z0.b }, p0, [x1]14; CHECK-NEXT: ret15 %c = load <vscale x 1 x i8>, ptr %a16 store <vscale x 1 x i8> %c, ptr %b17 ret void18}19 20define void @sve_load_store_nxv2i8(ptr %a, ptr %b) {21; CHECK-LABEL: sve_load_store_nxv2i8:22; CHECK: // %bb.0:23; CHECK-NEXT: ptrue p0.d24; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]25; CHECK-NEXT: st1b { z0.d }, p0, [x1]26; CHECK-NEXT: ret27 %c = load <vscale x 2 x i8>, ptr %a28 store <vscale x 2 x i8> %c, ptr %b29 ret void30}31 32define void @sve_load_store_nxv3i8(ptr %a, ptr %b) {33; CHECK-LABEL: sve_load_store_nxv3i8:34; CHECK: // %bb.0:35; CHECK-NEXT: rdvl x8, #136; CHECK-NEXT: mov w9, #3 // =0x337; CHECK-NEXT: lsr x8, x8, #438; CHECK-NEXT: mul x8, x8, x939; CHECK-NEXT: whilelo p0.s, xzr, x840; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0]41; CHECK-NEXT: st1b { z0.s }, p0, [x1]42; CHECK-NEXT: ret43 %c = load <vscale x 3 x i8>, ptr %a44 store <vscale x 3 x i8> %c, ptr %b45 ret void46}47 48define void @sve_load_store_nxv4i8(ptr %a, ptr %b) {49; CHECK-LABEL: sve_load_store_nxv4i8:50; CHECK: // %bb.0:51; CHECK-NEXT: ptrue p0.s52; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0]53; CHECK-NEXT: st1b { z0.s }, p0, [x1]54; CHECK-NEXT: ret55 %c = load <vscale x 4 x i8>, ptr %a56 store <vscale x 4 x i8> %c, ptr %b57 ret void58}59 60define void @sve_load_store_nxv5i8(ptr %a, ptr %b) {61; CHECK-LABEL: sve_load_store_nxv5i8:62; CHECK: // %bb.0:63; CHECK-NEXT: rdvl x8, #164; CHECK-NEXT: mov w9, #5 // =0x565; CHECK-NEXT: lsr x8, x8, #466; CHECK-NEXT: mul x8, x8, x967; CHECK-NEXT: whilelo p0.h, xzr, x868; CHECK-NEXT: ld1b { z0.h }, p0/z, [x0]69; CHECK-NEXT: st1b { z0.h }, p0, [x1]70; CHECK-NEXT: ret71 %c = load <vscale x 5 x i8>, ptr %a72 store <vscale x 5 x i8> %c, ptr %b73 ret void74}75 76define void @sve_load_store_nxv6i8(ptr %a, ptr %b) {77; CHECK-LABEL: sve_load_store_nxv6i8:78; CHECK: // %bb.0:79; CHECK-NEXT: ptrue p0.d80; CHECK-NEXT: ptrue p1.s81; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, #2, mul vl]82; CHECK-NEXT: ld1b { z1.s }, p1/z, [x0]83; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s84; CHECK-NEXT: st1b { z1.s }, p1, [x1]85; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h86; CHECK-NEXT: uunpkhi z0.s, z0.h87; CHECK-NEXT: uunpklo z0.d, z0.s88; CHECK-NEXT: st1b { z0.d }, p0, [x1, #2, mul vl]89; CHECK-NEXT: ret90 %c = load <vscale x 6 x i8>, ptr %a91 store <vscale x 6 x i8> %c, ptr %b92 ret void93}94 95define void @sve_load_store_nxv7i8(ptr %a, ptr %b) {96; CHECK-LABEL: sve_load_store_nxv7i8:97; CHECK: // %bb.0:98; CHECK-NEXT: rdvl x8, #199; CHECK-NEXT: mov w9, #7 // =0x7100; CHECK-NEXT: lsr x8, x8, #4101; CHECK-NEXT: mul x8, x8, x9102; CHECK-NEXT: whilelo p0.h, xzr, x8103; CHECK-NEXT: ld1b { z0.h }, p0/z, [x0]104; CHECK-NEXT: st1b { z0.h }, p0, [x1]105; CHECK-NEXT: ret106 %c = load <vscale x 7 x i8>, ptr %a107 store <vscale x 7 x i8> %c, ptr %b108 ret void109}110 111define void @sve_load_store_nxv8i8(ptr %a, ptr %b) {112; CHECK-LABEL: sve_load_store_nxv8i8:113; CHECK: // %bb.0:114; CHECK-NEXT: ptrue p0.h115; CHECK-NEXT: ld1b { z0.h }, p0/z, [x0]116; CHECK-NEXT: st1b { z0.h }, p0, [x1]117; CHECK-NEXT: ret118 %c = load <vscale x 8 x i8>, ptr %a119 store <vscale x 8 x i8> %c, ptr %b120 ret void121}122 123define void @sve_load_store_nxv9i8(ptr %a, ptr %b) {124; CHECK-LABEL: sve_load_store_nxv9i8:125; CHECK: // %bb.0:126; CHECK-NEXT: rdvl x8, #1127; CHECK-NEXT: mov w9, #9 // =0x9128; CHECK-NEXT: lsr x8, x8, #4129; CHECK-NEXT: mul x8, x8, x9130; CHECK-NEXT: whilelo p0.b, xzr, x8131; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]132; CHECK-NEXT: st1b { z0.b }, p0, [x1]133; CHECK-NEXT: ret134 %c = load <vscale x 9 x i8>, ptr %a135 store <vscale x 9 x i8> %c, ptr %b136 ret void137}138 139define void @sve_load_store_nxv10i8(ptr %a, ptr %b) {140; CHECK-LABEL: sve_load_store_nxv10i8:141; CHECK: // %bb.0:142; CHECK-NEXT: ptrue p0.d143; CHECK-NEXT: ptrue p1.h144; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, #4, mul vl]145; CHECK-NEXT: ld1b { z1.h }, p1/z, [x0]146; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s147; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h148; CHECK-NEXT: uzp1 z0.b, z1.b, z0.b149; CHECK-NEXT: uunpkhi z1.h, z0.b150; CHECK-NEXT: uunpklo z0.h, z0.b151; CHECK-NEXT: uunpklo z1.s, z1.h152; CHECK-NEXT: st1b { z0.h }, p1, [x1]153; CHECK-NEXT: uunpklo z1.d, z1.s154; CHECK-NEXT: st1b { z1.d }, p0, [x1, #4, mul vl]155; CHECK-NEXT: ret156 %c = load <vscale x 10 x i8>, ptr %a157 store <vscale x 10 x i8> %c, ptr %b158 ret void159}160 161define void @sve_load_store_nxv11i8(ptr %a, ptr %b) {162; CHECK-LABEL: sve_load_store_nxv11i8:163; CHECK: // %bb.0:164; CHECK-NEXT: rdvl x8, #1165; CHECK-NEXT: mov w9, #11 // =0xb166; CHECK-NEXT: lsr x8, x8, #4167; CHECK-NEXT: mul x8, x8, x9168; CHECK-NEXT: whilelo p0.b, xzr, x8169; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]170; CHECK-NEXT: st1b { z0.b }, p0, [x1]171; CHECK-NEXT: ret172 %c = load <vscale x 11 x i8>, ptr %a173 store <vscale x 11 x i8> %c, ptr %b174 ret void175}176 177define void @sve_load_store_nxv12i8(ptr %a, ptr %b) {178; CHECK-LABEL: sve_load_store_nxv12i8:179; CHECK: // %bb.0:180; CHECK-NEXT: ptrue p0.s181; CHECK-NEXT: ptrue p1.h182; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, #2, mul vl]183; CHECK-NEXT: ld1b { z1.h }, p1/z, [x0]184; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h185; CHECK-NEXT: uzp1 z0.b, z1.b, z0.b186; CHECK-NEXT: uunpkhi z1.h, z0.b187; CHECK-NEXT: uunpklo z0.h, z0.b188; CHECK-NEXT: uunpklo z1.s, z1.h189; CHECK-NEXT: st1b { z0.h }, p1, [x1]190; CHECK-NEXT: st1b { z1.s }, p0, [x1, #2, mul vl]191; CHECK-NEXT: ret192 %c = load <vscale x 12 x i8>, ptr %a193 store <vscale x 12 x i8> %c, ptr %b194 ret void195}196 197define void @sve_load_store_nxv13i8(ptr %a, ptr %b) {198; CHECK-LABEL: sve_load_store_nxv13i8:199; CHECK: // %bb.0:200; CHECK-NEXT: rdvl x8, #1201; CHECK-NEXT: mov w9, #13 // =0xd202; CHECK-NEXT: lsr x8, x8, #4203; CHECK-NEXT: mul x8, x8, x9204; CHECK-NEXT: whilelo p0.b, xzr, x8205; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]206; CHECK-NEXT: st1b { z0.b }, p0, [x1]207; CHECK-NEXT: ret208 %c = load <vscale x 13 x i8>, ptr %a209 store <vscale x 13 x i8> %c, ptr %b210 ret void211}212 213define void @sve_load_store_nxv14i8(ptr %a, ptr %b) {214; CHECK-LABEL: sve_load_store_nxv14i8:215; CHECK: // %bb.0:216; CHECK-NEXT: ptrue p0.d217; CHECK-NEXT: ptrue p1.s218; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, #6, mul vl]219; CHECK-NEXT: ptrue p2.h220; CHECK-NEXT: ld1b { z1.s }, p1/z, [x0, #2, mul vl]221; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s222; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h223; CHECK-NEXT: ld1b { z1.h }, p2/z, [x0]224; CHECK-NEXT: uzp1 z0.b, z1.b, z0.b225; CHECK-NEXT: uunpkhi z1.h, z0.b226; CHECK-NEXT: uunpklo z0.h, z0.b227; CHECK-NEXT: uunpkhi z2.s, z1.h228; CHECK-NEXT: uunpklo z1.s, z1.h229; CHECK-NEXT: st1b { z0.h }, p2, [x1]230; CHECK-NEXT: uunpklo z2.d, z2.s231; CHECK-NEXT: st1b { z1.s }, p1, [x1, #2, mul vl]232; CHECK-NEXT: st1b { z2.d }, p0, [x1, #6, mul vl]233; CHECK-NEXT: ret234 %c = load <vscale x 14 x i8>, ptr %a235 store <vscale x 14 x i8> %c, ptr %b236 ret void237}238 239define void @sve_load_store_nxv15i8(ptr %a, ptr %b) {240; CHECK-LABEL: sve_load_store_nxv15i8:241; CHECK: // %bb.0:242; CHECK-NEXT: rdvl x8, #1243; CHECK-NEXT: mov w9, #15 // =0xf244; CHECK-NEXT: lsr x8, x8, #4245; CHECK-NEXT: mul x8, x8, x9246; CHECK-NEXT: whilelo p0.b, xzr, x8247; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]248; CHECK-NEXT: st1b { z0.b }, p0, [x1]249; CHECK-NEXT: ret250 %c = load <vscale x 15 x i8>, ptr %a251 store <vscale x 15 x i8> %c, ptr %b252 ret void253}254 255define void @sve_load_store_nxv16i8(ptr %a, ptr %b) {256; CHECK-LABEL: sve_load_store_nxv16i8:257; CHECK: // %bb.0:258; CHECK-NEXT: ldr z0, [x0]259; CHECK-NEXT: str z0, [x1]260; CHECK-NEXT: ret261 %c = load <vscale x 16 x i8>, ptr %a262 store <vscale x 16 x i8> %c, ptr %b263 ret void264}265 266define void @sve_load_store_nxv17i8(ptr %a, ptr %b) {267; CHECK-LABEL: sve_load_store_nxv17i8:268; CHECK: // %bb.0:269; CHECK-NEXT: rdvl x8, #1270; CHECK-NEXT: mov w10, #17 // =0x11271; CHECK-NEXT: lsr x9, x8, #4272; CHECK-NEXT: mul x9, x9, x10273; CHECK-NEXT: whilelo p0.b, x8, x9274; CHECK-NEXT: whilelo p1.b, xzr, x9275; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, #1, mul vl]276; CHECK-NEXT: ld1b { z1.b }, p1/z, [x0]277; CHECK-NEXT: st1b { z0.b }, p0, [x1, #1, mul vl]278; CHECK-NEXT: st1b { z1.b }, p1, [x1]279; CHECK-NEXT: ret280 %c = load <vscale x 17 x i8>, ptr %a281 store <vscale x 17 x i8> %c, ptr %b282 ret void283}284 285define void @sve_load_store_nxv18i8(ptr %a, ptr %b) {286; CHECK-LABEL: sve_load_store_nxv18i8:287; CHECK: // %bb.0:288; CHECK-NEXT: ptrue p0.d289; CHECK-NEXT: rdvl x8, #1290; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, x8]291; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s292; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h293; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b294; CHECK-NEXT: uunpklo z0.h, z0.b295; CHECK-NEXT: uunpklo z1.s, z0.h296; CHECK-NEXT: uunpkhi z0.s, z0.h297; CHECK-NEXT: uunpklo z1.d, z1.s298; CHECK-NEXT: uzp1 z1.s, z1.s, z0.s299; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h300; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b301; CHECK-NEXT: uunpklo z0.h, z0.b302; CHECK-NEXT: uunpkhi z1.s, z0.h303; CHECK-NEXT: uunpklo z0.s, z0.h304; CHECK-NEXT: uunpkhi z1.d, z1.s305; CHECK-NEXT: uzp1 z1.s, z0.s, z1.s306; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h307; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b308; CHECK-NEXT: uunpklo z0.h, z0.b309; CHECK-NEXT: uunpkhi z1.s, z0.h310; CHECK-NEXT: uunpklo z0.s, z0.h311; CHECK-NEXT: uunpklo z1.d, z1.s312; CHECK-NEXT: uzp1 z1.s, z1.s, z0.s313; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h314; CHECK-NEXT: uzp1 z1.b, z0.b, z0.b315; CHECK-NEXT: uunpkhi z1.h, z1.b316; CHECK-NEXT: uunpklo z2.s, z1.h317; CHECK-NEXT: uunpkhi z1.s, z1.h318; CHECK-NEXT: uunpkhi z2.d, z2.s319; CHECK-NEXT: uzp1 z2.s, z0.s, z2.s320; CHECK-NEXT: uzp1 z1.h, z2.h, z1.h321; CHECK-NEXT: uzp1 z1.b, z0.b, z1.b322; CHECK-NEXT: uunpkhi z1.h, z1.b323; CHECK-NEXT: uunpklo z2.s, z1.h324; CHECK-NEXT: uunpkhi z1.s, z1.h325; CHECK-NEXT: uunpklo z2.d, z2.s326; CHECK-NEXT: uzp1 z2.s, z2.s, z0.s327; CHECK-NEXT: uzp1 z1.h, z2.h, z1.h328; CHECK-NEXT: uzp1 z1.b, z0.b, z1.b329; CHECK-NEXT: uunpkhi z1.h, z1.b330; CHECK-NEXT: uunpkhi z2.s, z1.h331; CHECK-NEXT: uunpklo z1.s, z1.h332; CHECK-NEXT: uunpkhi z2.d, z2.s333; CHECK-NEXT: uzp1 z2.s, z0.s, z2.s334; CHECK-NEXT: uzp1 z1.h, z1.h, z2.h335; CHECK-NEXT: uzp1 z1.b, z0.b, z1.b336; CHECK-NEXT: uunpkhi z1.h, z1.b337; CHECK-NEXT: uunpkhi z2.s, z1.h338; CHECK-NEXT: uunpklo z1.s, z1.h339; CHECK-NEXT: uunpklo z2.d, z2.s340; CHECK-NEXT: uzp1 z2.s, z2.s, z0.s341; CHECK-NEXT: uzp1 z1.h, z1.h, z2.h342; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b343; CHECK-NEXT: ldr z1, [x0]344; CHECK-NEXT: uunpklo z0.h, z0.b345; CHECK-NEXT: uunpklo z0.s, z0.h346; CHECK-NEXT: uunpklo z0.d, z0.s347; CHECK-NEXT: st1b { z0.d }, p0, [x1, x8]348; CHECK-NEXT: str z1, [x1]349; CHECK-NEXT: ret350 %c = load <vscale x 18 x i8>, ptr %a351 store <vscale x 18 x i8> %c, ptr %b352 ret void353}354 355define void @sve_load_store_nxv19i8(ptr %a, ptr %b) {356; CHECK-LABEL: sve_load_store_nxv19i8:357; CHECK: // %bb.0:358; CHECK-NEXT: rdvl x8, #1359; CHECK-NEXT: mov w10, #19 // =0x13360; CHECK-NEXT: lsr x9, x8, #4361; CHECK-NEXT: mul x9, x9, x10362; CHECK-NEXT: whilelo p0.b, x8, x9363; CHECK-NEXT: whilelo p1.b, xzr, x9364; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, #1, mul vl]365; CHECK-NEXT: ld1b { z1.b }, p1/z, [x0]366; CHECK-NEXT: st1b { z0.b }, p0, [x1, #1, mul vl]367; CHECK-NEXT: st1b { z1.b }, p1, [x1]368; CHECK-NEXT: ret369 %c = load <vscale x 19 x i8>, ptr %a370 store <vscale x 19 x i8> %c, ptr %b371 ret void372}373 374define void @sve_load_store_nxv20i8(ptr %a, ptr %b) {375; CHECK-LABEL: sve_load_store_nxv20i8:376; CHECK: // %bb.0:377; CHECK-NEXT: ptrue p0.s378; CHECK-NEXT: ldr z0, [x0]379; CHECK-NEXT: ld1b { z1.s }, p0/z, [x0, #4, mul vl]380; CHECK-NEXT: str z0, [x1]381; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h382; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b383; CHECK-NEXT: uunpklo z0.h, z0.b384; CHECK-NEXT: uunpklo z0.s, z0.h385; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h386; CHECK-NEXT: uzp1 z1.b, z0.b, z0.b387; CHECK-NEXT: uunpkhi z1.h, z1.b388; CHECK-NEXT: uunpkhi z1.s, z1.h389; CHECK-NEXT: uzp1 z1.h, z0.h, z1.h390; CHECK-NEXT: uzp1 z1.b, z0.b, z1.b391; CHECK-NEXT: uunpkhi z1.h, z1.b392; CHECK-NEXT: uunpklo z1.s, z1.h393; CHECK-NEXT: uzp1 z1.h, z1.h, z0.h394; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b395; CHECK-NEXT: uunpklo z0.h, z0.b396; CHECK-NEXT: uunpklo z0.s, z0.h397; CHECK-NEXT: st1b { z0.s }, p0, [x1, #4, mul vl]398; CHECK-NEXT: ret399 %c = load <vscale x 20 x i8>, ptr %a400 store <vscale x 20 x i8> %c, ptr %b401 ret void402}403 404define void @sve_load_store_nxv21i8(ptr %a, ptr %b) {405; CHECK-LABEL: sve_load_store_nxv21i8:406; CHECK: // %bb.0:407; CHECK-NEXT: rdvl x8, #1408; CHECK-NEXT: mov w10, #21 // =0x15409; CHECK-NEXT: lsr x9, x8, #4410; CHECK-NEXT: mul x9, x9, x10411; CHECK-NEXT: whilelo p0.b, x8, x9412; CHECK-NEXT: whilelo p1.b, xzr, x9413; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, #1, mul vl]414; CHECK-NEXT: ld1b { z1.b }, p1/z, [x0]415; CHECK-NEXT: st1b { z0.b }, p0, [x1, #1, mul vl]416; CHECK-NEXT: st1b { z1.b }, p1, [x1]417; CHECK-NEXT: ret418 %c = load <vscale x 21 x i8>, ptr %a419 store <vscale x 21 x i8> %c, ptr %b420 ret void421}422 423define void @sve_load_store_nxv22i8(ptr %a, ptr %b) {424; CHECK-LABEL: sve_load_store_nxv22i8:425; CHECK: // %bb.0:426; CHECK-NEXT: ptrue p0.s427; CHECK-NEXT: cntw x8, all, mul #5428; CHECK-NEXT: ldr z2, [x0]429; CHECK-NEXT: ptrue p1.d430; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, #4, mul vl]431; CHECK-NEXT: ld1b { z1.d }, p1/z, [x0, x8]432; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h433; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b434; CHECK-NEXT: uunpklo z0.h, z0.b435; CHECK-NEXT: uunpklo z0.s, z0.h436; CHECK-NEXT: uzp1 z1.s, z1.s, z0.s437; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h438; CHECK-NEXT: uzp1 z1.b, z0.b, z0.b439; CHECK-NEXT: uunpkhi z1.h, z1.b440; CHECK-NEXT: uunpkhi z1.s, z1.h441; CHECK-NEXT: uzp1 z1.h, z0.h, z1.h442; CHECK-NEXT: uzp1 z1.b, z0.b, z1.b443; CHECK-NEXT: uunpkhi z1.h, z1.b444; CHECK-NEXT: uunpklo z1.s, z1.h445; CHECK-NEXT: uzp1 z1.h, z1.h, z0.h446; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b447; CHECK-NEXT: uunpklo z0.h, z0.b448; CHECK-NEXT: uunpkhi z1.s, z0.h449; CHECK-NEXT: uunpklo z0.s, z0.h450; CHECK-NEXT: uunpklo z1.d, z1.s451; CHECK-NEXT: st1b { z1.d }, p1, [x1, x8]452; CHECK-NEXT: st1b { z0.s }, p0, [x1, #4, mul vl]453; CHECK-NEXT: str z2, [x1]454; CHECK-NEXT: ret455 %c = load <vscale x 22 x i8>, ptr %a456 store <vscale x 22 x i8> %c, ptr %b457 ret void458}459 460define void @sve_load_store_nxv23i8(ptr %a, ptr %b) {461; CHECK-LABEL: sve_load_store_nxv23i8:462; CHECK: // %bb.0:463; CHECK-NEXT: rdvl x8, #1464; CHECK-NEXT: mov w10, #23 // =0x17465; CHECK-NEXT: lsr x9, x8, #4466; CHECK-NEXT: mul x9, x9, x10467; CHECK-NEXT: whilelo p0.b, x8, x9468; CHECK-NEXT: whilelo p1.b, xzr, x9469; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, #1, mul vl]470; CHECK-NEXT: ld1b { z1.b }, p1/z, [x0]471; CHECK-NEXT: st1b { z0.b }, p0, [x1, #1, mul vl]472; CHECK-NEXT: st1b { z1.b }, p1, [x1]473; CHECK-NEXT: ret474 %c = load <vscale x 23 x i8>, ptr %a475 store <vscale x 23 x i8> %c, ptr %b476 ret void477}478 479define void @sve_load_store_nxv24i8(ptr %a, ptr %b) {480; CHECK-LABEL: sve_load_store_nxv24i8:481; CHECK: // %bb.0:482; CHECK-NEXT: ptrue p0.h483; CHECK-NEXT: ldr z0, [x0]484; CHECK-NEXT: ld1b { z1.h }, p0/z, [x0, #2, mul vl]485; CHECK-NEXT: str z0, [x1]486; CHECK-NEXT: uzp1 z0.b, z1.b, z0.b487; CHECK-NEXT: uunpklo z0.h, z0.b488; CHECK-NEXT: st1b { z0.h }, p0, [x1, #2, mul vl]489; CHECK-NEXT: ret490 %c = load <vscale x 24 x i8>, ptr %a491 store <vscale x 24 x i8> %c, ptr %b492 ret void493}494 495define void @sve_load_store_nxv25i8(ptr %a, ptr %b) {496; CHECK-LABEL: sve_load_store_nxv25i8:497; CHECK: // %bb.0:498; CHECK-NEXT: rdvl x8, #1499; CHECK-NEXT: mov w10, #25 // =0x19500; CHECK-NEXT: lsr x9, x8, #4501; CHECK-NEXT: mul x9, x9, x10502; CHECK-NEXT: whilelo p0.b, x8, x9503; CHECK-NEXT: whilelo p1.b, xzr, x9504; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, #1, mul vl]505; CHECK-NEXT: ld1b { z1.b }, p1/z, [x0]506; CHECK-NEXT: st1b { z0.b }, p0, [x1, #1, mul vl]507; CHECK-NEXT: st1b { z1.b }, p1, [x1]508; CHECK-NEXT: ret509 %c = load <vscale x 25 x i8>, ptr %a510 store <vscale x 25 x i8> %c, ptr %b511 ret void512}513 514define void @sve_load_store_nxv26i8(ptr %a, ptr %b) {515; CHECK-LABEL: sve_load_store_nxv26i8:516; CHECK: // %bb.0:517; CHECK-NEXT: ptrue p0.d518; CHECK-NEXT: cnth x8, all, mul #3519; CHECK-NEXT: ldr z2, [x0]520; CHECK-NEXT: ptrue p1.h521; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, x8]522; CHECK-NEXT: ld1b { z1.h }, p1/z, [x0, #2, mul vl]523; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s524; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h525; CHECK-NEXT: uzp1 z0.b, z1.b, z0.b526; CHECK-NEXT: uunpkhi z1.h, z0.b527; CHECK-NEXT: uunpklo z0.h, z0.b528; CHECK-NEXT: uunpklo z1.s, z1.h529; CHECK-NEXT: uunpklo z1.d, z1.s530; CHECK-NEXT: st1b { z1.d }, p0, [x1, x8]531; CHECK-NEXT: st1b { z0.h }, p1, [x1, #2, mul vl]532; CHECK-NEXT: str z2, [x1]533; CHECK-NEXT: ret534 %c = load <vscale x 26 x i8>, ptr %a535 store <vscale x 26 x i8> %c, ptr %b536 ret void537}538 539define void @sve_load_store_nxv27i8(ptr %a, ptr %b) {540; CHECK-LABEL: sve_load_store_nxv27i8:541; CHECK: // %bb.0:542; CHECK-NEXT: rdvl x8, #1543; CHECK-NEXT: mov w10, #27 // =0x1b544; CHECK-NEXT: lsr x9, x8, #4545; CHECK-NEXT: mul x9, x9, x10546; CHECK-NEXT: whilelo p0.b, x8, x9547; CHECK-NEXT: whilelo p1.b, xzr, x9548; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, #1, mul vl]549; CHECK-NEXT: ld1b { z1.b }, p1/z, [x0]550; CHECK-NEXT: st1b { z0.b }, p0, [x1, #1, mul vl]551; CHECK-NEXT: st1b { z1.b }, p1, [x1]552; CHECK-NEXT: ret553 %c = load <vscale x 27 x i8>, ptr %a554 store <vscale x 27 x i8> %c, ptr %b555 ret void556}557 558define void @sve_load_store_nxv28i8(ptr %a, ptr %b) {559; CHECK-LABEL: sve_load_store_nxv28i8:560; CHECK: // %bb.0:561; CHECK-NEXT: ptrue p0.s562; CHECK-NEXT: ldr z2, [x0]563; CHECK-NEXT: ptrue p1.h564; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, #6, mul vl]565; CHECK-NEXT: ld1b { z1.h }, p1/z, [x0, #2, mul vl]566; CHECK-NEXT: str z2, [x1]567; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h568; CHECK-NEXT: uzp1 z0.b, z1.b, z0.b569; CHECK-NEXT: uunpkhi z1.h, z0.b570; CHECK-NEXT: uunpklo z0.h, z0.b571; CHECK-NEXT: uunpklo z1.s, z1.h572; CHECK-NEXT: st1b { z0.h }, p1, [x1, #2, mul vl]573; CHECK-NEXT: st1b { z1.s }, p0, [x1, #6, mul vl]574; CHECK-NEXT: ret575 %c = load <vscale x 28 x i8>, ptr %a576 store <vscale x 28 x i8> %c, ptr %b577 ret void578}579 580define void @sve_load_store_nxv29i8(ptr %a, ptr %b) {581; CHECK-LABEL: sve_load_store_nxv29i8:582; CHECK: // %bb.0:583; CHECK-NEXT: rdvl x8, #1584; CHECK-NEXT: mov w10, #29 // =0x1d585; CHECK-NEXT: lsr x9, x8, #4586; CHECK-NEXT: mul x9, x9, x10587; CHECK-NEXT: whilelo p0.b, x8, x9588; CHECK-NEXT: whilelo p1.b, xzr, x9589; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, #1, mul vl]590; CHECK-NEXT: ld1b { z1.b }, p1/z, [x0]591; CHECK-NEXT: st1b { z0.b }, p0, [x1, #1, mul vl]592; CHECK-NEXT: st1b { z1.b }, p1, [x1]593; CHECK-NEXT: ret594 %c = load <vscale x 29 x i8>, ptr %a595 store <vscale x 29 x i8> %c, ptr %b596 ret void597}598 599define void @sve_load_store_nxv30i8(ptr %a, ptr %b) {600; CHECK-LABEL: sve_load_store_nxv30i8:601; CHECK: // %bb.0:602; CHECK-NEXT: ptrue p0.d603; CHECK-NEXT: cntw x8, all, mul #7604; CHECK-NEXT: ldr z3, [x0]605; CHECK-NEXT: ptrue p1.s606; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, x8]607; CHECK-NEXT: ptrue p2.h608; CHECK-NEXT: ld1b { z1.s }, p1/z, [x0, #6, mul vl]609; CHECK-NEXT: ld1b { z2.h }, p2/z, [x0, #2, mul vl]610; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s611; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h612; CHECK-NEXT: uzp1 z0.b, z2.b, z0.b613; CHECK-NEXT: uunpkhi z1.h, z0.b614; CHECK-NEXT: uunpklo z0.h, z0.b615; CHECK-NEXT: uunpkhi z2.s, z1.h616; CHECK-NEXT: uunpklo z1.s, z1.h617; CHECK-NEXT: uunpklo z2.d, z2.s618; CHECK-NEXT: st1b { z2.d }, p0, [x1, x8]619; CHECK-NEXT: st1b { z0.h }, p2, [x1, #2, mul vl]620; CHECK-NEXT: st1b { z1.s }, p1, [x1, #6, mul vl]621; CHECK-NEXT: str z3, [x1]622; CHECK-NEXT: ret623 %c = load <vscale x 30 x i8>, ptr %a624 store <vscale x 30 x i8> %c, ptr %b625 ret void626}627 628define void @sve_load_store_nxv31i8(ptr %a, ptr %b) {629; CHECK-LABEL: sve_load_store_nxv31i8:630; CHECK: // %bb.0:631; CHECK-NEXT: rdvl x8, #1632; CHECK-NEXT: mov w10, #31 // =0x1f633; CHECK-NEXT: lsr x9, x8, #4634; CHECK-NEXT: mul x9, x9, x10635; CHECK-NEXT: whilelo p0.b, x8, x9636; CHECK-NEXT: whilelo p1.b, xzr, x9637; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0, #1, mul vl]638; CHECK-NEXT: ld1b { z1.b }, p1/z, [x0]639; CHECK-NEXT: st1b { z0.b }, p0, [x1, #1, mul vl]640; CHECK-NEXT: st1b { z1.b }, p1, [x1]641; CHECK-NEXT: ret642 %c = load <vscale x 31 x i8>, ptr %a643 store <vscale x 31 x i8> %c, ptr %b644 ret void645}646 647define void @sve_load_store_nxv32i8(ptr %a, ptr %b) {648; CHECK-LABEL: sve_load_store_nxv32i8:649; CHECK: // %bb.0:650; CHECK-NEXT: ldr z0, [x0, #1, mul vl]651; CHECK-NEXT: ldr z1, [x0]652; CHECK-NEXT: str z0, [x1, #1, mul vl]653; CHECK-NEXT: str z1, [x1]654; CHECK-NEXT: ret655 %c = load <vscale x 32 x i8>, ptr %a656 store <vscale x 32 x i8> %c, ptr %b657 ret void658}659 660define void @sve_load_store_nxv1i16(ptr %a, ptr %b) {661; CHECK-LABEL: sve_load_store_nxv1i16:662; CHECK: // %bb.0:663; CHECK-NEXT: rdvl x8, #1664; CHECK-NEXT: lsr x8, x8, #4665; CHECK-NEXT: whilelo p0.h, xzr, x8666; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]667; CHECK-NEXT: st1h { z0.h }, p0, [x1]668; CHECK-NEXT: ret669 %c = load <vscale x 1 x i16>, ptr %a670 store <vscale x 1 x i16> %c, ptr %b671 ret void672}673 674define void @sve_load_store_nxv2i16(ptr %a, ptr %b) {675; CHECK-LABEL: sve_load_store_nxv2i16:676; CHECK: // %bb.0:677; CHECK-NEXT: ptrue p0.d678; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0]679; CHECK-NEXT: st1h { z0.d }, p0, [x1]680; CHECK-NEXT: ret681 %c = load <vscale x 2 x i16>, ptr %a682 store <vscale x 2 x i16> %c, ptr %b683 ret void684}685 686define void @sve_load_store_nxv3i16(ptr %a, ptr %b) {687; CHECK-LABEL: sve_load_store_nxv3i16:688; CHECK: // %bb.0:689; CHECK-NEXT: rdvl x8, #1690; CHECK-NEXT: mov w9, #3 // =0x3691; CHECK-NEXT: lsr x8, x8, #4692; CHECK-NEXT: mul x8, x8, x9693; CHECK-NEXT: whilelo p0.s, xzr, x8694; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0]695; CHECK-NEXT: st1h { z0.s }, p0, [x1]696; CHECK-NEXT: ret697 %c = load <vscale x 3 x i16>, ptr %a698 store <vscale x 3 x i16> %c, ptr %b699 ret void700}701 702define void @sve_load_store_nxv4i16(ptr %a, ptr %b) {703; CHECK-LABEL: sve_load_store_nxv4i16:704; CHECK: // %bb.0:705; CHECK-NEXT: ptrue p0.s706; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0]707; CHECK-NEXT: st1h { z0.s }, p0, [x1]708; CHECK-NEXT: ret709 %c = load <vscale x 4 x i16>, ptr %a710 store <vscale x 4 x i16> %c, ptr %b711 ret void712}713 714define void @sve_load_store_nxv5i16(ptr %a, ptr %b) {715; CHECK-LABEL: sve_load_store_nxv5i16:716; CHECK: // %bb.0:717; CHECK-NEXT: rdvl x8, #1718; CHECK-NEXT: mov w9, #5 // =0x5719; CHECK-NEXT: lsr x8, x8, #4720; CHECK-NEXT: mul x8, x8, x9721; CHECK-NEXT: whilelo p0.h, xzr, x8722; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]723; CHECK-NEXT: st1h { z0.h }, p0, [x1]724; CHECK-NEXT: ret725 %c = load <vscale x 5 x i16>, ptr %a726 store <vscale x 5 x i16> %c, ptr %b727 ret void728}729 730define void @sve_load_store_nxv6i16(ptr %a, ptr %b) {731; CHECK-LABEL: sve_load_store_nxv6i16:732; CHECK: // %bb.0:733; CHECK-NEXT: ptrue p0.d734; CHECK-NEXT: ptrue p1.s735; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, #2, mul vl]736; CHECK-NEXT: ld1h { z1.s }, p1/z, [x0]737; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s738; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h739; CHECK-NEXT: uunpkhi z1.s, z0.h740; CHECK-NEXT: uunpklo z0.s, z0.h741; CHECK-NEXT: uunpklo z1.d, z1.s742; CHECK-NEXT: st1h { z0.s }, p1, [x1]743; CHECK-NEXT: st1h { z1.d }, p0, [x1, #2, mul vl]744; CHECK-NEXT: ret745 %c = load <vscale x 6 x i16>, ptr %a746 store <vscale x 6 x i16> %c, ptr %b747 ret void748}749 750define void @sve_load_store_nxv7i16(ptr %a, ptr %b) {751; CHECK-LABEL: sve_load_store_nxv7i16:752; CHECK: // %bb.0:753; CHECK-NEXT: rdvl x8, #1754; CHECK-NEXT: mov w9, #7 // =0x7755; CHECK-NEXT: lsr x8, x8, #4756; CHECK-NEXT: mul x8, x8, x9757; CHECK-NEXT: whilelo p0.h, xzr, x8758; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]759; CHECK-NEXT: st1h { z0.h }, p0, [x1]760; CHECK-NEXT: ret761 %c = load <vscale x 7 x i16>, ptr %a762 store <vscale x 7 x i16> %c, ptr %b763 ret void764}765 766define void @sve_load_store_nxv8i16(ptr %a, ptr %b) {767; CHECK-LABEL: sve_load_store_nxv8i16:768; CHECK: // %bb.0:769; CHECK-NEXT: ldr z0, [x0]770; CHECK-NEXT: str z0, [x1]771; CHECK-NEXT: ret772 %c = load <vscale x 8 x i16>, ptr %a773 store <vscale x 8 x i16> %c, ptr %b774 ret void775}776 777define void @sve_load_store_nxv9i16(ptr %a, ptr %b) {778; CHECK-LABEL: sve_load_store_nxv9i16:779; CHECK: // %bb.0:780; CHECK-NEXT: rdvl x8, #1781; CHECK-NEXT: mov w9, #9 // =0x9782; CHECK-NEXT: lsr x8, x8, #4783; CHECK-NEXT: mul x8, x8, x9784; CHECK-NEXT: whilelo p0.b, xzr, x8785; CHECK-NEXT: punpkhi p1.h, p0.b786; CHECK-NEXT: punpklo p0.h, p0.b787; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0, #1, mul vl]788; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]789; CHECK-NEXT: st1h { z0.h }, p1, [x1, #1, mul vl]790; CHECK-NEXT: st1h { z1.h }, p0, [x1]791; CHECK-NEXT: ret792 %c = load <vscale x 9 x i16>, ptr %a793 store <vscale x 9 x i16> %c, ptr %b794 ret void795}796 797define void @sve_load_store_nxv10i16(ptr %a, ptr %b) {798; CHECK-LABEL: sve_load_store_nxv10i16:799; CHECK: // %bb.0:800; CHECK-NEXT: ptrue p0.d801; CHECK-NEXT: ldr z0, [x0]802; CHECK-NEXT: ld1h { z1.d }, p0/z, [x0, #4, mul vl]803; CHECK-NEXT: str z0, [x1]804; CHECK-NEXT: uzp1 z0.s, z1.s, z0.s805; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h806; CHECK-NEXT: uunpklo z0.s, z0.h807; CHECK-NEXT: uunpklo z0.d, z0.s808; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s809; CHECK-NEXT: uzp1 z1.h, z0.h, z0.h810; CHECK-NEXT: uunpkhi z1.s, z1.h811; CHECK-NEXT: uunpkhi z1.d, z1.s812; CHECK-NEXT: uzp1 z1.s, z0.s, z1.s813; CHECK-NEXT: uzp1 z1.h, z0.h, z1.h814; CHECK-NEXT: uunpkhi z1.s, z1.h815; CHECK-NEXT: uunpklo z1.d, z1.s816; CHECK-NEXT: uzp1 z1.s, z1.s, z0.s817; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h818; CHECK-NEXT: uunpklo z0.s, z0.h819; CHECK-NEXT: uunpklo z0.d, z0.s820; CHECK-NEXT: st1h { z0.d }, p0, [x1, #4, mul vl]821; CHECK-NEXT: ret822 %c = load <vscale x 10 x i16>, ptr %a823 store <vscale x 10 x i16> %c, ptr %b824 ret void825}826 827define void @sve_load_store_nxv11i16(ptr %a, ptr %b) {828; CHECK-LABEL: sve_load_store_nxv11i16:829; CHECK: // %bb.0:830; CHECK-NEXT: rdvl x8, #1831; CHECK-NEXT: mov w9, #11 // =0xb832; CHECK-NEXT: lsr x8, x8, #4833; CHECK-NEXT: mul x8, x8, x9834; CHECK-NEXT: whilelo p0.b, xzr, x8835; CHECK-NEXT: punpkhi p1.h, p0.b836; CHECK-NEXT: punpklo p0.h, p0.b837; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0, #1, mul vl]838; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]839; CHECK-NEXT: st1h { z0.h }, p1, [x1, #1, mul vl]840; CHECK-NEXT: st1h { z1.h }, p0, [x1]841; CHECK-NEXT: ret842 %c = load <vscale x 11 x i16>, ptr %a843 store <vscale x 11 x i16> %c, ptr %b844 ret void845}846 847define void @sve_load_store_nxv12i16(ptr %a, ptr %b) {848; CHECK-LABEL: sve_load_store_nxv12i16:849; CHECK: // %bb.0:850; CHECK-NEXT: ptrue p0.s851; CHECK-NEXT: ldr z0, [x0]852; CHECK-NEXT: ld1h { z1.s }, p0/z, [x0, #2, mul vl]853; CHECK-NEXT: str z0, [x1]854; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h855; CHECK-NEXT: uunpklo z0.s, z0.h856; CHECK-NEXT: st1h { z0.s }, p0, [x1, #2, mul vl]857; CHECK-NEXT: ret858 %c = load <vscale x 12 x i16>, ptr %a859 store <vscale x 12 x i16> %c, ptr %b860 ret void861}862 863define void @sve_load_store_nxv13i16(ptr %a, ptr %b) {864; CHECK-LABEL: sve_load_store_nxv13i16:865; CHECK: // %bb.0:866; CHECK-NEXT: rdvl x8, #1867; CHECK-NEXT: mov w9, #13 // =0xd868; CHECK-NEXT: lsr x8, x8, #4869; CHECK-NEXT: mul x8, x8, x9870; CHECK-NEXT: whilelo p0.b, xzr, x8871; CHECK-NEXT: punpkhi p1.h, p0.b872; CHECK-NEXT: punpklo p0.h, p0.b873; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0, #1, mul vl]874; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]875; CHECK-NEXT: st1h { z0.h }, p1, [x1, #1, mul vl]876; CHECK-NEXT: st1h { z1.h }, p0, [x1]877; CHECK-NEXT: ret878 %c = load <vscale x 13 x i16>, ptr %a879 store <vscale x 13 x i16> %c, ptr %b880 ret void881}882 883define void @sve_load_store_nxv14i16(ptr %a, ptr %b) {884; CHECK-LABEL: sve_load_store_nxv14i16:885; CHECK: // %bb.0:886; CHECK-NEXT: ptrue p0.d887; CHECK-NEXT: ldr z2, [x0]888; CHECK-NEXT: ptrue p1.s889; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, #6, mul vl]890; CHECK-NEXT: ld1h { z1.s }, p1/z, [x0, #2, mul vl]891; CHECK-NEXT: str z2, [x1]892; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s893; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h894; CHECK-NEXT: uunpkhi z1.s, z0.h895; CHECK-NEXT: uunpklo z0.s, z0.h896; CHECK-NEXT: uunpklo z1.d, z1.s897; CHECK-NEXT: st1h { z0.s }, p1, [x1, #2, mul vl]898; CHECK-NEXT: st1h { z1.d }, p0, [x1, #6, mul vl]899; CHECK-NEXT: ret900 %c = load <vscale x 14 x i16>, ptr %a901 store <vscale x 14 x i16> %c, ptr %b902 ret void903}904 905define void @sve_load_store_nxv15i16(ptr %a, ptr %b) {906; CHECK-LABEL: sve_load_store_nxv15i16:907; CHECK: // %bb.0:908; CHECK-NEXT: rdvl x8, #1909; CHECK-NEXT: mov w9, #15 // =0xf910; CHECK-NEXT: lsr x8, x8, #4911; CHECK-NEXT: mul x8, x8, x9912; CHECK-NEXT: whilelo p0.b, xzr, x8913; CHECK-NEXT: punpkhi p1.h, p0.b914; CHECK-NEXT: punpklo p0.h, p0.b915; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0, #1, mul vl]916; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]917; CHECK-NEXT: st1h { z0.h }, p1, [x1, #1, mul vl]918; CHECK-NEXT: st1h { z1.h }, p0, [x1]919; CHECK-NEXT: ret920 %c = load <vscale x 15 x i16>, ptr %a921 store <vscale x 15 x i16> %c, ptr %b922 ret void923}924 925define void @sve_load_store_nxv16i16(ptr %a, ptr %b) {926; CHECK-LABEL: sve_load_store_nxv16i16:927; CHECK: // %bb.0:928; CHECK-NEXT: ldr z0, [x0, #1, mul vl]929; CHECK-NEXT: ldr z1, [x0]930; CHECK-NEXT: str z0, [x1, #1, mul vl]931; CHECK-NEXT: str z1, [x1]932; CHECK-NEXT: ret933 %c = load <vscale x 16 x i16>, ptr %a934 store <vscale x 16 x i16> %c, ptr %b935 ret void936}937 938define void @sve_load_store_nxv1i32(ptr %a, ptr %b) {939; CHECK-LABEL: sve_load_store_nxv1i32:940; CHECK: // %bb.0:941; CHECK-NEXT: rdvl x8, #1942; CHECK-NEXT: lsr x8, x8, #4943; CHECK-NEXT: whilelo p0.s, xzr, x8944; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]945; CHECK-NEXT: st1w { z0.s }, p0, [x1]946; CHECK-NEXT: ret947 %c = load <vscale x 1 x i32>, ptr %a948 store <vscale x 1 x i32> %c, ptr %b949 ret void950}951 952define void @sve_load_store_nxv2i32(ptr %a, ptr %b) {953; CHECK-LABEL: sve_load_store_nxv2i32:954; CHECK: // %bb.0:955; CHECK-NEXT: ptrue p0.d956; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0]957; CHECK-NEXT: st1w { z0.d }, p0, [x1]958; CHECK-NEXT: ret959 %c = load <vscale x 2 x i32>, ptr %a960 store <vscale x 2 x i32> %c, ptr %b961 ret void962}963 964define void @sve_load_store_nxv3i32(ptr %a, ptr %b) {965; CHECK-LABEL: sve_load_store_nxv3i32:966; CHECK: // %bb.0:967; CHECK-NEXT: rdvl x8, #1968; CHECK-NEXT: mov w9, #3 // =0x3969; CHECK-NEXT: lsr x8, x8, #4970; CHECK-NEXT: mul x8, x8, x9971; CHECK-NEXT: whilelo p0.s, xzr, x8972; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]973; CHECK-NEXT: st1w { z0.s }, p0, [x1]974; CHECK-NEXT: ret975 %c = load <vscale x 3 x i32>, ptr %a976 store <vscale x 3 x i32> %c, ptr %b977 ret void978}979 980define void @sve_load_store_nxv4i32(ptr %a, ptr %b) {981; CHECK-LABEL: sve_load_store_nxv4i32:982; CHECK: // %bb.0:983; CHECK-NEXT: ldr z0, [x0]984; CHECK-NEXT: str z0, [x1]985; CHECK-NEXT: ret986 %c = load <vscale x 4 x i32>, ptr %a987 store <vscale x 4 x i32> %c, ptr %b988 ret void989}990 991define void @sve_load_store_nxv5i32(ptr %a, ptr %b) {992; CHECK-LABEL: sve_load_store_nxv5i32:993; CHECK: // %bb.0:994; CHECK-NEXT: rdvl x8, #1995; CHECK-NEXT: mov w9, #5 // =0x5996; CHECK-NEXT: lsr x8, x8, #4997; CHECK-NEXT: mul x8, x8, x9998; CHECK-NEXT: whilelo p0.h, xzr, x8999; CHECK-NEXT: punpkhi p1.h, p0.b1000; CHECK-NEXT: punpklo p0.h, p0.b1001; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl]1002; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]1003; CHECK-NEXT: st1w { z0.s }, p1, [x1, #1, mul vl]1004; CHECK-NEXT: st1w { z1.s }, p0, [x1]1005; CHECK-NEXT: ret1006 %c = load <vscale x 5 x i32>, ptr %a1007 store <vscale x 5 x i32> %c, ptr %b1008 ret void1009}1010 1011define void @sve_load_store_nxv6i32(ptr %a, ptr %b) {1012; CHECK-LABEL: sve_load_store_nxv6i32:1013; CHECK: // %bb.0:1014; CHECK-NEXT: ptrue p0.d1015; CHECK-NEXT: ldr z0, [x0]1016; CHECK-NEXT: ld1w { z1.d }, p0/z, [x0, #2, mul vl]1017; CHECK-NEXT: str z0, [x1]1018; CHECK-NEXT: uzp1 z0.s, z1.s, z0.s1019; CHECK-NEXT: uunpklo z0.d, z0.s1020; CHECK-NEXT: st1w { z0.d }, p0, [x1, #2, mul vl]1021; CHECK-NEXT: ret1022 %c = load <vscale x 6 x i32>, ptr %a1023 store <vscale x 6 x i32> %c, ptr %b1024 ret void1025}1026 1027define void @sve_load_store_nxv7i32(ptr %a, ptr %b) {1028; CHECK-LABEL: sve_load_store_nxv7i32:1029; CHECK: // %bb.0:1030; CHECK-NEXT: rdvl x8, #11031; CHECK-NEXT: mov w9, #7 // =0x71032; CHECK-NEXT: lsr x8, x8, #41033; CHECK-NEXT: mul x8, x8, x91034; CHECK-NEXT: whilelo p0.h, xzr, x81035; CHECK-NEXT: punpkhi p1.h, p0.b1036; CHECK-NEXT: punpklo p0.h, p0.b1037; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl]1038; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]1039; CHECK-NEXT: st1w { z0.s }, p1, [x1, #1, mul vl]1040; CHECK-NEXT: st1w { z1.s }, p0, [x1]1041; CHECK-NEXT: ret1042 %c = load <vscale x 7 x i32>, ptr %a1043 store <vscale x 7 x i32> %c, ptr %b1044 ret void1045}1046 1047define void @sve_load_store_nxv8i32(ptr %a, ptr %b) {1048; CHECK-LABEL: sve_load_store_nxv8i32:1049; CHECK: // %bb.0:1050; CHECK-NEXT: ldr z0, [x0, #1, mul vl]1051; CHECK-NEXT: ldr z1, [x0]1052; CHECK-NEXT: str z0, [x1, #1, mul vl]1053; CHECK-NEXT: str z1, [x1]1054; CHECK-NEXT: ret1055 %c = load <vscale x 8 x i32>, ptr %a1056 store <vscale x 8 x i32> %c, ptr %b1057 ret void1058}1059 1060define void @sve_load_store_nxv1i64(ptr %a, ptr %b) {1061; CHECK-LABEL: sve_load_store_nxv1i64:1062; CHECK: // %bb.0:1063; CHECK-NEXT: rdvl x8, #11064; CHECK-NEXT: lsr x8, x8, #41065; CHECK-NEXT: whilelo p0.d, xzr, x81066; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1067; CHECK-NEXT: st1d { z0.d }, p0, [x1]1068; CHECK-NEXT: ret1069 %c = load <vscale x 1 x i64>, ptr %a1070 store <vscale x 1 x i64> %c, ptr %b1071 ret void1072}1073 1074define void @sve_load_store_nxv2i64(ptr %a, ptr %b) {1075; CHECK-LABEL: sve_load_store_nxv2i64:1076; CHECK: // %bb.0:1077; CHECK-NEXT: ldr z0, [x0]1078; CHECK-NEXT: str z0, [x1]1079; CHECK-NEXT: ret1080 %c = load <vscale x 2 x i64>, ptr %a1081 store <vscale x 2 x i64> %c, ptr %b1082 ret void1083}1084 1085define void @sve_load_store_nxv3i64(ptr %a, ptr %b) {1086; CHECK-LABEL: sve_load_store_nxv3i64:1087; CHECK: // %bb.0:1088; CHECK-NEXT: rdvl x8, #11089; CHECK-NEXT: mov w9, #3 // =0x31090; CHECK-NEXT: lsr x8, x8, #41091; CHECK-NEXT: mul x8, x8, x91092; CHECK-NEXT: whilelo p0.s, xzr, x81093; CHECK-NEXT: punpkhi p1.h, p0.b1094; CHECK-NEXT: punpklo p0.h, p0.b1095; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #1, mul vl]1096; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]1097; CHECK-NEXT: st1d { z0.d }, p1, [x1, #1, mul vl]1098; CHECK-NEXT: st1d { z1.d }, p0, [x1]1099; CHECK-NEXT: ret1100 %c = load <vscale x 3 x i64>, ptr %a1101 store <vscale x 3 x i64> %c, ptr %b1102 ret void1103}1104 1105define void @sve_load_store_nxv4i64(ptr %a, ptr %b) {1106; CHECK-LABEL: sve_load_store_nxv4i64:1107; CHECK: // %bb.0:1108; CHECK-NEXT: ldr z0, [x0, #1, mul vl]1109; CHECK-NEXT: ldr z1, [x0]1110; CHECK-NEXT: str z0, [x1, #1, mul vl]1111; CHECK-NEXT: str z1, [x1]1112; CHECK-NEXT: ret1113 %c = load <vscale x 4 x i64>, ptr %a1114 store <vscale x 4 x i64> %c, ptr %b1115 ret void1116}1117 1118define void @sve_load_store_nxv1f16(ptr %a, ptr %b) {1119; CHECK-LABEL: sve_load_store_nxv1f16:1120; CHECK: // %bb.0:1121; CHECK-NEXT: rdvl x8, #11122; CHECK-NEXT: lsr x8, x8, #41123; CHECK-NEXT: whilelo p0.d, xzr, x81124; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0]1125; CHECK-NEXT: st1h { z0.d }, p0, [x1]1126; CHECK-NEXT: ret1127 %c = load <vscale x 1 x half>, ptr %a1128 store <vscale x 1 x half> %c, ptr %b1129 ret void1130}1131 1132define void @sve_load_store_nxv2f16(ptr %a, ptr %b) {1133; CHECK-LABEL: sve_load_store_nxv2f16:1134; CHECK: // %bb.0:1135; CHECK-NEXT: ptrue p0.d1136; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0]1137; CHECK-NEXT: st1h { z0.d }, p0, [x1]1138; CHECK-NEXT: ret1139 %c = load <vscale x 2 x half>, ptr %a1140 store <vscale x 2 x half> %c, ptr %b1141 ret void1142}1143 1144define void @sve_load_store_nxv3f16(ptr %a, ptr %b) {1145; CHECK-LABEL: sve_load_store_nxv3f16:1146; CHECK: // %bb.0:1147; CHECK-NEXT: rdvl x8, #11148; CHECK-NEXT: mov w9, #3 // =0x31149; CHECK-NEXT: lsr x8, x8, #41150; CHECK-NEXT: mul x8, x8, x91151; CHECK-NEXT: whilelo p0.s, xzr, x81152; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0]1153; CHECK-NEXT: st1h { z0.s }, p0, [x1]1154; CHECK-NEXT: ret1155 %c = load <vscale x 3 x half>, ptr %a1156 store <vscale x 3 x half> %c, ptr %b1157 ret void1158}1159 1160define void @sve_load_store_nxv4f16(ptr %a, ptr %b) {1161; CHECK-LABEL: sve_load_store_nxv4f16:1162; CHECK: // %bb.0:1163; CHECK-NEXT: ptrue p0.s1164; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0]1165; CHECK-NEXT: st1h { z0.s }, p0, [x1]1166; CHECK-NEXT: ret1167 %c = load <vscale x 4 x half>, ptr %a1168 store <vscale x 4 x half> %c, ptr %b1169 ret void1170}1171 1172define void @sve_load_store_nxv5f16(ptr %a, ptr %b) {1173; CHECK-LABEL: sve_load_store_nxv5f16:1174; CHECK: // %bb.0:1175; CHECK-NEXT: rdvl x8, #11176; CHECK-NEXT: mov w9, #5 // =0x51177; CHECK-NEXT: lsr x8, x8, #41178; CHECK-NEXT: mul x8, x8, x91179; CHECK-NEXT: whilelo p0.h, xzr, x81180; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1181; CHECK-NEXT: st1h { z0.h }, p0, [x1]1182; CHECK-NEXT: ret1183 %c = load <vscale x 5 x half>, ptr %a1184 store <vscale x 5 x half> %c, ptr %b1185 ret void1186}1187 1188define void @sve_load_store_nxv6f16(ptr %a, ptr %b) {1189; CHECK-LABEL: sve_load_store_nxv6f16:1190; CHECK: // %bb.0:1191; CHECK-NEXT: ptrue p0.d1192; CHECK-NEXT: ptrue p1.s1193; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, #2, mul vl]1194; CHECK-NEXT: ld1h { z1.s }, p1/z, [x0]1195; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s1196; CHECK-NEXT: st1h { z1.s }, p1, [x1]1197; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h1198; CHECK-NEXT: uunpkhi z0.s, z0.h1199; CHECK-NEXT: uunpklo z0.d, z0.s1200; CHECK-NEXT: st1h { z0.d }, p0, [x1, #2, mul vl]1201; CHECK-NEXT: ret1202 %c = load <vscale x 6 x half>, ptr %a1203 store <vscale x 6 x half> %c, ptr %b1204 ret void1205}1206 1207define void @sve_load_store_nxv7f16(ptr %a, ptr %b) {1208; CHECK-LABEL: sve_load_store_nxv7f16:1209; CHECK: // %bb.0:1210; CHECK-NEXT: rdvl x8, #11211; CHECK-NEXT: mov w9, #7 // =0x71212; CHECK-NEXT: lsr x8, x8, #41213; CHECK-NEXT: mul x8, x8, x91214; CHECK-NEXT: whilelo p0.h, xzr, x81215; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1216; CHECK-NEXT: st1h { z0.h }, p0, [x1]1217; CHECK-NEXT: ret1218 %c = load <vscale x 7 x half>, ptr %a1219 store <vscale x 7 x half> %c, ptr %b1220 ret void1221}1222 1223define void @sve_load_store_nxv8f16(ptr %a, ptr %b) {1224; CHECK-LABEL: sve_load_store_nxv8f16:1225; CHECK: // %bb.0:1226; CHECK-NEXT: ldr z0, [x0]1227; CHECK-NEXT: str z0, [x1]1228; CHECK-NEXT: ret1229 %c = load <vscale x 8 x half>, ptr %a1230 store <vscale x 8 x half> %c, ptr %b1231 ret void1232}1233 1234define void @sve_load_store_nxv9f16(ptr %a, ptr %b) {1235; CHECK-LABEL: sve_load_store_nxv9f16:1236; CHECK: // %bb.0:1237; CHECK-NEXT: rdvl x8, #11238; CHECK-NEXT: mov w9, #9 // =0x91239; CHECK-NEXT: lsr x8, x8, #41240; CHECK-NEXT: mul x8, x8, x91241; CHECK-NEXT: whilelo p0.b, xzr, x81242; CHECK-NEXT: punpkhi p1.h, p0.b1243; CHECK-NEXT: punpklo p0.h, p0.b1244; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0, #1, mul vl]1245; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]1246; CHECK-NEXT: st1h { z0.h }, p1, [x1, #1, mul vl]1247; CHECK-NEXT: st1h { z1.h }, p0, [x1]1248; CHECK-NEXT: ret1249 %c = load <vscale x 9 x half>, ptr %a1250 store <vscale x 9 x half> %c, ptr %b1251 ret void1252}1253 1254define void @sve_load_store_nxv10f16(ptr %a, ptr %b) {1255; CHECK-LABEL: sve_load_store_nxv10f16:1256; CHECK: // %bb.0:1257; CHECK-NEXT: ptrue p0.d1258; CHECK-NEXT: ldr z0, [x0]1259; CHECK-NEXT: ld1h { z1.d }, p0/z, [x0, #4, mul vl]1260; CHECK-NEXT: str z0, [x1]1261; CHECK-NEXT: st1h { z1.d }, p0, [x1, #4, mul vl]1262; CHECK-NEXT: ret1263 %c = load <vscale x 10 x half>, ptr %a1264 store <vscale x 10 x half> %c, ptr %b1265 ret void1266}1267 1268define void @sve_load_store_nxv11f16(ptr %a, ptr %b) {1269; CHECK-LABEL: sve_load_store_nxv11f16:1270; CHECK: // %bb.0:1271; CHECK-NEXT: rdvl x8, #11272; CHECK-NEXT: mov w9, #11 // =0xb1273; CHECK-NEXT: lsr x8, x8, #41274; CHECK-NEXT: mul x8, x8, x91275; CHECK-NEXT: whilelo p0.b, xzr, x81276; CHECK-NEXT: punpkhi p1.h, p0.b1277; CHECK-NEXT: punpklo p0.h, p0.b1278; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0, #1, mul vl]1279; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]1280; CHECK-NEXT: st1h { z0.h }, p1, [x1, #1, mul vl]1281; CHECK-NEXT: st1h { z1.h }, p0, [x1]1282; CHECK-NEXT: ret1283 %c = load <vscale x 11 x half>, ptr %a1284 store <vscale x 11 x half> %c, ptr %b1285 ret void1286}1287 1288define void @sve_load_store_nxv12f16(ptr %a, ptr %b) {1289; CHECK-LABEL: sve_load_store_nxv12f16:1290; CHECK: // %bb.0:1291; CHECK-NEXT: ptrue p0.s1292; CHECK-NEXT: ldr z0, [x0]1293; CHECK-NEXT: ld1h { z1.s }, p0/z, [x0, #2, mul vl]1294; CHECK-NEXT: str z0, [x1]1295; CHECK-NEXT: st1h { z1.s }, p0, [x1, #2, mul vl]1296; CHECK-NEXT: ret1297 %c = load <vscale x 12 x half>, ptr %a1298 store <vscale x 12 x half> %c, ptr %b1299 ret void1300}1301 1302define void @sve_load_store_nxv13f16(ptr %a, ptr %b) {1303; CHECK-LABEL: sve_load_store_nxv13f16:1304; CHECK: // %bb.0:1305; CHECK-NEXT: rdvl x8, #11306; CHECK-NEXT: mov w9, #13 // =0xd1307; CHECK-NEXT: lsr x8, x8, #41308; CHECK-NEXT: mul x8, x8, x91309; CHECK-NEXT: whilelo p0.b, xzr, x81310; CHECK-NEXT: punpkhi p1.h, p0.b1311; CHECK-NEXT: punpklo p0.h, p0.b1312; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0, #1, mul vl]1313; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]1314; CHECK-NEXT: st1h { z0.h }, p1, [x1, #1, mul vl]1315; CHECK-NEXT: st1h { z1.h }, p0, [x1]1316; CHECK-NEXT: ret1317 %c = load <vscale x 13 x half>, ptr %a1318 store <vscale x 13 x half> %c, ptr %b1319 ret void1320}1321 1322define void @sve_load_store_nxv14f16(ptr %a, ptr %b) {1323; CHECK-LABEL: sve_load_store_nxv14f16:1324; CHECK: // %bb.0:1325; CHECK-NEXT: ptrue p0.d1326; CHECK-NEXT: ldr z2, [x0]1327; CHECK-NEXT: ptrue p1.s1328; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, #6, mul vl]1329; CHECK-NEXT: ld1h { z1.s }, p1/z, [x0, #2, mul vl]1330; CHECK-NEXT: str z2, [x1]1331; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s1332; CHECK-NEXT: st1h { z1.s }, p1, [x1, #2, mul vl]1333; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h1334; CHECK-NEXT: uunpkhi z0.s, z0.h1335; CHECK-NEXT: uunpklo z0.d, z0.s1336; CHECK-NEXT: st1h { z0.d }, p0, [x1, #6, mul vl]1337; CHECK-NEXT: ret1338 %c = load <vscale x 14 x half>, ptr %a1339 store <vscale x 14 x half> %c, ptr %b1340 ret void1341}1342 1343define void @sve_load_store_nxv15f16(ptr %a, ptr %b) {1344; CHECK-LABEL: sve_load_store_nxv15f16:1345; CHECK: // %bb.0:1346; CHECK-NEXT: rdvl x8, #11347; CHECK-NEXT: mov w9, #15 // =0xf1348; CHECK-NEXT: lsr x8, x8, #41349; CHECK-NEXT: mul x8, x8, x91350; CHECK-NEXT: whilelo p0.b, xzr, x81351; CHECK-NEXT: punpkhi p1.h, p0.b1352; CHECK-NEXT: punpklo p0.h, p0.b1353; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0, #1, mul vl]1354; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]1355; CHECK-NEXT: st1h { z0.h }, p1, [x1, #1, mul vl]1356; CHECK-NEXT: st1h { z1.h }, p0, [x1]1357; CHECK-NEXT: ret1358 %c = load <vscale x 15 x half>, ptr %a1359 store <vscale x 15 x half> %c, ptr %b1360 ret void1361}1362 1363define void @sve_load_store_nxv16f16(ptr %a, ptr %b) {1364; CHECK-LABEL: sve_load_store_nxv16f16:1365; CHECK: // %bb.0:1366; CHECK-NEXT: ldr z0, [x0, #1, mul vl]1367; CHECK-NEXT: ldr z1, [x0]1368; CHECK-NEXT: str z0, [x1, #1, mul vl]1369; CHECK-NEXT: str z1, [x1]1370; CHECK-NEXT: ret1371 %c = load <vscale x 16 x half>, ptr %a1372 store <vscale x 16 x half> %c, ptr %b1373 ret void1374}1375 1376define void @sve_load_store_nxv1f32(ptr %a, ptr %b) {1377; CHECK-LABEL: sve_load_store_nxv1f32:1378; CHECK: // %bb.0:1379; CHECK-NEXT: rdvl x8, #11380; CHECK-NEXT: lsr x8, x8, #41381; CHECK-NEXT: whilelo p0.d, xzr, x81382; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0]1383; CHECK-NEXT: st1w { z0.d }, p0, [x1]1384; CHECK-NEXT: ret1385 %c = load <vscale x 1 x float>, ptr %a1386 store <vscale x 1 x float> %c, ptr %b1387 ret void1388}1389 1390define void @sve_load_store_nxv2f32(ptr %a, ptr %b) {1391; CHECK-LABEL: sve_load_store_nxv2f32:1392; CHECK: // %bb.0:1393; CHECK-NEXT: ptrue p0.d1394; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0]1395; CHECK-NEXT: st1w { z0.d }, p0, [x1]1396; CHECK-NEXT: ret1397 %c = load <vscale x 2 x float>, ptr %a1398 store <vscale x 2 x float> %c, ptr %b1399 ret void1400}1401 1402define void @sve_load_store_nxv3f32(ptr %a, ptr %b) {1403; CHECK-LABEL: sve_load_store_nxv3f32:1404; CHECK: // %bb.0:1405; CHECK-NEXT: rdvl x8, #11406; CHECK-NEXT: mov w9, #3 // =0x31407; CHECK-NEXT: lsr x8, x8, #41408; CHECK-NEXT: mul x8, x8, x91409; CHECK-NEXT: whilelo p0.s, xzr, x81410; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1411; CHECK-NEXT: st1w { z0.s }, p0, [x1]1412; CHECK-NEXT: ret1413 %c = load <vscale x 3 x float>, ptr %a1414 store <vscale x 3 x float> %c, ptr %b1415 ret void1416}1417 1418define void @sve_load_store_nxv4f32(ptr %a, ptr %b) {1419; CHECK-LABEL: sve_load_store_nxv4f32:1420; CHECK: // %bb.0:1421; CHECK-NEXT: ldr z0, [x0]1422; CHECK-NEXT: str z0, [x1]1423; CHECK-NEXT: ret1424 %c = load <vscale x 4 x float>, ptr %a1425 store <vscale x 4 x float> %c, ptr %b1426 ret void1427}1428 1429define void @sve_load_store_nxv5f32(ptr %a, ptr %b) {1430; CHECK-LABEL: sve_load_store_nxv5f32:1431; CHECK: // %bb.0:1432; CHECK-NEXT: rdvl x8, #11433; CHECK-NEXT: mov w9, #5 // =0x51434; CHECK-NEXT: lsr x8, x8, #41435; CHECK-NEXT: mul x8, x8, x91436; CHECK-NEXT: whilelo p0.h, xzr, x81437; CHECK-NEXT: punpkhi p1.h, p0.b1438; CHECK-NEXT: punpklo p0.h, p0.b1439; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl]1440; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]1441; CHECK-NEXT: st1w { z0.s }, p1, [x1, #1, mul vl]1442; CHECK-NEXT: st1w { z1.s }, p0, [x1]1443; CHECK-NEXT: ret1444 %c = load <vscale x 5 x float>, ptr %a1445 store <vscale x 5 x float> %c, ptr %b1446 ret void1447}1448 1449define void @sve_load_store_nxv6f32(ptr %a, ptr %b) {1450; CHECK-LABEL: sve_load_store_nxv6f32:1451; CHECK: // %bb.0:1452; CHECK-NEXT: ptrue p0.d1453; CHECK-NEXT: ldr z0, [x0]1454; CHECK-NEXT: ld1w { z1.d }, p0/z, [x0, #2, mul vl]1455; CHECK-NEXT: str z0, [x1]1456; CHECK-NEXT: st1w { z1.d }, p0, [x1, #2, mul vl]1457; CHECK-NEXT: ret1458 %c = load <vscale x 6 x float>, ptr %a1459 store <vscale x 6 x float> %c, ptr %b1460 ret void1461}1462 1463define void @sve_load_store_nxv7f32(ptr %a, ptr %b) {1464; CHECK-LABEL: sve_load_store_nxv7f32:1465; CHECK: // %bb.0:1466; CHECK-NEXT: rdvl x8, #11467; CHECK-NEXT: mov w9, #7 // =0x71468; CHECK-NEXT: lsr x8, x8, #41469; CHECK-NEXT: mul x8, x8, x91470; CHECK-NEXT: whilelo p0.h, xzr, x81471; CHECK-NEXT: punpkhi p1.h, p0.b1472; CHECK-NEXT: punpklo p0.h, p0.b1473; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0, #1, mul vl]1474; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0]1475; CHECK-NEXT: st1w { z0.s }, p1, [x1, #1, mul vl]1476; CHECK-NEXT: st1w { z1.s }, p0, [x1]1477; CHECK-NEXT: ret1478 %c = load <vscale x 7 x float>, ptr %a1479 store <vscale x 7 x float> %c, ptr %b1480 ret void1481}1482 1483define void @sve_load_store_nxv8f32(ptr %a, ptr %b) {1484; CHECK-LABEL: sve_load_store_nxv8f32:1485; CHECK: // %bb.0:1486; CHECK-NEXT: ldr z0, [x0, #1, mul vl]1487; CHECK-NEXT: ldr z1, [x0]1488; CHECK-NEXT: str z0, [x1, #1, mul vl]1489; CHECK-NEXT: str z1, [x1]1490; CHECK-NEXT: ret1491 %c = load <vscale x 8 x float>, ptr %a1492 store <vscale x 8 x float> %c, ptr %b1493 ret void1494}1495 1496define void @sve_load_store_nxv1f64(ptr %a, ptr %b) {1497; CHECK-LABEL: sve_load_store_nxv1f64:1498; CHECK: // %bb.0:1499; CHECK-NEXT: rdvl x8, #11500; CHECK-NEXT: lsr x8, x8, #41501; CHECK-NEXT: whilelo p0.d, xzr, x81502; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]1503; CHECK-NEXT: st1d { z0.d }, p0, [x1]1504; CHECK-NEXT: ret1505 %c = load <vscale x 1 x double>, ptr %a1506 store <vscale x 1 x double> %c, ptr %b1507 ret void1508}1509 1510define void @sve_load_store_nxv2f64(ptr %a, ptr %b) {1511; CHECK-LABEL: sve_load_store_nxv2f64:1512; CHECK: // %bb.0:1513; CHECK-NEXT: ldr z0, [x0]1514; CHECK-NEXT: str z0, [x1]1515; CHECK-NEXT: ret1516 %c = load <vscale x 2 x double>, ptr %a1517 store <vscale x 2 x double> %c, ptr %b1518 ret void1519}1520 1521define void @sve_load_store_nxv3f64(ptr %a, ptr %b) {1522; CHECK-LABEL: sve_load_store_nxv3f64:1523; CHECK: // %bb.0:1524; CHECK-NEXT: rdvl x8, #11525; CHECK-NEXT: mov w9, #3 // =0x31526; CHECK-NEXT: lsr x8, x8, #41527; CHECK-NEXT: mul x8, x8, x91528; CHECK-NEXT: whilelo p0.s, xzr, x81529; CHECK-NEXT: punpkhi p1.h, p0.b1530; CHECK-NEXT: punpklo p0.h, p0.b1531; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0, #1, mul vl]1532; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]1533; CHECK-NEXT: st1d { z0.d }, p1, [x1, #1, mul vl]1534; CHECK-NEXT: st1d { z1.d }, p0, [x1]1535; CHECK-NEXT: ret1536 %c = load <vscale x 3 x double>, ptr %a1537 store <vscale x 3 x double> %c, ptr %b1538 ret void1539}1540 1541define void @sve_load_store_nxv4f64(ptr %a, ptr %b) {1542; CHECK-LABEL: sve_load_store_nxv4f64:1543; CHECK: // %bb.0:1544; CHECK-NEXT: ldr z0, [x0, #1, mul vl]1545; CHECK-NEXT: ldr z1, [x0]1546; CHECK-NEXT: str z0, [x1, #1, mul vl]1547; CHECK-NEXT: str z1, [x1]1548; CHECK-NEXT: ret1549 %c = load <vscale x 4 x double>, ptr %a1550 store <vscale x 4 x double> %c, ptr %b1551 ret void1552}1553 1554define void @sve_load_store_nxv1bf16(ptr %a, ptr %b) {1555; CHECK-LABEL: sve_load_store_nxv1bf16:1556; CHECK: // %bb.0:1557; CHECK-NEXT: rdvl x8, #11558; CHECK-NEXT: lsr x8, x8, #41559; CHECK-NEXT: whilelo p0.d, xzr, x81560; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0]1561; CHECK-NEXT: st1h { z0.d }, p0, [x1]1562; CHECK-NEXT: ret1563 %c = load <vscale x 1 x bfloat>, ptr %a1564 store <vscale x 1 x bfloat> %c, ptr %b1565 ret void1566}1567 1568define void @sve_load_store_nxv2bf16(ptr %a, ptr %b) {1569; CHECK-LABEL: sve_load_store_nxv2bf16:1570; CHECK: // %bb.0:1571; CHECK-NEXT: ptrue p0.d1572; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0]1573; CHECK-NEXT: st1h { z0.d }, p0, [x1]1574; CHECK-NEXT: ret1575 %c = load <vscale x 2 x bfloat>, ptr %a1576 store <vscale x 2 x bfloat> %c, ptr %b1577 ret void1578}1579 1580define void @sve_load_store_nxv3bf16(ptr %a, ptr %b) {1581; CHECK-LABEL: sve_load_store_nxv3bf16:1582; CHECK: // %bb.0:1583; CHECK-NEXT: rdvl x8, #11584; CHECK-NEXT: mov w9, #3 // =0x31585; CHECK-NEXT: lsr x8, x8, #41586; CHECK-NEXT: mul x8, x8, x91587; CHECK-NEXT: whilelo p0.s, xzr, x81588; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0]1589; CHECK-NEXT: st1h { z0.s }, p0, [x1]1590; CHECK-NEXT: ret1591 %c = load <vscale x 3 x bfloat>, ptr %a1592 store <vscale x 3 x bfloat> %c, ptr %b1593 ret void1594}1595 1596define void @sve_load_store_nxv4bf16(ptr %a, ptr %b) {1597; CHECK-LABEL: sve_load_store_nxv4bf16:1598; CHECK: // %bb.0:1599; CHECK-NEXT: ptrue p0.s1600; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0]1601; CHECK-NEXT: st1h { z0.s }, p0, [x1]1602; CHECK-NEXT: ret1603 %c = load <vscale x 4 x bfloat>, ptr %a1604 store <vscale x 4 x bfloat> %c, ptr %b1605 ret void1606}1607 1608define void @sve_load_store_nxv5bf16(ptr %a, ptr %b) {1609; CHECK-LABEL: sve_load_store_nxv5bf16:1610; CHECK: // %bb.0:1611; CHECK-NEXT: rdvl x8, #11612; CHECK-NEXT: mov w9, #5 // =0x51613; CHECK-NEXT: lsr x8, x8, #41614; CHECK-NEXT: mul x8, x8, x91615; CHECK-NEXT: whilelo p0.h, xzr, x81616; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1617; CHECK-NEXT: st1h { z0.h }, p0, [x1]1618; CHECK-NEXT: ret1619 %c = load <vscale x 5 x bfloat>, ptr %a1620 store <vscale x 5 x bfloat> %c, ptr %b1621 ret void1622}1623 1624define void @sve_load_store_nxv6bf16(ptr %a, ptr %b) {1625; CHECK-LABEL: sve_load_store_nxv6bf16:1626; CHECK: // %bb.0:1627; CHECK-NEXT: ptrue p0.d1628; CHECK-NEXT: ptrue p1.s1629; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, #2, mul vl]1630; CHECK-NEXT: ld1h { z1.s }, p1/z, [x0]1631; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s1632; CHECK-NEXT: st1h { z1.s }, p1, [x1]1633; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h1634; CHECK-NEXT: uunpkhi z0.s, z0.h1635; CHECK-NEXT: uunpklo z0.d, z0.s1636; CHECK-NEXT: st1h { z0.d }, p0, [x1, #2, mul vl]1637; CHECK-NEXT: ret1638 %c = load <vscale x 6 x bfloat>, ptr %a1639 store <vscale x 6 x bfloat> %c, ptr %b1640 ret void1641}1642 1643define void @sve_load_store_nxv7bf16(ptr %a, ptr %b) {1644; CHECK-LABEL: sve_load_store_nxv7bf16:1645; CHECK: // %bb.0:1646; CHECK-NEXT: rdvl x8, #11647; CHECK-NEXT: mov w9, #7 // =0x71648; CHECK-NEXT: lsr x8, x8, #41649; CHECK-NEXT: mul x8, x8, x91650; CHECK-NEXT: whilelo p0.h, xzr, x81651; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]1652; CHECK-NEXT: st1h { z0.h }, p0, [x1]1653; CHECK-NEXT: ret1654 %c = load <vscale x 7 x bfloat>, ptr %a1655 store <vscale x 7 x bfloat> %c, ptr %b1656 ret void1657}1658 1659define void @sve_load_store_nxv8bf16(ptr %a, ptr %b) {1660; CHECK-LABEL: sve_load_store_nxv8bf16:1661; CHECK: // %bb.0:1662; CHECK-NEXT: ldr z0, [x0]1663; CHECK-NEXT: str z0, [x1]1664; CHECK-NEXT: ret1665 %c = load <vscale x 8 x bfloat>, ptr %a1666 store <vscale x 8 x bfloat> %c, ptr %b1667 ret void1668}1669 1670define void @sve_load_store_nxv9bf16(ptr %a, ptr %b) {1671; CHECK-LABEL: sve_load_store_nxv9bf16:1672; CHECK: // %bb.0:1673; CHECK-NEXT: rdvl x8, #11674; CHECK-NEXT: mov w9, #9 // =0x91675; CHECK-NEXT: lsr x8, x8, #41676; CHECK-NEXT: mul x8, x8, x91677; CHECK-NEXT: whilelo p0.b, xzr, x81678; CHECK-NEXT: punpkhi p1.h, p0.b1679; CHECK-NEXT: punpklo p0.h, p0.b1680; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0, #1, mul vl]1681; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]1682; CHECK-NEXT: st1h { z0.h }, p1, [x1, #1, mul vl]1683; CHECK-NEXT: st1h { z1.h }, p0, [x1]1684; CHECK-NEXT: ret1685 %c = load <vscale x 9 x bfloat>, ptr %a1686 store <vscale x 9 x bfloat> %c, ptr %b1687 ret void1688}1689 1690define void @sve_load_store_nxv10bf16(ptr %a, ptr %b) {1691; CHECK-LABEL: sve_load_store_nxv10bf16:1692; CHECK: // %bb.0:1693; CHECK-NEXT: ptrue p0.d1694; CHECK-NEXT: ldr z0, [x0]1695; CHECK-NEXT: ld1h { z1.d }, p0/z, [x0, #4, mul vl]1696; CHECK-NEXT: str z0, [x1]1697; CHECK-NEXT: st1h { z1.d }, p0, [x1, #4, mul vl]1698; CHECK-NEXT: ret1699 %c = load <vscale x 10 x bfloat>, ptr %a1700 store <vscale x 10 x bfloat> %c, ptr %b1701 ret void1702}1703 1704define void @sve_load_store_nxv11bf16(ptr %a, ptr %b) {1705; CHECK-LABEL: sve_load_store_nxv11bf16:1706; CHECK: // %bb.0:1707; CHECK-NEXT: rdvl x8, #11708; CHECK-NEXT: mov w9, #11 // =0xb1709; CHECK-NEXT: lsr x8, x8, #41710; CHECK-NEXT: mul x8, x8, x91711; CHECK-NEXT: whilelo p0.b, xzr, x81712; CHECK-NEXT: punpkhi p1.h, p0.b1713; CHECK-NEXT: punpklo p0.h, p0.b1714; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0, #1, mul vl]1715; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]1716; CHECK-NEXT: st1h { z0.h }, p1, [x1, #1, mul vl]1717; CHECK-NEXT: st1h { z1.h }, p0, [x1]1718; CHECK-NEXT: ret1719 %c = load <vscale x 11 x bfloat>, ptr %a1720 store <vscale x 11 x bfloat> %c, ptr %b1721 ret void1722}1723 1724define void @sve_load_store_nxv12bf16(ptr %a, ptr %b) {1725; CHECK-LABEL: sve_load_store_nxv12bf16:1726; CHECK: // %bb.0:1727; CHECK-NEXT: ptrue p0.s1728; CHECK-NEXT: ldr z0, [x0]1729; CHECK-NEXT: ld1h { z1.s }, p0/z, [x0, #2, mul vl]1730; CHECK-NEXT: str z0, [x1]1731; CHECK-NEXT: st1h { z1.s }, p0, [x1, #2, mul vl]1732; CHECK-NEXT: ret1733 %c = load <vscale x 12 x bfloat>, ptr %a1734 store <vscale x 12 x bfloat> %c, ptr %b1735 ret void1736}1737 1738define void @sve_load_store_nxv13bf16(ptr %a, ptr %b) {1739; CHECK-LABEL: sve_load_store_nxv13bf16:1740; CHECK: // %bb.0:1741; CHECK-NEXT: rdvl x8, #11742; CHECK-NEXT: mov w9, #13 // =0xd1743; CHECK-NEXT: lsr x8, x8, #41744; CHECK-NEXT: mul x8, x8, x91745; CHECK-NEXT: whilelo p0.b, xzr, x81746; CHECK-NEXT: punpkhi p1.h, p0.b1747; CHECK-NEXT: punpklo p0.h, p0.b1748; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0, #1, mul vl]1749; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]1750; CHECK-NEXT: st1h { z0.h }, p1, [x1, #1, mul vl]1751; CHECK-NEXT: st1h { z1.h }, p0, [x1]1752; CHECK-NEXT: ret1753 %c = load <vscale x 13 x bfloat>, ptr %a1754 store <vscale x 13 x bfloat> %c, ptr %b1755 ret void1756}1757 1758define void @sve_load_store_nxv14bf16(ptr %a, ptr %b) {1759; CHECK-LABEL: sve_load_store_nxv14bf16:1760; CHECK: // %bb.0:1761; CHECK-NEXT: ptrue p0.d1762; CHECK-NEXT: ldr z2, [x0]1763; CHECK-NEXT: ptrue p1.s1764; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, #6, mul vl]1765; CHECK-NEXT: ld1h { z1.s }, p1/z, [x0, #2, mul vl]1766; CHECK-NEXT: str z2, [x1]1767; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s1768; CHECK-NEXT: st1h { z1.s }, p1, [x1, #2, mul vl]1769; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h1770; CHECK-NEXT: uunpkhi z0.s, z0.h1771; CHECK-NEXT: uunpklo z0.d, z0.s1772; CHECK-NEXT: st1h { z0.d }, p0, [x1, #6, mul vl]1773; CHECK-NEXT: ret1774 %c = load <vscale x 14 x bfloat>, ptr %a1775 store <vscale x 14 x bfloat> %c, ptr %b1776 ret void1777}1778 1779define void @sve_load_store_nxv15bf16(ptr %a, ptr %b) {1780; CHECK-LABEL: sve_load_store_nxv15bf16:1781; CHECK: // %bb.0:1782; CHECK-NEXT: rdvl x8, #11783; CHECK-NEXT: mov w9, #15 // =0xf1784; CHECK-NEXT: lsr x8, x8, #41785; CHECK-NEXT: mul x8, x8, x91786; CHECK-NEXT: whilelo p0.b, xzr, x81787; CHECK-NEXT: punpkhi p1.h, p0.b1788; CHECK-NEXT: punpklo p0.h, p0.b1789; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0, #1, mul vl]1790; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]1791; CHECK-NEXT: st1h { z0.h }, p1, [x1, #1, mul vl]1792; CHECK-NEXT: st1h { z1.h }, p0, [x1]1793; CHECK-NEXT: ret1794 %c = load <vscale x 15 x bfloat>, ptr %a1795 store <vscale x 15 x bfloat> %c, ptr %b1796 ret void1797}1798 1799define void @sve_load_store_nxv16bf16(ptr %a, ptr %b) {1800; CHECK-LABEL: sve_load_store_nxv16bf16:1801; CHECK: // %bb.0:1802; CHECK-NEXT: ldr z0, [x0, #1, mul vl]1803; CHECK-NEXT: ldr z1, [x0]1804; CHECK-NEXT: str z0, [x1, #1, mul vl]1805; CHECK-NEXT: str z1, [x1]1806; CHECK-NEXT: ret1807 %c = load <vscale x 16 x bfloat>, ptr %a1808 store <vscale x 16 x bfloat> %c, ptr %b1809 ret void1810}1811 1812define <vscale x 1 x i16> @sve_sextload_nxv1i8(ptr %a, ptr %b) {1813; CHECK-LABEL: sve_sextload_nxv1i8:1814; CHECK: // %bb.0:1815; CHECK-NEXT: rdvl x8, #11816; CHECK-NEXT: lsr x8, x8, #41817; CHECK-NEXT: whilelo p0.h, xzr, x81818; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0]1819; CHECK-NEXT: ret1820 %c = load <vscale x 1 x i8>, ptr %a1821 %c.sext = sext <vscale x 1 x i8> %c to <vscale x 1 x i16>1822 ret <vscale x 1 x i16> %c.sext1823}1824 1825define <vscale x 2 x i16> @sve_sextload_nxv2i8(ptr %a, ptr %b) {1826; CHECK-LABEL: sve_sextload_nxv2i8:1827; CHECK: // %bb.0:1828; CHECK-NEXT: ptrue p0.d1829; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]1830; CHECK-NEXT: ret1831 %c = load <vscale x 2 x i8>, ptr %a1832 %c.sext = sext <vscale x 2 x i8> %c to <vscale x 2 x i16>1833 ret <vscale x 2 x i16> %c.sext1834}1835 1836define <vscale x 3 x i16> @sve_sextload_nxv3i8(ptr %a, ptr %b) {1837; CHECK-LABEL: sve_sextload_nxv3i8:1838; CHECK: // %bb.0:1839; CHECK-NEXT: rdvl x8, #11840; CHECK-NEXT: mov w9, #3 // =0x31841; CHECK-NEXT: lsr x8, x8, #41842; CHECK-NEXT: mul x8, x8, x91843; CHECK-NEXT: whilelo p0.s, xzr, x81844; CHECK-NEXT: ld1sb { z0.s }, p0/z, [x0]1845; CHECK-NEXT: ret1846 %c = load <vscale x 3 x i8>, ptr %a1847 %c.sext = sext <vscale x 3 x i8> %c to <vscale x 3 x i16>1848 ret <vscale x 3 x i16> %c.sext1849}1850 1851define <vscale x 4 x i16> @sve_sextload_nxv4i8(ptr %a, ptr %b) {1852; CHECK-LABEL: sve_sextload_nxv4i8:1853; CHECK: // %bb.0:1854; CHECK-NEXT: ptrue p0.s1855; CHECK-NEXT: ld1sb { z0.s }, p0/z, [x0]1856; CHECK-NEXT: ret1857 %c = load <vscale x 4 x i8>, ptr %a1858 %c.sext = sext <vscale x 4 x i8> %c to <vscale x 4 x i16>1859 ret <vscale x 4 x i16> %c.sext1860}1861 1862define <vscale x 5 x i16> @sve_sextload_nxv5i8(ptr %a, ptr %b) {1863; CHECK-LABEL: sve_sextload_nxv5i8:1864; CHECK: // %bb.0:1865; CHECK-NEXT: rdvl x8, #11866; CHECK-NEXT: mov w9, #5 // =0x51867; CHECK-NEXT: lsr x8, x8, #41868; CHECK-NEXT: mul x8, x8, x91869; CHECK-NEXT: whilelo p0.h, xzr, x81870; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0]1871; CHECK-NEXT: ret1872 %c = load <vscale x 5 x i8>, ptr %a1873 %c.sext = sext <vscale x 5 x i8> %c to <vscale x 5 x i16>1874 ret <vscale x 5 x i16> %c.sext1875}1876 1877define <vscale x 6 x i16> @sve_sextload_nxv6i8(ptr %a, ptr %b) {1878; CHECK-LABEL: sve_sextload_nxv6i8:1879; CHECK: // %bb.0:1880; CHECK-NEXT: cntd x8, all, mul #31881; CHECK-NEXT: whilelo p0.h, xzr, x81882; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0]1883; CHECK-NEXT: ret1884 %c = load <vscale x 6 x i8>, ptr %a1885 %c.sext = sext <vscale x 6 x i8> %c to <vscale x 6 x i16>1886 ret <vscale x 6 x i16> %c.sext1887}1888 1889define <vscale x 7 x i16> @sve_sextload_nxv7i8(ptr %a, ptr %b) {1890; CHECK-LABEL: sve_sextload_nxv7i8:1891; CHECK: // %bb.0:1892; CHECK-NEXT: rdvl x8, #11893; CHECK-NEXT: mov w9, #7 // =0x71894; CHECK-NEXT: lsr x8, x8, #41895; CHECK-NEXT: mul x8, x8, x91896; CHECK-NEXT: whilelo p0.h, xzr, x81897; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0]1898; CHECK-NEXT: ret1899 %c = load <vscale x 7 x i8>, ptr %a1900 %c.sext = sext <vscale x 7 x i8> %c to <vscale x 7 x i16>1901 ret <vscale x 7 x i16> %c.sext1902}1903 1904define <vscale x 8 x i16> @sve_sextload_nxv8i8(ptr %a, ptr %b) {1905; CHECK-LABEL: sve_sextload_nxv8i8:1906; CHECK: // %bb.0:1907; CHECK-NEXT: ptrue p0.h1908; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0]1909; CHECK-NEXT: ret1910 %c = load <vscale x 8 x i8>, ptr %a1911 %c.sext = sext <vscale x 8 x i8> %c to <vscale x 8 x i16>1912 ret <vscale x 8 x i16> %c.sext1913}1914 1915define <vscale x 9 x i16> @sve_sextload_nxv9i8(ptr %a, ptr %b) {1916; CHECK-LABEL: sve_sextload_nxv9i8:1917; CHECK: // %bb.0:1918; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1919; CHECK-NEXT: addvl sp, sp, #-21920; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG1921; CHECK-NEXT: .cfi_offset w29, -161922; CHECK-NEXT: rdvl x8, #11923; CHECK-NEXT: mov w9, #9 // =0x91924; CHECK-NEXT: lsr x8, x8, #41925; CHECK-NEXT: mul x8, x8, x91926; CHECK-NEXT: whilelo p0.b, xzr, x81927; CHECK-NEXT: punpkhi p1.h, p0.b1928; CHECK-NEXT: punpklo p0.h, p0.b1929; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0, #1, mul vl]1930; CHECK-NEXT: ld1sb { z1.h }, p0/z, [x0]1931; CHECK-NEXT: st1h { z0.h }, p1, [sp, #1, mul vl]1932; CHECK-NEXT: st1h { z1.h }, p0, [sp]1933; CHECK-NEXT: ldr z1, [sp, #1, mul vl]1934; CHECK-NEXT: ldr z0, [sp]1935; CHECK-NEXT: addvl sp, sp, #21936; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1937; CHECK-NEXT: ret1938 %c = load <vscale x 9 x i8>, ptr %a1939 %c.sext = sext <vscale x 9 x i8> %c to <vscale x 9 x i16>1940 ret <vscale x 9 x i16> %c.sext1941}1942 1943define <vscale x 10 x i16> @sve_sextload_nxv10i8(ptr %a, ptr %b) {1944; CHECK-LABEL: sve_sextload_nxv10i8:1945; CHECK: // %bb.0:1946; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1947; CHECK-NEXT: addvl sp, sp, #-21948; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG1949; CHECK-NEXT: .cfi_offset w29, -161950; CHECK-NEXT: cntd x8, all, mul #51951; CHECK-NEXT: whilelo p0.b, xzr, x81952; CHECK-NEXT: punpkhi p1.h, p0.b1953; CHECK-NEXT: punpklo p0.h, p0.b1954; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0, #1, mul vl]1955; CHECK-NEXT: ld1sb { z1.h }, p0/z, [x0]1956; CHECK-NEXT: ptrue p0.d1957; CHECK-NEXT: uunpklo z0.s, z0.h1958; CHECK-NEXT: str z1, [sp]1959; CHECK-NEXT: uunpklo z0.d, z0.s1960; CHECK-NEXT: st1h { z0.d }, p0, [sp, #4, mul vl]1961; CHECK-NEXT: ldr z0, [sp]1962; CHECK-NEXT: ldr z1, [sp, #1, mul vl]1963; CHECK-NEXT: addvl sp, sp, #21964; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1965; CHECK-NEXT: ret1966 %c = load <vscale x 10 x i8>, ptr %a1967 %c.sext = sext <vscale x 10 x i8> %c to <vscale x 10 x i16>1968 ret <vscale x 10 x i16> %c.sext1969}1970 1971define <vscale x 11 x i16> @sve_sextload_nxv11i8(ptr %a, ptr %b) {1972; CHECK-LABEL: sve_sextload_nxv11i8:1973; CHECK: // %bb.0:1974; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill1975; CHECK-NEXT: addvl sp, sp, #-21976; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG1977; CHECK-NEXT: .cfi_offset w29, -161978; CHECK-NEXT: rdvl x8, #11979; CHECK-NEXT: mov w9, #11 // =0xb1980; CHECK-NEXT: lsr x8, x8, #41981; CHECK-NEXT: mul x8, x8, x91982; CHECK-NEXT: whilelo p0.b, xzr, x81983; CHECK-NEXT: punpkhi p1.h, p0.b1984; CHECK-NEXT: punpklo p0.h, p0.b1985; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0, #1, mul vl]1986; CHECK-NEXT: ld1sb { z1.h }, p0/z, [x0]1987; CHECK-NEXT: st1h { z0.h }, p1, [sp, #1, mul vl]1988; CHECK-NEXT: st1h { z1.h }, p0, [sp]1989; CHECK-NEXT: ldr z1, [sp, #1, mul vl]1990; CHECK-NEXT: ldr z0, [sp]1991; CHECK-NEXT: addvl sp, sp, #21992; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload1993; CHECK-NEXT: ret1994 %c = load <vscale x 11 x i8>, ptr %a1995 %c.sext = sext <vscale x 11 x i8> %c to <vscale x 11 x i16>1996 ret <vscale x 11 x i16> %c.sext1997}1998 1999define <vscale x 12 x i16> @sve_sextload_nxv12i8(ptr %a, ptr %b) {2000; CHECK-LABEL: sve_sextload_nxv12i8:2001; CHECK: // %bb.0:2002; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2003; CHECK-NEXT: addvl sp, sp, #-22004; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2005; CHECK-NEXT: .cfi_offset w29, -162006; CHECK-NEXT: cntw x8, all, mul #32007; CHECK-NEXT: whilelo p0.b, xzr, x82008; CHECK-NEXT: punpkhi p1.h, p0.b2009; CHECK-NEXT: punpklo p0.h, p0.b2010; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0, #1, mul vl]2011; CHECK-NEXT: ptrue p1.s2012; CHECK-NEXT: ld1sb { z1.h }, p0/z, [x0]2013; CHECK-NEXT: uunpklo z0.s, z0.h2014; CHECK-NEXT: str z1, [sp]2015; CHECK-NEXT: st1h { z0.s }, p1, [sp, #2, mul vl]2016; CHECK-NEXT: ldr z0, [sp]2017; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2018; CHECK-NEXT: addvl sp, sp, #22019; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2020; CHECK-NEXT: ret2021 %c = load <vscale x 12 x i8>, ptr %a2022 %c.sext = sext <vscale x 12 x i8> %c to <vscale x 12 x i16>2023 ret <vscale x 12 x i16> %c.sext2024}2025 2026define <vscale x 13 x i16> @sve_sextload_nxv13i8(ptr %a, ptr %b) {2027; CHECK-LABEL: sve_sextload_nxv13i8:2028; CHECK: // %bb.0:2029; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2030; CHECK-NEXT: addvl sp, sp, #-22031; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2032; CHECK-NEXT: .cfi_offset w29, -162033; CHECK-NEXT: rdvl x8, #12034; CHECK-NEXT: mov w9, #13 // =0xd2035; CHECK-NEXT: lsr x8, x8, #42036; CHECK-NEXT: mul x8, x8, x92037; CHECK-NEXT: whilelo p0.b, xzr, x82038; CHECK-NEXT: punpkhi p1.h, p0.b2039; CHECK-NEXT: punpklo p0.h, p0.b2040; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0, #1, mul vl]2041; CHECK-NEXT: ld1sb { z1.h }, p0/z, [x0]2042; CHECK-NEXT: st1h { z0.h }, p1, [sp, #1, mul vl]2043; CHECK-NEXT: st1h { z1.h }, p0, [sp]2044; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2045; CHECK-NEXT: ldr z0, [sp]2046; CHECK-NEXT: addvl sp, sp, #22047; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2048; CHECK-NEXT: ret2049 %c = load <vscale x 13 x i8>, ptr %a2050 %c.sext = sext <vscale x 13 x i8> %c to <vscale x 13 x i16>2051 ret <vscale x 13 x i16> %c.sext2052}2053 2054define <vscale x 14 x i16> @sve_sextload_nxv14i8(ptr %a, ptr %b) {2055; CHECK-LABEL: sve_sextload_nxv14i8:2056; CHECK: // %bb.0:2057; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2058; CHECK-NEXT: addvl sp, sp, #-22059; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2060; CHECK-NEXT: .cfi_offset w29, -162061; CHECK-NEXT: cntd x8, all, mul #72062; CHECK-NEXT: whilelo p0.b, xzr, x82063; CHECK-NEXT: punpkhi p1.h, p0.b2064; CHECK-NEXT: punpklo p0.h, p0.b2065; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0, #1, mul vl]2066; CHECK-NEXT: ptrue p1.s2067; CHECK-NEXT: ld1sb { z2.h }, p0/z, [x0]2068; CHECK-NEXT: ptrue p0.d2069; CHECK-NEXT: uunpkhi z1.s, z0.h2070; CHECK-NEXT: uunpklo z0.s, z0.h2071; CHECK-NEXT: str z2, [sp]2072; CHECK-NEXT: uunpklo z1.d, z1.s2073; CHECK-NEXT: st1h { z0.s }, p1, [sp, #2, mul vl]2074; CHECK-NEXT: ldr z0, [sp]2075; CHECK-NEXT: st1h { z1.d }, p0, [sp, #6, mul vl]2076; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2077; CHECK-NEXT: addvl sp, sp, #22078; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2079; CHECK-NEXT: ret2080 %c = load <vscale x 14 x i8>, ptr %a2081 %c.sext = sext <vscale x 14 x i8> %c to <vscale x 14 x i16>2082 ret <vscale x 14 x i16> %c.sext2083}2084 2085define <vscale x 15 x i16> @sve_sextload_nxv15i8(ptr %a, ptr %b) {2086; CHECK-LABEL: sve_sextload_nxv15i8:2087; CHECK: // %bb.0:2088; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2089; CHECK-NEXT: addvl sp, sp, #-22090; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2091; CHECK-NEXT: .cfi_offset w29, -162092; CHECK-NEXT: rdvl x8, #12093; CHECK-NEXT: mov w9, #15 // =0xf2094; CHECK-NEXT: lsr x8, x8, #42095; CHECK-NEXT: mul x8, x8, x92096; CHECK-NEXT: whilelo p0.b, xzr, x82097; CHECK-NEXT: punpkhi p1.h, p0.b2098; CHECK-NEXT: punpklo p0.h, p0.b2099; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0, #1, mul vl]2100; CHECK-NEXT: ld1sb { z1.h }, p0/z, [x0]2101; CHECK-NEXT: st1h { z0.h }, p1, [sp, #1, mul vl]2102; CHECK-NEXT: st1h { z1.h }, p0, [sp]2103; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2104; CHECK-NEXT: ldr z0, [sp]2105; CHECK-NEXT: addvl sp, sp, #22106; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2107; CHECK-NEXT: ret2108 %c = load <vscale x 15 x i8>, ptr %a2109 %c.sext = sext <vscale x 15 x i8> %c to <vscale x 15 x i16>2110 ret <vscale x 15 x i16> %c.sext2111}2112 2113define <vscale x 16 x i16> @sve_sextload_nxv16i8(ptr %a, ptr %b) {2114; CHECK-LABEL: sve_sextload_nxv16i8:2115; CHECK: // %bb.0:2116; CHECK-NEXT: ptrue p0.h2117; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0]2118; CHECK-NEXT: ld1sb { z1.h }, p0/z, [x0, #1, mul vl]2119; CHECK-NEXT: ret2120 %c = load <vscale x 16 x i8>, ptr %a2121 %c.sext = sext <vscale x 16 x i8> %c to <vscale x 16 x i16>2122 ret <vscale x 16 x i16> %c.sext2123}2124 2125define <vscale x 1 x i32> @sve_sextload_nxv1i16(ptr %a, ptr %b) {2126; CHECK-LABEL: sve_sextload_nxv1i16:2127; CHECK: // %bb.0:2128; CHECK-NEXT: rdvl x8, #12129; CHECK-NEXT: lsr x8, x8, #42130; CHECK-NEXT: whilelo p0.s, xzr, x82131; CHECK-NEXT: ld1sh { z0.s }, p0/z, [x0]2132; CHECK-NEXT: ret2133 %c = load <vscale x 1 x i16>, ptr %a2134 %c.sext = sext <vscale x 1 x i16> %c to <vscale x 1 x i32>2135 ret <vscale x 1 x i32> %c.sext2136}2137 2138define <vscale x 2 x i32> @sve_sextload_nxv2i16(ptr %a, ptr %b) {2139; CHECK-LABEL: sve_sextload_nxv2i16:2140; CHECK: // %bb.0:2141; CHECK-NEXT: ptrue p0.d2142; CHECK-NEXT: ld1sh { z0.d }, p0/z, [x0]2143; CHECK-NEXT: ret2144 %c = load <vscale x 2 x i16>, ptr %a2145 %c.sext = sext <vscale x 2 x i16> %c to <vscale x 2 x i32>2146 ret <vscale x 2 x i32> %c.sext2147}2148 2149define <vscale x 3 x i32> @sve_sextload_nxv3i16(ptr %a, ptr %b) {2150; CHECK-LABEL: sve_sextload_nxv3i16:2151; CHECK: // %bb.0:2152; CHECK-NEXT: rdvl x8, #12153; CHECK-NEXT: mov w9, #3 // =0x32154; CHECK-NEXT: lsr x8, x8, #42155; CHECK-NEXT: mul x8, x8, x92156; CHECK-NEXT: whilelo p0.s, xzr, x82157; CHECK-NEXT: ld1sh { z0.s }, p0/z, [x0]2158; CHECK-NEXT: ret2159 %c = load <vscale x 3 x i16>, ptr %a2160 %c.sext = sext <vscale x 3 x i16> %c to <vscale x 3 x i32>2161 ret <vscale x 3 x i32> %c.sext2162}2163 2164define <vscale x 4 x i32> @sve_sextload_nxv4i16(ptr %a, ptr %b) {2165; CHECK-LABEL: sve_sextload_nxv4i16:2166; CHECK: // %bb.0:2167; CHECK-NEXT: ptrue p0.s2168; CHECK-NEXT: ld1sh { z0.s }, p0/z, [x0]2169; CHECK-NEXT: ret2170 %c = load <vscale x 4 x i16>, ptr %a2171 %c.sext = sext <vscale x 4 x i16> %c to <vscale x 4 x i32>2172 ret <vscale x 4 x i32> %c.sext2173}2174 2175define <vscale x 5 x i32> @sve_sextload_nxv5i16(ptr %a, ptr %b) {2176; CHECK-LABEL: sve_sextload_nxv5i16:2177; CHECK: // %bb.0:2178; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2179; CHECK-NEXT: addvl sp, sp, #-22180; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2181; CHECK-NEXT: .cfi_offset w29, -162182; CHECK-NEXT: rdvl x8, #12183; CHECK-NEXT: mov w9, #5 // =0x52184; CHECK-NEXT: lsr x8, x8, #42185; CHECK-NEXT: mul x8, x8, x92186; CHECK-NEXT: whilelo p0.h, xzr, x82187; CHECK-NEXT: punpkhi p1.h, p0.b2188; CHECK-NEXT: punpklo p0.h, p0.b2189; CHECK-NEXT: ld1sh { z0.s }, p1/z, [x0, #1, mul vl]2190; CHECK-NEXT: ld1sh { z1.s }, p0/z, [x0]2191; CHECK-NEXT: st1w { z0.s }, p1, [sp, #1, mul vl]2192; CHECK-NEXT: st1w { z1.s }, p0, [sp]2193; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2194; CHECK-NEXT: ldr z0, [sp]2195; CHECK-NEXT: addvl sp, sp, #22196; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2197; CHECK-NEXT: ret2198 %c = load <vscale x 5 x i16>, ptr %a2199 %c.sext = sext <vscale x 5 x i16> %c to <vscale x 5 x i32>2200 ret <vscale x 5 x i32> %c.sext2201}2202 2203define <vscale x 6 x i32> @sve_sextload_nxv6i16(ptr %a, ptr %b) {2204; CHECK-LABEL: sve_sextload_nxv6i16:2205; CHECK: // %bb.0:2206; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2207; CHECK-NEXT: addvl sp, sp, #-22208; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2209; CHECK-NEXT: .cfi_offset w29, -162210; CHECK-NEXT: cntd x8, all, mul #32211; CHECK-NEXT: whilelo p0.h, xzr, x82212; CHECK-NEXT: punpkhi p1.h, p0.b2213; CHECK-NEXT: punpklo p0.h, p0.b2214; CHECK-NEXT: ld1sh { z0.s }, p1/z, [x0, #1, mul vl]2215; CHECK-NEXT: ptrue p1.d2216; CHECK-NEXT: ld1sh { z1.s }, p0/z, [x0]2217; CHECK-NEXT: uunpklo z0.d, z0.s2218; CHECK-NEXT: str z1, [sp]2219; CHECK-NEXT: st1w { z0.d }, p1, [sp, #2, mul vl]2220; CHECK-NEXT: ldr z0, [sp]2221; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2222; CHECK-NEXT: addvl sp, sp, #22223; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2224; CHECK-NEXT: ret2225 %c = load <vscale x 6 x i16>, ptr %a2226 %c.sext = sext <vscale x 6 x i16> %c to <vscale x 6 x i32>2227 ret <vscale x 6 x i32> %c.sext2228}2229 2230define <vscale x 7 x i32> @sve_sextload_nxv7i16(ptr %a, ptr %b) {2231; CHECK-LABEL: sve_sextload_nxv7i16:2232; CHECK: // %bb.0:2233; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2234; CHECK-NEXT: addvl sp, sp, #-22235; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2236; CHECK-NEXT: .cfi_offset w29, -162237; CHECK-NEXT: rdvl x8, #12238; CHECK-NEXT: mov w9, #7 // =0x72239; CHECK-NEXT: lsr x8, x8, #42240; CHECK-NEXT: mul x8, x8, x92241; CHECK-NEXT: whilelo p0.h, xzr, x82242; CHECK-NEXT: punpkhi p1.h, p0.b2243; CHECK-NEXT: punpklo p0.h, p0.b2244; CHECK-NEXT: ld1sh { z0.s }, p1/z, [x0, #1, mul vl]2245; CHECK-NEXT: ld1sh { z1.s }, p0/z, [x0]2246; CHECK-NEXT: st1w { z0.s }, p1, [sp, #1, mul vl]2247; CHECK-NEXT: st1w { z1.s }, p0, [sp]2248; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2249; CHECK-NEXT: ldr z0, [sp]2250; CHECK-NEXT: addvl sp, sp, #22251; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2252; CHECK-NEXT: ret2253 %c = load <vscale x 7 x i16>, ptr %a2254 %c.sext = sext <vscale x 7 x i16> %c to <vscale x 7 x i32>2255 ret <vscale x 7 x i32> %c.sext2256}2257 2258define <vscale x 8 x i32> @sve_sextload_nxv8i16(ptr %a, ptr %b) {2259; CHECK-LABEL: sve_sextload_nxv8i16:2260; CHECK: // %bb.0:2261; CHECK-NEXT: ptrue p0.s2262; CHECK-NEXT: ld1sh { z0.s }, p0/z, [x0]2263; CHECK-NEXT: ld1sh { z1.s }, p0/z, [x0, #1, mul vl]2264; CHECK-NEXT: ret2265 %c = load <vscale x 8 x i16>, ptr %a2266 %c.sext = sext <vscale x 8 x i16> %c to <vscale x 8 x i32>2267 ret <vscale x 8 x i32> %c.sext2268}2269 2270define <vscale x 1 x i64> @sve_sextload_nxv1i32(ptr %a, ptr %b) {2271; CHECK-LABEL: sve_sextload_nxv1i32:2272; CHECK: // %bb.0:2273; CHECK-NEXT: rdvl x8, #12274; CHECK-NEXT: lsr x8, x8, #42275; CHECK-NEXT: whilelo p0.d, xzr, x82276; CHECK-NEXT: ld1sw { z0.d }, p0/z, [x0]2277; CHECK-NEXT: ret2278 %c = load <vscale x 1 x i32>, ptr %a2279 %c.sext = sext <vscale x 1 x i32> %c to <vscale x 1 x i64>2280 ret <vscale x 1 x i64> %c.sext2281}2282 2283define <vscale x 2 x i64> @sve_sextload_nxv2i32(ptr %a, ptr %b) {2284; CHECK-LABEL: sve_sextload_nxv2i32:2285; CHECK: // %bb.0:2286; CHECK-NEXT: ptrue p0.d2287; CHECK-NEXT: ld1sw { z0.d }, p0/z, [x0]2288; CHECK-NEXT: ret2289 %c = load <vscale x 2 x i32>, ptr %a2290 %c.sext = sext <vscale x 2 x i32> %c to <vscale x 2 x i64>2291 ret <vscale x 2 x i64> %c.sext2292}2293 2294define <vscale x 3 x i64> @sve_sextload_nxv3i32(ptr %a, ptr %b) {2295; CHECK-LABEL: sve_sextload_nxv3i32:2296; CHECK: // %bb.0:2297; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2298; CHECK-NEXT: addvl sp, sp, #-22299; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2300; CHECK-NEXT: .cfi_offset w29, -162301; CHECK-NEXT: rdvl x8, #12302; CHECK-NEXT: mov w9, #3 // =0x32303; CHECK-NEXT: lsr x8, x8, #42304; CHECK-NEXT: mul x8, x8, x92305; CHECK-NEXT: whilelo p0.s, xzr, x82306; CHECK-NEXT: punpkhi p1.h, p0.b2307; CHECK-NEXT: punpklo p0.h, p0.b2308; CHECK-NEXT: ld1sw { z0.d }, p1/z, [x0, #1, mul vl]2309; CHECK-NEXT: ld1sw { z1.d }, p0/z, [x0]2310; CHECK-NEXT: st1d { z0.d }, p1, [sp, #1, mul vl]2311; CHECK-NEXT: st1d { z1.d }, p0, [sp]2312; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2313; CHECK-NEXT: ldr z0, [sp]2314; CHECK-NEXT: addvl sp, sp, #22315; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2316; CHECK-NEXT: ret2317 %c = load <vscale x 3 x i32>, ptr %a2318 %c.sext = sext <vscale x 3 x i32> %c to <vscale x 3 x i64>2319 ret <vscale x 3 x i64> %c.sext2320}2321 2322define <vscale x 4 x i64> @sve_sextload_nxv4i32(ptr %a, ptr %b) {2323; CHECK-LABEL: sve_sextload_nxv4i32:2324; CHECK: // %bb.0:2325; CHECK-NEXT: ptrue p0.d2326; CHECK-NEXT: ld1sw { z0.d }, p0/z, [x0]2327; CHECK-NEXT: ld1sw { z1.d }, p0/z, [x0, #1, mul vl]2328; CHECK-NEXT: ret2329 %c = load <vscale x 4 x i32>, ptr %a2330 %c.sext = sext <vscale x 4 x i32> %c to <vscale x 4 x i64>2331 ret <vscale x 4 x i64> %c.sext2332}2333 2334define <vscale x 1 x i16> @sve_zextload_nxv1i8(ptr %a, ptr %b) {2335; CHECK-LABEL: sve_zextload_nxv1i8:2336; CHECK: // %bb.0:2337; CHECK-NEXT: rdvl x8, #12338; CHECK-NEXT: lsr x8, x8, #42339; CHECK-NEXT: whilelo p0.h, xzr, x82340; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0]2341; CHECK-NEXT: ret2342 %c = load <vscale x 1 x i8>, ptr %a2343 %c.zext = sext <vscale x 1 x i8> %c to <vscale x 1 x i16>2344 ret <vscale x 1 x i16> %c.zext2345}2346 2347define <vscale x 2 x i16> @sve_zextload_nxv2i8(ptr %a, ptr %b) {2348; CHECK-LABEL: sve_zextload_nxv2i8:2349; CHECK: // %bb.0:2350; CHECK-NEXT: ptrue p0.d2351; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0]2352; CHECK-NEXT: ret2353 %c = load <vscale x 2 x i8>, ptr %a2354 %c.zext = sext <vscale x 2 x i8> %c to <vscale x 2 x i16>2355 ret <vscale x 2 x i16> %c.zext2356}2357 2358define <vscale x 3 x i16> @sve_zextload_nxv3i8(ptr %a, ptr %b) {2359; CHECK-LABEL: sve_zextload_nxv3i8:2360; CHECK: // %bb.0:2361; CHECK-NEXT: rdvl x8, #12362; CHECK-NEXT: mov w9, #3 // =0x32363; CHECK-NEXT: lsr x8, x8, #42364; CHECK-NEXT: mul x8, x8, x92365; CHECK-NEXT: whilelo p0.s, xzr, x82366; CHECK-NEXT: ld1sb { z0.s }, p0/z, [x0]2367; CHECK-NEXT: ret2368 %c = load <vscale x 3 x i8>, ptr %a2369 %c.zext = sext <vscale x 3 x i8> %c to <vscale x 3 x i16>2370 ret <vscale x 3 x i16> %c.zext2371}2372 2373define <vscale x 4 x i16> @sve_zextload_nxv4i8(ptr %a, ptr %b) {2374; CHECK-LABEL: sve_zextload_nxv4i8:2375; CHECK: // %bb.0:2376; CHECK-NEXT: ptrue p0.s2377; CHECK-NEXT: ld1sb { z0.s }, p0/z, [x0]2378; CHECK-NEXT: ret2379 %c = load <vscale x 4 x i8>, ptr %a2380 %c.zext = sext <vscale x 4 x i8> %c to <vscale x 4 x i16>2381 ret <vscale x 4 x i16> %c.zext2382}2383 2384define <vscale x 5 x i16> @sve_zextload_nxv5i8(ptr %a, ptr %b) {2385; CHECK-LABEL: sve_zextload_nxv5i8:2386; CHECK: // %bb.0:2387; CHECK-NEXT: rdvl x8, #12388; CHECK-NEXT: mov w9, #5 // =0x52389; CHECK-NEXT: lsr x8, x8, #42390; CHECK-NEXT: mul x8, x8, x92391; CHECK-NEXT: whilelo p0.h, xzr, x82392; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0]2393; CHECK-NEXT: ret2394 %c = load <vscale x 5 x i8>, ptr %a2395 %c.zext = sext <vscale x 5 x i8> %c to <vscale x 5 x i16>2396 ret <vscale x 5 x i16> %c.zext2397}2398 2399define <vscale x 6 x i16> @sve_zextload_nxv6i8(ptr %a, ptr %b) {2400; CHECK-LABEL: sve_zextload_nxv6i8:2401; CHECK: // %bb.0:2402; CHECK-NEXT: cntd x8, all, mul #32403; CHECK-NEXT: whilelo p0.h, xzr, x82404; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0]2405; CHECK-NEXT: ret2406 %c = load <vscale x 6 x i8>, ptr %a2407 %c.zext = sext <vscale x 6 x i8> %c to <vscale x 6 x i16>2408 ret <vscale x 6 x i16> %c.zext2409}2410 2411define <vscale x 7 x i16> @sve_zextload_nxv7i8(ptr %a, ptr %b) {2412; CHECK-LABEL: sve_zextload_nxv7i8:2413; CHECK: // %bb.0:2414; CHECK-NEXT: rdvl x8, #12415; CHECK-NEXT: mov w9, #7 // =0x72416; CHECK-NEXT: lsr x8, x8, #42417; CHECK-NEXT: mul x8, x8, x92418; CHECK-NEXT: whilelo p0.h, xzr, x82419; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0]2420; CHECK-NEXT: ret2421 %c = load <vscale x 7 x i8>, ptr %a2422 %c.zext = sext <vscale x 7 x i8> %c to <vscale x 7 x i16>2423 ret <vscale x 7 x i16> %c.zext2424}2425 2426define <vscale x 8 x i16> @sve_zextload_nxv8i8(ptr %a, ptr %b) {2427; CHECK-LABEL: sve_zextload_nxv8i8:2428; CHECK: // %bb.0:2429; CHECK-NEXT: ptrue p0.h2430; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0]2431; CHECK-NEXT: ret2432 %c = load <vscale x 8 x i8>, ptr %a2433 %c.zext = sext <vscale x 8 x i8> %c to <vscale x 8 x i16>2434 ret <vscale x 8 x i16> %c.zext2435}2436 2437define <vscale x 9 x i16> @sve_zextload_nxv9i8(ptr %a, ptr %b) {2438; CHECK-LABEL: sve_zextload_nxv9i8:2439; CHECK: // %bb.0:2440; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2441; CHECK-NEXT: addvl sp, sp, #-22442; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2443; CHECK-NEXT: .cfi_offset w29, -162444; CHECK-NEXT: rdvl x8, #12445; CHECK-NEXT: mov w9, #9 // =0x92446; CHECK-NEXT: lsr x8, x8, #42447; CHECK-NEXT: mul x8, x8, x92448; CHECK-NEXT: whilelo p0.b, xzr, x82449; CHECK-NEXT: punpkhi p1.h, p0.b2450; CHECK-NEXT: punpklo p0.h, p0.b2451; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0, #1, mul vl]2452; CHECK-NEXT: ld1sb { z1.h }, p0/z, [x0]2453; CHECK-NEXT: st1h { z0.h }, p1, [sp, #1, mul vl]2454; CHECK-NEXT: st1h { z1.h }, p0, [sp]2455; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2456; CHECK-NEXT: ldr z0, [sp]2457; CHECK-NEXT: addvl sp, sp, #22458; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2459; CHECK-NEXT: ret2460 %c = load <vscale x 9 x i8>, ptr %a2461 %c.zext = sext <vscale x 9 x i8> %c to <vscale x 9 x i16>2462 ret <vscale x 9 x i16> %c.zext2463}2464 2465define <vscale x 10 x i16> @sve_zextload_nxv10i8(ptr %a, ptr %b) {2466; CHECK-LABEL: sve_zextload_nxv10i8:2467; CHECK: // %bb.0:2468; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2469; CHECK-NEXT: addvl sp, sp, #-22470; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2471; CHECK-NEXT: .cfi_offset w29, -162472; CHECK-NEXT: cntd x8, all, mul #52473; CHECK-NEXT: whilelo p0.b, xzr, x82474; CHECK-NEXT: punpkhi p1.h, p0.b2475; CHECK-NEXT: punpklo p0.h, p0.b2476; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0, #1, mul vl]2477; CHECK-NEXT: ld1sb { z1.h }, p0/z, [x0]2478; CHECK-NEXT: ptrue p0.d2479; CHECK-NEXT: uunpklo z0.s, z0.h2480; CHECK-NEXT: str z1, [sp]2481; CHECK-NEXT: uunpklo z0.d, z0.s2482; CHECK-NEXT: st1h { z0.d }, p0, [sp, #4, mul vl]2483; CHECK-NEXT: ldr z0, [sp]2484; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2485; CHECK-NEXT: addvl sp, sp, #22486; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2487; CHECK-NEXT: ret2488 %c = load <vscale x 10 x i8>, ptr %a2489 %c.zext = sext <vscale x 10 x i8> %c to <vscale x 10 x i16>2490 ret <vscale x 10 x i16> %c.zext2491}2492 2493define <vscale x 11 x i16> @sve_zextload_nxv11i8(ptr %a, ptr %b) {2494; CHECK-LABEL: sve_zextload_nxv11i8:2495; CHECK: // %bb.0:2496; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2497; CHECK-NEXT: addvl sp, sp, #-22498; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2499; CHECK-NEXT: .cfi_offset w29, -162500; CHECK-NEXT: rdvl x8, #12501; CHECK-NEXT: mov w9, #11 // =0xb2502; CHECK-NEXT: lsr x8, x8, #42503; CHECK-NEXT: mul x8, x8, x92504; CHECK-NEXT: whilelo p0.b, xzr, x82505; CHECK-NEXT: punpkhi p1.h, p0.b2506; CHECK-NEXT: punpklo p0.h, p0.b2507; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0, #1, mul vl]2508; CHECK-NEXT: ld1sb { z1.h }, p0/z, [x0]2509; CHECK-NEXT: st1h { z0.h }, p1, [sp, #1, mul vl]2510; CHECK-NEXT: st1h { z1.h }, p0, [sp]2511; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2512; CHECK-NEXT: ldr z0, [sp]2513; CHECK-NEXT: addvl sp, sp, #22514; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2515; CHECK-NEXT: ret2516 %c = load <vscale x 11 x i8>, ptr %a2517 %c.zext = sext <vscale x 11 x i8> %c to <vscale x 11 x i16>2518 ret <vscale x 11 x i16> %c.zext2519}2520 2521define <vscale x 12 x i16> @sve_zextload_nxv12i8(ptr %a, ptr %b) {2522; CHECK-LABEL: sve_zextload_nxv12i8:2523; CHECK: // %bb.0:2524; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2525; CHECK-NEXT: addvl sp, sp, #-22526; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2527; CHECK-NEXT: .cfi_offset w29, -162528; CHECK-NEXT: cntw x8, all, mul #32529; CHECK-NEXT: whilelo p0.b, xzr, x82530; CHECK-NEXT: punpkhi p1.h, p0.b2531; CHECK-NEXT: punpklo p0.h, p0.b2532; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0, #1, mul vl]2533; CHECK-NEXT: ptrue p1.s2534; CHECK-NEXT: ld1sb { z1.h }, p0/z, [x0]2535; CHECK-NEXT: uunpklo z0.s, z0.h2536; CHECK-NEXT: str z1, [sp]2537; CHECK-NEXT: st1h { z0.s }, p1, [sp, #2, mul vl]2538; CHECK-NEXT: ldr z0, [sp]2539; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2540; CHECK-NEXT: addvl sp, sp, #22541; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2542; CHECK-NEXT: ret2543 %c = load <vscale x 12 x i8>, ptr %a2544 %c.zext = sext <vscale x 12 x i8> %c to <vscale x 12 x i16>2545 ret <vscale x 12 x i16> %c.zext2546}2547 2548define <vscale x 13 x i16> @sve_zextload_nxv13i8(ptr %a, ptr %b) {2549; CHECK-LABEL: sve_zextload_nxv13i8:2550; CHECK: // %bb.0:2551; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2552; CHECK-NEXT: addvl sp, sp, #-22553; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2554; CHECK-NEXT: .cfi_offset w29, -162555; CHECK-NEXT: rdvl x8, #12556; CHECK-NEXT: mov w9, #13 // =0xd2557; CHECK-NEXT: lsr x8, x8, #42558; CHECK-NEXT: mul x8, x8, x92559; CHECK-NEXT: whilelo p0.b, xzr, x82560; CHECK-NEXT: punpkhi p1.h, p0.b2561; CHECK-NEXT: punpklo p0.h, p0.b2562; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0, #1, mul vl]2563; CHECK-NEXT: ld1sb { z1.h }, p0/z, [x0]2564; CHECK-NEXT: st1h { z0.h }, p1, [sp, #1, mul vl]2565; CHECK-NEXT: st1h { z1.h }, p0, [sp]2566; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2567; CHECK-NEXT: ldr z0, [sp]2568; CHECK-NEXT: addvl sp, sp, #22569; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2570; CHECK-NEXT: ret2571 %c = load <vscale x 13 x i8>, ptr %a2572 %c.zext = sext <vscale x 13 x i8> %c to <vscale x 13 x i16>2573 ret <vscale x 13 x i16> %c.zext2574}2575 2576define <vscale x 14 x i16> @sve_zextload_nxv14i8(ptr %a, ptr %b) {2577; CHECK-LABEL: sve_zextload_nxv14i8:2578; CHECK: // %bb.0:2579; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2580; CHECK-NEXT: addvl sp, sp, #-22581; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2582; CHECK-NEXT: .cfi_offset w29, -162583; CHECK-NEXT: cntd x8, all, mul #72584; CHECK-NEXT: whilelo p0.b, xzr, x82585; CHECK-NEXT: punpkhi p1.h, p0.b2586; CHECK-NEXT: punpklo p0.h, p0.b2587; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0, #1, mul vl]2588; CHECK-NEXT: ptrue p1.s2589; CHECK-NEXT: ld1sb { z2.h }, p0/z, [x0]2590; CHECK-NEXT: ptrue p0.d2591; CHECK-NEXT: uunpkhi z1.s, z0.h2592; CHECK-NEXT: uunpklo z0.s, z0.h2593; CHECK-NEXT: str z2, [sp]2594; CHECK-NEXT: uunpklo z1.d, z1.s2595; CHECK-NEXT: st1h { z0.s }, p1, [sp, #2, mul vl]2596; CHECK-NEXT: ldr z0, [sp]2597; CHECK-NEXT: st1h { z1.d }, p0, [sp, #6, mul vl]2598; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2599; CHECK-NEXT: addvl sp, sp, #22600; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2601; CHECK-NEXT: ret2602 %c = load <vscale x 14 x i8>, ptr %a2603 %c.zext = sext <vscale x 14 x i8> %c to <vscale x 14 x i16>2604 ret <vscale x 14 x i16> %c.zext2605}2606 2607define <vscale x 15 x i16> @sve_zextload_nxv15i8(ptr %a, ptr %b) {2608; CHECK-LABEL: sve_zextload_nxv15i8:2609; CHECK: // %bb.0:2610; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2611; CHECK-NEXT: addvl sp, sp, #-22612; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2613; CHECK-NEXT: .cfi_offset w29, -162614; CHECK-NEXT: rdvl x8, #12615; CHECK-NEXT: mov w9, #15 // =0xf2616; CHECK-NEXT: lsr x8, x8, #42617; CHECK-NEXT: mul x8, x8, x92618; CHECK-NEXT: whilelo p0.b, xzr, x82619; CHECK-NEXT: punpkhi p1.h, p0.b2620; CHECK-NEXT: punpklo p0.h, p0.b2621; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0, #1, mul vl]2622; CHECK-NEXT: ld1sb { z1.h }, p0/z, [x0]2623; CHECK-NEXT: st1h { z0.h }, p1, [sp, #1, mul vl]2624; CHECK-NEXT: st1h { z1.h }, p0, [sp]2625; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2626; CHECK-NEXT: ldr z0, [sp]2627; CHECK-NEXT: addvl sp, sp, #22628; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2629; CHECK-NEXT: ret2630 %c = load <vscale x 15 x i8>, ptr %a2631 %c.zext = sext <vscale x 15 x i8> %c to <vscale x 15 x i16>2632 ret <vscale x 15 x i16> %c.zext2633}2634 2635define <vscale x 16 x i16> @sve_zextload_nxv16i8(ptr %a, ptr %b) {2636; CHECK-LABEL: sve_zextload_nxv16i8:2637; CHECK: // %bb.0:2638; CHECK-NEXT: ptrue p0.h2639; CHECK-NEXT: ld1sb { z0.h }, p0/z, [x0]2640; CHECK-NEXT: ld1sb { z1.h }, p0/z, [x0, #1, mul vl]2641; CHECK-NEXT: ret2642 %c = load <vscale x 16 x i8>, ptr %a2643 %c.zext = sext <vscale x 16 x i8> %c to <vscale x 16 x i16>2644 ret <vscale x 16 x i16> %c.zext2645}2646 2647define <vscale x 1 x i32> @sve_zextload_nxv1i16(ptr %a, ptr %b) {2648; CHECK-LABEL: sve_zextload_nxv1i16:2649; CHECK: // %bb.0:2650; CHECK-NEXT: rdvl x8, #12651; CHECK-NEXT: lsr x8, x8, #42652; CHECK-NEXT: whilelo p0.s, xzr, x82653; CHECK-NEXT: ld1sh { z0.s }, p0/z, [x0]2654; CHECK-NEXT: ret2655 %c = load <vscale x 1 x i16>, ptr %a2656 %c.zext = sext <vscale x 1 x i16> %c to <vscale x 1 x i32>2657 ret <vscale x 1 x i32> %c.zext2658}2659 2660define <vscale x 2 x i32> @sve_zextload_nxv2i16(ptr %a, ptr %b) {2661; CHECK-LABEL: sve_zextload_nxv2i16:2662; CHECK: // %bb.0:2663; CHECK-NEXT: ptrue p0.d2664; CHECK-NEXT: ld1sh { z0.d }, p0/z, [x0]2665; CHECK-NEXT: ret2666 %c = load <vscale x 2 x i16>, ptr %a2667 %c.zext = sext <vscale x 2 x i16> %c to <vscale x 2 x i32>2668 ret <vscale x 2 x i32> %c.zext2669}2670 2671define <vscale x 3 x i32> @sve_zextload_nxv3i16(ptr %a, ptr %b) {2672; CHECK-LABEL: sve_zextload_nxv3i16:2673; CHECK: // %bb.0:2674; CHECK-NEXT: rdvl x8, #12675; CHECK-NEXT: mov w9, #3 // =0x32676; CHECK-NEXT: lsr x8, x8, #42677; CHECK-NEXT: mul x8, x8, x92678; CHECK-NEXT: whilelo p0.s, xzr, x82679; CHECK-NEXT: ld1sh { z0.s }, p0/z, [x0]2680; CHECK-NEXT: ret2681 %c = load <vscale x 3 x i16>, ptr %a2682 %c.zext = sext <vscale x 3 x i16> %c to <vscale x 3 x i32>2683 ret <vscale x 3 x i32> %c.zext2684}2685 2686define <vscale x 4 x i32> @sve_zextload_nxv4i16(ptr %a, ptr %b) {2687; CHECK-LABEL: sve_zextload_nxv4i16:2688; CHECK: // %bb.0:2689; CHECK-NEXT: ptrue p0.s2690; CHECK-NEXT: ld1sh { z0.s }, p0/z, [x0]2691; CHECK-NEXT: ret2692 %c = load <vscale x 4 x i16>, ptr %a2693 %c.zext = sext <vscale x 4 x i16> %c to <vscale x 4 x i32>2694 ret <vscale x 4 x i32> %c.zext2695}2696 2697define <vscale x 5 x i32> @sve_zextload_nxv5i16(ptr %a, ptr %b) {2698; CHECK-LABEL: sve_zextload_nxv5i16:2699; CHECK: // %bb.0:2700; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2701; CHECK-NEXT: addvl sp, sp, #-22702; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2703; CHECK-NEXT: .cfi_offset w29, -162704; CHECK-NEXT: rdvl x8, #12705; CHECK-NEXT: mov w9, #5 // =0x52706; CHECK-NEXT: lsr x8, x8, #42707; CHECK-NEXT: mul x8, x8, x92708; CHECK-NEXT: whilelo p0.h, xzr, x82709; CHECK-NEXT: punpkhi p1.h, p0.b2710; CHECK-NEXT: punpklo p0.h, p0.b2711; CHECK-NEXT: ld1sh { z0.s }, p1/z, [x0, #1, mul vl]2712; CHECK-NEXT: ld1sh { z1.s }, p0/z, [x0]2713; CHECK-NEXT: st1w { z0.s }, p1, [sp, #1, mul vl]2714; CHECK-NEXT: st1w { z1.s }, p0, [sp]2715; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2716; CHECK-NEXT: ldr z0, [sp]2717; CHECK-NEXT: addvl sp, sp, #22718; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2719; CHECK-NEXT: ret2720 %c = load <vscale x 5 x i16>, ptr %a2721 %c.zext = sext <vscale x 5 x i16> %c to <vscale x 5 x i32>2722 ret <vscale x 5 x i32> %c.zext2723}2724 2725define <vscale x 6 x i32> @sve_zextload_nxv6i16(ptr %a, ptr %b) {2726; CHECK-LABEL: sve_zextload_nxv6i16:2727; CHECK: // %bb.0:2728; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2729; CHECK-NEXT: addvl sp, sp, #-22730; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2731; CHECK-NEXT: .cfi_offset w29, -162732; CHECK-NEXT: cntd x8, all, mul #32733; CHECK-NEXT: whilelo p0.h, xzr, x82734; CHECK-NEXT: punpkhi p1.h, p0.b2735; CHECK-NEXT: punpklo p0.h, p0.b2736; CHECK-NEXT: ld1sh { z0.s }, p1/z, [x0, #1, mul vl]2737; CHECK-NEXT: ptrue p1.d2738; CHECK-NEXT: ld1sh { z1.s }, p0/z, [x0]2739; CHECK-NEXT: uunpklo z0.d, z0.s2740; CHECK-NEXT: str z1, [sp]2741; CHECK-NEXT: st1w { z0.d }, p1, [sp, #2, mul vl]2742; CHECK-NEXT: ldr z0, [sp]2743; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2744; CHECK-NEXT: addvl sp, sp, #22745; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2746; CHECK-NEXT: ret2747 %c = load <vscale x 6 x i16>, ptr %a2748 %c.zext = sext <vscale x 6 x i16> %c to <vscale x 6 x i32>2749 ret <vscale x 6 x i32> %c.zext2750}2751 2752define <vscale x 7 x i32> @sve_zextload_nxv7i16(ptr %a, ptr %b) {2753; CHECK-LABEL: sve_zextload_nxv7i16:2754; CHECK: // %bb.0:2755; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2756; CHECK-NEXT: addvl sp, sp, #-22757; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2758; CHECK-NEXT: .cfi_offset w29, -162759; CHECK-NEXT: rdvl x8, #12760; CHECK-NEXT: mov w9, #7 // =0x72761; CHECK-NEXT: lsr x8, x8, #42762; CHECK-NEXT: mul x8, x8, x92763; CHECK-NEXT: whilelo p0.h, xzr, x82764; CHECK-NEXT: punpkhi p1.h, p0.b2765; CHECK-NEXT: punpklo p0.h, p0.b2766; CHECK-NEXT: ld1sh { z0.s }, p1/z, [x0, #1, mul vl]2767; CHECK-NEXT: ld1sh { z1.s }, p0/z, [x0]2768; CHECK-NEXT: st1w { z0.s }, p1, [sp, #1, mul vl]2769; CHECK-NEXT: st1w { z1.s }, p0, [sp]2770; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2771; CHECK-NEXT: ldr z0, [sp]2772; CHECK-NEXT: addvl sp, sp, #22773; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2774; CHECK-NEXT: ret2775 %c = load <vscale x 7 x i16>, ptr %a2776 %c.zext = sext <vscale x 7 x i16> %c to <vscale x 7 x i32>2777 ret <vscale x 7 x i32> %c.zext2778}2779 2780define <vscale x 8 x i32> @sve_zextload_nxv8i16(ptr %a, ptr %b) {2781; CHECK-LABEL: sve_zextload_nxv8i16:2782; CHECK: // %bb.0:2783; CHECK-NEXT: ptrue p0.s2784; CHECK-NEXT: ld1sh { z0.s }, p0/z, [x0]2785; CHECK-NEXT: ld1sh { z1.s }, p0/z, [x0, #1, mul vl]2786; CHECK-NEXT: ret2787 %c = load <vscale x 8 x i16>, ptr %a2788 %c.zext = sext <vscale x 8 x i16> %c to <vscale x 8 x i32>2789 ret <vscale x 8 x i32> %c.zext2790}2791 2792define <vscale x 1 x i64> @sve_zextload_nxv1i32(ptr %a, ptr %b) {2793; CHECK-LABEL: sve_zextload_nxv1i32:2794; CHECK: // %bb.0:2795; CHECK-NEXT: rdvl x8, #12796; CHECK-NEXT: lsr x8, x8, #42797; CHECK-NEXT: whilelo p0.d, xzr, x82798; CHECK-NEXT: ld1sw { z0.d }, p0/z, [x0]2799; CHECK-NEXT: ret2800 %c = load <vscale x 1 x i32>, ptr %a2801 %c.zext = sext <vscale x 1 x i32> %c to <vscale x 1 x i64>2802 ret <vscale x 1 x i64> %c.zext2803}2804 2805define <vscale x 2 x i64> @sve_zextload_nxv2i32(ptr %a, ptr %b) {2806; CHECK-LABEL: sve_zextload_nxv2i32:2807; CHECK: // %bb.0:2808; CHECK-NEXT: ptrue p0.d2809; CHECK-NEXT: ld1sw { z0.d }, p0/z, [x0]2810; CHECK-NEXT: ret2811 %c = load <vscale x 2 x i32>, ptr %a2812 %c.zext = sext <vscale x 2 x i32> %c to <vscale x 2 x i64>2813 ret <vscale x 2 x i64> %c.zext2814}2815 2816define <vscale x 3 x i64> @sve_zextload_nxv3i32(ptr %a, ptr %b) {2817; CHECK-LABEL: sve_zextload_nxv3i32:2818; CHECK: // %bb.0:2819; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill2820; CHECK-NEXT: addvl sp, sp, #-22821; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG2822; CHECK-NEXT: .cfi_offset w29, -162823; CHECK-NEXT: rdvl x8, #12824; CHECK-NEXT: mov w9, #3 // =0x32825; CHECK-NEXT: lsr x8, x8, #42826; CHECK-NEXT: mul x8, x8, x92827; CHECK-NEXT: whilelo p0.s, xzr, x82828; CHECK-NEXT: punpkhi p1.h, p0.b2829; CHECK-NEXT: punpklo p0.h, p0.b2830; CHECK-NEXT: ld1sw { z0.d }, p1/z, [x0, #1, mul vl]2831; CHECK-NEXT: ld1sw { z1.d }, p0/z, [x0]2832; CHECK-NEXT: st1d { z0.d }, p1, [sp, #1, mul vl]2833; CHECK-NEXT: st1d { z1.d }, p0, [sp]2834; CHECK-NEXT: ldr z1, [sp, #1, mul vl]2835; CHECK-NEXT: ldr z0, [sp]2836; CHECK-NEXT: addvl sp, sp, #22837; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload2838; CHECK-NEXT: ret2839 %c = load <vscale x 3 x i32>, ptr %a2840 %c.zext = sext <vscale x 3 x i32> %c to <vscale x 3 x i64>2841 ret <vscale x 3 x i64> %c.zext2842}2843 2844define <vscale x 4 x i64> @sve_zextload_nxv4i32(ptr %a, ptr %b) {2845; CHECK-LABEL: sve_zextload_nxv4i32:2846; CHECK: // %bb.0:2847; CHECK-NEXT: ptrue p0.d2848; CHECK-NEXT: ld1sw { z0.d }, p0/z, [x0]2849; CHECK-NEXT: ld1sw { z1.d }, p0/z, [x0, #1, mul vl]2850; CHECK-NEXT: ret2851 %c = load <vscale x 4 x i32>, ptr %a2852 %c.zext = sext <vscale x 4 x i32> %c to <vscale x 4 x i64>2853 ret <vscale x 4 x i64> %c.zext2854}2855