1053 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2563; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5124; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5125 6target triple = "aarch64-unknown-linux-gnu"7 8;9; sext i1 -> i3210;11 12; NOTE: Covers the scenario where a SIGN_EXTEND_INREG is required, whose inreg13; type's element type is not byte based and thus cannot be lowered directly to14; an SVE instruction.15define void @sext_v8i1_v8i32(<8 x i1> %a, ptr %out) vscale_range(2,0) #0 {16; CHECK-LABEL: sext_v8i1_v8i32:17; CHECK: // %bb.0:18; CHECK-NEXT: // kill: def $d0 killed $d0 def $z019; CHECK-NEXT: ptrue p0.s, vl820; CHECK-NEXT: uunpklo z0.h, z0.b21; CHECK-NEXT: uunpklo z0.s, z0.h22; CHECK-NEXT: lsl z0.s, z0.s, #3123; CHECK-NEXT: asr z0.s, z0.s, #3124; CHECK-NEXT: st1w { z0.s }, p0, [x0]25; CHECK-NEXT: ret26 %b = sext <8 x i1> %a to <8 x i32>27 store <8 x i32> %b, ptr %out28 ret void29}30 31;32; sext i3 -> i6433;34 35; NOTE: Covers the scenario where a SIGN_EXTEND_INREG is required, whose inreg36; type's element type is not power-of-2 based and thus cannot be lowered37; directly to an SVE instruction.38define void @sext_v4i3_v4i64(<4 x i3> %a, ptr %out) vscale_range(2,0) #0 {39; CHECK-LABEL: sext_v4i3_v4i64:40; CHECK: // %bb.0:41; CHECK-NEXT: // kill: def $d0 killed $d0 def $z042; CHECK-NEXT: ptrue p0.d, vl443; CHECK-NEXT: uunpklo z0.s, z0.h44; CHECK-NEXT: uunpklo z0.d, z0.s45; CHECK-NEXT: lsl z0.d, z0.d, #6146; CHECK-NEXT: asr z0.d, z0.d, #6147; CHECK-NEXT: st1d { z0.d }, p0, [x0]48; CHECK-NEXT: ret49 %b = sext <4 x i3> %a to <4 x i64>50 store <4 x i64> %b, ptr %out51 ret void52}53 54;55; sext i8 -> i1656;57 58define void @sext_v16i8_v16i16(<16 x i8> %a, ptr %out) vscale_range(2,0) #0 {59; CHECK-LABEL: sext_v16i8_v16i16:60; CHECK: // %bb.0:61; CHECK-NEXT: // kill: def $q0 killed $q0 def $z062; CHECK-NEXT: ptrue p0.h, vl1663; CHECK-NEXT: sunpklo z0.h, z0.b64; CHECK-NEXT: st1h { z0.h }, p0, [x0]65; CHECK-NEXT: ret66 %b = sext <16 x i8> %a to <16 x i16>67 store <16 x i16>%b, ptr %out68 ret void69}70 71; NOTE: Extra 'add' is to prevent the extend being combined with the load.72define void @sext_v32i8_v32i16(ptr %in, ptr %out) #0 {73; VBITS_GE_256-LABEL: sext_v32i8_v32i16:74; VBITS_GE_256: // %bb.0:75; VBITS_GE_256-NEXT: ptrue p0.b, vl3276; VBITS_GE_256-NEXT: mov x8, #16 // =0x1077; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0]78; VBITS_GE_256-NEXT: ptrue p0.h, vl1679; VBITS_GE_256-NEXT: add z0.b, z0.b, z0.b80; VBITS_GE_256-NEXT: movprfx z1, z081; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #1682; VBITS_GE_256-NEXT: sunpklo z0.h, z0.b83; VBITS_GE_256-NEXT: sunpklo z1.h, z1.b84; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x1]85; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x1, x8, lsl #1]86; VBITS_GE_256-NEXT: ret87;88; VBITS_GE_512-LABEL: sext_v32i8_v32i16:89; VBITS_GE_512: // %bb.0:90; VBITS_GE_512-NEXT: ptrue p0.b, vl3291; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]92; VBITS_GE_512-NEXT: ptrue p0.h, vl3293; VBITS_GE_512-NEXT: add z0.b, z0.b, z0.b94; VBITS_GE_512-NEXT: sunpklo z0.h, z0.b95; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x1]96; VBITS_GE_512-NEXT: ret97 %a = load <32 x i8>, ptr %in98 %b = add <32 x i8> %a, %a99 %c = sext <32 x i8> %b to <32 x i16>100 store <32 x i16> %c, ptr %out101 ret void102}103 104define void @sext_v64i8_v64i16(ptr %in, ptr %out) vscale_range(8,0) #0 {105; CHECK-LABEL: sext_v64i8_v64i16:106; CHECK: // %bb.0:107; CHECK-NEXT: ptrue p0.b, vl64108; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]109; CHECK-NEXT: ptrue p0.h, vl64110; CHECK-NEXT: add z0.b, z0.b, z0.b111; CHECK-NEXT: sunpklo z0.h, z0.b112; CHECK-NEXT: st1h { z0.h }, p0, [x1]113; CHECK-NEXT: ret114 %a = load <64 x i8>, ptr %in115 %b = add <64 x i8> %a, %a116 %c = sext <64 x i8> %b to <64 x i16>117 store <64 x i16> %c, ptr %out118 ret void119}120 121define void @sext_v128i8_v128i16(ptr %in, ptr %out) vscale_range(16,0) #0 {122; CHECK-LABEL: sext_v128i8_v128i16:123; CHECK: // %bb.0:124; CHECK-NEXT: ptrue p0.b, vl128125; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]126; CHECK-NEXT: ptrue p0.h, vl128127; CHECK-NEXT: add z0.b, z0.b, z0.b128; CHECK-NEXT: sunpklo z0.h, z0.b129; CHECK-NEXT: st1h { z0.h }, p0, [x1]130; CHECK-NEXT: ret131 %a = load <128 x i8>, ptr %in132 %b = add <128 x i8> %a, %a133 %c = sext <128 x i8> %b to <128 x i16>134 store <128 x i16> %c, ptr %out135 ret void136}137 138;139; sext i8 -> i32140;141 142define void @sext_v8i8_v8i32(<8 x i8> %a, ptr %out) vscale_range(2,0) #0 {143; CHECK-LABEL: sext_v8i8_v8i32:144; CHECK: // %bb.0:145; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0146; CHECK-NEXT: ptrue p0.s, vl8147; CHECK-NEXT: sunpklo z0.h, z0.b148; CHECK-NEXT: sunpklo z0.s, z0.h149; CHECK-NEXT: st1w { z0.s }, p0, [x0]150; CHECK-NEXT: ret151 %b = sext <8 x i8> %a to <8 x i32>152 store <8 x i32>%b, ptr %out153 ret void154}155 156define void @sext_v16i8_v16i32(<16 x i8> %a, ptr %out) #0 {157; VBITS_GE_256-LABEL: sext_v16i8_v16i32:158; VBITS_GE_256: // %bb.0:159; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0160; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8161; VBITS_GE_256-NEXT: ptrue p0.s, vl8162; VBITS_GE_256-NEXT: mov x8, #8 // =0x8163; VBITS_GE_256-NEXT: sunpklo z0.h, z0.b164; VBITS_GE_256-NEXT: sunpklo z1.h, z1.b165; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h166; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h167; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0]168; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0, x8, lsl #2]169; VBITS_GE_256-NEXT: ret170;171; VBITS_GE_512-LABEL: sext_v16i8_v16i32:172; VBITS_GE_512: // %bb.0:173; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0174; VBITS_GE_512-NEXT: ptrue p0.s, vl16175; VBITS_GE_512-NEXT: sunpklo z0.h, z0.b176; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h177; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]178; VBITS_GE_512-NEXT: ret179 %b = sext <16 x i8> %a to <16 x i32>180 store <16 x i32> %b, ptr %out181 ret void182}183 184define void @sext_v32i8_v32i32(ptr %in, ptr %out) vscale_range(8,0) #0 {185; CHECK-LABEL: sext_v32i8_v32i32:186; CHECK: // %bb.0:187; CHECK-NEXT: ptrue p0.b, vl32188; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]189; CHECK-NEXT: ptrue p0.s, vl32190; CHECK-NEXT: add z0.b, z0.b, z0.b191; CHECK-NEXT: sunpklo z0.h, z0.b192; CHECK-NEXT: sunpklo z0.s, z0.h193; CHECK-NEXT: st1w { z0.s }, p0, [x1]194; CHECK-NEXT: ret195 %a = load <32 x i8>, ptr %in196 %b = add <32 x i8> %a, %a197 %c = sext <32 x i8> %b to <32 x i32>198 store <32 x i32> %c, ptr %out199 ret void200}201 202define void @sext_v64i8_v64i32(ptr %in, ptr %out) vscale_range(16,0) #0 {203; CHECK-LABEL: sext_v64i8_v64i32:204; CHECK: // %bb.0:205; CHECK-NEXT: ptrue p0.b, vl64206; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]207; CHECK-NEXT: ptrue p0.s, vl64208; CHECK-NEXT: add z0.b, z0.b, z0.b209; CHECK-NEXT: sunpklo z0.h, z0.b210; CHECK-NEXT: sunpklo z0.s, z0.h211; CHECK-NEXT: st1w { z0.s }, p0, [x1]212; CHECK-NEXT: ret213 %a = load <64 x i8>, ptr %in214 %b = add <64 x i8> %a, %a215 %c = sext <64 x i8> %b to <64 x i32>216 store <64 x i32> %c, ptr %out217 ret void218}219 220;221; sext i8 -> i64222;223 224; NOTE: v4i8 is an unpacked typed stored within a v4i16 container. The sign225; extend is a two step process where the container is any_extend'd with the226; result feeding an inreg sign extend.227define void @sext_v4i8_v4i64(<4 x i8> %a, ptr %out) vscale_range(2,0) #0 {228; CHECK-LABEL: sext_v4i8_v4i64:229; CHECK: // %bb.0:230; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0231; CHECK-NEXT: ptrue p0.d, vl4232; CHECK-NEXT: uunpklo z0.s, z0.h233; CHECK-NEXT: uunpklo z0.d, z0.s234; CHECK-NEXT: sxtb z0.d, p0/m, z0.d235; CHECK-NEXT: st1d { z0.d }, p0, [x0]236; CHECK-NEXT: ret237 %b = sext <4 x i8> %a to <4 x i64>238 store <4 x i64>%b, ptr %out239 ret void240}241 242define void @sext_v8i8_v8i64(<8 x i8> %a, ptr %out) #0 {243; VBITS_GE_256-LABEL: sext_v8i8_v8i64:244; VBITS_GE_256: // %bb.0:245; VBITS_GE_256-NEXT: sshll v0.8h, v0.8b, #0246; VBITS_GE_256-NEXT: ptrue p0.d, vl4247; VBITS_GE_256-NEXT: mov x8, #4 // =0x4248; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8249; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h250; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h251; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s252; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s253; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0]254; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0, x8, lsl #3]255; VBITS_GE_256-NEXT: ret256;257; VBITS_GE_512-LABEL: sext_v8i8_v8i64:258; VBITS_GE_512: // %bb.0:259; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 def $z0260; VBITS_GE_512-NEXT: ptrue p0.d, vl8261; VBITS_GE_512-NEXT: sunpklo z0.h, z0.b262; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h263; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s264; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]265; VBITS_GE_512-NEXT: ret266 %b = sext <8 x i8> %a to <8 x i64>267 store <8 x i64>%b, ptr %out268 ret void269}270 271define void @sext_v16i8_v16i64(<16 x i8> %a, ptr %out) vscale_range(8,0) #0 {272; CHECK-LABEL: sext_v16i8_v16i64:273; CHECK: // %bb.0:274; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0275; CHECK-NEXT: ptrue p0.d, vl16276; CHECK-NEXT: sunpklo z0.h, z0.b277; CHECK-NEXT: sunpklo z0.s, z0.h278; CHECK-NEXT: sunpklo z0.d, z0.s279; CHECK-NEXT: st1d { z0.d }, p0, [x0]280; CHECK-NEXT: ret281 %b = sext <16 x i8> %a to <16 x i64>282 store <16 x i64> %b, ptr %out283 ret void284}285 286define void @sext_v32i8_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 {287; CHECK-LABEL: sext_v32i8_v32i64:288; CHECK: // %bb.0:289; CHECK-NEXT: ptrue p0.b, vl32290; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]291; CHECK-NEXT: ptrue p0.d, vl32292; CHECK-NEXT: add z0.b, z0.b, z0.b293; CHECK-NEXT: sunpklo z0.h, z0.b294; CHECK-NEXT: sunpklo z0.s, z0.h295; CHECK-NEXT: sunpklo z0.d, z0.s296; CHECK-NEXT: st1d { z0.d }, p0, [x1]297; CHECK-NEXT: ret298 %a = load <32 x i8>, ptr %in299 %b = add <32 x i8> %a, %a300 %c = sext <32 x i8> %b to <32 x i64>301 store <32 x i64> %c, ptr %out302 ret void303}304 305;306; sext i16 -> i32307;308 309define void @sext_v8i16_v8i32(<8 x i16> %a, ptr %out) vscale_range(2,0) #0 {310; CHECK-LABEL: sext_v8i16_v8i32:311; CHECK: // %bb.0:312; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0313; CHECK-NEXT: ptrue p0.s, vl8314; CHECK-NEXT: sunpklo z0.s, z0.h315; CHECK-NEXT: st1w { z0.s }, p0, [x0]316; CHECK-NEXT: ret317 %b = sext <8 x i16> %a to <8 x i32>318 store <8 x i32>%b, ptr %out319 ret void320}321 322define void @sext_v16i16_v16i32(ptr %in, ptr %out) #0 {323; VBITS_GE_256-LABEL: sext_v16i16_v16i32:324; VBITS_GE_256: // %bb.0:325; VBITS_GE_256-NEXT: ptrue p0.h, vl16326; VBITS_GE_256-NEXT: mov x8, #8 // =0x8327; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]328; VBITS_GE_256-NEXT: ptrue p0.s, vl8329; VBITS_GE_256-NEXT: add z0.h, z0.h, z0.h330; VBITS_GE_256-NEXT: movprfx z1, z0331; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #16332; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h333; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h334; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x1]335; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x1, x8, lsl #2]336; VBITS_GE_256-NEXT: ret337;338; VBITS_GE_512-LABEL: sext_v16i16_v16i32:339; VBITS_GE_512: // %bb.0:340; VBITS_GE_512-NEXT: ptrue p0.h, vl16341; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]342; VBITS_GE_512-NEXT: ptrue p0.s, vl16343; VBITS_GE_512-NEXT: add z0.h, z0.h, z0.h344; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h345; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x1]346; VBITS_GE_512-NEXT: ret347 %a = load <16 x i16>, ptr %in348 %b = add <16 x i16> %a, %a349 %c = sext <16 x i16> %b to <16 x i32>350 store <16 x i32> %c, ptr %out351 ret void352}353 354define void @sext_v32i16_v32i32(ptr %in, ptr %out) vscale_range(8,0) #0 {355; CHECK-LABEL: sext_v32i16_v32i32:356; CHECK: // %bb.0:357; CHECK-NEXT: ptrue p0.h, vl32358; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]359; CHECK-NEXT: ptrue p0.s, vl32360; CHECK-NEXT: add z0.h, z0.h, z0.h361; CHECK-NEXT: sunpklo z0.s, z0.h362; CHECK-NEXT: st1w { z0.s }, p0, [x1]363; CHECK-NEXT: ret364 %a = load <32 x i16>, ptr %in365 %b = add <32 x i16> %a, %a366 %c = sext <32 x i16> %b to <32 x i32>367 store <32 x i32> %c, ptr %out368 ret void369}370 371define void @sext_v64i16_v64i32(ptr %in, ptr %out) vscale_range(16,0) #0 {372; CHECK-LABEL: sext_v64i16_v64i32:373; CHECK: // %bb.0:374; CHECK-NEXT: ptrue p0.h, vl64375; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]376; CHECK-NEXT: ptrue p0.s, vl64377; CHECK-NEXT: add z0.h, z0.h, z0.h378; CHECK-NEXT: sunpklo z0.s, z0.h379; CHECK-NEXT: st1w { z0.s }, p0, [x1]380; CHECK-NEXT: ret381 %a = load <64 x i16>, ptr %in382 %b = add <64 x i16> %a, %a383 %c = sext <64 x i16> %b to <64 x i32>384 store <64 x i32> %c, ptr %out385 ret void386}387 388;389; sext i16 -> i64390;391 392define void @sext_v4i16_v4i64(<4 x i16> %a, ptr %out) vscale_range(2,0) #0 {393; CHECK-LABEL: sext_v4i16_v4i64:394; CHECK: // %bb.0:395; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0396; CHECK-NEXT: ptrue p0.d, vl4397; CHECK-NEXT: sunpklo z0.s, z0.h398; CHECK-NEXT: sunpklo z0.d, z0.s399; CHECK-NEXT: st1d { z0.d }, p0, [x0]400; CHECK-NEXT: ret401 %b = sext <4 x i16> %a to <4 x i64>402 store <4 x i64>%b, ptr %out403 ret void404}405 406define void @sext_v8i16_v8i64(<8 x i16> %a, ptr %out) #0 {407; VBITS_GE_256-LABEL: sext_v8i16_v8i64:408; VBITS_GE_256: // %bb.0:409; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0410; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8411; VBITS_GE_256-NEXT: ptrue p0.d, vl4412; VBITS_GE_256-NEXT: mov x8, #4 // =0x4413; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h414; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h415; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s416; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s417; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0]418; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0, x8, lsl #3]419; VBITS_GE_256-NEXT: ret420;421; VBITS_GE_512-LABEL: sext_v8i16_v8i64:422; VBITS_GE_512: // %bb.0:423; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0424; VBITS_GE_512-NEXT: ptrue p0.d, vl8425; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h426; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s427; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]428; VBITS_GE_512-NEXT: ret429 %b = sext <8 x i16> %a to <8 x i64>430 store <8 x i64>%b, ptr %out431 ret void432}433 434define void @sext_v16i16_v16i64(ptr %in, ptr %out) vscale_range(8,0) #0 {435; CHECK-LABEL: sext_v16i16_v16i64:436; CHECK: // %bb.0:437; CHECK-NEXT: ptrue p0.h, vl16438; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]439; CHECK-NEXT: ptrue p0.d, vl16440; CHECK-NEXT: add z0.h, z0.h, z0.h441; CHECK-NEXT: sunpklo z0.s, z0.h442; CHECK-NEXT: sunpklo z0.d, z0.s443; CHECK-NEXT: st1d { z0.d }, p0, [x1]444; CHECK-NEXT: ret445 %a = load <16 x i16>, ptr %in446 %b = add <16 x i16> %a, %a447 %c = sext <16 x i16> %b to <16 x i64>448 store <16 x i64> %c, ptr %out449 ret void450}451 452define void @sext_v32i16_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 {453; CHECK-LABEL: sext_v32i16_v32i64:454; CHECK: // %bb.0:455; CHECK-NEXT: ptrue p0.h, vl32456; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]457; CHECK-NEXT: ptrue p0.d, vl32458; CHECK-NEXT: add z0.h, z0.h, z0.h459; CHECK-NEXT: sunpklo z0.s, z0.h460; CHECK-NEXT: sunpklo z0.d, z0.s461; CHECK-NEXT: st1d { z0.d }, p0, [x1]462; CHECK-NEXT: ret463 %a = load <32 x i16>, ptr %in464 %b = add <32 x i16> %a, %a465 %c = sext <32 x i16> %b to <32 x i64>466 store <32 x i64> %c, ptr %out467 ret void468}469 470;471; sext i32 -> i64472;473 474define void @sext_v4i32_v4i64(<4 x i32> %a, ptr %out) vscale_range(2,0) #0 {475; CHECK-LABEL: sext_v4i32_v4i64:476; CHECK: // %bb.0:477; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0478; CHECK-NEXT: ptrue p0.d, vl4479; CHECK-NEXT: sunpklo z0.d, z0.s480; CHECK-NEXT: st1d { z0.d }, p0, [x0]481; CHECK-NEXT: ret482 %b = sext <4 x i32> %a to <4 x i64>483 store <4 x i64>%b, ptr %out484 ret void485}486 487define void @sext_v8i32_v8i64(ptr %in, ptr %out) #0 {488; VBITS_GE_256-LABEL: sext_v8i32_v8i64:489; VBITS_GE_256: // %bb.0:490; VBITS_GE_256-NEXT: ptrue p0.s, vl8491; VBITS_GE_256-NEXT: mov x8, #4 // =0x4492; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0]493; VBITS_GE_256-NEXT: ptrue p0.d, vl4494; VBITS_GE_256-NEXT: add z0.s, z0.s, z0.s495; VBITS_GE_256-NEXT: movprfx z1, z0496; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #16497; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s498; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s499; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x1]500; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x1, x8, lsl #3]501; VBITS_GE_256-NEXT: ret502;503; VBITS_GE_512-LABEL: sext_v8i32_v8i64:504; VBITS_GE_512: // %bb.0:505; VBITS_GE_512-NEXT: ptrue p0.s, vl8506; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]507; VBITS_GE_512-NEXT: ptrue p0.d, vl8508; VBITS_GE_512-NEXT: add z0.s, z0.s, z0.s509; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s510; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x1]511; VBITS_GE_512-NEXT: ret512 %a = load <8 x i32>, ptr %in513 %b = add <8 x i32> %a, %a514 %c = sext <8 x i32> %b to <8 x i64>515 store <8 x i64> %c, ptr %out516 ret void517}518 519define void @sext_v16i32_v16i64(ptr %in, ptr %out) vscale_range(8,0) #0 {520; CHECK-LABEL: sext_v16i32_v16i64:521; CHECK: // %bb.0:522; CHECK-NEXT: ptrue p0.s, vl16523; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]524; CHECK-NEXT: ptrue p0.d, vl16525; CHECK-NEXT: add z0.s, z0.s, z0.s526; CHECK-NEXT: sunpklo z0.d, z0.s527; CHECK-NEXT: st1d { z0.d }, p0, [x1]528; CHECK-NEXT: ret529 %a = load <16 x i32>, ptr %in530 %b = add <16 x i32> %a, %a531 %c = sext <16 x i32> %b to <16 x i64>532 store <16 x i64> %c, ptr %out533 ret void534}535 536define void @sext_v32i32_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 {537; CHECK-LABEL: sext_v32i32_v32i64:538; CHECK: // %bb.0:539; CHECK-NEXT: ptrue p0.s, vl32540; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]541; CHECK-NEXT: ptrue p0.d, vl32542; CHECK-NEXT: add z0.s, z0.s, z0.s543; CHECK-NEXT: sunpklo z0.d, z0.s544; CHECK-NEXT: st1d { z0.d }, p0, [x1]545; CHECK-NEXT: ret546 %a = load <32 x i32>, ptr %in547 %b = add <32 x i32> %a, %a548 %c = sext <32 x i32> %b to <32 x i64>549 store <32 x i64> %c, ptr %out550 ret void551}552 553;554; zext i8 -> i16555;556 557define void @zext_v16i8_v16i16(<16 x i8> %a, ptr %out) vscale_range(2,0) #0 {558; CHECK-LABEL: zext_v16i8_v16i16:559; CHECK: // %bb.0:560; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0561; CHECK-NEXT: ptrue p0.h, vl16562; CHECK-NEXT: uunpklo z0.h, z0.b563; CHECK-NEXT: st1h { z0.h }, p0, [x0]564; CHECK-NEXT: ret565 %b = zext <16 x i8> %a to <16 x i16>566 store <16 x i16>%b, ptr %out567 ret void568}569 570; NOTE: Extra 'add' is to prevent the extend being combined with the load.571define void @zext_v32i8_v32i16(ptr %in, ptr %out) #0 {572; VBITS_GE_256-LABEL: zext_v32i8_v32i16:573; VBITS_GE_256: // %bb.0:574; VBITS_GE_256-NEXT: ptrue p0.b, vl32575; VBITS_GE_256-NEXT: mov x8, #16 // =0x10576; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0]577; VBITS_GE_256-NEXT: ptrue p0.h, vl16578; VBITS_GE_256-NEXT: add z0.b, z0.b, z0.b579; VBITS_GE_256-NEXT: movprfx z1, z0580; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #16581; VBITS_GE_256-NEXT: uunpklo z0.h, z0.b582; VBITS_GE_256-NEXT: uunpklo z1.h, z1.b583; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x1]584; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x1, x8, lsl #1]585; VBITS_GE_256-NEXT: ret586;587; VBITS_GE_512-LABEL: zext_v32i8_v32i16:588; VBITS_GE_512: // %bb.0:589; VBITS_GE_512-NEXT: ptrue p0.b, vl32590; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]591; VBITS_GE_512-NEXT: ptrue p0.h, vl32592; VBITS_GE_512-NEXT: add z0.b, z0.b, z0.b593; VBITS_GE_512-NEXT: uunpklo z0.h, z0.b594; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x1]595; VBITS_GE_512-NEXT: ret596 %a = load <32 x i8>, ptr %in597 %b = add <32 x i8> %a, %a598 %c = zext <32 x i8> %b to <32 x i16>599 store <32 x i16> %c, ptr %out600 ret void601}602 603define void @zext_v64i8_v64i16(ptr %in, ptr %out) vscale_range(8,0) #0 {604; CHECK-LABEL: zext_v64i8_v64i16:605; CHECK: // %bb.0:606; CHECK-NEXT: ptrue p0.b, vl64607; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]608; CHECK-NEXT: ptrue p0.h, vl64609; CHECK-NEXT: add z0.b, z0.b, z0.b610; CHECK-NEXT: uunpklo z0.h, z0.b611; CHECK-NEXT: st1h { z0.h }, p0, [x1]612; CHECK-NEXT: ret613 %a = load <64 x i8>, ptr %in614 %b = add <64 x i8> %a, %a615 %c = zext <64 x i8> %b to <64 x i16>616 store <64 x i16> %c, ptr %out617 ret void618}619 620define void @zext_v128i8_v128i16(ptr %in, ptr %out) vscale_range(16,0) #0 {621; CHECK-LABEL: zext_v128i8_v128i16:622; CHECK: // %bb.0:623; CHECK-NEXT: ptrue p0.b, vl128624; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]625; CHECK-NEXT: ptrue p0.h, vl128626; CHECK-NEXT: add z0.b, z0.b, z0.b627; CHECK-NEXT: uunpklo z0.h, z0.b628; CHECK-NEXT: st1h { z0.h }, p0, [x1]629; CHECK-NEXT: ret630 %a = load <128 x i8>, ptr %in631 %b = add <128 x i8> %a, %a632 %c = zext <128 x i8> %b to <128 x i16>633 store <128 x i16> %c, ptr %out634 ret void635}636 637;638; zext i8 -> i32639;640 641define void @zext_v8i8_v8i32(<8 x i8> %a, ptr %out) vscale_range(2,0) #0 {642; CHECK-LABEL: zext_v8i8_v8i32:643; CHECK: // %bb.0:644; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0645; CHECK-NEXT: ptrue p0.s, vl8646; CHECK-NEXT: uunpklo z0.h, z0.b647; CHECK-NEXT: uunpklo z0.s, z0.h648; CHECK-NEXT: st1w { z0.s }, p0, [x0]649; CHECK-NEXT: ret650 %b = zext <8 x i8> %a to <8 x i32>651 store <8 x i32>%b, ptr %out652 ret void653}654 655define void @zext_v16i8_v16i32(<16 x i8> %a, ptr %out) #0 {656; VBITS_GE_256-LABEL: zext_v16i8_v16i32:657; VBITS_GE_256: // %bb.0:658; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0659; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8660; VBITS_GE_256-NEXT: ptrue p0.s, vl8661; VBITS_GE_256-NEXT: mov x8, #8 // =0x8662; VBITS_GE_256-NEXT: uunpklo z0.h, z0.b663; VBITS_GE_256-NEXT: uunpklo z1.h, z1.b664; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h665; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h666; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0]667; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0, x8, lsl #2]668; VBITS_GE_256-NEXT: ret669;670; VBITS_GE_512-LABEL: zext_v16i8_v16i32:671; VBITS_GE_512: // %bb.0:672; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0673; VBITS_GE_512-NEXT: ptrue p0.s, vl16674; VBITS_GE_512-NEXT: uunpklo z0.h, z0.b675; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h676; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]677; VBITS_GE_512-NEXT: ret678 %b = zext <16 x i8> %a to <16 x i32>679 store <16 x i32> %b, ptr %out680 ret void681}682 683define void @zext_v32i8_v32i32(ptr %in, ptr %out) vscale_range(8,0) #0 {684; CHECK-LABEL: zext_v32i8_v32i32:685; CHECK: // %bb.0:686; CHECK-NEXT: ptrue p0.b, vl32687; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]688; CHECK-NEXT: ptrue p0.s, vl32689; CHECK-NEXT: add z0.b, z0.b, z0.b690; CHECK-NEXT: uunpklo z0.h, z0.b691; CHECK-NEXT: uunpklo z0.s, z0.h692; CHECK-NEXT: st1w { z0.s }, p0, [x1]693; CHECK-NEXT: ret694 %a = load <32 x i8>, ptr %in695 %b = add <32 x i8> %a, %a696 %c = zext <32 x i8> %b to <32 x i32>697 store <32 x i32> %c, ptr %out698 ret void699}700 701define void @zext_v64i8_v64i32(ptr %in, ptr %out) vscale_range(16,0) #0 {702; CHECK-LABEL: zext_v64i8_v64i32:703; CHECK: // %bb.0:704; CHECK-NEXT: ptrue p0.b, vl64705; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]706; CHECK-NEXT: ptrue p0.s, vl64707; CHECK-NEXT: add z0.b, z0.b, z0.b708; CHECK-NEXT: uunpklo z0.h, z0.b709; CHECK-NEXT: uunpklo z0.s, z0.h710; CHECK-NEXT: st1w { z0.s }, p0, [x1]711; CHECK-NEXT: ret712 %a = load <64 x i8>, ptr %in713 %b = add <64 x i8> %a, %a714 %c = zext <64 x i8> %b to <64 x i32>715 store <64 x i32> %c, ptr %out716 ret void717}718 719;720; zext i8 -> i64721;722 723; NOTE: v4i8 is an unpacked typed stored within a v4i16 container. The zero724; extend is a two step process where the container is zero_extend_inreg'd with725; the result feeding a normal zero extend from halfs to doublewords.726define void @zext_v4i8_v4i64(<4 x i8> %a, ptr %out) vscale_range(2,0) #0 {727; CHECK-LABEL: zext_v4i8_v4i64:728; CHECK: // %bb.0:729; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0730; CHECK-NEXT: ptrue p0.d, vl4731; CHECK-NEXT: bic v0.4h, #255, lsl #8732; CHECK-NEXT: uunpklo z0.s, z0.h733; CHECK-NEXT: uunpklo z0.d, z0.s734; CHECK-NEXT: st1d { z0.d }, p0, [x0]735; CHECK-NEXT: ret736 %b = zext <4 x i8> %a to <4 x i64>737 store <4 x i64>%b, ptr %out738 ret void739}740 741define void @zext_v8i8_v8i64(<8 x i8> %a, ptr %out) #0 {742; VBITS_GE_256-LABEL: zext_v8i8_v8i64:743; VBITS_GE_256: // %bb.0:744; VBITS_GE_256-NEXT: ushll v0.8h, v0.8b, #0745; VBITS_GE_256-NEXT: ptrue p0.d, vl4746; VBITS_GE_256-NEXT: mov x8, #4 // =0x4747; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8748; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h749; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h750; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s751; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s752; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0]753; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0, x8, lsl #3]754; VBITS_GE_256-NEXT: ret755;756; VBITS_GE_512-LABEL: zext_v8i8_v8i64:757; VBITS_GE_512: // %bb.0:758; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 def $z0759; VBITS_GE_512-NEXT: ptrue p0.d, vl8760; VBITS_GE_512-NEXT: uunpklo z0.h, z0.b761; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h762; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s763; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]764; VBITS_GE_512-NEXT: ret765 %b = zext <8 x i8> %a to <8 x i64>766 store <8 x i64>%b, ptr %out767 ret void768}769 770define void @zext_v16i8_v16i64(<16 x i8> %a, ptr %out) vscale_range(8,0) #0 {771; CHECK-LABEL: zext_v16i8_v16i64:772; CHECK: // %bb.0:773; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0774; CHECK-NEXT: ptrue p0.d, vl16775; CHECK-NEXT: uunpklo z0.h, z0.b776; CHECK-NEXT: uunpklo z0.s, z0.h777; CHECK-NEXT: uunpklo z0.d, z0.s778; CHECK-NEXT: st1d { z0.d }, p0, [x0]779; CHECK-NEXT: ret780 %b = zext <16 x i8> %a to <16 x i64>781 store <16 x i64> %b, ptr %out782 ret void783}784 785define void @zext_v32i8_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 {786; CHECK-LABEL: zext_v32i8_v32i64:787; CHECK: // %bb.0:788; CHECK-NEXT: ptrue p0.b, vl32789; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]790; CHECK-NEXT: ptrue p0.d, vl32791; CHECK-NEXT: add z0.b, z0.b, z0.b792; CHECK-NEXT: uunpklo z0.h, z0.b793; CHECK-NEXT: uunpklo z0.s, z0.h794; CHECK-NEXT: uunpklo z0.d, z0.s795; CHECK-NEXT: st1d { z0.d }, p0, [x1]796; CHECK-NEXT: ret797 %a = load <32 x i8>, ptr %in798 %b = add <32 x i8> %a, %a799 %c = zext <32 x i8> %b to <32 x i64>800 store <32 x i64> %c, ptr %out801 ret void802}803 804;805; zext i16 -> i32806;807 808define void @zext_v8i16_v8i32(<8 x i16> %a, ptr %out) vscale_range(2,0) #0 {809; CHECK-LABEL: zext_v8i16_v8i32:810; CHECK: // %bb.0:811; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0812; CHECK-NEXT: ptrue p0.s, vl8813; CHECK-NEXT: uunpklo z0.s, z0.h814; CHECK-NEXT: st1w { z0.s }, p0, [x0]815; CHECK-NEXT: ret816 %b = zext <8 x i16> %a to <8 x i32>817 store <8 x i32>%b, ptr %out818 ret void819}820 821define void @zext_v16i16_v16i32(ptr %in, ptr %out) #0 {822; VBITS_GE_256-LABEL: zext_v16i16_v16i32:823; VBITS_GE_256: // %bb.0:824; VBITS_GE_256-NEXT: ptrue p0.h, vl16825; VBITS_GE_256-NEXT: mov x8, #8 // =0x8826; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]827; VBITS_GE_256-NEXT: ptrue p0.s, vl8828; VBITS_GE_256-NEXT: add z0.h, z0.h, z0.h829; VBITS_GE_256-NEXT: movprfx z1, z0830; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #16831; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h832; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h833; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x1]834; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x1, x8, lsl #2]835; VBITS_GE_256-NEXT: ret836;837; VBITS_GE_512-LABEL: zext_v16i16_v16i32:838; VBITS_GE_512: // %bb.0:839; VBITS_GE_512-NEXT: ptrue p0.h, vl16840; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]841; VBITS_GE_512-NEXT: ptrue p0.s, vl16842; VBITS_GE_512-NEXT: add z0.h, z0.h, z0.h843; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h844; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x1]845; VBITS_GE_512-NEXT: ret846 %a = load <16 x i16>, ptr %in847 %b = add <16 x i16> %a, %a848 %c = zext <16 x i16> %b to <16 x i32>849 store <16 x i32> %c, ptr %out850 ret void851}852 853define void @zext_v32i16_v32i32(ptr %in, ptr %out) vscale_range(8,0) #0 {854; CHECK-LABEL: zext_v32i16_v32i32:855; CHECK: // %bb.0:856; CHECK-NEXT: ptrue p0.h, vl32857; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]858; CHECK-NEXT: ptrue p0.s, vl32859; CHECK-NEXT: add z0.h, z0.h, z0.h860; CHECK-NEXT: uunpklo z0.s, z0.h861; CHECK-NEXT: st1w { z0.s }, p0, [x1]862; CHECK-NEXT: ret863 %a = load <32 x i16>, ptr %in864 %b = add <32 x i16> %a, %a865 %c = zext <32 x i16> %b to <32 x i32>866 store <32 x i32> %c, ptr %out867 ret void868}869 870define void @zext_v64i16_v64i32(ptr %in, ptr %out) vscale_range(16,0) #0 {871; CHECK-LABEL: zext_v64i16_v64i32:872; CHECK: // %bb.0:873; CHECK-NEXT: ptrue p0.h, vl64874; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]875; CHECK-NEXT: ptrue p0.s, vl64876; CHECK-NEXT: add z0.h, z0.h, z0.h877; CHECK-NEXT: uunpklo z0.s, z0.h878; CHECK-NEXT: st1w { z0.s }, p0, [x1]879; CHECK-NEXT: ret880 %a = load <64 x i16>, ptr %in881 %b = add <64 x i16> %a, %a882 %c = zext <64 x i16> %b to <64 x i32>883 store <64 x i32> %c, ptr %out884 ret void885}886 887;888; zext i16 -> i64889;890 891define void @zext_v4i16_v4i64(<4 x i16> %a, ptr %out) vscale_range(2,0) #0 {892; CHECK-LABEL: zext_v4i16_v4i64:893; CHECK: // %bb.0:894; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0895; CHECK-NEXT: ptrue p0.d, vl4896; CHECK-NEXT: uunpklo z0.s, z0.h897; CHECK-NEXT: uunpklo z0.d, z0.s898; CHECK-NEXT: st1d { z0.d }, p0, [x0]899; CHECK-NEXT: ret900 %b = zext <4 x i16> %a to <4 x i64>901 store <4 x i64>%b, ptr %out902 ret void903}904 905define void @zext_v8i16_v8i64(<8 x i16> %a, ptr %out) #0 {906; VBITS_GE_256-LABEL: zext_v8i16_v8i64:907; VBITS_GE_256: // %bb.0:908; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0909; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8910; VBITS_GE_256-NEXT: ptrue p0.d, vl4911; VBITS_GE_256-NEXT: mov x8, #4 // =0x4912; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h913; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h914; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s915; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s916; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0]917; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0, x8, lsl #3]918; VBITS_GE_256-NEXT: ret919;920; VBITS_GE_512-LABEL: zext_v8i16_v8i64:921; VBITS_GE_512: // %bb.0:922; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0923; VBITS_GE_512-NEXT: ptrue p0.d, vl8924; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h925; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s926; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]927; VBITS_GE_512-NEXT: ret928 %b = zext <8 x i16> %a to <8 x i64>929 store <8 x i64>%b, ptr %out930 ret void931}932 933define void @zext_v16i16_v16i64(ptr %in, ptr %out) vscale_range(8,0) #0 {934; CHECK-LABEL: zext_v16i16_v16i64:935; CHECK: // %bb.0:936; CHECK-NEXT: ptrue p0.h, vl16937; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]938; CHECK-NEXT: ptrue p0.d, vl16939; CHECK-NEXT: add z0.h, z0.h, z0.h940; CHECK-NEXT: uunpklo z0.s, z0.h941; CHECK-NEXT: uunpklo z0.d, z0.s942; CHECK-NEXT: st1d { z0.d }, p0, [x1]943; CHECK-NEXT: ret944 %a = load <16 x i16>, ptr %in945 %b = add <16 x i16> %a, %a946 %c = zext <16 x i16> %b to <16 x i64>947 store <16 x i64> %c, ptr %out948 ret void949}950 951define void @zext_v32i16_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 {952; CHECK-LABEL: zext_v32i16_v32i64:953; CHECK: // %bb.0:954; CHECK-NEXT: ptrue p0.h, vl32955; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]956; CHECK-NEXT: ptrue p0.d, vl32957; CHECK-NEXT: add z0.h, z0.h, z0.h958; CHECK-NEXT: uunpklo z0.s, z0.h959; CHECK-NEXT: uunpklo z0.d, z0.s960; CHECK-NEXT: st1d { z0.d }, p0, [x1]961; CHECK-NEXT: ret962 %a = load <32 x i16>, ptr %in963 %b = add <32 x i16> %a, %a964 %c = zext <32 x i16> %b to <32 x i64>965 store <32 x i64> %c, ptr %out966 ret void967}968 969;970; zext i32 -> i64971;972 973define void @zext_v4i32_v4i64(<4 x i32> %a, ptr %out) vscale_range(2,0) #0 {974; CHECK-LABEL: zext_v4i32_v4i64:975; CHECK: // %bb.0:976; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0977; CHECK-NEXT: ptrue p0.d, vl4978; CHECK-NEXT: uunpklo z0.d, z0.s979; CHECK-NEXT: st1d { z0.d }, p0, [x0]980; CHECK-NEXT: ret981 %b = zext <4 x i32> %a to <4 x i64>982 store <4 x i64>%b, ptr %out983 ret void984}985 986define void @zext_v8i32_v8i64(ptr %in, ptr %out) #0 {987; VBITS_GE_256-LABEL: zext_v8i32_v8i64:988; VBITS_GE_256: // %bb.0:989; VBITS_GE_256-NEXT: ptrue p0.s, vl8990; VBITS_GE_256-NEXT: mov x8, #4 // =0x4991; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0]992; VBITS_GE_256-NEXT: ptrue p0.d, vl4993; VBITS_GE_256-NEXT: add z0.s, z0.s, z0.s994; VBITS_GE_256-NEXT: movprfx z1, z0995; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #16996; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s997; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s998; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x1]999; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x1, x8, lsl #3]1000; VBITS_GE_256-NEXT: ret1001;1002; VBITS_GE_512-LABEL: zext_v8i32_v8i64:1003; VBITS_GE_512: // %bb.0:1004; VBITS_GE_512-NEXT: ptrue p0.s, vl81005; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]1006; VBITS_GE_512-NEXT: ptrue p0.d, vl81007; VBITS_GE_512-NEXT: add z0.s, z0.s, z0.s1008; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s1009; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x1]1010; VBITS_GE_512-NEXT: ret1011 %a = load <8 x i32>, ptr %in1012 %b = add <8 x i32> %a, %a1013 %c = zext <8 x i32> %b to <8 x i64>1014 store <8 x i64> %c, ptr %out1015 ret void1016}1017 1018define void @zext_v16i32_v16i64(ptr %in, ptr %out) vscale_range(8,0) #0 {1019; CHECK-LABEL: zext_v16i32_v16i64:1020; CHECK: // %bb.0:1021; CHECK-NEXT: ptrue p0.s, vl161022; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1023; CHECK-NEXT: ptrue p0.d, vl161024; CHECK-NEXT: add z0.s, z0.s, z0.s1025; CHECK-NEXT: uunpklo z0.d, z0.s1026; CHECK-NEXT: st1d { z0.d }, p0, [x1]1027; CHECK-NEXT: ret1028 %a = load <16 x i32>, ptr %in1029 %b = add <16 x i32> %a, %a1030 %c = zext <16 x i32> %b to <16 x i64>1031 store <16 x i64> %c, ptr %out1032 ret void1033}1034 1035define void @zext_v32i32_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 {1036; CHECK-LABEL: zext_v32i32_v32i64:1037; CHECK: // %bb.0:1038; CHECK-NEXT: ptrue p0.s, vl321039; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]1040; CHECK-NEXT: ptrue p0.d, vl321041; CHECK-NEXT: add z0.s, z0.s, z0.s1042; CHECK-NEXT: uunpklo z0.d, z0.s1043; CHECK-NEXT: st1d { z0.d }, p0, [x1]1044; CHECK-NEXT: ret1045 %a = load <32 x i32>, ptr %in1046 %b = add <32 x i32> %a, %a1047 %c = zext <32 x i32> %b to <32 x i64>1048 store <32 x i64> %c, ptr %out1049 ret void1050}1051 1052attributes #0 = { nounwind "target-features"="+sve" }1053