1591 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2563; RUN: llc -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5124; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_5125 6target triple = "aarch64-unknown-linux-gnu"7 8;9; Masked Loads10;11 12define <2 x half> @masked_load_v2f16(ptr %ap, ptr %bp) vscale_range(2,0) #0 {13; CHECK-LABEL: masked_load_v2f16:14; CHECK: // %bb.0:15; CHECK-NEXT: ldr s1, [x0]16; CHECK-NEXT: ldr s2, [x1]17; CHECK-NEXT: movi v0.2d, #000000000000000018; CHECK-NEXT: ptrue p0.h, vl419; CHECK-NEXT: fcmeq v1.4h, v1.4h, v2.4h20; CHECK-NEXT: sshll v1.4s, v1.4h, #021; CHECK-NEXT: mov v0.h[0], v1.h[0]22; CHECK-NEXT: mov w8, v1.s[1]23; CHECK-NEXT: mov v0.h[1], w824; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #025; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]26; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z027; CHECK-NEXT: ret28 %a = load <2 x half>, ptr %ap29 %b = load <2 x half>, ptr %bp30 %mask = fcmp oeq <2 x half> %a, %b31 %load = call <2 x half> @llvm.masked.load.v2f16(ptr %ap, i32 8, <2 x i1> %mask, <2 x half> zeroinitializer)32 ret <2 x half> %load33}34 35define <2 x float> @masked_load_v2f32(ptr %ap, ptr %bp) vscale_range(2,0) #0 {36; CHECK-LABEL: masked_load_v2f32:37; CHECK: // %bb.0:38; CHECK-NEXT: ldr d0, [x0]39; CHECK-NEXT: ldr d1, [x1]40; CHECK-NEXT: ptrue p0.s, vl241; CHECK-NEXT: fcmeq v0.2s, v0.2s, v1.2s42; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #043; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]44; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z045; CHECK-NEXT: ret46 %a = load <2 x float>, ptr %ap47 %b = load <2 x float>, ptr %bp48 %mask = fcmp oeq <2 x float> %a, %b49 %load = call <2 x float> @llvm.masked.load.v2f32(ptr %ap, i32 8, <2 x i1> %mask, <2 x float> zeroinitializer)50 ret <2 x float> %load51}52 53define <4 x float> @masked_load_v4f32(ptr %ap, ptr %bp) vscale_range(1,0) #0 {54; CHECK-LABEL: masked_load_v4f32:55; CHECK: // %bb.0:56; CHECK-NEXT: ldr q0, [x0]57; CHECK-NEXT: ldr q1, [x1]58; CHECK-NEXT: ptrue p0.s, vl459; CHECK-NEXT: fcmeq v0.4s, v0.4s, v1.4s60; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #061; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]62; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z063; CHECK-NEXT: ret64 %a = load <4 x float>, ptr %ap65 %b = load <4 x float>, ptr %bp66 %mask = fcmp oeq <4 x float> %a, %b67 %load = call <4 x float> @llvm.masked.load.v4f32(ptr %ap, i32 8, <4 x i1> %mask, <4 x float> zeroinitializer)68 ret <4 x float> %load69}70 71define void @masked_load_v8f32(ptr %ap, ptr %bp, ptr %c) vscale_range(2,0) #0 {72; CHECK-LABEL: masked_load_v8f32:73; CHECK: // %bb.0:74; CHECK-NEXT: ptrue p0.s, vl875; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]76; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]77; CHECK-NEXT: fcmeq p1.s, p0/z, z0.s, z1.s78; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0]79; CHECK-NEXT: st1w { z0.s }, p0, [x2]80; CHECK-NEXT: ret81 %a = load <8 x float>, ptr %ap82 %b = load <8 x float>, ptr %bp83 %mask = fcmp oeq <8 x float> %a, %b84 %load = call <8 x float> @llvm.masked.load.v8f32(ptr %ap, i32 8, <8 x i1> %mask, <8 x float> zeroinitializer)85 store <8 x float> %load, ptr %c86 ret void87}88 89define void @masked_load_v16f32(ptr %ap, ptr %bp, ptr %c) #0 {90; VBITS_GE_256-LABEL: masked_load_v16f32:91; VBITS_GE_256: // %bb.0:92; VBITS_GE_256-NEXT: ptrue p0.s, vl893; VBITS_GE_256-NEXT: mov x8, #8 // =0x894; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]95; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]96; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]97; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]98; VBITS_GE_256-NEXT: fcmeq p1.s, p0/z, z0.s, z1.s99; VBITS_GE_256-NEXT: fcmeq p2.s, p0/z, z2.s, z3.s100; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x0, x8, lsl #2]101; VBITS_GE_256-NEXT: ld1w { z1.s }, p2/z, [x0]102; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2, x8, lsl #2]103; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2]104; VBITS_GE_256-NEXT: ret105;106; VBITS_GE_512-LABEL: masked_load_v16f32:107; VBITS_GE_512: // %bb.0:108; VBITS_GE_512-NEXT: ptrue p0.s, vl16109; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]110; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]111; VBITS_GE_512-NEXT: fcmeq p1.s, p0/z, z0.s, z1.s112; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x0]113; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]114; VBITS_GE_512-NEXT: ret115 %a = load <16 x float>, ptr %ap116 %b = load <16 x float>, ptr %bp117 %mask = fcmp oeq <16 x float> %a, %b118 %load = call <16 x float> @llvm.masked.load.v16f32(ptr %ap, i32 8, <16 x i1> %mask, <16 x float> zeroinitializer)119 store <16 x float> %load, ptr %c120 ret void121}122 123define void @masked_load_v32f32(ptr %ap, ptr %bp, ptr %c) vscale_range(8,0) #0 {124; CHECK-LABEL: masked_load_v32f32:125; CHECK: // %bb.0:126; CHECK-NEXT: ptrue p0.s, vl32127; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]128; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]129; CHECK-NEXT: fcmeq p1.s, p0/z, z0.s, z1.s130; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0]131; CHECK-NEXT: st1w { z0.s }, p0, [x2]132; CHECK-NEXT: ret133 %a = load <32 x float>, ptr %ap134 %b = load <32 x float>, ptr %bp135 %mask = fcmp oeq <32 x float> %a, %b136 %load = call <32 x float> @llvm.masked.load.v32f32(ptr %ap, i32 8, <32 x i1> %mask, <32 x float> zeroinitializer)137 store <32 x float> %load, ptr %c138 ret void139}140 141define void @masked_load_v64f32(ptr %ap, ptr %bp, ptr %c) vscale_range(16,0) #0 {142; CHECK-LABEL: masked_load_v64f32:143; CHECK: // %bb.0:144; CHECK-NEXT: ptrue p0.s, vl64145; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]146; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1]147; CHECK-NEXT: fcmeq p1.s, p0/z, z0.s, z1.s148; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0]149; CHECK-NEXT: st1w { z0.s }, p0, [x2]150; CHECK-NEXT: ret151 %a = load <64 x float>, ptr %ap152 %b = load <64 x float>, ptr %bp153 %mask = fcmp oeq <64 x float> %a, %b154 %load = call <64 x float> @llvm.masked.load.v64f32(ptr %ap, i32 8, <64 x i1> %mask, <64 x float> zeroinitializer)155 store <64 x float> %load, ptr %c156 ret void157}158 159define void @masked_load_v64i8(ptr %ap, ptr %bp, ptr %c) #0 {160; VBITS_GE_256-LABEL: masked_load_v64i8:161; VBITS_GE_256: // %bb.0:162; VBITS_GE_256-NEXT: ptrue p0.b, vl32163; VBITS_GE_256-NEXT: mov w8, #32 // =0x20164; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]165; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x1, x8]166; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0]167; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x1]168; VBITS_GE_256-NEXT: cmpeq p1.b, p0/z, z0.b, z1.b169; VBITS_GE_256-NEXT: cmpeq p2.b, p0/z, z2.b, z3.b170; VBITS_GE_256-NEXT: ld1b { z0.b }, p1/z, [x0, x8]171; VBITS_GE_256-NEXT: ld1b { z1.b }, p2/z, [x0]172; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x2, x8]173; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x2]174; VBITS_GE_256-NEXT: ret175;176; VBITS_GE_512-LABEL: masked_load_v64i8:177; VBITS_GE_512: // %bb.0:178; VBITS_GE_512-NEXT: ptrue p0.b, vl64179; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]180; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x1]181; VBITS_GE_512-NEXT: cmpeq p1.b, p0/z, z0.b, z1.b182; VBITS_GE_512-NEXT: ld1b { z0.b }, p1/z, [x0]183; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x2]184; VBITS_GE_512-NEXT: ret185 %a = load <64 x i8>, ptr %ap186 %b = load <64 x i8>, ptr %bp187 %mask = icmp eq <64 x i8> %a, %b188 %load = call <64 x i8> @llvm.masked.load.v64i8(ptr %ap, i32 8, <64 x i1> %mask, <64 x i8> poison)189 store <64 x i8> %load, ptr %c190 ret void191}192 193define void @masked_load_v32i16(ptr %ap, ptr %bp, ptr %c) #0 {194; VBITS_GE_256-LABEL: masked_load_v32i16:195; VBITS_GE_256: // %bb.0:196; VBITS_GE_256-NEXT: ptrue p0.h, vl16197; VBITS_GE_256-NEXT: mov x8, #16 // =0x10198; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]199; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1, x8, lsl #1]200; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0]201; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1]202; VBITS_GE_256-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h203; VBITS_GE_256-NEXT: cmpeq p2.h, p0/z, z2.h, z3.h204; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x0, x8, lsl #1]205; VBITS_GE_256-NEXT: ld1h { z1.h }, p2/z, [x0]206; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x2, x8, lsl #1]207; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x2]208; VBITS_GE_256-NEXT: ret209;210; VBITS_GE_512-LABEL: masked_load_v32i16:211; VBITS_GE_512: // %bb.0:212; VBITS_GE_512-NEXT: ptrue p0.h, vl32213; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0]214; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1]215; VBITS_GE_512-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h216; VBITS_GE_512-NEXT: ld1h { z0.h }, p1/z, [x0]217; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x2]218; VBITS_GE_512-NEXT: ret219 %a = load <32 x i16>, ptr %ap220 %b = load <32 x i16>, ptr %bp221 %mask = icmp eq <32 x i16> %a, %b222 %load = call <32 x i16> @llvm.masked.load.v32i16(ptr %ap, i32 8, <32 x i1> %mask, <32 x i16> poison)223 store <32 x i16> %load, ptr %c224 ret void225}226 227define void @masked_load_v16i32(ptr %ap, ptr %bp, ptr %c) #0 {228; VBITS_GE_256-LABEL: masked_load_v16i32:229; VBITS_GE_256: // %bb.0:230; VBITS_GE_256-NEXT: ptrue p0.s, vl8231; VBITS_GE_256-NEXT: mov x8, #8 // =0x8232; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]233; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1, x8, lsl #2]234; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]235; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1]236; VBITS_GE_256-NEXT: cmpeq p1.s, p0/z, z0.s, z1.s237; VBITS_GE_256-NEXT: cmpeq p2.s, p0/z, z2.s, z3.s238; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x0, x8, lsl #2]239; VBITS_GE_256-NEXT: ld1w { z1.s }, p2/z, [x0]240; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2, x8, lsl #2]241; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2]242; VBITS_GE_256-NEXT: ret243;244; VBITS_GE_512-LABEL: masked_load_v16i32:245; VBITS_GE_512: // %bb.0:246; VBITS_GE_512-NEXT: ptrue p0.s, vl16247; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]248; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1]249; VBITS_GE_512-NEXT: cmpeq p1.s, p0/z, z0.s, z1.s250; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x0]251; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]252; VBITS_GE_512-NEXT: ret253 %a = load <16 x i32>, ptr %ap254 %b = load <16 x i32>, ptr %bp255 %mask = icmp eq <16 x i32> %a, %b256 %load = call <16 x i32> @llvm.masked.load.v16i32(ptr %ap, i32 8, <16 x i1> %mask, <16 x i32> poison)257 store <16 x i32> %load, ptr %c258 ret void259}260 261define void @masked_load_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {262; VBITS_GE_256-LABEL: masked_load_v8i64:263; VBITS_GE_256: // %bb.0:264; VBITS_GE_256-NEXT: ptrue p0.d, vl4265; VBITS_GE_256-NEXT: mov x8, #4 // =0x4266; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]267; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]268; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]269; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]270; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d271; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z2.d, z3.d272; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [x0, x8, lsl #3]273; VBITS_GE_256-NEXT: ld1d { z1.d }, p2/z, [x0]274; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2, x8, lsl #3]275; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2]276; VBITS_GE_256-NEXT: ret277;278; VBITS_GE_512-LABEL: masked_load_v8i64:279; VBITS_GE_512: // %bb.0:280; VBITS_GE_512-NEXT: ptrue p0.d, vl8281; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]282; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]283; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d284; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [x0]285; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]286; VBITS_GE_512-NEXT: ret287 %a = load <8 x i64>, ptr %ap288 %b = load <8 x i64>, ptr %bp289 %mask = icmp eq <8 x i64> %a, %b290 %load = call <8 x i64> @llvm.masked.load.v8i64(ptr %ap, i32 8, <8 x i1> %mask, <8 x i64> poison)291 store <8 x i64> %load, ptr %c292 ret void293}294 295define void @masked_load_passthru_v8i64(ptr %ap, ptr %bp, ptr %c) #0 {296; VBITS_GE_256-LABEL: masked_load_passthru_v8i64:297; VBITS_GE_256: // %bb.0:298; VBITS_GE_256-NEXT: ptrue p0.d, vl4299; VBITS_GE_256-NEXT: mov x8, #4 // =0x4300; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]301; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]302; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]303; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]304; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d305; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z2.d, z3.d306; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [x0, x8, lsl #3]307; VBITS_GE_256-NEXT: ld1d { z2.d }, p2/z, [x0]308; VBITS_GE_256-NEXT: sel z0.d, p1, z0.d, z1.d309; VBITS_GE_256-NEXT: sel z1.d, p2, z2.d, z3.d310; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2, x8, lsl #3]311; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2]312; VBITS_GE_256-NEXT: ret313;314; VBITS_GE_512-LABEL: masked_load_passthru_v8i64:315; VBITS_GE_512: // %bb.0:316; VBITS_GE_512-NEXT: ptrue p0.d, vl8317; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]318; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]319; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d320; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [x0]321; VBITS_GE_512-NEXT: sel z0.d, p1, z0.d, z1.d322; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]323; VBITS_GE_512-NEXT: ret324 %a = load <8 x i64>, ptr %ap325 %b = load <8 x i64>, ptr %bp326 %mask = icmp eq <8 x i64> %a, %b327 %load = call <8 x i64> @llvm.masked.load.v8i64(ptr %ap, i32 8, <8 x i1> %mask, <8 x i64> %b)328 store <8 x i64> %load, ptr %c329 ret void330}331 332define void @masked_load_passthru_v8f64(ptr %ap, ptr %bp, ptr %c) #0 {333; VBITS_GE_256-LABEL: masked_load_passthru_v8f64:334; VBITS_GE_256: // %bb.0:335; VBITS_GE_256-NEXT: ptrue p0.d, vl4336; VBITS_GE_256-NEXT: mov x8, #4 // =0x4337; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]338; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3]339; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0]340; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1]341; VBITS_GE_256-NEXT: fcmeq p1.d, p0/z, z0.d, z1.d342; VBITS_GE_256-NEXT: fcmeq p2.d, p0/z, z2.d, z3.d343; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [x0, x8, lsl #3]344; VBITS_GE_256-NEXT: ld1d { z2.d }, p2/z, [x0]345; VBITS_GE_256-NEXT: sel z0.d, p1, z0.d, z1.d346; VBITS_GE_256-NEXT: sel z1.d, p2, z2.d, z3.d347; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2, x8, lsl #3]348; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2]349; VBITS_GE_256-NEXT: ret350;351; VBITS_GE_512-LABEL: masked_load_passthru_v8f64:352; VBITS_GE_512: // %bb.0:353; VBITS_GE_512-NEXT: ptrue p0.d, vl8354; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]355; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1]356; VBITS_GE_512-NEXT: fcmeq p1.d, p0/z, z0.d, z1.d357; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [x0]358; VBITS_GE_512-NEXT: sel z0.d, p1, z0.d, z1.d359; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]360; VBITS_GE_512-NEXT: ret361 %a = load <8 x double>, ptr %ap362 %b = load <8 x double>, ptr %bp363 %mask = fcmp oeq <8 x double> %a, %b364 %load = call <8 x double> @llvm.masked.load.v8f64(ptr %ap, i32 8, <8 x i1> %mask, <8 x double> %b)365 store <8 x double> %load, ptr %c366 ret void367}368 369define void @masked_load_sext_v32i8i16(ptr %ap, ptr %bp, ptr %c) #0 {370; VBITS_GE_256-LABEL: masked_load_sext_v32i8i16:371; VBITS_GE_256: // %bb.0:372; VBITS_GE_256-NEXT: ptrue p0.b, vl32373; VBITS_GE_256-NEXT: mov x8, #16 // =0x10374; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x1]375; VBITS_GE_256-NEXT: cmpeq p0.b, p0/z, z0.b, #0376; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0]377; VBITS_GE_256-NEXT: ptrue p0.h, vl16378; VBITS_GE_256-NEXT: movprfx z1, z0379; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #16380; VBITS_GE_256-NEXT: sunpklo z0.h, z0.b381; VBITS_GE_256-NEXT: sunpklo z1.h, z1.b382; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x2]383; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x2, x8, lsl #1]384; VBITS_GE_256-NEXT: ret385;386; VBITS_GE_512-LABEL: masked_load_sext_v32i8i16:387; VBITS_GE_512: // %bb.0:388; VBITS_GE_512-NEXT: ptrue p0.h, vl32389; VBITS_GE_512-NEXT: ld1b { z0.h }, p0/z, [x1]390; VBITS_GE_512-NEXT: cmpeq p1.h, p0/z, z0.h, #0391; VBITS_GE_512-NEXT: ld1sb { z0.h }, p1/z, [x0]392; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x2]393; VBITS_GE_512-NEXT: ret394 %b = load <32 x i8>, ptr %bp395 %mask = icmp eq <32 x i8> %b, zeroinitializer396 %load = call <32 x i8> @llvm.masked.load.v32i8(ptr %ap, i32 8, <32 x i1> %mask, <32 x i8> poison)397 %ext = sext <32 x i8> %load to <32 x i16>398 store <32 x i16> %ext, ptr %c399 ret void400}401 402define void @masked_load_sext_v16i8i32(ptr %ap, ptr %bp, ptr %c) #0 {403; VBITS_GE_256-LABEL: masked_load_sext_v16i8i32:404; VBITS_GE_256: // %bb.0:405; VBITS_GE_256-NEXT: ptrue p0.b, vl16406; VBITS_GE_256-NEXT: ldr q0, [x1]407; VBITS_GE_256-NEXT: mov x8, #8 // =0x8408; VBITS_GE_256-NEXT: cmpeq p0.b, p0/z, z0.b, #0409; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0]410; VBITS_GE_256-NEXT: ptrue p0.s, vl8411; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8412; VBITS_GE_256-NEXT: sunpklo z0.h, z0.b413; VBITS_GE_256-NEXT: sunpklo z1.h, z1.b414; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h415; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h416; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]417; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]418; VBITS_GE_256-NEXT: ret419;420; VBITS_GE_512-LABEL: masked_load_sext_v16i8i32:421; VBITS_GE_512: // %bb.0:422; VBITS_GE_512-NEXT: ptrue p0.s, vl16423; VBITS_GE_512-NEXT: ld1b { z0.s }, p0/z, [x1]424; VBITS_GE_512-NEXT: cmpeq p1.s, p0/z, z0.s, #0425; VBITS_GE_512-NEXT: ld1sb { z0.s }, p1/z, [x0]426; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]427; VBITS_GE_512-NEXT: ret428 %b = load <16 x i8>, ptr %bp429 %mask = icmp eq <16 x i8> %b, zeroinitializer430 %load = call <16 x i8> @llvm.masked.load.v16i8(ptr %ap, i32 8, <16 x i1> %mask, <16 x i8> poison)431 %ext = sext <16 x i8> %load to <16 x i32>432 store <16 x i32> %ext, ptr %c433 ret void434}435 436define void @masked_load_sext_v8i8i64(ptr %ap, ptr %bp, ptr %c) #0 {437; VBITS_GE_256-LABEL: masked_load_sext_v8i8i64:438; VBITS_GE_256: // %bb.0:439; VBITS_GE_256-NEXT: ptrue p0.b, vl8440; VBITS_GE_256-NEXT: ldr d0, [x1]441; VBITS_GE_256-NEXT: mov x8, #4 // =0x4442; VBITS_GE_256-NEXT: cmpeq p0.b, p0/z, z0.b, #0443; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0]444; VBITS_GE_256-NEXT: ptrue p0.d, vl4445; VBITS_GE_256-NEXT: sshll v0.8h, v0.8b, #0446; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8447; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h448; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h449; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s450; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s451; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]452; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]453; VBITS_GE_256-NEXT: ret454;455; VBITS_GE_512-LABEL: masked_load_sext_v8i8i64:456; VBITS_GE_512: // %bb.0:457; VBITS_GE_512-NEXT: ptrue p0.d, vl8458; VBITS_GE_512-NEXT: ld1b { z0.d }, p0/z, [x1]459; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0460; VBITS_GE_512-NEXT: ld1sb { z0.d }, p1/z, [x0]461; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]462; VBITS_GE_512-NEXT: ret463 %b = load <8 x i8>, ptr %bp464 %mask = icmp eq <8 x i8> %b, zeroinitializer465 %load = call <8 x i8> @llvm.masked.load.v8i8(ptr %ap, i32 8, <8 x i1> %mask, <8 x i8> poison)466 %ext = sext <8 x i8> %load to <8 x i64>467 store <8 x i64> %ext, ptr %c468 ret void469}470 471define void @masked_load_sext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 {472; VBITS_GE_256-LABEL: masked_load_sext_v16i16i32:473; VBITS_GE_256: // %bb.0:474; VBITS_GE_256-NEXT: ptrue p0.h, vl16475; VBITS_GE_256-NEXT: mov x8, #8 // =0x8476; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x1]477; VBITS_GE_256-NEXT: cmpeq p0.h, p0/z, z0.h, #0478; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]479; VBITS_GE_256-NEXT: ptrue p0.s, vl8480; VBITS_GE_256-NEXT: movprfx z1, z0481; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #16482; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h483; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h484; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]485; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]486; VBITS_GE_256-NEXT: ret487;488; VBITS_GE_512-LABEL: masked_load_sext_v16i16i32:489; VBITS_GE_512: // %bb.0:490; VBITS_GE_512-NEXT: ptrue p0.s, vl16491; VBITS_GE_512-NEXT: ld1h { z0.s }, p0/z, [x1]492; VBITS_GE_512-NEXT: cmpeq p1.s, p0/z, z0.s, #0493; VBITS_GE_512-NEXT: ld1sh { z0.s }, p1/z, [x0]494; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]495; VBITS_GE_512-NEXT: ret496 %b = load <16 x i16>, ptr %bp497 %mask = icmp eq <16 x i16> %b, zeroinitializer498 %load = call <16 x i16> @llvm.masked.load.v16i16(ptr %ap, i32 8, <16 x i1> %mask, <16 x i16> poison)499 %ext = sext <16 x i16> %load to <16 x i32>500 store <16 x i32> %ext, ptr %c501 ret void502}503 504define void @masked_load_sext_v8i16i64(ptr %ap, ptr %bp, ptr %c) #0 {505; VBITS_GE_256-LABEL: masked_load_sext_v8i16i64:506; VBITS_GE_256: // %bb.0:507; VBITS_GE_256-NEXT: ptrue p0.h, vl8508; VBITS_GE_256-NEXT: ldr q0, [x1]509; VBITS_GE_256-NEXT: mov x8, #4 // =0x4510; VBITS_GE_256-NEXT: cmpeq p0.h, p0/z, z0.h, #0511; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]512; VBITS_GE_256-NEXT: ptrue p0.d, vl4513; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8514; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h515; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h516; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s517; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s518; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]519; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]520; VBITS_GE_256-NEXT: ret521;522; VBITS_GE_512-LABEL: masked_load_sext_v8i16i64:523; VBITS_GE_512: // %bb.0:524; VBITS_GE_512-NEXT: ptrue p0.d, vl8525; VBITS_GE_512-NEXT: ld1h { z0.d }, p0/z, [x1]526; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0527; VBITS_GE_512-NEXT: ld1sh { z0.d }, p1/z, [x0]528; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]529; VBITS_GE_512-NEXT: ret530 %b = load <8 x i16>, ptr %bp531 %mask = icmp eq <8 x i16> %b, zeroinitializer532 %load = call <8 x i16> @llvm.masked.load.v8i16(ptr %ap, i32 8, <8 x i1> %mask, <8 x i16> poison)533 %ext = sext <8 x i16> %load to <8 x i64>534 store <8 x i64> %ext, ptr %c535 ret void536}537 538define void @masked_load_sext_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {539; VBITS_GE_256-LABEL: masked_load_sext_v8i32i64:540; VBITS_GE_256: // %bb.0:541; VBITS_GE_256-NEXT: ptrue p0.s, vl8542; VBITS_GE_256-NEXT: mov x8, #4 // =0x4543; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1]544; VBITS_GE_256-NEXT: cmpeq p0.s, p0/z, z0.s, #0545; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0]546; VBITS_GE_256-NEXT: ptrue p0.d, vl4547; VBITS_GE_256-NEXT: movprfx z1, z0548; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #16549; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s550; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s551; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]552; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]553; VBITS_GE_256-NEXT: ret554;555; VBITS_GE_512-LABEL: masked_load_sext_v8i32i64:556; VBITS_GE_512: // %bb.0:557; VBITS_GE_512-NEXT: ptrue p0.d, vl8558; VBITS_GE_512-NEXT: ld1w { z0.d }, p0/z, [x1]559; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0560; VBITS_GE_512-NEXT: ld1sw { z0.d }, p1/z, [x0]561; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]562; VBITS_GE_512-NEXT: ret563 %b = load <8 x i32>, ptr %bp564 %mask = icmp eq <8 x i32> %b, zeroinitializer565 %load = call <8 x i32> @llvm.masked.load.v8i32(ptr %ap, i32 8, <8 x i1> %mask, <8 x i32> poison)566 %ext = sext <8 x i32> %load to <8 x i64>567 store <8 x i64> %ext, ptr %c568 ret void569}570 571define void @masked_load_zext_v32i8i16(ptr %ap, ptr %bp, ptr %c) #0 {572; VBITS_GE_256-LABEL: masked_load_zext_v32i8i16:573; VBITS_GE_256: // %bb.0:574; VBITS_GE_256-NEXT: ptrue p0.b, vl32575; VBITS_GE_256-NEXT: mov x8, #16 // =0x10576; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x1]577; VBITS_GE_256-NEXT: cmpeq p0.b, p0/z, z0.b, #0578; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0]579; VBITS_GE_256-NEXT: ptrue p0.h, vl16580; VBITS_GE_256-NEXT: movprfx z1, z0581; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #16582; VBITS_GE_256-NEXT: uunpklo z0.h, z0.b583; VBITS_GE_256-NEXT: uunpklo z1.h, z1.b584; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x2]585; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x2, x8, lsl #1]586; VBITS_GE_256-NEXT: ret587;588; VBITS_GE_512-LABEL: masked_load_zext_v32i8i16:589; VBITS_GE_512: // %bb.0:590; VBITS_GE_512-NEXT: ptrue p0.h, vl32591; VBITS_GE_512-NEXT: ld1b { z0.h }, p0/z, [x1]592; VBITS_GE_512-NEXT: cmpeq p1.h, p0/z, z0.h, #0593; VBITS_GE_512-NEXT: ld1b { z0.h }, p1/z, [x0]594; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x2]595; VBITS_GE_512-NEXT: ret596 %b = load <32 x i8>, ptr %bp597 %mask = icmp eq <32 x i8> %b, zeroinitializer598 %load = call <32 x i8> @llvm.masked.load.v32i8(ptr %ap, i32 8, <32 x i1> %mask, <32 x i8> poison)599 %ext = zext <32 x i8> %load to <32 x i16>600 store <32 x i16> %ext, ptr %c601 ret void602}603 604define void @masked_load_zext_v16i8i32(ptr %ap, ptr %bp, ptr %c) #0 {605; VBITS_GE_256-LABEL: masked_load_zext_v16i8i32:606; VBITS_GE_256: // %bb.0:607; VBITS_GE_256-NEXT: ptrue p0.b, vl16608; VBITS_GE_256-NEXT: ldr q0, [x1]609; VBITS_GE_256-NEXT: mov x8, #8 // =0x8610; VBITS_GE_256-NEXT: cmpeq p0.b, p0/z, z0.b, #0611; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0]612; VBITS_GE_256-NEXT: ptrue p0.s, vl8613; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8614; VBITS_GE_256-NEXT: uunpklo z0.h, z0.b615; VBITS_GE_256-NEXT: uunpklo z1.h, z1.b616; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h617; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h618; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]619; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]620; VBITS_GE_256-NEXT: ret621;622; VBITS_GE_512-LABEL: masked_load_zext_v16i8i32:623; VBITS_GE_512: // %bb.0:624; VBITS_GE_512-NEXT: ptrue p0.s, vl16625; VBITS_GE_512-NEXT: ld1b { z0.s }, p0/z, [x1]626; VBITS_GE_512-NEXT: cmpeq p1.s, p0/z, z0.s, #0627; VBITS_GE_512-NEXT: ld1b { z0.s }, p1/z, [x0]628; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]629; VBITS_GE_512-NEXT: ret630 %b = load <16 x i8>, ptr %bp631 %mask = icmp eq <16 x i8> %b, zeroinitializer632 %load = call <16 x i8> @llvm.masked.load.v16i8(ptr %ap, i32 8, <16 x i1> %mask, <16 x i8> poison)633 %ext = zext <16 x i8> %load to <16 x i32>634 store <16 x i32> %ext, ptr %c635 ret void636}637 638define void @masked_load_zext_v8i8i64(ptr %ap, ptr %bp, ptr %c) #0 {639; VBITS_GE_256-LABEL: masked_load_zext_v8i8i64:640; VBITS_GE_256: // %bb.0:641; VBITS_GE_256-NEXT: ptrue p0.b, vl8642; VBITS_GE_256-NEXT: ldr d0, [x1]643; VBITS_GE_256-NEXT: mov x8, #4 // =0x4644; VBITS_GE_256-NEXT: cmpeq p0.b, p0/z, z0.b, #0645; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0]646; VBITS_GE_256-NEXT: ptrue p0.d, vl4647; VBITS_GE_256-NEXT: ushll v0.8h, v0.8b, #0648; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8649; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h650; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h651; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s652; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s653; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]654; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]655; VBITS_GE_256-NEXT: ret656;657; VBITS_GE_512-LABEL: masked_load_zext_v8i8i64:658; VBITS_GE_512: // %bb.0:659; VBITS_GE_512-NEXT: ptrue p0.d, vl8660; VBITS_GE_512-NEXT: ld1b { z0.d }, p0/z, [x1]661; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0662; VBITS_GE_512-NEXT: ld1b { z0.d }, p1/z, [x0]663; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]664; VBITS_GE_512-NEXT: ret665 %b = load <8 x i8>, ptr %bp666 %mask = icmp eq <8 x i8> %b, zeroinitializer667 %load = call <8 x i8> @llvm.masked.load.v8i8(ptr %ap, i32 8, <8 x i1> %mask, <8 x i8> poison)668 %ext = zext <8 x i8> %load to <8 x i64>669 store <8 x i64> %ext, ptr %c670 ret void671}672 673define void @masked_load_zext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 {674; VBITS_GE_256-LABEL: masked_load_zext_v16i16i32:675; VBITS_GE_256: // %bb.0:676; VBITS_GE_256-NEXT: ptrue p0.h, vl16677; VBITS_GE_256-NEXT: mov x8, #8 // =0x8678; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x1]679; VBITS_GE_256-NEXT: cmpeq p0.h, p0/z, z0.h, #0680; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]681; VBITS_GE_256-NEXT: ptrue p0.s, vl8682; VBITS_GE_256-NEXT: movprfx z1, z0683; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #16684; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h685; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h686; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]687; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]688; VBITS_GE_256-NEXT: ret689;690; VBITS_GE_512-LABEL: masked_load_zext_v16i16i32:691; VBITS_GE_512: // %bb.0:692; VBITS_GE_512-NEXT: ptrue p0.s, vl16693; VBITS_GE_512-NEXT: ld1h { z0.s }, p0/z, [x1]694; VBITS_GE_512-NEXT: cmpeq p1.s, p0/z, z0.s, #0695; VBITS_GE_512-NEXT: ld1h { z0.s }, p1/z, [x0]696; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]697; VBITS_GE_512-NEXT: ret698 %b = load <16 x i16>, ptr %bp699 %mask = icmp eq <16 x i16> %b, zeroinitializer700 %load = call <16 x i16> @llvm.masked.load.v16i16(ptr %ap, i32 8, <16 x i1> %mask, <16 x i16> poison)701 %ext = zext <16 x i16> %load to <16 x i32>702 store <16 x i32> %ext, ptr %c703 ret void704}705 706define void @masked_load_zext_v8i16i64(ptr %ap, ptr %bp, ptr %c) #0 {707; VBITS_GE_256-LABEL: masked_load_zext_v8i16i64:708; VBITS_GE_256: // %bb.0:709; VBITS_GE_256-NEXT: ptrue p0.h, vl8710; VBITS_GE_256-NEXT: ldr q0, [x1]711; VBITS_GE_256-NEXT: mov x8, #4 // =0x4712; VBITS_GE_256-NEXT: cmpeq p0.h, p0/z, z0.h, #0713; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0]714; VBITS_GE_256-NEXT: ptrue p0.d, vl4715; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8716; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h717; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h718; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s719; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s720; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]721; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]722; VBITS_GE_256-NEXT: ret723;724; VBITS_GE_512-LABEL: masked_load_zext_v8i16i64:725; VBITS_GE_512: // %bb.0:726; VBITS_GE_512-NEXT: ptrue p0.d, vl8727; VBITS_GE_512-NEXT: ld1h { z0.d }, p0/z, [x1]728; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0729; VBITS_GE_512-NEXT: ld1h { z0.d }, p1/z, [x0]730; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]731; VBITS_GE_512-NEXT: ret732 %b = load <8 x i16>, ptr %bp733 %mask = icmp eq <8 x i16> %b, zeroinitializer734 %load = call <8 x i16> @llvm.masked.load.v8i16(ptr %ap, i32 8, <8 x i1> %mask, <8 x i16> poison)735 %ext = zext <8 x i16> %load to <8 x i64>736 store <8 x i64> %ext, ptr %c737 ret void738}739 740define void @masked_load_zext_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {741; VBITS_GE_256-LABEL: masked_load_zext_v8i32i64:742; VBITS_GE_256: // %bb.0:743; VBITS_GE_256-NEXT: ptrue p0.s, vl8744; VBITS_GE_256-NEXT: mov x8, #4 // =0x4745; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1]746; VBITS_GE_256-NEXT: cmpeq p0.s, p0/z, z0.s, #0747; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0]748; VBITS_GE_256-NEXT: ptrue p0.d, vl4749; VBITS_GE_256-NEXT: movprfx z1, z0750; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #16751; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s752; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s753; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]754; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]755; VBITS_GE_256-NEXT: ret756;757; VBITS_GE_512-LABEL: masked_load_zext_v8i32i64:758; VBITS_GE_512: // %bb.0:759; VBITS_GE_512-NEXT: ptrue p0.d, vl8760; VBITS_GE_512-NEXT: ld1w { z0.d }, p0/z, [x1]761; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0762; VBITS_GE_512-NEXT: ld1w { z0.d }, p1/z, [x0]763; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]764; VBITS_GE_512-NEXT: ret765 %b = load <8 x i32>, ptr %bp766 %mask = icmp eq <8 x i32> %b, zeroinitializer767 %load = call <8 x i32> @llvm.masked.load.v8i32(ptr %ap, i32 8, <8 x i1> %mask, <8 x i32> poison)768 %ext = zext <8 x i32> %load to <8 x i64>769 store <8 x i64> %ext, ptr %c770 ret void771}772 773define void @masked_load_sext_v32i8i16_m16(ptr %ap, ptr %bp, ptr %c) #0 {774; VBITS_GE_256-LABEL: masked_load_sext_v32i8i16_m16:775; VBITS_GE_256: // %bb.0:776; VBITS_GE_256-NEXT: ptrue p0.h, vl16777; VBITS_GE_256-NEXT: mov x8, #16 // =0x10778; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x1, x8, lsl #1]779; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1]780; VBITS_GE_256-NEXT: cmpeq p1.h, p0/z, z0.h, #0781; VBITS_GE_256-NEXT: cmpeq p2.h, p0/z, z1.h, #0782; VBITS_GE_256-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff783; VBITS_GE_256-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff784; VBITS_GE_256-NEXT: ptrue p1.b, vl16785; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b786; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b787; VBITS_GE_256-NEXT: splice z1.b, p1, z1.b, z0.b788; VBITS_GE_256-NEXT: ptrue p1.b, vl32789; VBITS_GE_256-NEXT: cmpne p1.b, p1/z, z1.b, #0790; VBITS_GE_256-NEXT: ld1b { z0.b }, p1/z, [x0]791; VBITS_GE_256-NEXT: movprfx z1, z0792; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #16793; VBITS_GE_256-NEXT: sunpklo z0.h, z0.b794; VBITS_GE_256-NEXT: sunpklo z1.h, z1.b795; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x2]796; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x2, x8, lsl #1]797; VBITS_GE_256-NEXT: ret798;799; VBITS_GE_512-LABEL: masked_load_sext_v32i8i16_m16:800; VBITS_GE_512: // %bb.0:801; VBITS_GE_512-NEXT: ptrue p0.h, vl32802; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x1]803; VBITS_GE_512-NEXT: cmpeq p1.h, p0/z, z0.h, #0804; VBITS_GE_512-NEXT: ld1sb { z0.h }, p1/z, [x0]805; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x2]806; VBITS_GE_512-NEXT: ret807 %b = load <32 x i16>, ptr %bp808 %mask = icmp eq <32 x i16> %b, zeroinitializer809 %load = call <32 x i8> @llvm.masked.load.v32i8(ptr %ap, i32 8, <32 x i1> %mask, <32 x i8> poison)810 %ext = sext <32 x i8> %load to <32 x i16>811 store <32 x i16> %ext, ptr %c812 ret void813}814 815define void @masked_load_sext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {816; VBITS_GE_256-LABEL: masked_load_sext_v16i8i32_m32:817; VBITS_GE_256: // %bb.0:818; VBITS_GE_256-NEXT: ptrue p0.s, vl8819; VBITS_GE_256-NEXT: mov x8, #8 // =0x8820; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1, x8, lsl #2]821; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1]822; VBITS_GE_256-NEXT: cmpeq p1.s, p0/z, z0.s, #0823; VBITS_GE_256-NEXT: cmpeq p2.s, p0/z, z1.s, #0824; VBITS_GE_256-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff825; VBITS_GE_256-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff826; VBITS_GE_256-NEXT: ptrue p1.b, vl16827; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h828; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h829; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b830; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b831; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0]832; VBITS_GE_256-NEXT: cmpne p1.b, p1/z, z1.b, #0833; VBITS_GE_256-NEXT: ld1b { z0.b }, p1/z, [x0]834; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8835; VBITS_GE_256-NEXT: sunpklo z0.h, z0.b836; VBITS_GE_256-NEXT: sunpklo z1.h, z1.b837; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h838; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h839; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]840; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]841; VBITS_GE_256-NEXT: ret842;843; VBITS_GE_512-LABEL: masked_load_sext_v16i8i32_m32:844; VBITS_GE_512: // %bb.0:845; VBITS_GE_512-NEXT: ptrue p0.s, vl16846; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x1]847; VBITS_GE_512-NEXT: cmpeq p1.s, p0/z, z0.s, #0848; VBITS_GE_512-NEXT: ld1sb { z0.s }, p1/z, [x0]849; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]850; VBITS_GE_512-NEXT: ret851 %b = load <16 x i32>, ptr %bp852 %mask = icmp eq <16 x i32> %b, zeroinitializer853 %load = call <16 x i8> @llvm.masked.load.v16i8(ptr %ap, i32 8, <16 x i1> %mask, <16 x i8> poison)854 %ext = sext <16 x i8> %load to <16 x i32>855 store <16 x i32> %ext, ptr %c856 ret void857}858 859define void @masked_load_sext_v8i8i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {860; VBITS_GE_256-LABEL: masked_load_sext_v8i8i64_m64:861; VBITS_GE_256: // %bb.0:862; VBITS_GE_256-NEXT: ptrue p0.d, vl4863; VBITS_GE_256-NEXT: mov x8, #4 // =0x4864; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]865; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]866; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #0867; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z1.d, #0868; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff869; VBITS_GE_256-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff870; VBITS_GE_256-NEXT: ptrue p1.s, vl4871; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s872; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s873; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s874; VBITS_GE_256-NEXT: ptrue p1.b, vl8875; VBITS_GE_256-NEXT: uzp1 z0.h, z1.h, z1.h876; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b877; VBITS_GE_256-NEXT: cmpne p1.b, p1/z, z0.b, #0878; VBITS_GE_256-NEXT: ld1b { z0.b }, p1/z, [x0]879; VBITS_GE_256-NEXT: sshll v0.8h, v0.8b, #0880; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8881; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h882; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h883; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s884; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s885; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]886; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]887; VBITS_GE_256-NEXT: ret888;889; VBITS_GE_512-LABEL: masked_load_sext_v8i8i64_m64:890; VBITS_GE_512: // %bb.0:891; VBITS_GE_512-NEXT: ptrue p0.d, vl8892; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]893; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0894; VBITS_GE_512-NEXT: ld1sb { z0.d }, p1/z, [x0]895; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]896; VBITS_GE_512-NEXT: ret897 %b = load <8 x i64>, ptr %bp898 %mask = icmp eq <8 x i64> %b, zeroinitializer899 %load = call <8 x i8> @llvm.masked.load.v8i8(ptr %ap, i32 8, <8 x i1> %mask, <8 x i8> poison)900 %ext = sext <8 x i8> %load to <8 x i64>901 store <8 x i64> %ext, ptr %c902 ret void903}904 905define void @masked_load_sext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {906; VBITS_GE_256-LABEL: masked_load_sext_v16i16i32_m32:907; VBITS_GE_256: // %bb.0:908; VBITS_GE_256-NEXT: ptrue p0.s, vl8909; VBITS_GE_256-NEXT: mov x8, #8 // =0x8910; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1, x8, lsl #2]911; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1]912; VBITS_GE_256-NEXT: cmpeq p1.s, p0/z, z0.s, #0913; VBITS_GE_256-NEXT: cmpeq p2.s, p0/z, z1.s, #0914; VBITS_GE_256-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff915; VBITS_GE_256-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff916; VBITS_GE_256-NEXT: ptrue p1.h, vl16917; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h918; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h919; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b920; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b921; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0]922; VBITS_GE_256-NEXT: sunpklo z0.h, z1.b923; VBITS_GE_256-NEXT: cmpne p1.h, p1/z, z0.h, #0924; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x0]925; VBITS_GE_256-NEXT: movprfx z1, z0926; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #16927; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h928; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h929; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]930; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]931; VBITS_GE_256-NEXT: ret932;933; VBITS_GE_512-LABEL: masked_load_sext_v16i16i32_m32:934; VBITS_GE_512: // %bb.0:935; VBITS_GE_512-NEXT: ptrue p0.s, vl16936; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x1]937; VBITS_GE_512-NEXT: cmpeq p1.s, p0/z, z0.s, #0938; VBITS_GE_512-NEXT: ld1sh { z0.s }, p1/z, [x0]939; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]940; VBITS_GE_512-NEXT: ret941 %b = load <16 x i32>, ptr %bp942 %mask = icmp eq <16 x i32> %b, zeroinitializer943 %load = call <16 x i16> @llvm.masked.load.v16i16(ptr %ap, i32 8, <16 x i1> %mask, <16 x i16> poison)944 %ext = sext <16 x i16> %load to <16 x i32>945 store <16 x i32> %ext, ptr %c946 ret void947}948 949define void @masked_load_sext_v8i16i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {950; VBITS_GE_256-LABEL: masked_load_sext_v8i16i64_m64:951; VBITS_GE_256: // %bb.0:952; VBITS_GE_256-NEXT: ptrue p0.d, vl4953; VBITS_GE_256-NEXT: mov x8, #4 // =0x4954; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]955; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]956; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #0957; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z1.d, #0958; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff959; VBITS_GE_256-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff960; VBITS_GE_256-NEXT: ptrue p1.s, vl4961; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s962; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s963; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s964; VBITS_GE_256-NEXT: ptrue p1.h, vl8965; VBITS_GE_256-NEXT: uzp1 z0.h, z1.h, z1.h966; VBITS_GE_256-NEXT: cmpne p1.h, p1/z, z0.h, #0967; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x0]968; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #8969; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h970; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h971; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s972; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s973; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]974; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]975; VBITS_GE_256-NEXT: ret976;977; VBITS_GE_512-LABEL: masked_load_sext_v8i16i64_m64:978; VBITS_GE_512: // %bb.0:979; VBITS_GE_512-NEXT: ptrue p0.d, vl8980; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]981; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0982; VBITS_GE_512-NEXT: ld1sh { z0.d }, p1/z, [x0]983; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]984; VBITS_GE_512-NEXT: ret985 %b = load <8 x i64>, ptr %bp986 %mask = icmp eq <8 x i64> %b, zeroinitializer987 %load = call <8 x i16> @llvm.masked.load.v8i16(ptr %ap, i32 8, <8 x i1> %mask, <8 x i16> poison)988 %ext = sext <8 x i16> %load to <8 x i64>989 store <8 x i64> %ext, ptr %c990 ret void991}992 993define void @masked_load_sext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {994; VBITS_GE_256-LABEL: masked_load_sext_v8i32i64_m64:995; VBITS_GE_256: // %bb.0:996; VBITS_GE_256-NEXT: ptrue p0.d, vl4997; VBITS_GE_256-NEXT: mov x8, #4 // =0x4998; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]999; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]1000; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #01001; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z1.d, #01002; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff1003; VBITS_GE_256-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff1004; VBITS_GE_256-NEXT: ptrue p1.s, vl41005; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s1006; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s1007; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s1008; VBITS_GE_256-NEXT: ptrue p1.s, vl81009; VBITS_GE_256-NEXT: cmpne p1.s, p1/z, z1.s, #01010; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x0]1011; VBITS_GE_256-NEXT: movprfx z1, z01012; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #161013; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s1014; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s1015; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]1016; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]1017; VBITS_GE_256-NEXT: ret1018;1019; VBITS_GE_512-LABEL: masked_load_sext_v8i32i64_m64:1020; VBITS_GE_512: // %bb.0:1021; VBITS_GE_512-NEXT: ptrue p0.d, vl81022; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]1023; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #01024; VBITS_GE_512-NEXT: ld1sw { z0.d }, p1/z, [x0]1025; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]1026; VBITS_GE_512-NEXT: ret1027 %b = load <8 x i64>, ptr %bp1028 %mask = icmp eq <8 x i64> %b, zeroinitializer1029 %load = call <8 x i32> @llvm.masked.load.v8i32(ptr %ap, i32 8, <8 x i1> %mask, <8 x i32> poison)1030 %ext = sext <8 x i32> %load to <8 x i64>1031 store <8 x i64> %ext, ptr %c1032 ret void1033}1034 1035define void @masked_load_zext_v32i8i16_m16(ptr %ap, ptr %bp, ptr %c) #0 {1036; VBITS_GE_256-LABEL: masked_load_zext_v32i8i16_m16:1037; VBITS_GE_256: // %bb.0:1038; VBITS_GE_256-NEXT: ptrue p0.h, vl161039; VBITS_GE_256-NEXT: mov x8, #16 // =0x101040; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x1, x8, lsl #1]1041; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x1]1042; VBITS_GE_256-NEXT: cmpeq p1.h, p0/z, z0.h, #01043; VBITS_GE_256-NEXT: cmpeq p2.h, p0/z, z1.h, #01044; VBITS_GE_256-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff1045; VBITS_GE_256-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff1046; VBITS_GE_256-NEXT: ptrue p1.b, vl161047; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b1048; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b1049; VBITS_GE_256-NEXT: splice z1.b, p1, z1.b, z0.b1050; VBITS_GE_256-NEXT: ptrue p1.b, vl321051; VBITS_GE_256-NEXT: cmpne p1.b, p1/z, z1.b, #01052; VBITS_GE_256-NEXT: ld1b { z0.b }, p1/z, [x0]1053; VBITS_GE_256-NEXT: movprfx z1, z01054; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #161055; VBITS_GE_256-NEXT: uunpklo z0.h, z0.b1056; VBITS_GE_256-NEXT: uunpklo z1.h, z1.b1057; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x2]1058; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x2, x8, lsl #1]1059; VBITS_GE_256-NEXT: ret1060;1061; VBITS_GE_512-LABEL: masked_load_zext_v32i8i16_m16:1062; VBITS_GE_512: // %bb.0:1063; VBITS_GE_512-NEXT: ptrue p0.h, vl321064; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x1]1065; VBITS_GE_512-NEXT: cmpeq p1.h, p0/z, z0.h, #01066; VBITS_GE_512-NEXT: ld1b { z0.h }, p1/z, [x0]1067; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x2]1068; VBITS_GE_512-NEXT: ret1069 %b = load <32 x i16>, ptr %bp1070 %mask = icmp eq <32 x i16> %b, zeroinitializer1071 %load = call <32 x i8> @llvm.masked.load.v32i8(ptr %ap, i32 8, <32 x i1> %mask, <32 x i8> poison)1072 %ext = zext <32 x i8> %load to <32 x i16>1073 store <32 x i16> %ext, ptr %c1074 ret void1075}1076 1077define void @masked_load_zext_v16i8i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {1078; VBITS_GE_256-LABEL: masked_load_zext_v16i8i32_m32:1079; VBITS_GE_256: // %bb.0:1080; VBITS_GE_256-NEXT: ptrue p0.s, vl81081; VBITS_GE_256-NEXT: mov x8, #8 // =0x81082; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1, x8, lsl #2]1083; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1]1084; VBITS_GE_256-NEXT: cmpeq p1.s, p0/z, z0.s, #01085; VBITS_GE_256-NEXT: cmpeq p2.s, p0/z, z1.s, #01086; VBITS_GE_256-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff1087; VBITS_GE_256-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff1088; VBITS_GE_256-NEXT: ptrue p1.b, vl161089; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h1090; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h1091; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b1092; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b1093; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0]1094; VBITS_GE_256-NEXT: cmpne p1.b, p1/z, z1.b, #01095; VBITS_GE_256-NEXT: ld1b { z0.b }, p1/z, [x0]1096; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #81097; VBITS_GE_256-NEXT: uunpklo z0.h, z0.b1098; VBITS_GE_256-NEXT: uunpklo z1.h, z1.b1099; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h1100; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h1101; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]1102; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]1103; VBITS_GE_256-NEXT: ret1104;1105; VBITS_GE_512-LABEL: masked_load_zext_v16i8i32_m32:1106; VBITS_GE_512: // %bb.0:1107; VBITS_GE_512-NEXT: ptrue p0.s, vl161108; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x1]1109; VBITS_GE_512-NEXT: cmpeq p1.s, p0/z, z0.s, #01110; VBITS_GE_512-NEXT: ld1b { z0.s }, p1/z, [x0]1111; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]1112; VBITS_GE_512-NEXT: ret1113 %b = load <16 x i32>, ptr %bp1114 %mask = icmp eq <16 x i32> %b, zeroinitializer1115 %load = call <16 x i8> @llvm.masked.load.v16i8(ptr %ap, i32 8, <16 x i1> %mask, <16 x i8> poison)1116 %ext = zext <16 x i8> %load to <16 x i32>1117 store <16 x i32> %ext, ptr %c1118 ret void1119}1120 1121define void @masked_load_zext_v8i8i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {1122; VBITS_GE_256-LABEL: masked_load_zext_v8i8i64_m64:1123; VBITS_GE_256: // %bb.0:1124; VBITS_GE_256-NEXT: ptrue p0.d, vl41125; VBITS_GE_256-NEXT: mov x8, #4 // =0x41126; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]1127; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]1128; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #01129; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z1.d, #01130; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff1131; VBITS_GE_256-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff1132; VBITS_GE_256-NEXT: ptrue p1.s, vl41133; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s1134; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s1135; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s1136; VBITS_GE_256-NEXT: ptrue p1.b, vl81137; VBITS_GE_256-NEXT: uzp1 z0.h, z1.h, z1.h1138; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b1139; VBITS_GE_256-NEXT: cmpne p1.b, p1/z, z0.b, #01140; VBITS_GE_256-NEXT: ld1b { z0.b }, p1/z, [x0]1141; VBITS_GE_256-NEXT: ushll v0.8h, v0.8b, #01142; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #81143; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h1144; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h1145; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s1146; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s1147; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]1148; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]1149; VBITS_GE_256-NEXT: ret1150;1151; VBITS_GE_512-LABEL: masked_load_zext_v8i8i64_m64:1152; VBITS_GE_512: // %bb.0:1153; VBITS_GE_512-NEXT: ptrue p0.d, vl81154; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]1155; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #01156; VBITS_GE_512-NEXT: ld1b { z0.d }, p1/z, [x0]1157; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]1158; VBITS_GE_512-NEXT: ret1159 %b = load <8 x i64>, ptr %bp1160 %mask = icmp eq <8 x i64> %b, zeroinitializer1161 %load = call <8 x i8> @llvm.masked.load.v8i8(ptr %ap, i32 8, <8 x i1> %mask, <8 x i8> poison)1162 %ext = zext <8 x i8> %load to <8 x i64>1163 store <8 x i64> %ext, ptr %c1164 ret void1165}1166 1167define void @masked_load_zext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {1168; VBITS_GE_256-LABEL: masked_load_zext_v16i16i32_m32:1169; VBITS_GE_256: // %bb.0:1170; VBITS_GE_256-NEXT: ptrue p0.s, vl81171; VBITS_GE_256-NEXT: mov x8, #8 // =0x81172; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1, x8, lsl #2]1173; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x1]1174; VBITS_GE_256-NEXT: cmpeq p1.s, p0/z, z0.s, #01175; VBITS_GE_256-NEXT: cmpeq p2.s, p0/z, z1.s, #01176; VBITS_GE_256-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff1177; VBITS_GE_256-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff1178; VBITS_GE_256-NEXT: ptrue p1.h, vl161179; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h1180; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h1181; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b1182; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b1183; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0]1184; VBITS_GE_256-NEXT: sunpklo z0.h, z1.b1185; VBITS_GE_256-NEXT: cmpne p1.h, p1/z, z0.h, #01186; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x0]1187; VBITS_GE_256-NEXT: movprfx z1, z01188; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #161189; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h1190; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h1191; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x2]1192; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2, x8, lsl #2]1193; VBITS_GE_256-NEXT: ret1194;1195; VBITS_GE_512-LABEL: masked_load_zext_v16i16i32_m32:1196; VBITS_GE_512: // %bb.0:1197; VBITS_GE_512-NEXT: ptrue p0.s, vl161198; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x1]1199; VBITS_GE_512-NEXT: cmpeq p1.s, p0/z, z0.s, #01200; VBITS_GE_512-NEXT: ld1h { z0.s }, p1/z, [x0]1201; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2]1202; VBITS_GE_512-NEXT: ret1203 %b = load <16 x i32>, ptr %bp1204 %mask = icmp eq <16 x i32> %b, zeroinitializer1205 %load = call <16 x i16> @llvm.masked.load.v16i16(ptr %ap, i32 8, <16 x i1> %mask, <16 x i16> poison)1206 %ext = zext <16 x i16> %load to <16 x i32>1207 store <16 x i32> %ext, ptr %c1208 ret void1209}1210 1211define void @masked_load_zext_v8i16i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {1212; VBITS_GE_256-LABEL: masked_load_zext_v8i16i64_m64:1213; VBITS_GE_256: // %bb.0:1214; VBITS_GE_256-NEXT: ptrue p0.d, vl41215; VBITS_GE_256-NEXT: mov x8, #4 // =0x41216; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]1217; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]1218; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #01219; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z1.d, #01220; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff1221; VBITS_GE_256-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff1222; VBITS_GE_256-NEXT: ptrue p1.s, vl41223; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s1224; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s1225; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s1226; VBITS_GE_256-NEXT: ptrue p1.h, vl81227; VBITS_GE_256-NEXT: uzp1 z0.h, z1.h, z1.h1228; VBITS_GE_256-NEXT: cmpne p1.h, p1/z, z0.h, #01229; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x0]1230; VBITS_GE_256-NEXT: ext v1.16b, v0.16b, v0.16b, #81231; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h1232; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h1233; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s1234; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s1235; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]1236; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]1237; VBITS_GE_256-NEXT: ret1238;1239; VBITS_GE_512-LABEL: masked_load_zext_v8i16i64_m64:1240; VBITS_GE_512: // %bb.0:1241; VBITS_GE_512-NEXT: ptrue p0.d, vl81242; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]1243; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #01244; VBITS_GE_512-NEXT: ld1h { z0.d }, p1/z, [x0]1245; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]1246; VBITS_GE_512-NEXT: ret1247 %b = load <8 x i64>, ptr %bp1248 %mask = icmp eq <8 x i64> %b, zeroinitializer1249 %load = call <8 x i16> @llvm.masked.load.v8i16(ptr %ap, i32 8, <8 x i1> %mask, <8 x i16> poison)1250 %ext = zext <8 x i16> %load to <8 x i64>1251 store <8 x i64> %ext, ptr %c1252 ret void1253}1254 1255define void @masked_load_zext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {1256; VBITS_GE_256-LABEL: masked_load_zext_v8i32i64_m64:1257; VBITS_GE_256: // %bb.0:1258; VBITS_GE_256-NEXT: ptrue p0.d, vl41259; VBITS_GE_256-NEXT: mov x8, #4 // =0x41260; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3]1261; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1]1262; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #01263; VBITS_GE_256-NEXT: cmpeq p2.d, p0/z, z1.d, #01264; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff1265; VBITS_GE_256-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff1266; VBITS_GE_256-NEXT: ptrue p1.s, vl41267; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s1268; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s1269; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s1270; VBITS_GE_256-NEXT: ptrue p1.s, vl81271; VBITS_GE_256-NEXT: cmpne p1.s, p1/z, z1.s, #01272; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x0]1273; VBITS_GE_256-NEXT: movprfx z1, z01274; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #161275; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s1276; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s1277; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]1278; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]1279; VBITS_GE_256-NEXT: ret1280;1281; VBITS_GE_512-LABEL: masked_load_zext_v8i32i64_m64:1282; VBITS_GE_512: // %bb.0:1283; VBITS_GE_512-NEXT: ptrue p0.d, vl81284; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1]1285; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #01286; VBITS_GE_512-NEXT: ld1w { z0.d }, p1/z, [x0]1287; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]1288; VBITS_GE_512-NEXT: ret1289 %b = load <8 x i64>, ptr %bp1290 %mask = icmp eq <8 x i64> %b, zeroinitializer1291 %load = call <8 x i32> @llvm.masked.load.v8i32(ptr %ap, i32 8, <8 x i1> %mask, <8 x i32> poison)1292 %ext = zext <8 x i32> %load to <8 x i64>1293 store <8 x i64> %ext, ptr %c1294 ret void1295}1296 1297define void @masked_load_sext_v128i8i16(ptr %ap, ptr %bp, ptr %c) vscale_range(16,0) #0 {1298; CHECK-LABEL: masked_load_sext_v128i8i16:1299; CHECK: // %bb.0:1300; CHECK-NEXT: ptrue p0.h, vl1281301; CHECK-NEXT: ld1b { z0.h }, p0/z, [x1]1302; CHECK-NEXT: cmpeq p1.h, p0/z, z0.h, #01303; CHECK-NEXT: ld1sb { z0.h }, p1/z, [x0]1304; CHECK-NEXT: st1h { z0.h }, p0, [x2]1305; CHECK-NEXT: ret1306 %b = load <128 x i8>, ptr %bp1307 %mask = icmp eq <128 x i8> %b, zeroinitializer1308 %load = call <128 x i8> @llvm.masked.load.v128i8(ptr %ap, i32 8, <128 x i1> %mask, <128 x i8> poison)1309 %ext = sext <128 x i8> %load to <128 x i16>1310 store <128 x i16> %ext, ptr %c1311 ret void1312}1313 1314define void @masked_load_sext_v64i8i32(ptr %ap, ptr %bp, ptr %c) vscale_range(16,0) #0 {1315; CHECK-LABEL: masked_load_sext_v64i8i32:1316; CHECK: // %bb.0:1317; CHECK-NEXT: ptrue p0.s, vl641318; CHECK-NEXT: ld1b { z0.s }, p0/z, [x1]1319; CHECK-NEXT: cmpeq p1.s, p0/z, z0.s, #01320; CHECK-NEXT: ld1sb { z0.s }, p1/z, [x0]1321; CHECK-NEXT: st1w { z0.s }, p0, [x2]1322; CHECK-NEXT: ret1323 %b = load <64 x i8>, ptr %bp1324 %mask = icmp eq <64 x i8> %b, zeroinitializer1325 %load = call <64 x i8> @llvm.masked.load.v64i8(ptr %ap, i32 8, <64 x i1> %mask, <64 x i8> poison)1326 %ext = sext <64 x i8> %load to <64 x i32>1327 store <64 x i32> %ext, ptr %c1328 ret void1329}1330 1331define void @masked_load_sext_v32i8i64(ptr %ap, ptr %bp, ptr %c) vscale_range(16,0) #0 {1332; CHECK-LABEL: masked_load_sext_v32i8i64:1333; CHECK: // %bb.0:1334; CHECK-NEXT: ptrue p0.d, vl321335; CHECK-NEXT: ld1b { z0.d }, p0/z, [x1]1336; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #01337; CHECK-NEXT: ld1sb { z0.d }, p1/z, [x0]1338; CHECK-NEXT: st1d { z0.d }, p0, [x2]1339; CHECK-NEXT: ret1340 %b = load <32 x i8>, ptr %bp1341 %mask = icmp eq <32 x i8> %b, zeroinitializer1342 %load = call <32 x i8> @llvm.masked.load.v32i8(ptr %ap, i32 8, <32 x i1> %mask, <32 x i8> poison)1343 %ext = sext <32 x i8> %load to <32 x i64>1344 store <32 x i64> %ext, ptr %c1345 ret void1346}1347 1348define void @masked_load_sext_v64i16i32(ptr %ap, ptr %bp, ptr %c) vscale_range(16,0) #0 {1349; CHECK-LABEL: masked_load_sext_v64i16i32:1350; CHECK: // %bb.0:1351; CHECK-NEXT: ptrue p0.s, vl641352; CHECK-NEXT: ld1h { z0.s }, p0/z, [x1]1353; CHECK-NEXT: cmpeq p1.s, p0/z, z0.s, #01354; CHECK-NEXT: ld1sh { z0.s }, p1/z, [x0]1355; CHECK-NEXT: st1w { z0.s }, p0, [x2]1356; CHECK-NEXT: ret1357 %b = load <64 x i16>, ptr %bp1358 %mask = icmp eq <64 x i16> %b, zeroinitializer1359 %load = call <64 x i16> @llvm.masked.load.v64i16(ptr %ap, i32 8, <64 x i1> %mask, <64 x i16> poison)1360 %ext = sext <64 x i16> %load to <64 x i32>1361 store <64 x i32> %ext, ptr %c1362 ret void1363}1364 1365define void @masked_load_sext_v32i16i64(ptr %ap, ptr %bp, ptr %c) vscale_range(16,0) #0 {1366; CHECK-LABEL: masked_load_sext_v32i16i64:1367; CHECK: // %bb.0:1368; CHECK-NEXT: ptrue p0.d, vl321369; CHECK-NEXT: ld1h { z0.d }, p0/z, [x1]1370; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #01371; CHECK-NEXT: ld1sh { z0.d }, p1/z, [x0]1372; CHECK-NEXT: st1d { z0.d }, p0, [x2]1373; CHECK-NEXT: ret1374 %b = load <32 x i16>, ptr %bp1375 %mask = icmp eq <32 x i16> %b, zeroinitializer1376 %load = call <32 x i16> @llvm.masked.load.v32i16(ptr %ap, i32 8, <32 x i1> %mask, <32 x i16> poison)1377 %ext = sext <32 x i16> %load to <32 x i64>1378 store <32 x i64> %ext, ptr %c1379 ret void1380}1381 1382define void @masked_load_sext_v32i32i64(ptr %ap, ptr %bp, ptr %c) vscale_range(16,0) #0 {1383; CHECK-LABEL: masked_load_sext_v32i32i64:1384; CHECK: // %bb.0:1385; CHECK-NEXT: ptrue p0.d, vl321386; CHECK-NEXT: ld1w { z0.d }, p0/z, [x1]1387; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #01388; CHECK-NEXT: ld1sw { z0.d }, p1/z, [x0]1389; CHECK-NEXT: st1d { z0.d }, p0, [x2]1390; CHECK-NEXT: ret1391 %b = load <32 x i32>, ptr %bp1392 %mask = icmp eq <32 x i32> %b, zeroinitializer1393 %load = call <32 x i32> @llvm.masked.load.v32i32(ptr %ap, i32 8, <32 x i1> %mask, <32 x i32> poison)1394 %ext = sext <32 x i32> %load to <32 x i64>1395 store <32 x i64> %ext, ptr %c1396 ret void1397}1398 1399define void @masked_load_zext_v128i8i16(ptr %ap, ptr %bp, ptr %c) vscale_range(16,0) #0 {1400; CHECK-LABEL: masked_load_zext_v128i8i16:1401; CHECK: // %bb.0:1402; CHECK-NEXT: ptrue p0.h, vl1281403; CHECK-NEXT: ld1b { z0.h }, p0/z, [x1]1404; CHECK-NEXT: cmpeq p1.h, p0/z, z0.h, #01405; CHECK-NEXT: ld1b { z0.h }, p1/z, [x0]1406; CHECK-NEXT: st1h { z0.h }, p0, [x2]1407; CHECK-NEXT: ret1408 %b = load <128 x i8>, ptr %bp1409 %mask = icmp eq <128 x i8> %b, zeroinitializer1410 %load = call <128 x i8> @llvm.masked.load.v128i8(ptr %ap, i32 8, <128 x i1> %mask, <128 x i8> poison)1411 %ext = zext <128 x i8> %load to <128 x i16>1412 store <128 x i16> %ext, ptr %c1413 ret void1414}1415 1416define void @masked_load_zext_v64i8i32(ptr %ap, ptr %bp, ptr %c) vscale_range(16,0) #0 {1417; CHECK-LABEL: masked_load_zext_v64i8i32:1418; CHECK: // %bb.0:1419; CHECK-NEXT: ptrue p0.s, vl641420; CHECK-NEXT: ld1b { z0.s }, p0/z, [x1]1421; CHECK-NEXT: cmpeq p1.s, p0/z, z0.s, #01422; CHECK-NEXT: ld1b { z0.s }, p1/z, [x0]1423; CHECK-NEXT: st1w { z0.s }, p0, [x2]1424; CHECK-NEXT: ret1425 %b = load <64 x i8>, ptr %bp1426 %mask = icmp eq <64 x i8> %b, zeroinitializer1427 %load = call <64 x i8> @llvm.masked.load.v64i8(ptr %ap, i32 8, <64 x i1> %mask, <64 x i8> poison)1428 %ext = zext <64 x i8> %load to <64 x i32>1429 store <64 x i32> %ext, ptr %c1430 ret void1431}1432 1433define void @masked_load_zext_v32i8i64(ptr %ap, ptr %bp, ptr %c) vscale_range(16,0) #0 {1434; CHECK-LABEL: masked_load_zext_v32i8i64:1435; CHECK: // %bb.0:1436; CHECK-NEXT: ptrue p0.d, vl321437; CHECK-NEXT: ld1b { z0.d }, p0/z, [x1]1438; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #01439; CHECK-NEXT: ld1b { z0.d }, p1/z, [x0]1440; CHECK-NEXT: st1d { z0.d }, p0, [x2]1441; CHECK-NEXT: ret1442 %b = load <32 x i8>, ptr %bp1443 %mask = icmp eq <32 x i8> %b, zeroinitializer1444 %load = call <32 x i8> @llvm.masked.load.v32i8(ptr %ap, i32 8, <32 x i1> %mask, <32 x i8> poison)1445 %ext = zext <32 x i8> %load to <32 x i64>1446 store <32 x i64> %ext, ptr %c1447 ret void1448}1449 1450define void @masked_load_zext_v64i16i32(ptr %ap, ptr %bp, ptr %c) vscale_range(16,0) #0 {1451; CHECK-LABEL: masked_load_zext_v64i16i32:1452; CHECK: // %bb.0:1453; CHECK-NEXT: ptrue p0.s, vl641454; CHECK-NEXT: ld1h { z0.s }, p0/z, [x1]1455; CHECK-NEXT: cmpeq p1.s, p0/z, z0.s, #01456; CHECK-NEXT: ld1h { z0.s }, p1/z, [x0]1457; CHECK-NEXT: st1w { z0.s }, p0, [x2]1458; CHECK-NEXT: ret1459 %b = load <64 x i16>, ptr %bp1460 %mask = icmp eq <64 x i16> %b, zeroinitializer1461 %load = call <64 x i16> @llvm.masked.load.v64i16(ptr %ap, i32 8, <64 x i1> %mask, <64 x i16> poison)1462 %ext = zext <64 x i16> %load to <64 x i32>1463 store <64 x i32> %ext, ptr %c1464 ret void1465}1466 1467define void @masked_load_zext_v32i16i64(ptr %ap, ptr %bp, ptr %c) vscale_range(16,0) #0 {1468; CHECK-LABEL: masked_load_zext_v32i16i64:1469; CHECK: // %bb.0:1470; CHECK-NEXT: ptrue p0.d, vl321471; CHECK-NEXT: ld1h { z0.d }, p0/z, [x1]1472; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #01473; CHECK-NEXT: ld1h { z0.d }, p1/z, [x0]1474; CHECK-NEXT: st1d { z0.d }, p0, [x2]1475; CHECK-NEXT: ret1476 %b = load <32 x i16>, ptr %bp1477 %mask = icmp eq <32 x i16> %b, zeroinitializer1478 %load = call <32 x i16> @llvm.masked.load.v32i16(ptr %ap, i32 8, <32 x i1> %mask, <32 x i16> poison)1479 %ext = zext <32 x i16> %load to <32 x i64>1480 store <32 x i64> %ext, ptr %c1481 ret void1482}1483 1484define void @masked_load_zext_v32i32i64(ptr %ap, ptr %bp, ptr %c) vscale_range(16,0) #0 {1485; CHECK-LABEL: masked_load_zext_v32i32i64:1486; CHECK: // %bb.0:1487; CHECK-NEXT: ptrue p0.d, vl321488; CHECK-NEXT: ld1w { z0.d }, p0/z, [x1]1489; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #01490; CHECK-NEXT: ld1w { z0.d }, p1/z, [x0]1491; CHECK-NEXT: st1d { z0.d }, p0, [x2]1492; CHECK-NEXT: ret1493 %b = load <32 x i32>, ptr %bp1494 %mask = icmp eq <32 x i32> %b, zeroinitializer1495 %load = call <32 x i32> @llvm.masked.load.v32i32(ptr %ap, i32 8, <32 x i1> %mask, <32 x i32> poison)1496 %ext = zext <32 x i32> %load to <32 x i64>1497 store <32 x i64> %ext, ptr %c1498 ret void1499}1500 1501define void @masked_load_sext_ugt_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {1502; VBITS_GE_256-LABEL: masked_load_sext_ugt_v8i32i64:1503; VBITS_GE_256: // %bb.0:1504; VBITS_GE_256-NEXT: ptrue p0.s, vl81505; VBITS_GE_256-NEXT: mov x8, #4 // =0x41506; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1]1507; VBITS_GE_256-NEXT: cmpne p0.s, p0/z, z0.s, #01508; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0]1509; VBITS_GE_256-NEXT: ptrue p0.d, vl41510; VBITS_GE_256-NEXT: movprfx z1, z01511; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #161512; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s1513; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s1514; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]1515; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]1516; VBITS_GE_256-NEXT: ret1517;1518; VBITS_GE_512-LABEL: masked_load_sext_ugt_v8i32i64:1519; VBITS_GE_512: // %bb.0:1520; VBITS_GE_512-NEXT: ptrue p0.d, vl81521; VBITS_GE_512-NEXT: ld1w { z0.d }, p0/z, [x1]1522; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z0.d, #01523; VBITS_GE_512-NEXT: ld1sw { z0.d }, p1/z, [x0]1524; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]1525; VBITS_GE_512-NEXT: ret1526 %b = load <8 x i32>, ptr %bp1527 %mask = icmp ugt <8 x i32> %b, zeroinitializer1528 %load = call <8 x i32> @llvm.masked.load.v8i32(ptr %ap, i32 8, <8 x i1> %mask, <8 x i32> poison)1529 %ext = sext <8 x i32> %load to <8 x i64>1530 store <8 x i64> %ext, ptr %c1531 ret void1532}1533 1534define void @masked_load_zext_sgt_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {1535; VBITS_GE_256-LABEL: masked_load_zext_sgt_v8i32i64:1536; VBITS_GE_256: // %bb.0:1537; VBITS_GE_256-NEXT: ptrue p0.s, vl81538; VBITS_GE_256-NEXT: mov x8, #4 // =0x41539; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1]1540; VBITS_GE_256-NEXT: cmpgt p0.s, p0/z, z0.s, #01541; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0]1542; VBITS_GE_256-NEXT: ptrue p0.d, vl41543; VBITS_GE_256-NEXT: movprfx z1, z01544; VBITS_GE_256-NEXT: ext z1.b, z1.b, z0.b, #161545; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s1546; VBITS_GE_256-NEXT: uunpklo z1.d, z1.s1547; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x2]1548; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x2, x8, lsl #3]1549; VBITS_GE_256-NEXT: ret1550;1551; VBITS_GE_512-LABEL: masked_load_zext_sgt_v8i32i64:1552; VBITS_GE_512: // %bb.0:1553; VBITS_GE_512-NEXT: ptrue p0.d, vl81554; VBITS_GE_512-NEXT: ld1sw { z0.d }, p0/z, [x1]1555; VBITS_GE_512-NEXT: cmpgt p1.d, p0/z, z0.d, #01556; VBITS_GE_512-NEXT: ld1w { z0.d }, p1/z, [x0]1557; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2]1558; VBITS_GE_512-NEXT: ret1559 %b = load <8 x i32>, ptr %bp1560 %mask = icmp sgt <8 x i32> %b, zeroinitializer1561 %load = call <8 x i32> @llvm.masked.load.v8i32(ptr %ap, i32 8, <8 x i1> %mask, <8 x i32> poison)1562 %ext = zext <8 x i32> %load to <8 x i64>1563 store <8 x i64> %ext, ptr %c1564 ret void1565}1566 1567declare <2 x half> @llvm.masked.load.v2f16(ptr, i32, <2 x i1>, <2 x half>)1568declare <2 x float> @llvm.masked.load.v2f32(ptr, i32, <2 x i1>, <2 x float>)1569declare <4 x float> @llvm.masked.load.v4f32(ptr, i32, <4 x i1>, <4 x float>)1570declare <8 x float> @llvm.masked.load.v8f32(ptr, i32, <8 x i1>, <8 x float>)1571declare <16 x float> @llvm.masked.load.v16f32(ptr, i32, <16 x i1>, <16 x float>)1572declare <32 x float> @llvm.masked.load.v32f32(ptr, i32, <32 x i1>, <32 x float>)1573declare <64 x float> @llvm.masked.load.v64f32(ptr, i32, <64 x i1>, <64 x float>)1574 1575declare <128 x i8> @llvm.masked.load.v128i8(ptr, i32, <128 x i1>, <128 x i8>)1576declare <64 x i8> @llvm.masked.load.v64i8(ptr, i32, <64 x i1>, <64 x i8>)1577declare <32 x i8> @llvm.masked.load.v32i8(ptr, i32, <32 x i1>, <32 x i8>)1578declare <16 x i8> @llvm.masked.load.v16i8(ptr, i32, <16 x i1>, <16 x i8>)1579declare <16 x i16> @llvm.masked.load.v16i16(ptr, i32, <16 x i1>, <16 x i16>)1580declare <8 x i8> @llvm.masked.load.v8i8(ptr, i32, <8 x i1>, <8 x i8>)1581declare <8 x i16> @llvm.masked.load.v8i16(ptr, i32, <8 x i1>, <8 x i16>)1582declare <8 x i32> @llvm.masked.load.v8i32(ptr, i32, <8 x i1>, <8 x i32>)1583declare <32 x i32> @llvm.masked.load.v32i32(ptr, i32, <32 x i1>, <32 x i32>)1584declare <32 x i16> @llvm.masked.load.v32i16(ptr, i32, <32 x i1>, <32 x i16>)1585declare <64 x i16> @llvm.masked.load.v64i16(ptr, i32, <64 x i1>, <64 x i16>)1586declare <16 x i32> @llvm.masked.load.v16i32(ptr, i32, <16 x i1>, <16 x i32>)1587declare <8 x i64> @llvm.masked.load.v8i64(ptr, i32, <8 x i1>, <8 x i64>)1588declare <8 x double> @llvm.masked.load.v8f64(ptr, i32, <8 x i1>, <8 x double>)1589 1590attributes #0 = { "target-features"="+sve" }1591