1154 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc < %s -mtriple=aarch64-- -mattr=+sve | FileCheck %s -check-prefix=SVE3 4define void @test_masked_store_success_v4i8(<4 x i8> %x, ptr %ptr, <4 x i1> %mask) {5; SVE-LABEL: test_masked_store_success_v4i8:6; SVE: // %bb.0:7; SVE-NEXT: shl v1.4h, v1.4h, #158; SVE-NEXT: ldr s2, [x0]9; SVE-NEXT: zip1 v2.8b, v2.8b, v2.8b10; SVE-NEXT: cmlt v1.4h, v1.4h, #011; SVE-NEXT: bif v0.8b, v2.8b, v1.8b12; SVE-NEXT: uzp1 v0.8b, v0.8b, v0.8b13; SVE-NEXT: str s0, [x0]14; SVE-NEXT: ret15 %load = load <4 x i8>, ptr %ptr, align 3216 %sel = select <4 x i1> %mask, <4 x i8> %x, <4 x i8> %load17 store <4 x i8> %sel, ptr %ptr, align 3218 ret void19}20 21define void @test_masked_store_success_v4i16(<4 x i16> %x, ptr %ptr, <4 x i1> %mask) {22; SVE-LABEL: test_masked_store_success_v4i16:23; SVE: // %bb.0:24; SVE-NEXT: shl v1.4h, v1.4h, #1525; SVE-NEXT: ptrue p0.h, vl426; SVE-NEXT: // kill: def $d0 killed $d0 def $z027; SVE-NEXT: cmpne p0.h, p0/z, z1.h, #028; SVE-NEXT: st1h { z0.h }, p0, [x0]29; SVE-NEXT: ret30 %load = load <4 x i16>, ptr %ptr, align 3231 %sel = select <4 x i1> %mask, <4 x i16> %x, <4 x i16> %load32 store <4 x i16> %sel, ptr %ptr, align 3233 ret void34}35 36define void @test_masked_store_success_v4i32(<4 x i32> %x, ptr %ptr, <4 x i1> %mask) {37; SVE-LABEL: test_masked_store_success_v4i32:38; SVE: // %bb.0:39; SVE-NEXT: ushll v1.4s, v1.4h, #040; SVE-NEXT: ptrue p0.s, vl441; SVE-NEXT: // kill: def $q0 killed $q0 def $z042; SVE-NEXT: shl v1.4s, v1.4s, #3143; SVE-NEXT: cmpne p0.s, p0/z, z1.s, #044; SVE-NEXT: st1w { z0.s }, p0, [x0]45; SVE-NEXT: ret46 %load = load <4 x i32>, ptr %ptr, align 3247 %sel = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> %load48 store <4 x i32> %sel, ptr %ptr, align 3249 ret void50}51 52define void @test_masked_store_success_v4i64(<4 x i64> %x, ptr %ptr, <4 x i1> %mask) {53; SVE-LABEL: test_masked_store_success_v4i64:54; SVE: // %bb.0:55; SVE-NEXT: ushll v2.4s, v2.4h, #056; SVE-NEXT: ptrue p0.d, vl257; SVE-NEXT: mov x8, #2 // =0x258; SVE-NEXT: // kill: def $q1 killed $q1 def $z159; SVE-NEXT: // kill: def $q0 killed $q0 def $z060; SVE-NEXT: ushll2 v3.2d, v2.4s, #061; SVE-NEXT: ushll v2.2d, v2.2s, #062; SVE-NEXT: shl v3.2d, v3.2d, #6363; SVE-NEXT: shl v2.2d, v2.2d, #6364; SVE-NEXT: cmpne p1.d, p0/z, z3.d, #065; SVE-NEXT: cmpne p0.d, p0/z, z2.d, #066; SVE-NEXT: st1d { z1.d }, p1, [x0, x8, lsl #3]67; SVE-NEXT: st1d { z0.d }, p0, [x0]68; SVE-NEXT: ret69 %load = load <4 x i64>, ptr %ptr, align 3270 %sel = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> %load71 store <4 x i64> %sel, ptr %ptr, align 3272 ret void73}74 75define void @test_masked_store_success_v4f16(<4 x half> %x, ptr %ptr, <4 x i1> %mask) {76; SVE-LABEL: test_masked_store_success_v4f16:77; SVE: // %bb.0:78; SVE-NEXT: shl v1.4h, v1.4h, #1579; SVE-NEXT: ptrue p0.h, vl480; SVE-NEXT: // kill: def $d0 killed $d0 def $z081; SVE-NEXT: cmpne p0.h, p0/z, z1.h, #082; SVE-NEXT: st1h { z0.h }, p0, [x0]83; SVE-NEXT: ret84 %load = load <4 x half>, ptr %ptr, align 3285 %sel = select <4 x i1> %mask, <4 x half> %x, <4 x half> %load86 store <4 x half> %sel, ptr %ptr, align 3287 ret void88}89 90define void @test_masked_store_success_v4f32(<4 x float> %x, ptr %ptr, <4 x i1> %mask) {91; SVE-LABEL: test_masked_store_success_v4f32:92; SVE: // %bb.0:93; SVE-NEXT: ushll v1.4s, v1.4h, #094; SVE-NEXT: ptrue p0.s, vl495; SVE-NEXT: // kill: def $q0 killed $q0 def $z096; SVE-NEXT: shl v1.4s, v1.4s, #3197; SVE-NEXT: cmpne p0.s, p0/z, z1.s, #098; SVE-NEXT: st1w { z0.s }, p0, [x0]99; SVE-NEXT: ret100 %load = load <4 x float>, ptr %ptr, align 32101 %sel = select <4 x i1> %mask, <4 x float> %x, <4 x float> %load102 store <4 x float> %sel, ptr %ptr, align 32103 ret void104}105 106define void @test_masked_store_success_v4f64(<4 x double> %x, ptr %ptr, <4 x i1> %mask) {107; SVE-LABEL: test_masked_store_success_v4f64:108; SVE: // %bb.0:109; SVE-NEXT: ushll v2.4s, v2.4h, #0110; SVE-NEXT: ptrue p0.d, vl2111; SVE-NEXT: mov x8, #2 // =0x2112; SVE-NEXT: // kill: def $q1 killed $q1 def $z1113; SVE-NEXT: // kill: def $q0 killed $q0 def $z0114; SVE-NEXT: ushll2 v3.2d, v2.4s, #0115; SVE-NEXT: ushll v2.2d, v2.2s, #0116; SVE-NEXT: shl v3.2d, v3.2d, #63117; SVE-NEXT: shl v2.2d, v2.2d, #63118; SVE-NEXT: cmpne p1.d, p0/z, z3.d, #0119; SVE-NEXT: cmpne p0.d, p0/z, z2.d, #0120; SVE-NEXT: st1d { z1.d }, p1, [x0, x8, lsl #3]121; SVE-NEXT: st1d { z0.d }, p0, [x0]122; SVE-NEXT: ret123 %load = load <4 x double>, ptr %ptr, align 32124 %sel = select <4 x i1> %mask, <4 x double> %x, <4 x double> %load125 store <4 x double> %sel, ptr %ptr, align 32126 ret void127}128 129define void @test_masked_store_success_v8i8(<8 x i8> %x, ptr %ptr, <8 x i1> %mask) {130; SVE-LABEL: test_masked_store_success_v8i8:131; SVE: // %bb.0:132; SVE-NEXT: shl v1.8b, v1.8b, #7133; SVE-NEXT: ptrue p0.b, vl8134; SVE-NEXT: // kill: def $d0 killed $d0 def $z0135; SVE-NEXT: cmpne p0.b, p0/z, z1.b, #0136; SVE-NEXT: st1b { z0.b }, p0, [x0]137; SVE-NEXT: ret138 %load = load <8 x i8>, ptr %ptr, align 32139 %sel = select <8 x i1> %mask, <8 x i8> %x, <8 x i8> %load140 store <8 x i8> %sel, ptr %ptr, align 32141 ret void142}143 144define void @test_masked_store_success_v8i16(<8 x i16> %x, ptr %ptr, <8 x i1> %mask) {145; SVE-LABEL: test_masked_store_success_v8i16:146; SVE: // %bb.0:147; SVE-NEXT: ushll v1.8h, v1.8b, #0148; SVE-NEXT: ptrue p0.h, vl8149; SVE-NEXT: // kill: def $q0 killed $q0 def $z0150; SVE-NEXT: shl v1.8h, v1.8h, #15151; SVE-NEXT: cmpne p0.h, p0/z, z1.h, #0152; SVE-NEXT: st1h { z0.h }, p0, [x0]153; SVE-NEXT: ret154 %load = load <8 x i16>, ptr %ptr, align 32155 %sel = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> %load156 store <8 x i16> %sel, ptr %ptr, align 32157 ret void158}159 160define void @test_masked_store_success_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {161; SVE-LABEL: test_masked_store_success_v8i32:162; SVE: // %bb.0:163; SVE-NEXT: // kill: def $q0 killed $q0 def $z0164; SVE-NEXT: zip2 v3.8b, v2.8b, v0.8b165; SVE-NEXT: zip1 v2.8b, v2.8b, v0.8b166; SVE-NEXT: mov x8, #4 // =0x4167; SVE-NEXT: ptrue p0.s, vl4168; SVE-NEXT: // kill: def $q1 killed $q1 def $z1169; SVE-NEXT: ushll v3.4s, v3.4h, #0170; SVE-NEXT: ushll v2.4s, v2.4h, #0171; SVE-NEXT: shl v3.4s, v3.4s, #31172; SVE-NEXT: shl v2.4s, v2.4s, #31173; SVE-NEXT: cmpne p1.s, p0/z, z3.s, #0174; SVE-NEXT: cmpne p0.s, p0/z, z2.s, #0175; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]176; SVE-NEXT: st1w { z0.s }, p0, [x0]177; SVE-NEXT: ret178 %load = load <8 x i32>, ptr %ptr, align 32179 %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load180 store <8 x i32> %sel, ptr %ptr, align 32181 ret void182}183 184define void @test_masked_store_success_v8i64(<8 x i64> %x, ptr %ptr, <8 x i1> %mask) {185; SVE-LABEL: test_masked_store_success_v8i64:186; SVE: // %bb.0:187; SVE-NEXT: // kill: def $d4 killed $d4 def $q4188; SVE-NEXT: mov b5, v4.b[4]189; SVE-NEXT: mov b6, v4.b[6]190; SVE-NEXT: mov x8, #4 // =0x4191; SVE-NEXT: mov b7, v4.b[2]192; SVE-NEXT: mov b16, v4.b[0]193; SVE-NEXT: // kill: def $q2 killed $q2 def $z2194; SVE-NEXT: mov x9, #6 // =0x6195; SVE-NEXT: ptrue p0.d, vl2196; SVE-NEXT: // kill: def $q3 killed $q3 def $z3197; SVE-NEXT: // kill: def $q1 killed $q1 def $z1198; SVE-NEXT: // kill: def $q0 killed $q0 def $z0199; SVE-NEXT: mov v5.b[4], v4.b[5]200; SVE-NEXT: mov v6.b[4], v4.b[7]201; SVE-NEXT: mov v7.b[4], v4.b[3]202; SVE-NEXT: mov v16.b[4], v4.b[1]203; SVE-NEXT: ushll v4.2d, v5.2s, #0204; SVE-NEXT: ushll v5.2d, v6.2s, #0205; SVE-NEXT: ushll v6.2d, v7.2s, #0206; SVE-NEXT: ushll v7.2d, v16.2s, #0207; SVE-NEXT: shl v4.2d, v4.2d, #63208; SVE-NEXT: shl v5.2d, v5.2d, #63209; SVE-NEXT: shl v6.2d, v6.2d, #63210; SVE-NEXT: cmpne p1.d, p0/z, z4.d, #0211; SVE-NEXT: shl v4.2d, v7.2d, #63212; SVE-NEXT: cmpne p2.d, p0/z, z5.d, #0213; SVE-NEXT: cmpne p3.d, p0/z, z6.d, #0214; SVE-NEXT: cmpne p0.d, p0/z, z4.d, #0215; SVE-NEXT: st1d { z2.d }, p1, [x0, x8, lsl #3]216; SVE-NEXT: mov x8, #2 // =0x2217; SVE-NEXT: st1d { z3.d }, p2, [x0, x9, lsl #3]218; SVE-NEXT: st1d { z1.d }, p3, [x0, x8, lsl #3]219; SVE-NEXT: st1d { z0.d }, p0, [x0]220; SVE-NEXT: ret221 %load = load <8 x i64>, ptr %ptr, align 32222 %sel = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> %load223 store <8 x i64> %sel, ptr %ptr, align 32224 ret void225}226 227define void @test_masked_store_success_v8f16(<8 x half> %x, ptr %ptr, <8 x i1> %mask) {228; SVE-LABEL: test_masked_store_success_v8f16:229; SVE: // %bb.0:230; SVE-NEXT: ushll v1.8h, v1.8b, #0231; SVE-NEXT: ptrue p0.h, vl8232; SVE-NEXT: // kill: def $q0 killed $q0 def $z0233; SVE-NEXT: shl v1.8h, v1.8h, #15234; SVE-NEXT: cmpne p0.h, p0/z, z1.h, #0235; SVE-NEXT: st1h { z0.h }, p0, [x0]236; SVE-NEXT: ret237 %load = load <8 x half>, ptr %ptr, align 32238 %sel = select <8 x i1> %mask, <8 x half> %x, <8 x half> %load239 store <8 x half> %sel, ptr %ptr, align 32240 ret void241}242 243define void @test_masked_store_success_v8f32(<8 x float> %x, ptr %ptr, <8 x i1> %mask) {244; SVE-LABEL: test_masked_store_success_v8f32:245; SVE: // %bb.0:246; SVE-NEXT: // kill: def $q0 killed $q0 def $z0247; SVE-NEXT: zip2 v3.8b, v2.8b, v0.8b248; SVE-NEXT: zip1 v2.8b, v2.8b, v0.8b249; SVE-NEXT: mov x8, #4 // =0x4250; SVE-NEXT: ptrue p0.s, vl4251; SVE-NEXT: // kill: def $q1 killed $q1 def $z1252; SVE-NEXT: ushll v3.4s, v3.4h, #0253; SVE-NEXT: ushll v2.4s, v2.4h, #0254; SVE-NEXT: shl v3.4s, v3.4s, #31255; SVE-NEXT: shl v2.4s, v2.4s, #31256; SVE-NEXT: cmpne p1.s, p0/z, z3.s, #0257; SVE-NEXT: cmpne p0.s, p0/z, z2.s, #0258; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]259; SVE-NEXT: st1w { z0.s }, p0, [x0]260; SVE-NEXT: ret261 %load = load <8 x float>, ptr %ptr, align 32262 %sel = select <8 x i1> %mask, <8 x float> %x, <8 x float> %load263 store <8 x float> %sel, ptr %ptr, align 32264 ret void265}266 267define void @test_masked_store_success_v8f64(<8 x double> %x, ptr %ptr, <8 x i1> %mask) {268; SVE-LABEL: test_masked_store_success_v8f64:269; SVE: // %bb.0:270; SVE-NEXT: // kill: def $d4 killed $d4 def $q4271; SVE-NEXT: mov b5, v4.b[4]272; SVE-NEXT: mov b6, v4.b[6]273; SVE-NEXT: mov x8, #4 // =0x4274; SVE-NEXT: mov b7, v4.b[2]275; SVE-NEXT: mov b16, v4.b[0]276; SVE-NEXT: // kill: def $q2 killed $q2 def $z2277; SVE-NEXT: mov x9, #6 // =0x6278; SVE-NEXT: ptrue p0.d, vl2279; SVE-NEXT: // kill: def $q3 killed $q3 def $z3280; SVE-NEXT: // kill: def $q1 killed $q1 def $z1281; SVE-NEXT: // kill: def $q0 killed $q0 def $z0282; SVE-NEXT: mov v5.b[4], v4.b[5]283; SVE-NEXT: mov v6.b[4], v4.b[7]284; SVE-NEXT: mov v7.b[4], v4.b[3]285; SVE-NEXT: mov v16.b[4], v4.b[1]286; SVE-NEXT: ushll v4.2d, v5.2s, #0287; SVE-NEXT: ushll v5.2d, v6.2s, #0288; SVE-NEXT: ushll v6.2d, v7.2s, #0289; SVE-NEXT: ushll v7.2d, v16.2s, #0290; SVE-NEXT: shl v4.2d, v4.2d, #63291; SVE-NEXT: shl v5.2d, v5.2d, #63292; SVE-NEXT: shl v6.2d, v6.2d, #63293; SVE-NEXT: cmpne p1.d, p0/z, z4.d, #0294; SVE-NEXT: shl v4.2d, v7.2d, #63295; SVE-NEXT: cmpne p2.d, p0/z, z5.d, #0296; SVE-NEXT: cmpne p3.d, p0/z, z6.d, #0297; SVE-NEXT: cmpne p0.d, p0/z, z4.d, #0298; SVE-NEXT: st1d { z2.d }, p1, [x0, x8, lsl #3]299; SVE-NEXT: mov x8, #2 // =0x2300; SVE-NEXT: st1d { z3.d }, p2, [x0, x9, lsl #3]301; SVE-NEXT: st1d { z1.d }, p3, [x0, x8, lsl #3]302; SVE-NEXT: st1d { z0.d }, p0, [x0]303; SVE-NEXT: ret304 %load = load <8 x double>, ptr %ptr, align 32305 %sel = select <8 x i1> %mask, <8 x double> %x, <8 x double> %load306 store <8 x double> %sel, ptr %ptr, align 32307 ret void308}309 310define void @test_masked_store_success_v16i8(<16 x i8> %x, ptr %ptr, <16 x i1> %mask) {311; SVE-LABEL: test_masked_store_success_v16i8:312; SVE: // %bb.0:313; SVE-NEXT: shl v1.16b, v1.16b, #7314; SVE-NEXT: ptrue p0.b, vl16315; SVE-NEXT: // kill: def $q0 killed $q0 def $z0316; SVE-NEXT: cmpne p0.b, p0/z, z1.b, #0317; SVE-NEXT: st1b { z0.b }, p0, [x0]318; SVE-NEXT: ret319 %load = load <16 x i8>, ptr %ptr, align 32320 %sel = select <16 x i1> %mask, <16 x i8> %x, <16 x i8> %load321 store <16 x i8> %sel, ptr %ptr, align 32322 ret void323}324 325define void @test_masked_store_success_v16i16(<16 x i16> %x, ptr %ptr, <16 x i1> %mask) {326; SVE-LABEL: test_masked_store_success_v16i16:327; SVE: // %bb.0:328; SVE-NEXT: ushll2 v3.8h, v2.16b, #0329; SVE-NEXT: ushll v2.8h, v2.8b, #0330; SVE-NEXT: mov x8, #8 // =0x8331; SVE-NEXT: ptrue p0.h, vl8332; SVE-NEXT: // kill: def $q1 killed $q1 def $z1333; SVE-NEXT: // kill: def $q0 killed $q0 def $z0334; SVE-NEXT: shl v3.8h, v3.8h, #15335; SVE-NEXT: shl v2.8h, v2.8h, #15336; SVE-NEXT: cmpne p1.h, p0/z, z3.h, #0337; SVE-NEXT: cmpne p0.h, p0/z, z2.h, #0338; SVE-NEXT: st1h { z1.h }, p1, [x0, x8, lsl #1]339; SVE-NEXT: st1h { z0.h }, p0, [x0]340; SVE-NEXT: ret341 %load = load <16 x i16>, ptr %ptr, align 32342 %sel = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> %load343 store <16 x i16> %sel, ptr %ptr, align 32344 ret void345}346 347define void @test_masked_store_success_v16i32(<16 x i32> %x, ptr %ptr, <16 x i1> %mask) {348; SVE-LABEL: test_masked_store_success_v16i32:349; SVE: // %bb.0:350; SVE-NEXT: ext v5.16b, v4.16b, v4.16b, #8351; SVE-NEXT: // kill: def $q0 killed $q0 def $z0352; SVE-NEXT: zip2 v6.8b, v4.8b, v0.8b353; SVE-NEXT: mov x8, #4 // =0x4354; SVE-NEXT: zip1 v4.8b, v4.8b, v0.8b355; SVE-NEXT: ptrue p0.s, vl4356; SVE-NEXT: // kill: def $q1 killed $q1 def $z1357; SVE-NEXT: mov x9, #8 // =0x8358; SVE-NEXT: // kill: def $q3 killed $q3 def $z3359; SVE-NEXT: // kill: def $q2 killed $q2 def $z2360; SVE-NEXT: zip1 v7.8b, v5.8b, v0.8b361; SVE-NEXT: zip2 v5.8b, v5.8b, v0.8b362; SVE-NEXT: ushll v6.4s, v6.4h, #0363; SVE-NEXT: ushll v4.4s, v4.4h, #0364; SVE-NEXT: shl v6.4s, v6.4s, #31365; SVE-NEXT: ushll v7.4s, v7.4h, #0366; SVE-NEXT: ushll v5.4s, v5.4h, #0367; SVE-NEXT: shl v4.4s, v4.4s, #31368; SVE-NEXT: cmpne p1.s, p0/z, z6.s, #0369; SVE-NEXT: shl v7.4s, v7.4s, #31370; SVE-NEXT: shl v5.4s, v5.4s, #31371; SVE-NEXT: cmpne p2.s, p0/z, z7.s, #0372; SVE-NEXT: cmpne p3.s, p0/z, z5.s, #0373; SVE-NEXT: cmpne p0.s, p0/z, z4.s, #0374; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]375; SVE-NEXT: mov x8, #12 // =0xc376; SVE-NEXT: st1w { z2.s }, p2, [x0, x9, lsl #2]377; SVE-NEXT: st1w { z3.s }, p3, [x0, x8, lsl #2]378; SVE-NEXT: st1w { z0.s }, p0, [x0]379; SVE-NEXT: ret380 %load = load <16 x i32>, ptr %ptr, align 32381 %sel = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %load382 store <16 x i32> %sel, ptr %ptr, align 32383 ret void384}385 386define void @test_masked_store_success_v32i8(<32 x i8> %x, ptr %ptr, <32 x i1> %mask) {387; SVE-LABEL: test_masked_store_success_v32i8:388; SVE: // %bb.0:389; SVE-NEXT: ldr w8, [sp, #72]390; SVE-NEXT: fmov s2, w1391; SVE-NEXT: ldr w9, [sp, #80]392; SVE-NEXT: ptrue p0.b, vl16393; SVE-NEXT: // kill: def $q1 killed $q1 def $z1394; SVE-NEXT: // kill: def $q0 killed $q0 def $z0395; SVE-NEXT: fmov s3, w8396; SVE-NEXT: ldr w8, [sp, #88]397; SVE-NEXT: mov v2.b[1], w2398; SVE-NEXT: mov v3.b[1], w9399; SVE-NEXT: ldr w9, [sp]400; SVE-NEXT: mov v2.b[2], w3401; SVE-NEXT: mov v3.b[2], w8402; SVE-NEXT: ldr w8, [sp, #96]403; SVE-NEXT: mov v2.b[3], w4404; SVE-NEXT: mov v3.b[3], w8405; SVE-NEXT: ldr w8, [sp, #104]406; SVE-NEXT: mov v2.b[4], w5407; SVE-NEXT: mov v3.b[4], w8408; SVE-NEXT: ldr w8, [sp, #112]409; SVE-NEXT: mov v2.b[5], w6410; SVE-NEXT: mov v3.b[5], w8411; SVE-NEXT: ldr w8, [sp, #120]412; SVE-NEXT: mov v2.b[6], w7413; SVE-NEXT: mov v3.b[6], w8414; SVE-NEXT: ldr w8, [sp, #128]415; SVE-NEXT: mov v2.b[7], w9416; SVE-NEXT: ldr w9, [sp, #8]417; SVE-NEXT: mov v3.b[7], w8418; SVE-NEXT: ldr w8, [sp, #136]419; SVE-NEXT: mov v2.b[8], w9420; SVE-NEXT: ldr w9, [sp, #16]421; SVE-NEXT: mov v3.b[8], w8422; SVE-NEXT: ldr w8, [sp, #144]423; SVE-NEXT: mov v2.b[9], w9424; SVE-NEXT: ldr w9, [sp, #24]425; SVE-NEXT: mov v3.b[9], w8426; SVE-NEXT: ldr w8, [sp, #152]427; SVE-NEXT: mov v2.b[10], w9428; SVE-NEXT: ldr w9, [sp, #32]429; SVE-NEXT: mov v3.b[10], w8430; SVE-NEXT: ldr w8, [sp, #160]431; SVE-NEXT: mov v2.b[11], w9432; SVE-NEXT: ldr w9, [sp, #40]433; SVE-NEXT: mov v3.b[11], w8434; SVE-NEXT: ldr w8, [sp, #168]435; SVE-NEXT: mov v2.b[12], w9436; SVE-NEXT: ldr w9, [sp, #48]437; SVE-NEXT: mov v3.b[12], w8438; SVE-NEXT: ldr w8, [sp, #176]439; SVE-NEXT: mov v2.b[13], w9440; SVE-NEXT: ldr w9, [sp, #56]441; SVE-NEXT: mov v3.b[13], w8442; SVE-NEXT: ldr w8, [sp, #184]443; SVE-NEXT: mov v2.b[14], w9444; SVE-NEXT: ldr w9, [sp, #64]445; SVE-NEXT: mov v3.b[14], w8446; SVE-NEXT: ldr w8, [sp, #192]447; SVE-NEXT: mov v2.b[15], w9448; SVE-NEXT: mov v3.b[15], w8449; SVE-NEXT: mov w8, #16 // =0x10450; SVE-NEXT: shl v2.16b, v2.16b, #7451; SVE-NEXT: shl v3.16b, v3.16b, #7452; SVE-NEXT: cmpne p1.b, p0/z, z3.b, #0453; SVE-NEXT: cmpne p0.b, p0/z, z2.b, #0454; SVE-NEXT: st1b { z1.b }, p1, [x0, x8]455; SVE-NEXT: st1b { z0.b }, p0, [x0]456; SVE-NEXT: ret457 %load = load <32 x i8>, ptr %ptr, align 32458 %sel = select <32 x i1> %mask, <32 x i8> %x, <32 x i8> %load459 store <32 x i8> %sel, ptr %ptr, align 32460 ret void461}462 463define void @test_masked_store_success_v32i16(<32 x i16> %x, ptr %ptr, <32 x i1> %mask) {464; SVE-LABEL: test_masked_store_success_v32i16:465; SVE: // %bb.0:466; SVE-NEXT: ldr w9, [sp, #72]467; SVE-NEXT: ldr w11, [sp, #136]468; SVE-NEXT: fmov s7, w1469; SVE-NEXT: ldr w8, [sp, #80]470; SVE-NEXT: ldr w10, [sp, #144]471; SVE-NEXT: ptrue p0.h, vl8472; SVE-NEXT: fmov s4, w9473; SVE-NEXT: ldr w9, [sp, #8]474; SVE-NEXT: fmov s5, w11475; SVE-NEXT: mov v7.b[1], w2476; SVE-NEXT: // kill: def $q2 killed $q2 def $z2477; SVE-NEXT: // kill: def $q3 killed $q3 def $z3478; SVE-NEXT: // kill: def $q1 killed $q1 def $z1479; SVE-NEXT: // kill: def $q0 killed $q0 def $z0480; SVE-NEXT: fmov s6, w9481; SVE-NEXT: ldr w9, [sp, #152]482; SVE-NEXT: mov v4.b[1], w8483; SVE-NEXT: ldr w8, [sp, #16]484; SVE-NEXT: mov v5.b[1], w10485; SVE-NEXT: mov v6.b[1], w8486; SVE-NEXT: ldr w8, [sp, #88]487; SVE-NEXT: mov v7.b[2], w3488; SVE-NEXT: mov v4.b[2], w8489; SVE-NEXT: ldr w8, [sp, #24]490; SVE-NEXT: mov v5.b[2], w9491; SVE-NEXT: ldr w9, [sp, #160]492; SVE-NEXT: mov v6.b[2], w8493; SVE-NEXT: ldr w8, [sp, #96]494; SVE-NEXT: mov v7.b[3], w4495; SVE-NEXT: mov v4.b[3], w8496; SVE-NEXT: ldr w8, [sp, #32]497; SVE-NEXT: mov v5.b[3], w9498; SVE-NEXT: ldr w9, [sp, #168]499; SVE-NEXT: mov v6.b[3], w8500; SVE-NEXT: ldr w8, [sp, #104]501; SVE-NEXT: mov v7.b[4], w5502; SVE-NEXT: mov v4.b[4], w8503; SVE-NEXT: ldr w8, [sp, #40]504; SVE-NEXT: mov v5.b[4], w9505; SVE-NEXT: ldr w9, [sp, #176]506; SVE-NEXT: mov v6.b[4], w8507; SVE-NEXT: ldr w8, [sp, #112]508; SVE-NEXT: mov v7.b[5], w6509; SVE-NEXT: mov v4.b[5], w8510; SVE-NEXT: ldr w8, [sp, #48]511; SVE-NEXT: mov v5.b[5], w9512; SVE-NEXT: ldr w9, [sp, #184]513; SVE-NEXT: mov v6.b[5], w8514; SVE-NEXT: ldr w8, [sp, #120]515; SVE-NEXT: mov v7.b[6], w7516; SVE-NEXT: mov v4.b[6], w8517; SVE-NEXT: ldr w8, [sp, #56]518; SVE-NEXT: mov v5.b[6], w9519; SVE-NEXT: ldr w9, [sp, #192]520; SVE-NEXT: mov v6.b[6], w8521; SVE-NEXT: ldr w8, [sp, #128]522; SVE-NEXT: mov v4.b[7], w8523; SVE-NEXT: ldr w8, [sp, #64]524; SVE-NEXT: mov v5.b[7], w9525; SVE-NEXT: ldr w9, [sp]526; SVE-NEXT: mov v6.b[7], w8527; SVE-NEXT: mov x8, #16 // =0x10528; SVE-NEXT: mov v7.b[7], w9529; SVE-NEXT: ushll v4.8h, v4.8b, #0530; SVE-NEXT: ushll v5.8h, v5.8b, #0531; SVE-NEXT: ushll v6.8h, v6.8b, #0532; SVE-NEXT: ushll v7.8h, v7.8b, #0533; SVE-NEXT: shl v4.8h, v4.8h, #15534; SVE-NEXT: shl v5.8h, v5.8h, #15535; SVE-NEXT: shl v6.8h, v6.8h, #15536; SVE-NEXT: cmpne p1.h, p0/z, z4.h, #0537; SVE-NEXT: shl v4.8h, v7.8h, #15538; SVE-NEXT: cmpne p2.h, p0/z, z5.h, #0539; SVE-NEXT: cmpne p3.h, p0/z, z6.h, #0540; SVE-NEXT: cmpne p0.h, p0/z, z4.h, #0541; SVE-NEXT: st1h { z2.h }, p1, [x0, x8, lsl #1]542; SVE-NEXT: mov x8, #24 // =0x18543; SVE-NEXT: st1h { z3.h }, p2, [x0, x8, lsl #1]544; SVE-NEXT: mov x8, #8 // =0x8545; SVE-NEXT: st1h { z1.h }, p3, [x0, x8, lsl #1]546; SVE-NEXT: st1h { z0.h }, p0, [x0]547; SVE-NEXT: ret548 %load = load <32 x i16>, ptr %ptr, align 32549 %sel = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> %load550 store <32 x i16> %sel, ptr %ptr, align 32551 ret void552}553 554define void @test_masked_store_success_v64i8(<64 x i8> %x, ptr %ptr, <64 x i1> %mask) {555; SVE-LABEL: test_masked_store_success_v64i8:556; SVE: // %bb.0:557; SVE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill558; SVE-NEXT: .cfi_def_cfa_offset 16559; SVE-NEXT: .cfi_offset w29, -16560; SVE-NEXT: ldr w8, [sp, #216]561; SVE-NEXT: ldr w9, [sp, #344]562; SVE-NEXT: fmov s6, w1563; SVE-NEXT: ldr w11, [sp, #88]564; SVE-NEXT: ldr w10, [sp, #224]565; SVE-NEXT: ptrue p0.b, vl16566; SVE-NEXT: fmov s5, w8567; SVE-NEXT: fmov s4, w9568; SVE-NEXT: ldr w8, [sp, #352]569; SVE-NEXT: fmov s7, w11570; SVE-NEXT: ldr w9, [sp, #96]571; SVE-NEXT: mov v6.b[1], w2572; SVE-NEXT: // kill: def $q2 killed $q2 def $z2573; SVE-NEXT: // kill: def $q3 killed $q3 def $z3574; SVE-NEXT: // kill: def $q1 killed $q1 def $z1575; SVE-NEXT: // kill: def $q0 killed $q0 def $z0576; SVE-NEXT: mov v5.b[1], w10577; SVE-NEXT: mov v4.b[1], w8578; SVE-NEXT: ldr w8, [sp, #232]579; SVE-NEXT: mov v7.b[1], w9580; SVE-NEXT: ldr w9, [sp, #360]581; SVE-NEXT: ldr w10, [sp, #112]582; SVE-NEXT: mov v6.b[2], w3583; SVE-NEXT: mov v5.b[2], w8584; SVE-NEXT: ldr w8, [sp, #104]585; SVE-NEXT: mov v4.b[2], w9586; SVE-NEXT: ldr w9, [sp, #368]587; SVE-NEXT: mov v7.b[2], w8588; SVE-NEXT: ldr w8, [sp, #240]589; SVE-NEXT: mov v6.b[3], w4590; SVE-NEXT: mov v5.b[3], w8591; SVE-NEXT: mov v4.b[3], w9592; SVE-NEXT: ldr w8, [sp, #248]593; SVE-NEXT: ldr w9, [sp, #376]594; SVE-NEXT: mov v7.b[3], w10595; SVE-NEXT: ldr w10, [sp, #120]596; SVE-NEXT: mov v6.b[4], w5597; SVE-NEXT: mov v5.b[4], w8598; SVE-NEXT: mov v4.b[4], w9599; SVE-NEXT: ldr w8, [sp, #256]600; SVE-NEXT: ldr w9, [sp, #384]601; SVE-NEXT: mov v7.b[4], w10602; SVE-NEXT: ldr w10, [sp, #128]603; SVE-NEXT: mov v6.b[5], w6604; SVE-NEXT: mov v5.b[5], w8605; SVE-NEXT: mov v4.b[5], w9606; SVE-NEXT: ldr w8, [sp, #264]607; SVE-NEXT: ldr w9, [sp, #392]608; SVE-NEXT: mov v7.b[5], w10609; SVE-NEXT: ldr w10, [sp, #136]610; SVE-NEXT: mov v6.b[6], w7611; SVE-NEXT: mov v5.b[6], w8612; SVE-NEXT: mov v4.b[6], w9613; SVE-NEXT: ldr w8, [sp, #272]614; SVE-NEXT: ldr w9, [sp, #400]615; SVE-NEXT: mov v7.b[6], w10616; SVE-NEXT: ldr w10, [sp, #144]617; SVE-NEXT: mov v5.b[7], w8618; SVE-NEXT: ldr w8, [sp, #16]619; SVE-NEXT: mov v4.b[7], w9620; SVE-NEXT: ldr w9, [sp, #280]621; SVE-NEXT: mov v7.b[7], w10622; SVE-NEXT: mov v6.b[7], w8623; SVE-NEXT: ldr w10, [sp, #408]624; SVE-NEXT: ldr w8, [sp, #152]625; SVE-NEXT: mov v5.b[8], w9626; SVE-NEXT: ldr w9, [sp, #24]627; SVE-NEXT: mov v4.b[8], w10628; SVE-NEXT: ldr w10, [sp, #288]629; SVE-NEXT: mov v7.b[8], w8630; SVE-NEXT: mov v6.b[8], w9631; SVE-NEXT: ldr w8, [sp, #416]632; SVE-NEXT: ldr w9, [sp, #160]633; SVE-NEXT: mov v5.b[9], w10634; SVE-NEXT: ldr w10, [sp, #32]635; SVE-NEXT: mov v4.b[9], w8636; SVE-NEXT: ldr w8, [sp, #296]637; SVE-NEXT: mov v7.b[9], w9638; SVE-NEXT: mov v6.b[9], w10639; SVE-NEXT: ldr w9, [sp, #424]640; SVE-NEXT: ldr w10, [sp, #168]641; SVE-NEXT: mov v5.b[10], w8642; SVE-NEXT: ldr w8, [sp, #40]643; SVE-NEXT: mov v4.b[10], w9644; SVE-NEXT: ldr w9, [sp, #304]645; SVE-NEXT: mov v7.b[10], w10646; SVE-NEXT: mov v6.b[10], w8647; SVE-NEXT: ldr w10, [sp, #432]648; SVE-NEXT: ldr w8, [sp, #176]649; SVE-NEXT: mov v5.b[11], w9650; SVE-NEXT: ldr w9, [sp, #48]651; SVE-NEXT: mov v4.b[11], w10652; SVE-NEXT: ldr w10, [sp, #312]653; SVE-NEXT: mov v7.b[11], w8654; SVE-NEXT: mov v6.b[11], w9655; SVE-NEXT: ldr w8, [sp, #440]656; SVE-NEXT: ldr w9, [sp, #184]657; SVE-NEXT: mov v5.b[12], w10658; SVE-NEXT: ldr w10, [sp, #56]659; SVE-NEXT: mov v4.b[12], w8660; SVE-NEXT: ldr w8, [sp, #320]661; SVE-NEXT: mov v7.b[12], w9662; SVE-NEXT: mov v6.b[12], w10663; SVE-NEXT: ldr w9, [sp, #448]664; SVE-NEXT: ldr w10, [sp, #192]665; SVE-NEXT: mov v5.b[13], w8666; SVE-NEXT: ldr w8, [sp, #64]667; SVE-NEXT: mov v4.b[13], w9668; SVE-NEXT: ldr w9, [sp, #328]669; SVE-NEXT: mov v7.b[13], w10670; SVE-NEXT: mov v6.b[13], w8671; SVE-NEXT: ldr w10, [sp, #456]672; SVE-NEXT: ldr w8, [sp, #200]673; SVE-NEXT: mov v5.b[14], w9674; SVE-NEXT: ldr w9, [sp, #72]675; SVE-NEXT: mov v4.b[14], w10676; SVE-NEXT: ldr w10, [sp, #336]677; SVE-NEXT: mov v7.b[14], w8678; SVE-NEXT: mov v6.b[14], w9679; SVE-NEXT: ldr w8, [sp, #464]680; SVE-NEXT: ldr w9, [sp, #208]681; SVE-NEXT: mov v5.b[15], w10682; SVE-NEXT: ldr w10, [sp, #80]683; SVE-NEXT: mov v4.b[15], w8684; SVE-NEXT: mov w8, #32 // =0x20685; SVE-NEXT: mov v7.b[15], w9686; SVE-NEXT: mov v6.b[15], w10687; SVE-NEXT: mov w9, #48 // =0x30688; SVE-NEXT: shl v5.16b, v5.16b, #7689; SVE-NEXT: shl v4.16b, v4.16b, #7690; SVE-NEXT: shl v7.16b, v7.16b, #7691; SVE-NEXT: cmpne p1.b, p0/z, z5.b, #0692; SVE-NEXT: shl v5.16b, v6.16b, #7693; SVE-NEXT: cmpne p2.b, p0/z, z4.b, #0694; SVE-NEXT: cmpne p3.b, p0/z, z7.b, #0695; SVE-NEXT: cmpne p0.b, p0/z, z5.b, #0696; SVE-NEXT: st1b { z2.b }, p1, [x0, x8]697; SVE-NEXT: mov w8, #16 // =0x10698; SVE-NEXT: st1b { z3.b }, p2, [x0, x9]699; SVE-NEXT: st1b { z1.b }, p3, [x0, x8]700; SVE-NEXT: st1b { z0.b }, p0, [x0]701; SVE-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload702; SVE-NEXT: ret703 %load = load <64 x i8>, ptr %ptr, align 32704 %sel = select <64 x i1> %mask, <64 x i8> %x, <64 x i8> %load705 store <64 x i8> %sel, ptr %ptr, align 32706 ret void707}708 709define void @test_masked_store_success_invert_mask_v4i32(<4 x i32> %x, ptr %ptr, <4 x i1> %mask) {710; SVE-LABEL: test_masked_store_success_invert_mask_v4i32:711; SVE: // %bb.0:712; SVE-NEXT: movi v2.4h, #1713; SVE-NEXT: ptrue p0.s, vl4714; SVE-NEXT: // kill: def $q0 killed $q0 def $z0715; SVE-NEXT: eor v1.8b, v1.8b, v2.8b716; SVE-NEXT: ushll v1.4s, v1.4h, #0717; SVE-NEXT: shl v1.4s, v1.4s, #31718; SVE-NEXT: cmpne p0.s, p0/z, z1.s, #0719; SVE-NEXT: st1w { z0.s }, p0, [x0]720; SVE-NEXT: ret721 %load = load <4 x i32>, ptr %ptr, align 32722 %sel = select <4 x i1> %mask, <4 x i32> %load, <4 x i32> %x723 store <4 x i32> %sel, ptr %ptr, align 32724 ret void725}726 727define void @test_masked_store_success_invert_mask_v8i32(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {728; SVE-LABEL: test_masked_store_success_invert_mask_v8i32:729; SVE: // %bb.0:730; SVE-NEXT: // kill: def $q0 killed $q0 def $z0731; SVE-NEXT: zip2 v3.8b, v2.8b, v0.8b732; SVE-NEXT: zip1 v2.8b, v2.8b, v0.8b733; SVE-NEXT: mov x8, #4 // =0x4734; SVE-NEXT: ptrue p0.s, vl4735; SVE-NEXT: // kill: def $q1 killed $q1 def $z1736; SVE-NEXT: ushll v3.4s, v3.4h, #0737; SVE-NEXT: ushll v2.4s, v2.4h, #0738; SVE-NEXT: shl v3.4s, v3.4s, #31739; SVE-NEXT: shl v2.4s, v2.4s, #31740; SVE-NEXT: cmpge p1.s, p0/z, z3.s, #0741; SVE-NEXT: cmpge p0.s, p0/z, z2.s, #0742; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]743; SVE-NEXT: st1w { z0.s }, p0, [x0]744; SVE-NEXT: ret745 %load = load <8 x i32>, ptr %ptr, align 32746 %sel = select <8 x i1> %mask, <8 x i32> %load, <8 x i32> %x747 store <8 x i32> %sel, ptr %ptr, align 32748 ret void749}750 751define void @test_masked_store_success_invert_mask_v16i32(<16 x i32> %x, ptr %ptr, <16 x i1> %mask) {752; SVE-LABEL: test_masked_store_success_invert_mask_v16i32:753; SVE: // %bb.0:754; SVE-NEXT: ext v5.16b, v4.16b, v4.16b, #8755; SVE-NEXT: // kill: def $q0 killed $q0 def $z0756; SVE-NEXT: zip2 v6.8b, v4.8b, v0.8b757; SVE-NEXT: mov x8, #4 // =0x4758; SVE-NEXT: zip1 v4.8b, v4.8b, v0.8b759; SVE-NEXT: ptrue p0.s, vl4760; SVE-NEXT: // kill: def $q1 killed $q1 def $z1761; SVE-NEXT: // kill: def $q2 killed $q2 def $z2762; SVE-NEXT: // kill: def $q3 killed $q3 def $z3763; SVE-NEXT: zip1 v7.8b, v5.8b, v0.8b764; SVE-NEXT: zip2 v5.8b, v5.8b, v0.8b765; SVE-NEXT: ushll v6.4s, v6.4h, #0766; SVE-NEXT: ushll v4.4s, v4.4h, #0767; SVE-NEXT: shl v6.4s, v6.4s, #31768; SVE-NEXT: ushll v7.4s, v7.4h, #0769; SVE-NEXT: ushll v5.4s, v5.4h, #0770; SVE-NEXT: shl v4.4s, v4.4s, #31771; SVE-NEXT: cmpge p1.s, p0/z, z6.s, #0772; SVE-NEXT: shl v7.4s, v7.4s, #31773; SVE-NEXT: shl v5.4s, v5.4s, #31774; SVE-NEXT: cmpge p2.s, p0/z, z7.s, #0775; SVE-NEXT: cmpge p3.s, p0/z, z5.s, #0776; SVE-NEXT: cmpge p0.s, p0/z, z4.s, #0777; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]778; SVE-NEXT: mov x8, #8 // =0x8779; SVE-NEXT: st1w { z2.s }, p2, [x0, x8, lsl #2]780; SVE-NEXT: mov x8, #12 // =0xc781; SVE-NEXT: st1w { z3.s }, p3, [x0, x8, lsl #2]782; SVE-NEXT: st1w { z0.s }, p0, [x0]783; SVE-NEXT: ret784 %load = load <16 x i32>, ptr %ptr, align 32785 %sel = select <16 x i1> %mask, <16 x i32> %load, <16 x i32> %x786 store <16 x i32> %sel, ptr %ptr, align 32787 ret void788}789 790define void @test_masked_store_zextload(<4 x i64> %x, ptr %ptr, <4 x i1> %mask) {791; SVE-LABEL: test_masked_store_zextload:792; SVE: // %bb.0:793; SVE-NEXT: ushll v2.4s, v2.4h, #0794; SVE-NEXT: ldr q4, [x0]795; SVE-NEXT: ushll2 v5.2d, v4.4s, #0796; SVE-NEXT: ushll v4.2d, v4.2s, #0797; SVE-NEXT: ushll2 v3.2d, v2.4s, #0798; SVE-NEXT: ushll v2.2d, v2.2s, #0799; SVE-NEXT: shl v3.2d, v3.2d, #63800; SVE-NEXT: shl v2.2d, v2.2d, #63801; SVE-NEXT: cmlt v3.2d, v3.2d, #0802; SVE-NEXT: cmlt v2.2d, v2.2d, #0803; SVE-NEXT: bif v1.16b, v5.16b, v3.16b804; SVE-NEXT: bif v0.16b, v4.16b, v2.16b805; SVE-NEXT: stp q0, q1, [x0]806; SVE-NEXT: ret807 %load = load <4 x i32>, ptr %ptr, align 32808 %zext = zext <4 x i32> %load to <4 x i64>809 %masked = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> %zext810 store <4 x i64> %masked, ptr %ptr, align 32811 ret void812}813 814define void @test_masked_store_volatile_load(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {815; SVE-LABEL: test_masked_store_volatile_load:816; SVE: // %bb.0:817; SVE-NEXT: zip1 v3.8b, v2.8b, v0.8b818; SVE-NEXT: zip2 v2.8b, v2.8b, v0.8b819; SVE-NEXT: ldr q4, [x0]820; SVE-NEXT: ldr q5, [x0, #16]821; SVE-NEXT: ushll v3.4s, v3.4h, #0822; SVE-NEXT: ushll v2.4s, v2.4h, #0823; SVE-NEXT: shl v3.4s, v3.4s, #31824; SVE-NEXT: shl v2.4s, v2.4s, #31825; SVE-NEXT: cmlt v3.4s, v3.4s, #0826; SVE-NEXT: cmlt v2.4s, v2.4s, #0827; SVE-NEXT: bif v0.16b, v4.16b, v3.16b828; SVE-NEXT: bif v1.16b, v5.16b, v2.16b829; SVE-NEXT: stp q0, q1, [x0]830; SVE-NEXT: ret831 %load = load volatile <8 x i32>, ptr %ptr, align 32832 %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load833 store <8 x i32> %sel, ptr %ptr, align 32834 ret void835}836 837define void @test_masked_store_volatile_store(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) {838; SVE-LABEL: test_masked_store_volatile_store:839; SVE: // %bb.0:840; SVE-NEXT: zip1 v3.8b, v2.8b, v0.8b841; SVE-NEXT: zip2 v2.8b, v2.8b, v0.8b842; SVE-NEXT: ldp q4, q5, [x0]843; SVE-NEXT: ushll v3.4s, v3.4h, #0844; SVE-NEXT: ushll v2.4s, v2.4h, #0845; SVE-NEXT: shl v3.4s, v3.4s, #31846; SVE-NEXT: shl v2.4s, v2.4s, #31847; SVE-NEXT: cmlt v3.4s, v3.4s, #0848; SVE-NEXT: cmlt v2.4s, v2.4s, #0849; SVE-NEXT: bif v0.16b, v4.16b, v3.16b850; SVE-NEXT: bif v1.16b, v5.16b, v2.16b851; SVE-NEXT: str q0, [x0]852; SVE-NEXT: str q1, [x0, #16]853; SVE-NEXT: ret854 %load = load <8 x i32>, ptr %ptr, align 32855 %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load856 store volatile <8 x i32> %sel, ptr %ptr, align 32857 ret void858}859 860declare void @use_vec(<8 x i32>)861 862define void @test_masked_store_intervening(<8 x i32> %x, ptr %ptr, <8 x i1> %mask) nounwind {863; SVE-LABEL: test_masked_store_intervening:864; SVE: // %bb.0:865; SVE-NEXT: sub sp, sp, #96866; SVE-NEXT: stp q1, q0, [sp, #32] // 32-byte Folded Spill867; SVE-NEXT: ldp q1, q3, [x0]868; SVE-NEXT: movi v0.2d, #0000000000000000869; SVE-NEXT: str d8, [sp, #64] // 8-byte Spill870; SVE-NEXT: fmov d8, d2871; SVE-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill872; SVE-NEXT: mov x19, x0873; SVE-NEXT: stp q1, q3, [sp] // 32-byte Folded Spill874; SVE-NEXT: movi v1.2d, #0000000000000000875; SVE-NEXT: stp q0, q0, [x0]876; SVE-NEXT: bl use_vec877; SVE-NEXT: zip2 v0.8b, v8.8b, v0.8b878; SVE-NEXT: ldp q3, q2, [sp, #16] // 32-byte Folded Reload879; SVE-NEXT: zip1 v1.8b, v8.8b, v0.8b880; SVE-NEXT: ushll v0.4s, v0.4h, #0881; SVE-NEXT: ldr d8, [sp, #64] // 8-byte Reload882; SVE-NEXT: shl v0.4s, v0.4s, #31883; SVE-NEXT: ushll v1.4s, v1.4h, #0884; SVE-NEXT: cmlt v0.4s, v0.4s, #0885; SVE-NEXT: shl v1.4s, v1.4s, #31886; SVE-NEXT: bsl v0.16b, v2.16b, v3.16b887; SVE-NEXT: ldr q2, [sp, #48] // 16-byte Reload888; SVE-NEXT: ldr q3, [sp] // 16-byte Reload889; SVE-NEXT: cmlt v1.4s, v1.4s, #0890; SVE-NEXT: bsl v1.16b, v2.16b, v3.16b891; SVE-NEXT: stp q1, q0, [x19]892; SVE-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload893; SVE-NEXT: add sp, sp, #96894; SVE-NEXT: ret895 %load = load <8 x i32>, ptr %ptr, align 32896 store <8 x i32> zeroinitializer, ptr %ptr, align 32897 %tmp = load <8 x i32>, ptr %ptr898 call void @use_vec(<8 x i32> %tmp)899 %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load900 store <8 x i32> %sel, ptr %ptr, align 32901 ret void902}903 904 905define void @test_masked_store_multiple_v8i32(<8 x i32> %x, <8 x i32> %y, ptr %ptr1, ptr %ptr2, <8 x i1> %mask, <8 x i1> %mask2) {906; SVE-LABEL: test_masked_store_multiple_v8i32:907; SVE: // %bb.0:908; SVE-NEXT: // kill: def $q0 killed $q0 def $z0909; SVE-NEXT: zip1 v6.8b, v5.8b, v0.8b910; SVE-NEXT: zip2 v7.8b, v4.8b, v0.8b911; SVE-NEXT: mov x8, #4 // =0x4912; SVE-NEXT: zip2 v5.8b, v5.8b, v0.8b913; SVE-NEXT: zip1 v4.8b, v4.8b, v0.8b914; SVE-NEXT: // kill: def $q1 killed $q1 def $z1915; SVE-NEXT: ptrue p0.s, vl4916; SVE-NEXT: ushll v6.4s, v6.4h, #0917; SVE-NEXT: ushll v7.4s, v7.4h, #0918; SVE-NEXT: ushll v5.4s, v5.4h, #0919; SVE-NEXT: ushll v4.4s, v4.4h, #0920; SVE-NEXT: shl v6.4s, v6.4s, #31921; SVE-NEXT: shl v7.4s, v7.4s, #31922; SVE-NEXT: shl v5.4s, v5.4s, #31923; SVE-NEXT: shl v4.4s, v4.4s, #31924; SVE-NEXT: cmlt v6.4s, v6.4s, #0925; SVE-NEXT: cmpne p1.s, p0/z, z7.s, #0926; SVE-NEXT: cmlt v5.4s, v5.4s, #0927; SVE-NEXT: ldp q7, q16, [x1]928; SVE-NEXT: cmpne p0.s, p0/z, z4.s, #0929; SVE-NEXT: bif v2.16b, v7.16b, v6.16b930; SVE-NEXT: bif v3.16b, v16.16b, v5.16b931; SVE-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2]932; SVE-NEXT: st1w { z0.s }, p0, [x0]933; SVE-NEXT: stp q2, q3, [x1]934; SVE-NEXT: ret935 %load = load <8 x i32>, ptr %ptr1, align 32936 %load2 = load <8 x i32>, ptr %ptr2, align 32937 %sel = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> %load938 %sel2 = select <8 x i1> %mask2, <8 x i32> %y, <8 x i32> %load2939 store <8 x i32> %sel, ptr %ptr1, align 32940 store <8 x i32> %sel2, ptr %ptr2, align 32941 ret void942}943 944define void @test_masked_store_multiple_v8i64(<8 x i64> %x, <8 x i64> %y, ptr %ptr1, ptr %ptr2, <8 x i1> %mask, <8 x i1> %mask2) {945; SVE-LABEL: test_masked_store_multiple_v8i64:946; SVE: // %bb.0:947; SVE-NEXT: ldp d16, d17, [sp]948; SVE-NEXT: ptrue p1.d, vl2949; SVE-NEXT: mov x9, #4 // =0x4950; SVE-NEXT: // kill: def $q3 killed $q3 def $z3951; SVE-NEXT: // kill: def $q2 killed $q2 def $z2952; SVE-NEXT: // kill: def $q1 killed $q1 def $z1953; SVE-NEXT: mov x8, #6 // =0x6954; SVE-NEXT: // kill: def $q0 killed $q0 def $z0955; SVE-NEXT: mov b18, v16.b[0]956; SVE-NEXT: mov b19, v16.b[4]957; SVE-NEXT: mov b20, v17.b[4]958; SVE-NEXT: mov b21, v16.b[6]959; SVE-NEXT: mov b22, v16.b[2]960; SVE-NEXT: mov b23, v17.b[6]961; SVE-NEXT: mov b24, v17.b[2]962; SVE-NEXT: mov b25, v17.b[0]963; SVE-NEXT: mov v18.b[4], v16.b[1]964; SVE-NEXT: mov v19.b[4], v16.b[5]965; SVE-NEXT: mov v20.b[4], v17.b[5]966; SVE-NEXT: mov v21.b[4], v16.b[7]967; SVE-NEXT: mov v22.b[4], v16.b[3]968; SVE-NEXT: mov v23.b[4], v17.b[7]969; SVE-NEXT: mov v24.b[4], v17.b[3]970; SVE-NEXT: mov v25.b[4], v17.b[1]971; SVE-NEXT: ushll v18.2d, v18.2s, #0972; SVE-NEXT: ushll v17.2d, v19.2s, #0973; SVE-NEXT: ushll v19.2d, v21.2s, #0974; SVE-NEXT: ushll v21.2d, v25.2s, #0975; SVE-NEXT: shl v16.2d, v18.2d, #63976; SVE-NEXT: ushll v18.2d, v20.2s, #0977; SVE-NEXT: ushll v20.2d, v23.2s, #0978; SVE-NEXT: shl v17.2d, v17.2d, #63979; SVE-NEXT: shl v19.2d, v19.2d, #63980; SVE-NEXT: shl v21.2d, v21.2d, #63981; SVE-NEXT: cmpne p0.d, p1/z, z16.d, #0982; SVE-NEXT: ushll v16.2d, v22.2s, #0983; SVE-NEXT: shl v18.2d, v18.2d, #63984; SVE-NEXT: ushll v22.2d, v24.2s, #0985; SVE-NEXT: shl v20.2d, v20.2d, #63986; SVE-NEXT: cmpne p2.d, p1/z, z17.d, #0987; SVE-NEXT: cmpne p3.d, p1/z, z19.d, #0988; SVE-NEXT: shl v16.2d, v16.2d, #63989; SVE-NEXT: cmlt v17.2d, v18.2d, #0990; SVE-NEXT: shl v18.2d, v22.2d, #63991; SVE-NEXT: ldp q19, q22, [x1, #32]992; SVE-NEXT: cmpne p1.d, p1/z, z16.d, #0993; SVE-NEXT: cmlt v16.2d, v20.2d, #0994; SVE-NEXT: cmlt v20.2d, v21.2d, #0995; SVE-NEXT: cmlt v18.2d, v18.2d, #0996; SVE-NEXT: bif v6.16b, v19.16b, v17.16b997; SVE-NEXT: ldp q17, q19, [x1]998; SVE-NEXT: st1d { z2.d }, p2, [x0, x9, lsl #3]999; SVE-NEXT: mov x9, #2 // =0x21000; SVE-NEXT: st1d { z3.d }, p3, [x0, x8, lsl #3]1001; SVE-NEXT: mov v2.16b, v16.16b1002; SVE-NEXT: mov v3.16b, v20.16b1003; SVE-NEXT: st1d { z1.d }, p1, [x0, x9, lsl #3]1004; SVE-NEXT: mov v1.16b, v18.16b1005; SVE-NEXT: st1d { z0.d }, p0, [x0]1006; SVE-NEXT: bsl v2.16b, v7.16b, v22.16b1007; SVE-NEXT: bsl v3.16b, v4.16b, v17.16b1008; SVE-NEXT: bsl v1.16b, v5.16b, v19.16b1009; SVE-NEXT: stp q6, q2, [x1, #32]1010; SVE-NEXT: stp q3, q1, [x1]1011; SVE-NEXT: ret1012 %load = load <8 x i64>, ptr %ptr1, align 321013 %load2 = load <8 x i64>, ptr %ptr2, align 321014 %sel = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> %load1015 %sel2 = select <8 x i1> %mask2, <8 x i64> %y, <8 x i64> %load21016 store <8 x i64> %sel, ptr %ptr1, align 321017 store <8 x i64> %sel2, ptr %ptr2, align 321018 ret void1019}1020 1021define void @test_masked_store_unaligned_v4i32(<4 x i32> %data, ptr %ptr, <4 x i1> %mask) {1022; SVE-LABEL: test_masked_store_unaligned_v4i32:1023; SVE: // %bb.0:1024; SVE-NEXT: ushll v1.4s, v1.4h, #01025; SVE-NEXT: ptrue p0.s, vl41026; SVE-NEXT: add x8, x0, #11027; SVE-NEXT: // kill: def $q0 killed $q0 def $z01028; SVE-NEXT: shl v1.4s, v1.4s, #311029; SVE-NEXT: cmpne p0.s, p0/z, z1.s, #01030; SVE-NEXT: st1w { z0.s }, p0, [x8]1031; SVE-NEXT: ret1032 %ptr_i8 = getelementptr i8, ptr %ptr, i32 11033 %ptr_vec = bitcast ptr %ptr_i8 to ptr1034 %load = load <4 x i32>, ptr %ptr_vec, align 11035 %sel = select <4 x i1> %mask, <4 x i32> %data, <4 x i32> %load1036 store <4 x i32> %sel, ptr %ptr_vec, align 11037 ret void1038}1039 1040define void @test_masked_store_unaligned_v4i64(<4 x i64> %data, ptr %ptr, <4 x i1> %mask) {1041; SVE-LABEL: test_masked_store_unaligned_v4i64:1042; SVE: // %bb.0:1043; SVE-NEXT: ushll v2.4s, v2.4h, #01044; SVE-NEXT: ptrue p0.d, vl21045; SVE-NEXT: add x8, x0, #171046; SVE-NEXT: add x9, x0, #11047; SVE-NEXT: // kill: def $q1 killed $q1 def $z11048; SVE-NEXT: // kill: def $q0 killed $q0 def $z01049; SVE-NEXT: ushll2 v3.2d, v2.4s, #01050; SVE-NEXT: ushll v2.2d, v2.2s, #01051; SVE-NEXT: shl v3.2d, v3.2d, #631052; SVE-NEXT: shl v2.2d, v2.2d, #631053; SVE-NEXT: cmpne p1.d, p0/z, z3.d, #01054; SVE-NEXT: cmpne p0.d, p0/z, z2.d, #01055; SVE-NEXT: st1d { z1.d }, p1, [x8]1056; SVE-NEXT: st1d { z0.d }, p0, [x9]1057; SVE-NEXT: ret1058 %ptr_i8 = getelementptr i8, ptr %ptr, i64 11059 %ptr_vec = bitcast ptr %ptr_i8 to ptr1060 %load = load <4 x i64>, ptr %ptr_vec, align 11061 %sel = select <4 x i1> %mask, <4 x i64> %data, <4 x i64> %load1062 store <4 x i64> %sel, ptr %ptr_vec, align 11063 ret void1064}1065 1066define void @test_masked_store_unaligned_v8i32(<8 x i32> %data, ptr %ptr, <8 x i1> %mask) {1067; SVE-LABEL: test_masked_store_unaligned_v8i32:1068; SVE: // %bb.0:1069; SVE-NEXT: // kill: def $q0 killed $q0 def $z01070; SVE-NEXT: zip1 v3.8b, v2.8b, v0.8b1071; SVE-NEXT: zip2 v2.8b, v2.8b, v0.8b1072; SVE-NEXT: add x8, x0, #11073; SVE-NEXT: ptrue p0.s, vl41074; SVE-NEXT: add x9, x0, #171075; SVE-NEXT: // kill: def $q1 killed $q1 def $z11076; SVE-NEXT: ushll v3.4s, v3.4h, #01077; SVE-NEXT: ushll v2.4s, v2.4h, #01078; SVE-NEXT: shl v3.4s, v3.4s, #311079; SVE-NEXT: shl v2.4s, v2.4s, #311080; SVE-NEXT: cmpne p1.s, p0/z, z3.s, #01081; SVE-NEXT: cmpne p0.s, p0/z, z2.s, #01082; SVE-NEXT: st1w { z0.s }, p1, [x8]1083; SVE-NEXT: st1w { z1.s }, p0, [x9]1084; SVE-NEXT: ret1085 %ptr_i8 = getelementptr i8, ptr %ptr, i32 11086 %ptr_vec = bitcast ptr %ptr_i8 to ptr1087 %load = load <8 x i32>, ptr %ptr_vec, align 11088 %sel = select <8 x i1> %mask, <8 x i32> %data, <8 x i32> %load1089 store <8 x i32> %sel, ptr %ptr_vec, align 11090 ret void1091}1092 1093define void @test_masked_store_unaligned_v8i64(<8 x i64> %data, ptr %ptr, <8 x i1> %mask) {1094; SVE-LABEL: test_masked_store_unaligned_v8i64:1095; SVE: // %bb.0:1096; SVE-NEXT: // kill: def $d4 killed $d4 def $q41097; SVE-NEXT: mov b5, v4.b[4]1098; SVE-NEXT: mov b6, v4.b[6]1099; SVE-NEXT: add x8, x0, #331100; SVE-NEXT: mov b7, v4.b[0]1101; SVE-NEXT: mov b16, v4.b[2]1102; SVE-NEXT: add x9, x0, #491103; SVE-NEXT: ptrue p0.d, vl21104; SVE-NEXT: // kill: def $q3 killed $q3 def $z31105; SVE-NEXT: // kill: def $q2 killed $q2 def $z21106; SVE-NEXT: // kill: def $q1 killed $q1 def $z11107; SVE-NEXT: // kill: def $q0 killed $q0 def $z01108; SVE-NEXT: mov v5.b[4], v4.b[5]1109; SVE-NEXT: mov v6.b[4], v4.b[7]1110; SVE-NEXT: mov v7.b[4], v4.b[1]1111; SVE-NEXT: mov v16.b[4], v4.b[3]1112; SVE-NEXT: ushll v4.2d, v5.2s, #01113; SVE-NEXT: ushll v5.2d, v6.2s, #01114; SVE-NEXT: ushll v6.2d, v7.2s, #01115; SVE-NEXT: ushll v7.2d, v16.2s, #01116; SVE-NEXT: shl v4.2d, v4.2d, #631117; SVE-NEXT: shl v5.2d, v5.2d, #631118; SVE-NEXT: shl v6.2d, v6.2d, #631119; SVE-NEXT: cmpne p1.d, p0/z, z4.d, #01120; SVE-NEXT: shl v4.2d, v7.2d, #631121; SVE-NEXT: cmpne p2.d, p0/z, z5.d, #01122; SVE-NEXT: cmpne p3.d, p0/z, z6.d, #01123; SVE-NEXT: cmpne p0.d, p0/z, z4.d, #01124; SVE-NEXT: st1d { z2.d }, p1, [x8]1125; SVE-NEXT: add x8, x0, #11126; SVE-NEXT: st1d { z3.d }, p2, [x9]1127; SVE-NEXT: add x9, x0, #171128; SVE-NEXT: st1d { z0.d }, p3, [x8]1129; SVE-NEXT: st1d { z1.d }, p0, [x9]1130; SVE-NEXT: ret1131 %ptr_i8 = getelementptr i8, ptr %ptr, i64 11132 %ptr_vec = bitcast ptr %ptr_i8 to ptr1133 %load = load <8 x i64>, ptr %ptr_vec, align 11134 %sel = select <8 x i1> %mask, <8 x i64> %data, <8 x i64> %load1135 store <8 x i64> %sel, ptr %ptr_vec, align 11136 ret void1137}1138 1139define void @PR159912(<1 x i1> %arg, ptr %ptr) #0 {1140; SVE-LABEL: PR159912:1141; SVE: // %bb.0:1142; SVE-NEXT: tst w0, #0x11143; SVE-NEXT: ldr d0, [x1]1144; SVE-NEXT: csetm x8, ne1145; SVE-NEXT: fmov d1, x81146; SVE-NEXT: bic v0.8b, v0.8b, v1.8b1147; SVE-NEXT: str d0, [x1]1148; SVE-NEXT: ret1149 %load = load <1 x i64>, ptr %ptr, align 81150 %select = select <1 x i1> %arg, <1 x i64> zeroinitializer, <1 x i64> %load1151 store <1 x i64> %select, ptr %ptr, align 81152 ret void1153}1154