1126 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE3; RUN: llc -mtriple=thumbebv8.1m.main-none-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE4 5define void @foo_v4i32_v4i32(ptr %dest, ptr %mask, ptr %src) {6; CHECK-LABEL: foo_v4i32_v4i32:7; CHECK: @ %bb.0: @ %entry8; CHECK-NEXT: vldrw.u32 q0, [r1]9; CHECK-NEXT: vptt.s32 gt, q0, zr10; CHECK-NEXT: vldrwt.u32 q0, [r2]11; CHECK-NEXT: vstrwt.32 q0, [r0]12; CHECK-NEXT: bx lr13entry:14 %0 = load <4 x i32>, ptr %mask, align 415 %1 = icmp sgt <4 x i32> %0, zeroinitializer16 %2 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %src, i32 4, <4 x i1> %1, <4 x i32> undef)17 call void @llvm.masked.store.v4i32.p0(<4 x i32> %2, ptr %dest, i32 4, <4 x i1> %1)18 ret void19}20 21define void @foo_sext_v4i32_v4i8(ptr %dest, ptr %mask, ptr %src) {22; CHECK-LABEL: foo_sext_v4i32_v4i8:23; CHECK: @ %bb.0: @ %entry24; CHECK-NEXT: vldrw.u32 q0, [r1]25; CHECK-NEXT: vptt.s32 gt, q0, zr26; CHECK-NEXT: vldrbt.s32 q0, [r2]27; CHECK-NEXT: vstrwt.32 q0, [r0]28; CHECK-NEXT: bx lr29entry:30 %0 = load <4 x i32>, ptr %mask, align 431 %1 = icmp sgt <4 x i32> %0, zeroinitializer32 %2 = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr %src, i32 1, <4 x i1> %1, <4 x i8> undef)33 %3 = sext <4 x i8> %2 to <4 x i32>34 call void @llvm.masked.store.v4i32.p0(<4 x i32> %3, ptr %dest, i32 4, <4 x i1> %1)35 ret void36}37 38define void @foo_sext_v4i32_v4i16(ptr %dest, ptr %mask, ptr %src) {39; CHECK-LABEL: foo_sext_v4i32_v4i16:40; CHECK: @ %bb.0: @ %entry41; CHECK-NEXT: vldrw.u32 q0, [r1]42; CHECK-NEXT: vptt.s32 gt, q0, zr43; CHECK-NEXT: vldrht.s32 q0, [r2]44; CHECK-NEXT: vstrwt.32 q0, [r0]45; CHECK-NEXT: bx lr46entry:47 %0 = load <4 x i32>, ptr %mask, align 448 %1 = icmp sgt <4 x i32> %0, zeroinitializer49 %2 = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr %src, i32 2, <4 x i1> %1, <4 x i16> undef)50 %3 = sext <4 x i16> %2 to <4 x i32>51 call void @llvm.masked.store.v4i32.p0(<4 x i32> %3, ptr %dest, i32 4, <4 x i1> %1)52 ret void53}54 55define void @foo_zext_v4i32_v4i8(ptr %dest, ptr %mask, ptr %src) {56; CHECK-LABEL: foo_zext_v4i32_v4i8:57; CHECK: @ %bb.0: @ %entry58; CHECK-NEXT: vldrw.u32 q0, [r1]59; CHECK-NEXT: vptt.s32 gt, q0, zr60; CHECK-NEXT: vldrbt.u32 q0, [r2]61; CHECK-NEXT: vstrwt.32 q0, [r0]62; CHECK-NEXT: bx lr63entry:64 %0 = load <4 x i32>, ptr %mask, align 465 %1 = icmp sgt <4 x i32> %0, zeroinitializer66 %2 = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr %src, i32 1, <4 x i1> %1, <4 x i8> undef)67 %3 = zext <4 x i8> %2 to <4 x i32>68 call void @llvm.masked.store.v4i32.p0(<4 x i32> %3, ptr %dest, i32 4, <4 x i1> %1)69 ret void70}71 72define void @foo_zext_v4i32_v4i16(ptr %dest, ptr %mask, ptr %src) {73; CHECK-LABEL: foo_zext_v4i32_v4i16:74; CHECK: @ %bb.0: @ %entry75; CHECK-NEXT: vldrw.u32 q0, [r1]76; CHECK-NEXT: vptt.s32 gt, q0, zr77; CHECK-NEXT: vldrht.u32 q0, [r2]78; CHECK-NEXT: vstrwt.32 q0, [r0]79; CHECK-NEXT: bx lr80entry:81 %0 = load <4 x i32>, ptr %mask, align 482 %1 = icmp sgt <4 x i32> %0, zeroinitializer83 %2 = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr %src, i32 2, <4 x i1> %1, <4 x i16> undef)84 %3 = zext <4 x i16> %2 to <4 x i32>85 call void @llvm.masked.store.v4i32.p0(<4 x i32> %3, ptr %dest, i32 4, <4 x i1> %1)86 ret void87}88 89define void @foo_sext_v2i64_v2i32(ptr %dest, ptr %mask, ptr %src) {90; CHECK-LE-LABEL: foo_sext_v2i64_v2i32:91; CHECK-LE: @ %bb.0: @ %entry92; CHECK-LE-NEXT: .save {r4, r5, r7, lr}93; CHECK-LE-NEXT: push {r4, r5, r7, lr}94; CHECK-LE-NEXT: .pad #495; CHECK-LE-NEXT: sub sp, #496; CHECK-LE-NEXT: ldrd r12, lr, [r1]97; CHECK-LE-NEXT: movs r1, #098; CHECK-LE-NEXT: @ implicit-def: $q199; CHECK-LE-NEXT: rsbs.w r3, r12, #0100; CHECK-LE-NEXT: vmov q0[2], q0[0], r12, lr101; CHECK-LE-NEXT: sbcs.w r3, r1, r12, asr #31102; CHECK-LE-NEXT: csetm r3, lt103; CHECK-LE-NEXT: rsbs.w r4, lr, #0104; CHECK-LE-NEXT: sbcs.w r4, r1, lr, asr #31105; CHECK-LE-NEXT: bfi r1, r3, #0, #1106; CHECK-LE-NEXT: csetm r3, lt107; CHECK-LE-NEXT: bfi r1, r3, #1, #1108; CHECK-LE-NEXT: lsls r3, r1, #31109; CHECK-LE-NEXT: itt ne110; CHECK-LE-NEXT: ldrne r3, [r2]111; CHECK-LE-NEXT: vmovne.32 q1[0], r3112; CHECK-LE-NEXT: lsls r1, r1, #30113; CHECK-LE-NEXT: itt mi114; CHECK-LE-NEXT: ldrmi r1, [r2, #4]115; CHECK-LE-NEXT: vmovmi.32 q1[2], r1116; CHECK-LE-NEXT: vmov r2, s6117; CHECK-LE-NEXT: movs r1, #0118; CHECK-LE-NEXT: vmov r3, s0119; CHECK-LE-NEXT: vmov r4, s4120; CHECK-LE-NEXT: vmov q1[2], q1[0], r4, r2121; CHECK-LE-NEXT: rsbs r5, r3, #0122; CHECK-LE-NEXT: asr.w r12, r2, #31123; CHECK-LE-NEXT: sbcs.w r2, r1, r3, asr #31124; CHECK-LE-NEXT: vmov r3, s2125; CHECK-LE-NEXT: csetm r2, lt126; CHECK-LE-NEXT: asr.w lr, r4, #31127; CHECK-LE-NEXT: vmov q1[3], q1[1], lr, r12128; CHECK-LE-NEXT: rsbs r5, r3, #0129; CHECK-LE-NEXT: sbcs.w r3, r1, r3, asr #31130; CHECK-LE-NEXT: bfi r1, r2, #0, #1131; CHECK-LE-NEXT: csetm r2, lt132; CHECK-LE-NEXT: bfi r1, r2, #1, #1133; CHECK-LE-NEXT: lsls r2, r1, #31134; CHECK-LE-NEXT: it ne135; CHECK-LE-NEXT: vstrne d2, [r0]136; CHECK-LE-NEXT: lsls r1, r1, #30137; CHECK-LE-NEXT: it mi138; CHECK-LE-NEXT: vstrmi d3, [r0, #8]139; CHECK-LE-NEXT: add sp, #4140; CHECK-LE-NEXT: pop {r4, r5, r7, pc}141;142; CHECK-BE-LABEL: foo_sext_v2i64_v2i32:143; CHECK-BE: @ %bb.0: @ %entry144; CHECK-BE-NEXT: .save {r4, r5, r7, lr}145; CHECK-BE-NEXT: push {r4, r5, r7, lr}146; CHECK-BE-NEXT: .pad #4147; CHECK-BE-NEXT: sub sp, #4148; CHECK-BE-NEXT: ldrd r12, lr, [r1]149; CHECK-BE-NEXT: rsbs.w r3, lr, #0150; CHECK-BE-NEXT: mov.w r1, #0151; CHECK-BE-NEXT: sbcs.w r3, r1, lr, asr #31152; CHECK-BE-NEXT: vmov q0[3], q0[1], r12, lr153; CHECK-BE-NEXT: csetm lr, lt154; CHECK-BE-NEXT: rsbs.w r3, r12, #0155; CHECK-BE-NEXT: @ implicit-def: $q2156; CHECK-BE-NEXT: sbcs.w r3, r1, r12, asr #31157; CHECK-BE-NEXT: bfi r1, lr, #0, #1158; CHECK-BE-NEXT: csetm r3, lt159; CHECK-BE-NEXT: bfi r1, r3, #1, #1160; CHECK-BE-NEXT: lsls r3, r1, #30161; CHECK-BE-NEXT: bpl .LBB5_2162; CHECK-BE-NEXT: @ %bb.1: @ %cond.load163; CHECK-BE-NEXT: ldr r3, [r2]164; CHECK-BE-NEXT: vmov.32 q1[1], r3165; CHECK-BE-NEXT: vrev64.32 q2, q1166; CHECK-BE-NEXT: .LBB5_2: @ %else167; CHECK-BE-NEXT: vrev64.32 q1, q0168; CHECK-BE-NEXT: lsls r1, r1, #31169; CHECK-BE-NEXT: beq .LBB5_4170; CHECK-BE-NEXT: @ %bb.3: @ %cond.load1171; CHECK-BE-NEXT: ldr r1, [r2, #4]172; CHECK-BE-NEXT: vrev64.32 q0, q2173; CHECK-BE-NEXT: vmov.32 q0[3], r1174; CHECK-BE-NEXT: vrev64.32 q2, q0175; CHECK-BE-NEXT: .LBB5_4: @ %else2176; CHECK-BE-NEXT: vrev64.32 q0, q2177; CHECK-BE-NEXT: vrev64.32 q2, q1178; CHECK-BE-NEXT: vmov r2, s3179; CHECK-BE-NEXT: movs r1, #0180; CHECK-BE-NEXT: vmov r3, s1181; CHECK-BE-NEXT: vmov r4, s11182; CHECK-BE-NEXT: asr.w r12, r2, #31183; CHECK-BE-NEXT: asr.w lr, r3, #31184; CHECK-BE-NEXT: rsbs r5, r4, #0185; CHECK-BE-NEXT: vmov q1[2], q1[0], lr, r12186; CHECK-BE-NEXT: sbcs.w r4, r1, r4, asr #31187; CHECK-BE-NEXT: vmov q1[3], q1[1], r3, r2188; CHECK-BE-NEXT: vmov r3, s9189; CHECK-BE-NEXT: csetm r2, lt190; CHECK-BE-NEXT: vrev64.32 q0, q1191; CHECK-BE-NEXT: rsbs r5, r3, #0192; CHECK-BE-NEXT: sbcs.w r3, r1, r3, asr #31193; CHECK-BE-NEXT: bfi r1, r2, #0, #1194; CHECK-BE-NEXT: csetm r2, lt195; CHECK-BE-NEXT: bfi r1, r2, #1, #1196; CHECK-BE-NEXT: lsls r2, r1, #30197; CHECK-BE-NEXT: it mi198; CHECK-BE-NEXT: vstrmi d0, [r0]199; CHECK-BE-NEXT: lsls r1, r1, #31200; CHECK-BE-NEXT: it ne201; CHECK-BE-NEXT: vstrne d1, [r0, #8]202; CHECK-BE-NEXT: add sp, #4203; CHECK-BE-NEXT: pop {r4, r5, r7, pc}204entry:205 %0 = load <2 x i32>, ptr %mask, align 4206 %1 = icmp sgt <2 x i32> %0, zeroinitializer207 %2 = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %src, i32 4, <2 x i1> %1, <2 x i32> undef)208 %3 = sext <2 x i32> %2 to <2 x i64>209 call void @llvm.masked.store.v2i64.p0(<2 x i64> %3, ptr %dest, i32 8, <2 x i1> %1)210 ret void211}212 213define void @foo_sext_v2i64_v2i32_unaligned(ptr %dest, ptr %mask, ptr %src) {214; CHECK-LE-LABEL: foo_sext_v2i64_v2i32_unaligned:215; CHECK-LE: @ %bb.0: @ %entry216; CHECK-LE-NEXT: .save {r4, r5, r7, lr}217; CHECK-LE-NEXT: push {r4, r5, r7, lr}218; CHECK-LE-NEXT: .pad #4219; CHECK-LE-NEXT: sub sp, #4220; CHECK-LE-NEXT: ldrd r12, lr, [r1]221; CHECK-LE-NEXT: movs r1, #0222; CHECK-LE-NEXT: @ implicit-def: $q0223; CHECK-LE-NEXT: rsbs.w r3, r12, #0224; CHECK-LE-NEXT: vmov q1[2], q1[0], r12, lr225; CHECK-LE-NEXT: sbcs.w r3, r1, r12, asr #31226; CHECK-LE-NEXT: csetm r3, lt227; CHECK-LE-NEXT: rsbs.w r4, lr, #0228; CHECK-LE-NEXT: sbcs.w r4, r1, lr, asr #31229; CHECK-LE-NEXT: bfi r1, r3, #0, #1230; CHECK-LE-NEXT: csetm r3, lt231; CHECK-LE-NEXT: bfi r1, r3, #1, #1232; CHECK-LE-NEXT: lsls r3, r1, #31233; CHECK-LE-NEXT: itt ne234; CHECK-LE-NEXT: ldrne r3, [r2]235; CHECK-LE-NEXT: vmovne.32 q0[0], r3236; CHECK-LE-NEXT: lsls r1, r1, #30237; CHECK-LE-NEXT: itt mi238; CHECK-LE-NEXT: ldrmi r1, [r2, #4]239; CHECK-LE-NEXT: vmovmi.32 q0[2], r1240; CHECK-LE-NEXT: vmov r2, s2241; CHECK-LE-NEXT: movs r1, #0242; CHECK-LE-NEXT: vmov r3, s4243; CHECK-LE-NEXT: vmov r4, s0244; CHECK-LE-NEXT: vmov q0[2], q0[0], r4, r2245; CHECK-LE-NEXT: rsbs r5, r3, #0246; CHECK-LE-NEXT: asr.w r12, r2, #31247; CHECK-LE-NEXT: sbcs.w r2, r1, r3, asr #31248; CHECK-LE-NEXT: vmov r3, s6249; CHECK-LE-NEXT: csetm r2, lt250; CHECK-LE-NEXT: asr.w lr, r4, #31251; CHECK-LE-NEXT: vmov q0[3], q0[1], lr, r12252; CHECK-LE-NEXT: rsbs r5, r3, #0253; CHECK-LE-NEXT: sbcs.w r3, r1, r3, asr #31254; CHECK-LE-NEXT: bfi r1, r2, #0, #1255; CHECK-LE-NEXT: csetm r2, lt256; CHECK-LE-NEXT: bfi r1, r2, #1, #1257; CHECK-LE-NEXT: lsls r2, r1, #31258; CHECK-LE-NEXT: itt ne259; CHECK-LE-NEXT: vmovne r2, r3, d0260; CHECK-LE-NEXT: strdne r2, r3, [r0]261; CHECK-LE-NEXT: lsls r1, r1, #30262; CHECK-LE-NEXT: itt mi263; CHECK-LE-NEXT: vmovmi r1, r2, d1264; CHECK-LE-NEXT: strdmi r1, r2, [r0, #8]265; CHECK-LE-NEXT: add sp, #4266; CHECK-LE-NEXT: pop {r4, r5, r7, pc}267;268; CHECK-BE-LABEL: foo_sext_v2i64_v2i32_unaligned:269; CHECK-BE: @ %bb.0: @ %entry270; CHECK-BE-NEXT: .save {r4, r5, r7, lr}271; CHECK-BE-NEXT: push {r4, r5, r7, lr}272; CHECK-BE-NEXT: .pad #4273; CHECK-BE-NEXT: sub sp, #4274; CHECK-BE-NEXT: ldrd r12, lr, [r1]275; CHECK-BE-NEXT: rsbs.w r3, lr, #0276; CHECK-BE-NEXT: mov.w r1, #0277; CHECK-BE-NEXT: sbcs.w r3, r1, lr, asr #31278; CHECK-BE-NEXT: vmov q0[3], q0[1], r12, lr279; CHECK-BE-NEXT: csetm lr, lt280; CHECK-BE-NEXT: rsbs.w r3, r12, #0281; CHECK-BE-NEXT: @ implicit-def: $q2282; CHECK-BE-NEXT: sbcs.w r3, r1, r12, asr #31283; CHECK-BE-NEXT: bfi r1, lr, #0, #1284; CHECK-BE-NEXT: csetm r3, lt285; CHECK-BE-NEXT: bfi r1, r3, #1, #1286; CHECK-BE-NEXT: lsls r3, r1, #30287; CHECK-BE-NEXT: bpl .LBB6_2288; CHECK-BE-NEXT: @ %bb.1: @ %cond.load289; CHECK-BE-NEXT: ldr r3, [r2]290; CHECK-BE-NEXT: vmov.32 q1[1], r3291; CHECK-BE-NEXT: vrev64.32 q2, q1292; CHECK-BE-NEXT: .LBB6_2: @ %else293; CHECK-BE-NEXT: vrev64.32 q1, q0294; CHECK-BE-NEXT: lsls r1, r1, #31295; CHECK-BE-NEXT: beq .LBB6_4296; CHECK-BE-NEXT: @ %bb.3: @ %cond.load1297; CHECK-BE-NEXT: ldr r1, [r2, #4]298; CHECK-BE-NEXT: vrev64.32 q0, q2299; CHECK-BE-NEXT: vmov.32 q0[3], r1300; CHECK-BE-NEXT: vrev64.32 q2, q0301; CHECK-BE-NEXT: .LBB6_4: @ %else2302; CHECK-BE-NEXT: vrev64.32 q0, q2303; CHECK-BE-NEXT: vrev64.32 q2, q1304; CHECK-BE-NEXT: vmov r2, s3305; CHECK-BE-NEXT: movs r1, #0306; CHECK-BE-NEXT: vmov r3, s1307; CHECK-BE-NEXT: vmov r4, s11308; CHECK-BE-NEXT: asr.w r12, r2, #31309; CHECK-BE-NEXT: asr.w lr, r3, #31310; CHECK-BE-NEXT: rsbs r5, r4, #0311; CHECK-BE-NEXT: vmov q1[2], q1[0], lr, r12312; CHECK-BE-NEXT: sbcs.w r4, r1, r4, asr #31313; CHECK-BE-NEXT: vmov q1[3], q1[1], r3, r2314; CHECK-BE-NEXT: vmov r3, s9315; CHECK-BE-NEXT: csetm r2, lt316; CHECK-BE-NEXT: vrev64.32 q0, q1317; CHECK-BE-NEXT: rsbs r5, r3, #0318; CHECK-BE-NEXT: sbcs.w r3, r1, r3, asr #31319; CHECK-BE-NEXT: bfi r1, r2, #0, #1320; CHECK-BE-NEXT: csetm r2, lt321; CHECK-BE-NEXT: bfi r1, r2, #1, #1322; CHECK-BE-NEXT: lsls r2, r1, #30323; CHECK-BE-NEXT: itt mi324; CHECK-BE-NEXT: vmovmi r2, r3, d0325; CHECK-BE-NEXT: strdmi r3, r2, [r0]326; CHECK-BE-NEXT: lsls r1, r1, #31327; CHECK-BE-NEXT: itt ne328; CHECK-BE-NEXT: vmovne r1, r2, d1329; CHECK-BE-NEXT: strdne r2, r1, [r0, #8]330; CHECK-BE-NEXT: add sp, #4331; CHECK-BE-NEXT: pop {r4, r5, r7, pc}332entry:333 %0 = load <2 x i32>, ptr %mask, align 4334 %1 = icmp sgt <2 x i32> %0, zeroinitializer335 %2 = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %src, i32 2, <2 x i1> %1, <2 x i32> undef)336 %3 = sext <2 x i32> %2 to <2 x i64>337 call void @llvm.masked.store.v2i64.p0(<2 x i64> %3, ptr %dest, i32 4, <2 x i1> %1)338 ret void339}340 341define void @foo_zext_v2i64_v2i32(ptr %dest, ptr %mask, ptr %src) {342; CHECK-LE-LABEL: foo_zext_v2i64_v2i32:343; CHECK-LE: @ %bb.0: @ %entry344; CHECK-LE-NEXT: .save {r4, lr}345; CHECK-LE-NEXT: push {r4, lr}346; CHECK-LE-NEXT: .pad #4347; CHECK-LE-NEXT: sub sp, #4348; CHECK-LE-NEXT: ldrd r12, lr, [r1]349; CHECK-LE-NEXT: movs r1, #0350; CHECK-LE-NEXT: @ implicit-def: $q0351; CHECK-LE-NEXT: vmov.i64 q2, #0xffffffff352; CHECK-LE-NEXT: rsbs.w r3, r12, #0353; CHECK-LE-NEXT: vmov q1[2], q1[0], r12, lr354; CHECK-LE-NEXT: sbcs.w r3, r1, r12, asr #31355; CHECK-LE-NEXT: csetm r3, lt356; CHECK-LE-NEXT: rsbs.w r4, lr, #0357; CHECK-LE-NEXT: sbcs.w r4, r1, lr, asr #31358; CHECK-LE-NEXT: bfi r1, r3, #0, #1359; CHECK-LE-NEXT: csetm r3, lt360; CHECK-LE-NEXT: bfi r1, r3, #1, #1361; CHECK-LE-NEXT: lsls r3, r1, #31362; CHECK-LE-NEXT: itt ne363; CHECK-LE-NEXT: ldrne r3, [r2]364; CHECK-LE-NEXT: vmovne.32 q0[0], r3365; CHECK-LE-NEXT: lsls r1, r1, #30366; CHECK-LE-NEXT: itt mi367; CHECK-LE-NEXT: ldrmi r1, [r2, #4]368; CHECK-LE-NEXT: vmovmi.32 q0[2], r1369; CHECK-LE-NEXT: vmov r2, s4370; CHECK-LE-NEXT: movs r1, #0371; CHECK-LE-NEXT: vand q0, q0, q2372; CHECK-LE-NEXT: rsbs r3, r2, #0373; CHECK-LE-NEXT: vmov r3, s6374; CHECK-LE-NEXT: sbcs.w r2, r1, r2, asr #31375; CHECK-LE-NEXT: csetm r2, lt376; CHECK-LE-NEXT: rsbs r4, r3, #0377; CHECK-LE-NEXT: sbcs.w r3, r1, r3, asr #31378; CHECK-LE-NEXT: bfi r1, r2, #0, #1379; CHECK-LE-NEXT: csetm r2, lt380; CHECK-LE-NEXT: bfi r1, r2, #1, #1381; CHECK-LE-NEXT: lsls r2, r1, #31382; CHECK-LE-NEXT: it ne383; CHECK-LE-NEXT: vstrne d0, [r0]384; CHECK-LE-NEXT: lsls r1, r1, #30385; CHECK-LE-NEXT: it mi386; CHECK-LE-NEXT: vstrmi d1, [r0, #8]387; CHECK-LE-NEXT: add sp, #4388; CHECK-LE-NEXT: pop {r4, pc}389;390; CHECK-BE-LABEL: foo_zext_v2i64_v2i32:391; CHECK-BE: @ %bb.0: @ %entry392; CHECK-BE-NEXT: .save {r7, lr}393; CHECK-BE-NEXT: push {r7, lr}394; CHECK-BE-NEXT: .pad #4395; CHECK-BE-NEXT: sub sp, #4396; CHECK-BE-NEXT: ldrd r12, lr, [r1]397; CHECK-BE-NEXT: rsbs.w r3, lr, #0398; CHECK-BE-NEXT: mov.w r1, #0399; CHECK-BE-NEXT: sbcs.w r3, r1, lr, asr #31400; CHECK-BE-NEXT: vmov q1[3], q1[1], r12, lr401; CHECK-BE-NEXT: csetm lr, lt402; CHECK-BE-NEXT: rsbs.w r3, r12, #0403; CHECK-BE-NEXT: @ implicit-def: $q0404; CHECK-BE-NEXT: sbcs.w r3, r1, r12, asr #31405; CHECK-BE-NEXT: bfi r1, lr, #0, #1406; CHECK-BE-NEXT: csetm r3, lt407; CHECK-BE-NEXT: bfi r1, r3, #1, #1408; CHECK-BE-NEXT: lsls r3, r1, #30409; CHECK-BE-NEXT: bpl .LBB7_2410; CHECK-BE-NEXT: @ %bb.1: @ %cond.load411; CHECK-BE-NEXT: ldr r3, [r2]412; CHECK-BE-NEXT: vmov.32 q2[1], r3413; CHECK-BE-NEXT: vrev64.32 q0, q2414; CHECK-BE-NEXT: .LBB7_2: @ %else415; CHECK-BE-NEXT: vrev64.32 q2, q1416; CHECK-BE-NEXT: lsls r1, r1, #31417; CHECK-BE-NEXT: beq .LBB7_4418; CHECK-BE-NEXT: @ %bb.3: @ %cond.load1419; CHECK-BE-NEXT: ldr r1, [r2, #4]420; CHECK-BE-NEXT: vrev64.32 q1, q0421; CHECK-BE-NEXT: vmov.32 q1[3], r1422; CHECK-BE-NEXT: vrev64.32 q0, q1423; CHECK-BE-NEXT: .LBB7_4: @ %else2424; CHECK-BE-NEXT: vmov.i64 q1, #0xffffffff00000000425; CHECK-BE-NEXT: movs r1, #0426; CHECK-BE-NEXT: vrev64.32 q3, q1427; CHECK-BE-NEXT: vrev64.32 q1, q2428; CHECK-BE-NEXT: vmov r2, s7429; CHECK-BE-NEXT: vand q0, q0, q3430; CHECK-BE-NEXT: rsbs r3, r2, #0431; CHECK-BE-NEXT: vmov r3, s5432; CHECK-BE-NEXT: sbcs.w r2, r1, r2, asr #31433; CHECK-BE-NEXT: csetm r12, lt434; CHECK-BE-NEXT: rsbs r2, r3, #0435; CHECK-BE-NEXT: sbcs.w r2, r1, r3, asr #31436; CHECK-BE-NEXT: bfi r1, r12, #0, #1437; CHECK-BE-NEXT: csetm r2, lt438; CHECK-BE-NEXT: bfi r1, r2, #1, #1439; CHECK-BE-NEXT: lsls r2, r1, #30440; CHECK-BE-NEXT: it mi441; CHECK-BE-NEXT: vstrmi d0, [r0]442; CHECK-BE-NEXT: lsls r1, r1, #31443; CHECK-BE-NEXT: it ne444; CHECK-BE-NEXT: vstrne d1, [r0, #8]445; CHECK-BE-NEXT: add sp, #4446; CHECK-BE-NEXT: pop {r7, pc}447entry:448 %0 = load <2 x i32>, ptr %mask, align 4449 %1 = icmp sgt <2 x i32> %0, zeroinitializer450 %2 = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %src, i32 4, <2 x i1> %1, <2 x i32> undef)451 %3 = zext <2 x i32> %2 to <2 x i64>452 call void @llvm.masked.store.v2i64.p0(<2 x i64> %3, ptr %dest, i32 8, <2 x i1> %1)453 ret void454}455 456define void @foo_zext_v2i64_v2i32_unaligned(ptr %dest, ptr %mask, ptr %src) {457; CHECK-LE-LABEL: foo_zext_v2i64_v2i32_unaligned:458; CHECK-LE: @ %bb.0: @ %entry459; CHECK-LE-NEXT: .save {r4, lr}460; CHECK-LE-NEXT: push {r4, lr}461; CHECK-LE-NEXT: .pad #4462; CHECK-LE-NEXT: sub sp, #4463; CHECK-LE-NEXT: ldrd r12, lr, [r1]464; CHECK-LE-NEXT: movs r1, #0465; CHECK-LE-NEXT: @ implicit-def: $q0466; CHECK-LE-NEXT: vmov.i64 q2, #0xffffffff467; CHECK-LE-NEXT: rsbs.w r3, r12, #0468; CHECK-LE-NEXT: vmov q1[2], q1[0], r12, lr469; CHECK-LE-NEXT: sbcs.w r3, r1, r12, asr #31470; CHECK-LE-NEXT: csetm r3, lt471; CHECK-LE-NEXT: rsbs.w r4, lr, #0472; CHECK-LE-NEXT: sbcs.w r4, r1, lr, asr #31473; CHECK-LE-NEXT: bfi r1, r3, #0, #1474; CHECK-LE-NEXT: csetm r3, lt475; CHECK-LE-NEXT: bfi r1, r3, #1, #1476; CHECK-LE-NEXT: lsls r3, r1, #31477; CHECK-LE-NEXT: itt ne478; CHECK-LE-NEXT: ldrne r3, [r2]479; CHECK-LE-NEXT: vmovne.32 q0[0], r3480; CHECK-LE-NEXT: lsls r1, r1, #30481; CHECK-LE-NEXT: itt mi482; CHECK-LE-NEXT: ldrmi r1, [r2, #4]483; CHECK-LE-NEXT: vmovmi.32 q0[2], r1484; CHECK-LE-NEXT: vmov r2, s4485; CHECK-LE-NEXT: movs r1, #0486; CHECK-LE-NEXT: vand q0, q0, q2487; CHECK-LE-NEXT: rsbs r3, r2, #0488; CHECK-LE-NEXT: vmov r3, s6489; CHECK-LE-NEXT: sbcs.w r2, r1, r2, asr #31490; CHECK-LE-NEXT: csetm r2, lt491; CHECK-LE-NEXT: rsbs r4, r3, #0492; CHECK-LE-NEXT: sbcs.w r3, r1, r3, asr #31493; CHECK-LE-NEXT: bfi r1, r2, #0, #1494; CHECK-LE-NEXT: csetm r2, lt495; CHECK-LE-NEXT: bfi r1, r2, #1, #1496; CHECK-LE-NEXT: lsls r2, r1, #31497; CHECK-LE-NEXT: itt ne498; CHECK-LE-NEXT: vmovne r2, r3, d0499; CHECK-LE-NEXT: strdne r2, r3, [r0]500; CHECK-LE-NEXT: lsls r1, r1, #30501; CHECK-LE-NEXT: itt mi502; CHECK-LE-NEXT: vmovmi r1, r2, d1503; CHECK-LE-NEXT: strdmi r1, r2, [r0, #8]504; CHECK-LE-NEXT: add sp, #4505; CHECK-LE-NEXT: pop {r4, pc}506;507; CHECK-BE-LABEL: foo_zext_v2i64_v2i32_unaligned:508; CHECK-BE: @ %bb.0: @ %entry509; CHECK-BE-NEXT: .save {r7, lr}510; CHECK-BE-NEXT: push {r7, lr}511; CHECK-BE-NEXT: .pad #4512; CHECK-BE-NEXT: sub sp, #4513; CHECK-BE-NEXT: ldrd r12, lr, [r1]514; CHECK-BE-NEXT: rsbs.w r3, lr, #0515; CHECK-BE-NEXT: mov.w r1, #0516; CHECK-BE-NEXT: sbcs.w r3, r1, lr, asr #31517; CHECK-BE-NEXT: vmov q1[3], q1[1], r12, lr518; CHECK-BE-NEXT: csetm lr, lt519; CHECK-BE-NEXT: rsbs.w r3, r12, #0520; CHECK-BE-NEXT: @ implicit-def: $q0521; CHECK-BE-NEXT: sbcs.w r3, r1, r12, asr #31522; CHECK-BE-NEXT: bfi r1, lr, #0, #1523; CHECK-BE-NEXT: csetm r3, lt524; CHECK-BE-NEXT: bfi r1, r3, #1, #1525; CHECK-BE-NEXT: lsls r3, r1, #30526; CHECK-BE-NEXT: bpl .LBB8_2527; CHECK-BE-NEXT: @ %bb.1: @ %cond.load528; CHECK-BE-NEXT: ldr r3, [r2]529; CHECK-BE-NEXT: vmov.32 q2[1], r3530; CHECK-BE-NEXT: vrev64.32 q0, q2531; CHECK-BE-NEXT: .LBB8_2: @ %else532; CHECK-BE-NEXT: vrev64.32 q2, q1533; CHECK-BE-NEXT: lsls r1, r1, #31534; CHECK-BE-NEXT: beq .LBB8_4535; CHECK-BE-NEXT: @ %bb.3: @ %cond.load1536; CHECK-BE-NEXT: ldr r1, [r2, #4]537; CHECK-BE-NEXT: vrev64.32 q1, q0538; CHECK-BE-NEXT: vmov.32 q1[3], r1539; CHECK-BE-NEXT: vrev64.32 q0, q1540; CHECK-BE-NEXT: .LBB8_4: @ %else2541; CHECK-BE-NEXT: vmov.i64 q1, #0xffffffff00000000542; CHECK-BE-NEXT: movs r1, #0543; CHECK-BE-NEXT: vrev64.32 q3, q1544; CHECK-BE-NEXT: vrev64.32 q1, q2545; CHECK-BE-NEXT: vmov r2, s7546; CHECK-BE-NEXT: vand q0, q0, q3547; CHECK-BE-NEXT: rsbs r3, r2, #0548; CHECK-BE-NEXT: vmov r3, s5549; CHECK-BE-NEXT: sbcs.w r2, r1, r2, asr #31550; CHECK-BE-NEXT: csetm r12, lt551; CHECK-BE-NEXT: rsbs r2, r3, #0552; CHECK-BE-NEXT: sbcs.w r2, r1, r3, asr #31553; CHECK-BE-NEXT: bfi r1, r12, #0, #1554; CHECK-BE-NEXT: csetm r2, lt555; CHECK-BE-NEXT: bfi r1, r2, #1, #1556; CHECK-BE-NEXT: lsls r2, r1, #30557; CHECK-BE-NEXT: itt mi558; CHECK-BE-NEXT: vmovmi r2, r3, d0559; CHECK-BE-NEXT: strdmi r3, r2, [r0]560; CHECK-BE-NEXT: lsls r1, r1, #31561; CHECK-BE-NEXT: itt ne562; CHECK-BE-NEXT: vmovne r1, r2, d1563; CHECK-BE-NEXT: strdne r2, r1, [r0, #8]564; CHECK-BE-NEXT: add sp, #4565; CHECK-BE-NEXT: pop {r7, pc}566entry:567 %0 = load <2 x i32>, ptr %mask, align 4568 %1 = icmp sgt <2 x i32> %0, zeroinitializer569 %2 = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %src, i32 2, <2 x i1> %1, <2 x i32> undef)570 %3 = zext <2 x i32> %2 to <2 x i64>571 call void @llvm.masked.store.v2i64.p0(<2 x i64> %3, ptr %dest, i32 4, <2 x i1> %1)572 ret void573}574 575define void @foo_v8i16_v8i16(ptr %dest, ptr %mask, ptr %src) {576; CHECK-LABEL: foo_v8i16_v8i16:577; CHECK: @ %bb.0: @ %entry578; CHECK-NEXT: vldrh.u16 q0, [r1]579; CHECK-NEXT: vptt.s16 gt, q0, zr580; CHECK-NEXT: vldrht.u16 q0, [r2]581; CHECK-NEXT: vstrht.16 q0, [r0]582; CHECK-NEXT: bx lr583entry:584 %0 = load <8 x i16>, ptr %mask, align 2585 %1 = icmp sgt <8 x i16> %0, zeroinitializer586 %2 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %src, i32 2, <8 x i1> %1, <8 x i16> undef)587 call void @llvm.masked.store.v8i16.p0(<8 x i16> %2, ptr %dest, i32 2, <8 x i1> %1)588 ret void589}590 591define void @foo_sext_v8i16_v8i8(ptr %dest, ptr %mask, ptr %src) {592; CHECK-LABEL: foo_sext_v8i16_v8i8:593; CHECK: @ %bb.0: @ %entry594; CHECK-NEXT: vldrh.u16 q0, [r1]595; CHECK-NEXT: vptt.s16 gt, q0, zr596; CHECK-NEXT: vldrbt.s16 q0, [r2]597; CHECK-NEXT: vstrht.16 q0, [r0]598; CHECK-NEXT: bx lr599entry:600 %0 = load <8 x i16>, ptr %mask, align 2601 %1 = icmp sgt <8 x i16> %0, zeroinitializer602 %2 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr %src, i32 1, <8 x i1> %1, <8 x i8> undef)603 %3 = sext <8 x i8> %2 to <8 x i16>604 call void @llvm.masked.store.v8i16.p0(<8 x i16> %3, ptr %dest, i32 2, <8 x i1> %1)605 ret void606}607 608define void @foo_zext_v8i16_v8i8(ptr %dest, ptr %mask, ptr %src) {609; CHECK-LABEL: foo_zext_v8i16_v8i8:610; CHECK: @ %bb.0: @ %entry611; CHECK-NEXT: vldrh.u16 q0, [r1]612; CHECK-NEXT: vptt.s16 gt, q0, zr613; CHECK-NEXT: vldrbt.u16 q0, [r2]614; CHECK-NEXT: vstrht.16 q0, [r0]615; CHECK-NEXT: bx lr616entry:617 %0 = load <8 x i16>, ptr %mask, align 2618 %1 = icmp sgt <8 x i16> %0, zeroinitializer619 %2 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr %src, i32 1, <8 x i1> %1, <8 x i8> undef)620 %3 = zext <8 x i8> %2 to <8 x i16>621 call void @llvm.masked.store.v8i16.p0(<8 x i16> %3, ptr %dest, i32 2, <8 x i1> %1)622 ret void623}624 625define void @foo_v16i8_v16i8(ptr %dest, ptr %mask, ptr %src) {626; CHECK-LABEL: foo_v16i8_v16i8:627; CHECK: @ %bb.0: @ %entry628; CHECK-NEXT: vldrb.u8 q0, [r1]629; CHECK-NEXT: vptt.s8 gt, q0, zr630; CHECK-NEXT: vldrbt.u8 q0, [r2]631; CHECK-NEXT: vstrbt.8 q0, [r0]632; CHECK-NEXT: bx lr633entry:634 %0 = load <16 x i8>, ptr %mask, align 1635 %1 = icmp sgt <16 x i8> %0, zeroinitializer636 %2 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %src, i32 1, <16 x i1> %1, <16 x i8> undef)637 call void @llvm.masked.store.v16i8.p0(<16 x i8> %2, ptr %dest, i32 1, <16 x i1> %1)638 ret void639}640 641define void @foo_trunc_v8i8_v8i16(ptr %dest, ptr %mask, ptr %src) {642; CHECK-LABEL: foo_trunc_v8i8_v8i16:643; CHECK: @ %bb.0: @ %entry644; CHECK-NEXT: vldrh.u16 q0, [r1]645; CHECK-NEXT: vptt.s16 gt, q0, zr646; CHECK-NEXT: vldrht.u16 q0, [r2]647; CHECK-NEXT: vstrbt.16 q0, [r0]648; CHECK-NEXT: bx lr649entry:650 %0 = load <8 x i16>, ptr %mask, align 2651 %1 = icmp sgt <8 x i16> %0, zeroinitializer652 %2 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %src, i32 2, <8 x i1> %1, <8 x i16> undef)653 %3 = trunc <8 x i16> %2 to <8 x i8>654 call void @llvm.masked.store.v8i8.p0(<8 x i8> %3, ptr %dest, i32 1, <8 x i1> %1)655 ret void656}657 658define void @foo_trunc_v4i8_v4i32(ptr %dest, ptr %mask, ptr %src) {659; CHECK-LABEL: foo_trunc_v4i8_v4i32:660; CHECK: @ %bb.0: @ %entry661; CHECK-NEXT: vldrw.u32 q0, [r1]662; CHECK-NEXT: vptt.s32 gt, q0, zr663; CHECK-NEXT: vldrwt.u32 q0, [r2]664; CHECK-NEXT: vstrbt.32 q0, [r0]665; CHECK-NEXT: bx lr666entry:667 %0 = load <4 x i32>, ptr %mask, align 4668 %1 = icmp sgt <4 x i32> %0, zeroinitializer669 %2 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %src, i32 4, <4 x i1> %1, <4 x i32> undef)670 %3 = trunc <4 x i32> %2 to <4 x i8>671 call void @llvm.masked.store.v4i8.p0(<4 x i8> %3, ptr %dest, i32 1, <4 x i1> %1)672 ret void673}674 675define void @foo_trunc_v4i16_v4i32(ptr %dest, ptr %mask, ptr %src) {676; CHECK-LABEL: foo_trunc_v4i16_v4i32:677; CHECK: @ %bb.0: @ %entry678; CHECK-NEXT: vldrw.u32 q0, [r1]679; CHECK-NEXT: vptt.s32 gt, q0, zr680; CHECK-NEXT: vldrwt.u32 q0, [r2]681; CHECK-NEXT: vstrht.32 q0, [r0]682; CHECK-NEXT: bx lr683entry:684 %0 = load <4 x i32>, ptr %mask, align 4685 %1 = icmp sgt <4 x i32> %0, zeroinitializer686 %2 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %src, i32 4, <4 x i1> %1, <4 x i32> undef)687 %3 = trunc <4 x i32> %2 to <4 x i16>688 call void @llvm.masked.store.v4i16.p0(<4 x i16> %3, ptr %dest, i32 2, <4 x i1> %1)689 ret void690}691 692define void @foo_v4f32_v4f32(ptr %dest, ptr %mask, ptr %src) {693; CHECK-LABEL: foo_v4f32_v4f32:694; CHECK: @ %bb.0: @ %entry695; CHECK-NEXT: vldrw.u32 q0, [r1]696; CHECK-NEXT: vptt.s32 gt, q0, zr697; CHECK-NEXT: vldrwt.u32 q0, [r2]698; CHECK-NEXT: vstrwt.32 q0, [r0]699; CHECK-NEXT: bx lr700entry:701 %0 = load <4 x i32>, ptr %mask, align 4702 %1 = icmp sgt <4 x i32> %0, zeroinitializer703 %2 = call <4 x float> @llvm.masked.load.v4f32.p0(ptr %src, i32 4, <4 x i1> %1, <4 x float> undef)704 call void @llvm.masked.store.v4f32.p0(<4 x float> %2, ptr %dest, i32 4, <4 x i1> %1)705 ret void706}707 708define void @foo_v8f16_v8f16(ptr %dest, ptr %mask, ptr %src) {709; CHECK-LABEL: foo_v8f16_v8f16:710; CHECK: @ %bb.0: @ %entry711; CHECK-NEXT: vldrh.u16 q0, [r1]712; CHECK-NEXT: vptt.s16 gt, q0, zr713; CHECK-NEXT: vldrht.u16 q0, [r2]714; CHECK-NEXT: vstrht.16 q0, [r0]715; CHECK-NEXT: bx lr716entry:717 %0 = load <8 x i16>, ptr %mask, align 2718 %1 = icmp sgt <8 x i16> %0, zeroinitializer719 %2 = call <8 x half> @llvm.masked.load.v8f16.p0(ptr %src, i32 2, <8 x i1> %1, <8 x half> undef)720 call void @llvm.masked.store.v8f16.p0(<8 x half> %2, ptr %dest, i32 2, <8 x i1> %1)721 ret void722}723 724define void @foo_v4f32_v4f16(ptr %dest, ptr %mask, ptr %src) {725; CHECK-LE-LABEL: foo_v4f32_v4f16:726; CHECK-LE: @ %bb.0: @ %entry727; CHECK-LE-NEXT: .save {r7, lr}728; CHECK-LE-NEXT: push {r7, lr}729; CHECK-LE-NEXT: .pad #4730; CHECK-LE-NEXT: sub sp, #4731; CHECK-LE-NEXT: vldrh.s32 q0, [r1]732; CHECK-LE-NEXT: vcmp.s32 gt, q0, zr733; CHECK-LE-NEXT: @ implicit-def: $q0734; CHECK-LE-NEXT: vmrs lr, p0735; CHECK-LE-NEXT: and r1, lr, #1736; CHECK-LE-NEXT: ubfx r3, lr, #4, #1737; CHECK-LE-NEXT: rsb.w r12, r1, #0738; CHECK-LE-NEXT: movs r1, #0739; CHECK-LE-NEXT: rsbs r3, r3, #0740; CHECK-LE-NEXT: bfi r1, r12, #0, #1741; CHECK-LE-NEXT: bfi r1, r3, #1, #1742; CHECK-LE-NEXT: ubfx r3, lr, #8, #1743; CHECK-LE-NEXT: rsbs r3, r3, #0744; CHECK-LE-NEXT: bfi r1, r3, #2, #1745; CHECK-LE-NEXT: ubfx r3, lr, #12, #1746; CHECK-LE-NEXT: rsbs r3, r3, #0747; CHECK-LE-NEXT: bfi r1, r3, #3, #1748; CHECK-LE-NEXT: lsls r3, r1, #31749; CHECK-LE-NEXT: bne .LBB18_6750; CHECK-LE-NEXT: @ %bb.1: @ %else751; CHECK-LE-NEXT: lsls r3, r1, #30752; CHECK-LE-NEXT: bmi .LBB18_7753; CHECK-LE-NEXT: .LBB18_2: @ %else2754; CHECK-LE-NEXT: lsls r3, r1, #29755; CHECK-LE-NEXT: bmi .LBB18_8756; CHECK-LE-NEXT: .LBB18_3: @ %else5757; CHECK-LE-NEXT: lsls r1, r1, #28758; CHECK-LE-NEXT: bpl .LBB18_5759; CHECK-LE-NEXT: .LBB18_4: @ %cond.load7760; CHECK-LE-NEXT: vldr.16 s2, [r2, #6]761; CHECK-LE-NEXT: vins.f16 s1, s2762; CHECK-LE-NEXT: .LBB18_5: @ %else8763; CHECK-LE-NEXT: vmrs r2, p0764; CHECK-LE-NEXT: movs r1, #0765; CHECK-LE-NEXT: vcvtt.f32.f16 s6, s0766; CHECK-LE-NEXT: vcvtb.f32.f16 s0, s0767; CHECK-LE-NEXT: vcvtt.f32.f16 s2, s1768; CHECK-LE-NEXT: vcvtb.f32.f16 s4, s1769; CHECK-LE-NEXT: and r3, r2, #1770; CHECK-LE-NEXT: rsbs r3, r3, #0771; CHECK-LE-NEXT: bfi r1, r3, #0, #1772; CHECK-LE-NEXT: ubfx r3, r2, #4, #1773; CHECK-LE-NEXT: rsbs r3, r3, #0774; CHECK-LE-NEXT: bfi r1, r3, #1, #1775; CHECK-LE-NEXT: ubfx r3, r2, #8, #1776; CHECK-LE-NEXT: ubfx r2, r2, #12, #1777; CHECK-LE-NEXT: rsbs r3, r3, #0778; CHECK-LE-NEXT: bfi r1, r3, #2, #1779; CHECK-LE-NEXT: rsbs r2, r2, #0780; CHECK-LE-NEXT: bfi r1, r2, #3, #1781; CHECK-LE-NEXT: lsls r2, r1, #31782; CHECK-LE-NEXT: itt ne783; CHECK-LE-NEXT: vmovne r2, s0784; CHECK-LE-NEXT: strne r2, [r0]785; CHECK-LE-NEXT: lsls r2, r1, #30786; CHECK-LE-NEXT: itt mi787; CHECK-LE-NEXT: vmovmi r2, s6788; CHECK-LE-NEXT: strmi r2, [r0, #4]789; CHECK-LE-NEXT: lsls r2, r1, #29790; CHECK-LE-NEXT: itt mi791; CHECK-LE-NEXT: vmovmi r2, s4792; CHECK-LE-NEXT: strmi r2, [r0, #8]793; CHECK-LE-NEXT: lsls r1, r1, #28794; CHECK-LE-NEXT: itt mi795; CHECK-LE-NEXT: vmovmi r1, s2796; CHECK-LE-NEXT: strmi r1, [r0, #12]797; CHECK-LE-NEXT: add sp, #4798; CHECK-LE-NEXT: pop {r7, pc}799; CHECK-LE-NEXT: .LBB18_6: @ %cond.load800; CHECK-LE-NEXT: vldr.16 s0, [r2]801; CHECK-LE-NEXT: lsls r3, r1, #30802; CHECK-LE-NEXT: bpl .LBB18_2803; CHECK-LE-NEXT: .LBB18_7: @ %cond.load1804; CHECK-LE-NEXT: vldr.16 s2, [r2, #2]805; CHECK-LE-NEXT: vins.f16 s0, s2806; CHECK-LE-NEXT: lsls r3, r1, #29807; CHECK-LE-NEXT: bpl .LBB18_3808; CHECK-LE-NEXT: .LBB18_8: @ %cond.load4809; CHECK-LE-NEXT: vldr.16 s1, [r2, #4]810; CHECK-LE-NEXT: vmovx.f16 s2, s0811; CHECK-LE-NEXT: vins.f16 s1, s2812; CHECK-LE-NEXT: lsls r1, r1, #28813; CHECK-LE-NEXT: bmi .LBB18_4814; CHECK-LE-NEXT: b .LBB18_5815;816; CHECK-BE-LABEL: foo_v4f32_v4f16:817; CHECK-BE: @ %bb.0: @ %entry818; CHECK-BE-NEXT: .save {r7, lr}819; CHECK-BE-NEXT: push {r7, lr}820; CHECK-BE-NEXT: .pad #4821; CHECK-BE-NEXT: sub sp, #4822; CHECK-BE-NEXT: vldrh.s32 q0, [r1]823; CHECK-BE-NEXT: vcmp.s32 gt, q0, zr824; CHECK-BE-NEXT: @ implicit-def: $q0825; CHECK-BE-NEXT: vmrs lr, p0826; CHECK-BE-NEXT: ubfx r1, lr, #12, #1827; CHECK-BE-NEXT: ubfx r3, lr, #8, #1828; CHECK-BE-NEXT: rsb.w r12, r1, #0829; CHECK-BE-NEXT: movs r1, #0830; CHECK-BE-NEXT: bfi r1, r12, #0, #1831; CHECK-BE-NEXT: rsbs r3, r3, #0832; CHECK-BE-NEXT: bfi r1, r3, #1, #1833; CHECK-BE-NEXT: ubfx r3, lr, #4, #1834; CHECK-BE-NEXT: rsbs r3, r3, #0835; CHECK-BE-NEXT: bfi r1, r3, #2, #1836; CHECK-BE-NEXT: and r3, lr, #1837; CHECK-BE-NEXT: rsbs r3, r3, #0838; CHECK-BE-NEXT: bfi r1, r3, #3, #1839; CHECK-BE-NEXT: lsls r3, r1, #28840; CHECK-BE-NEXT: bmi .LBB18_6841; CHECK-BE-NEXT: @ %bb.1: @ %else842; CHECK-BE-NEXT: lsls r3, r1, #29843; CHECK-BE-NEXT: bmi .LBB18_7844; CHECK-BE-NEXT: .LBB18_2: @ %else2845; CHECK-BE-NEXT: lsls r3, r1, #30846; CHECK-BE-NEXT: bmi .LBB18_8847; CHECK-BE-NEXT: .LBB18_3: @ %else5848; CHECK-BE-NEXT: lsls r1, r1, #31849; CHECK-BE-NEXT: beq .LBB18_5850; CHECK-BE-NEXT: .LBB18_4: @ %cond.load7851; CHECK-BE-NEXT: vldr.16 s2, [r2, #6]852; CHECK-BE-NEXT: vins.f16 s1, s2853; CHECK-BE-NEXT: .LBB18_5: @ %else8854; CHECK-BE-NEXT: vmrs r2, p0855; CHECK-BE-NEXT: movs r1, #0856; CHECK-BE-NEXT: vcvtt.f32.f16 s6, s0857; CHECK-BE-NEXT: vcvtb.f32.f16 s0, s0858; CHECK-BE-NEXT: vcvtt.f32.f16 s2, s1859; CHECK-BE-NEXT: vcvtb.f32.f16 s4, s1860; CHECK-BE-NEXT: ubfx r3, r2, #12, #1861; CHECK-BE-NEXT: rsbs r3, r3, #0862; CHECK-BE-NEXT: bfi r1, r3, #0, #1863; CHECK-BE-NEXT: ubfx r3, r2, #8, #1864; CHECK-BE-NEXT: rsbs r3, r3, #0865; CHECK-BE-NEXT: bfi r1, r3, #1, #1866; CHECK-BE-NEXT: ubfx r3, r2, #4, #1867; CHECK-BE-NEXT: and r2, r2, #1868; CHECK-BE-NEXT: rsbs r3, r3, #0869; CHECK-BE-NEXT: bfi r1, r3, #2, #1870; CHECK-BE-NEXT: rsbs r2, r2, #0871; CHECK-BE-NEXT: bfi r1, r2, #3, #1872; CHECK-BE-NEXT: lsls r2, r1, #28873; CHECK-BE-NEXT: itt mi874; CHECK-BE-NEXT: vmovmi r2, s0875; CHECK-BE-NEXT: strmi r2, [r0]876; CHECK-BE-NEXT: lsls r2, r1, #29877; CHECK-BE-NEXT: itt mi878; CHECK-BE-NEXT: vmovmi r2, s6879; CHECK-BE-NEXT: strmi r2, [r0, #4]880; CHECK-BE-NEXT: lsls r2, r1, #30881; CHECK-BE-NEXT: itt mi882; CHECK-BE-NEXT: vmovmi r2, s4883; CHECK-BE-NEXT: strmi r2, [r0, #8]884; CHECK-BE-NEXT: lsls r1, r1, #31885; CHECK-BE-NEXT: itt ne886; CHECK-BE-NEXT: vmovne r1, s2887; CHECK-BE-NEXT: strne r1, [r0, #12]888; CHECK-BE-NEXT: add sp, #4889; CHECK-BE-NEXT: pop {r7, pc}890; CHECK-BE-NEXT: .LBB18_6: @ %cond.load891; CHECK-BE-NEXT: vldr.16 s0, [r2]892; CHECK-BE-NEXT: lsls r3, r1, #29893; CHECK-BE-NEXT: bpl .LBB18_2894; CHECK-BE-NEXT: .LBB18_7: @ %cond.load1895; CHECK-BE-NEXT: vldr.16 s2, [r2, #2]896; CHECK-BE-NEXT: vins.f16 s0, s2897; CHECK-BE-NEXT: lsls r3, r1, #30898; CHECK-BE-NEXT: bpl .LBB18_3899; CHECK-BE-NEXT: .LBB18_8: @ %cond.load4900; CHECK-BE-NEXT: vldr.16 s1, [r2, #4]901; CHECK-BE-NEXT: vmovx.f16 s2, s0902; CHECK-BE-NEXT: vins.f16 s1, s2903; CHECK-BE-NEXT: lsls r1, r1, #31904; CHECK-BE-NEXT: bne .LBB18_4905; CHECK-BE-NEXT: b .LBB18_5906entry:907 %0 = load <4 x i16>, ptr %mask, align 2908 %1 = icmp sgt <4 x i16> %0, zeroinitializer909 %2 = call <4 x half> @llvm.masked.load.v4f16.p0(ptr %src, i32 2, <4 x i1> %1, <4 x half> undef)910 %3 = fpext <4 x half> %2 to <4 x float>911 call void @llvm.masked.store.v4f32.p0(<4 x float> %3, ptr %dest, i32 2, <4 x i1> %1)912 ret void913}914 915define void @foo_v4f32_v4f16_unaligned(ptr %dest, ptr %mask, ptr %src) {916; CHECK-LE-LABEL: foo_v4f32_v4f16_unaligned:917; CHECK-LE: @ %bb.0: @ %entry918; CHECK-LE-NEXT: .save {r7, lr}919; CHECK-LE-NEXT: push {r7, lr}920; CHECK-LE-NEXT: .pad #4921; CHECK-LE-NEXT: sub sp, #4922; CHECK-LE-NEXT: vldrh.s32 q0, [r1]923; CHECK-LE-NEXT: vcmp.s32 gt, q0, zr924; CHECK-LE-NEXT: @ implicit-def: $q0925; CHECK-LE-NEXT: vmrs lr, p0926; CHECK-LE-NEXT: and r1, lr, #1927; CHECK-LE-NEXT: ubfx r3, lr, #4, #1928; CHECK-LE-NEXT: rsb.w r12, r1, #0929; CHECK-LE-NEXT: movs r1, #0930; CHECK-LE-NEXT: rsbs r3, r3, #0931; CHECK-LE-NEXT: bfi r1, r12, #0, #1932; CHECK-LE-NEXT: bfi r1, r3, #1, #1933; CHECK-LE-NEXT: ubfx r3, lr, #8, #1934; CHECK-LE-NEXT: rsbs r3, r3, #0935; CHECK-LE-NEXT: bfi r1, r3, #2, #1936; CHECK-LE-NEXT: ubfx r3, lr, #12, #1937; CHECK-LE-NEXT: rsbs r3, r3, #0938; CHECK-LE-NEXT: bfi r1, r3, #3, #1939; CHECK-LE-NEXT: lsls r3, r1, #31940; CHECK-LE-NEXT: bne .LBB19_6941; CHECK-LE-NEXT: @ %bb.1: @ %else942; CHECK-LE-NEXT: lsls r3, r1, #30943; CHECK-LE-NEXT: bmi .LBB19_7944; CHECK-LE-NEXT: .LBB19_2: @ %else2945; CHECK-LE-NEXT: lsls r3, r1, #29946; CHECK-LE-NEXT: bmi .LBB19_8947; CHECK-LE-NEXT: .LBB19_3: @ %else5948; CHECK-LE-NEXT: lsls r1, r1, #28949; CHECK-LE-NEXT: bpl .LBB19_5950; CHECK-LE-NEXT: .LBB19_4: @ %cond.load7951; CHECK-LE-NEXT: vldr.16 s2, [r2, #6]952; CHECK-LE-NEXT: vins.f16 s1, s2953; CHECK-LE-NEXT: .LBB19_5: @ %else8954; CHECK-LE-NEXT: vmrs r2, p0955; CHECK-LE-NEXT: movs r1, #0956; CHECK-LE-NEXT: vcvtt.f32.f16 s6, s0957; CHECK-LE-NEXT: vcvtb.f32.f16 s0, s0958; CHECK-LE-NEXT: vcvtt.f32.f16 s2, s1959; CHECK-LE-NEXT: vcvtb.f32.f16 s4, s1960; CHECK-LE-NEXT: and r3, r2, #1961; CHECK-LE-NEXT: rsbs r3, r3, #0962; CHECK-LE-NEXT: bfi r1, r3, #0, #1963; CHECK-LE-NEXT: ubfx r3, r2, #4, #1964; CHECK-LE-NEXT: rsbs r3, r3, #0965; CHECK-LE-NEXT: bfi r1, r3, #1, #1966; CHECK-LE-NEXT: ubfx r3, r2, #8, #1967; CHECK-LE-NEXT: ubfx r2, r2, #12, #1968; CHECK-LE-NEXT: rsbs r3, r3, #0969; CHECK-LE-NEXT: bfi r1, r3, #2, #1970; CHECK-LE-NEXT: rsbs r2, r2, #0971; CHECK-LE-NEXT: bfi r1, r2, #3, #1972; CHECK-LE-NEXT: lsls r2, r1, #31973; CHECK-LE-NEXT: itt ne974; CHECK-LE-NEXT: vmovne r2, s0975; CHECK-LE-NEXT: strne r2, [r0]976; CHECK-LE-NEXT: lsls r2, r1, #30977; CHECK-LE-NEXT: itt mi978; CHECK-LE-NEXT: vmovmi r2, s6979; CHECK-LE-NEXT: strmi r2, [r0, #4]980; CHECK-LE-NEXT: lsls r2, r1, #29981; CHECK-LE-NEXT: itt mi982; CHECK-LE-NEXT: vmovmi r2, s4983; CHECK-LE-NEXT: strmi r2, [r0, #8]984; CHECK-LE-NEXT: lsls r1, r1, #28985; CHECK-LE-NEXT: itt mi986; CHECK-LE-NEXT: vmovmi r1, s2987; CHECK-LE-NEXT: strmi r1, [r0, #12]988; CHECK-LE-NEXT: add sp, #4989; CHECK-LE-NEXT: pop {r7, pc}990; CHECK-LE-NEXT: .LBB19_6: @ %cond.load991; CHECK-LE-NEXT: vldr.16 s0, [r2]992; CHECK-LE-NEXT: lsls r3, r1, #30993; CHECK-LE-NEXT: bpl .LBB19_2994; CHECK-LE-NEXT: .LBB19_7: @ %cond.load1995; CHECK-LE-NEXT: vldr.16 s2, [r2, #2]996; CHECK-LE-NEXT: vins.f16 s0, s2997; CHECK-LE-NEXT: lsls r3, r1, #29998; CHECK-LE-NEXT: bpl .LBB19_3999; CHECK-LE-NEXT: .LBB19_8: @ %cond.load41000; CHECK-LE-NEXT: vldr.16 s1, [r2, #4]1001; CHECK-LE-NEXT: vmovx.f16 s2, s01002; CHECK-LE-NEXT: vins.f16 s1, s21003; CHECK-LE-NEXT: lsls r1, r1, #281004; CHECK-LE-NEXT: bmi .LBB19_41005; CHECK-LE-NEXT: b .LBB19_51006;1007; CHECK-BE-LABEL: foo_v4f32_v4f16_unaligned:1008; CHECK-BE: @ %bb.0: @ %entry1009; CHECK-BE-NEXT: .save {r7, lr}1010; CHECK-BE-NEXT: push {r7, lr}1011; CHECK-BE-NEXT: .pad #41012; CHECK-BE-NEXT: sub sp, #41013; CHECK-BE-NEXT: vldrh.s32 q0, [r1]1014; CHECK-BE-NEXT: vcmp.s32 gt, q0, zr1015; CHECK-BE-NEXT: @ implicit-def: $q01016; CHECK-BE-NEXT: vmrs lr, p01017; CHECK-BE-NEXT: ubfx r1, lr, #12, #11018; CHECK-BE-NEXT: ubfx r3, lr, #8, #11019; CHECK-BE-NEXT: rsb.w r12, r1, #01020; CHECK-BE-NEXT: movs r1, #01021; CHECK-BE-NEXT: bfi r1, r12, #0, #11022; CHECK-BE-NEXT: rsbs r3, r3, #01023; CHECK-BE-NEXT: bfi r1, r3, #1, #11024; CHECK-BE-NEXT: ubfx r3, lr, #4, #11025; CHECK-BE-NEXT: rsbs r3, r3, #01026; CHECK-BE-NEXT: bfi r1, r3, #2, #11027; CHECK-BE-NEXT: and r3, lr, #11028; CHECK-BE-NEXT: rsbs r3, r3, #01029; CHECK-BE-NEXT: bfi r1, r3, #3, #11030; CHECK-BE-NEXT: lsls r3, r1, #281031; CHECK-BE-NEXT: bmi .LBB19_61032; CHECK-BE-NEXT: @ %bb.1: @ %else1033; CHECK-BE-NEXT: lsls r3, r1, #291034; CHECK-BE-NEXT: bmi .LBB19_71035; CHECK-BE-NEXT: .LBB19_2: @ %else21036; CHECK-BE-NEXT: lsls r3, r1, #301037; CHECK-BE-NEXT: bmi .LBB19_81038; CHECK-BE-NEXT: .LBB19_3: @ %else51039; CHECK-BE-NEXT: lsls r1, r1, #311040; CHECK-BE-NEXT: beq .LBB19_51041; CHECK-BE-NEXT: .LBB19_4: @ %cond.load71042; CHECK-BE-NEXT: vldr.16 s2, [r2, #6]1043; CHECK-BE-NEXT: vins.f16 s1, s21044; CHECK-BE-NEXT: .LBB19_5: @ %else81045; CHECK-BE-NEXT: vmrs r2, p01046; CHECK-BE-NEXT: movs r1, #01047; CHECK-BE-NEXT: vcvtt.f32.f16 s6, s01048; CHECK-BE-NEXT: vcvtb.f32.f16 s0, s01049; CHECK-BE-NEXT: vcvtt.f32.f16 s2, s11050; CHECK-BE-NEXT: vcvtb.f32.f16 s4, s11051; CHECK-BE-NEXT: ubfx r3, r2, #12, #11052; CHECK-BE-NEXT: rsbs r3, r3, #01053; CHECK-BE-NEXT: bfi r1, r3, #0, #11054; CHECK-BE-NEXT: ubfx r3, r2, #8, #11055; CHECK-BE-NEXT: rsbs r3, r3, #01056; CHECK-BE-NEXT: bfi r1, r3, #1, #11057; CHECK-BE-NEXT: ubfx r3, r2, #4, #11058; CHECK-BE-NEXT: and r2, r2, #11059; CHECK-BE-NEXT: rsbs r3, r3, #01060; CHECK-BE-NEXT: bfi r1, r3, #2, #11061; CHECK-BE-NEXT: rsbs r2, r2, #01062; CHECK-BE-NEXT: bfi r1, r2, #3, #11063; CHECK-BE-NEXT: lsls r2, r1, #281064; CHECK-BE-NEXT: itt mi1065; CHECK-BE-NEXT: vmovmi r2, s01066; CHECK-BE-NEXT: strmi r2, [r0]1067; CHECK-BE-NEXT: lsls r2, r1, #291068; CHECK-BE-NEXT: itt mi1069; CHECK-BE-NEXT: vmovmi r2, s61070; CHECK-BE-NEXT: strmi r2, [r0, #4]1071; CHECK-BE-NEXT: lsls r2, r1, #301072; CHECK-BE-NEXT: itt mi1073; CHECK-BE-NEXT: vmovmi r2, s41074; CHECK-BE-NEXT: strmi r2, [r0, #8]1075; CHECK-BE-NEXT: lsls r1, r1, #311076; CHECK-BE-NEXT: itt ne1077; CHECK-BE-NEXT: vmovne r1, s21078; CHECK-BE-NEXT: strne r1, [r0, #12]1079; CHECK-BE-NEXT: add sp, #41080; CHECK-BE-NEXT: pop {r7, pc}1081; CHECK-BE-NEXT: .LBB19_6: @ %cond.load1082; CHECK-BE-NEXT: vldr.16 s0, [r2]1083; CHECK-BE-NEXT: lsls r3, r1, #291084; CHECK-BE-NEXT: bpl .LBB19_21085; CHECK-BE-NEXT: .LBB19_7: @ %cond.load11086; CHECK-BE-NEXT: vldr.16 s2, [r2, #2]1087; CHECK-BE-NEXT: vins.f16 s0, s21088; CHECK-BE-NEXT: lsls r3, r1, #301089; CHECK-BE-NEXT: bpl .LBB19_31090; CHECK-BE-NEXT: .LBB19_8: @ %cond.load41091; CHECK-BE-NEXT: vldr.16 s1, [r2, #4]1092; CHECK-BE-NEXT: vmovx.f16 s2, s01093; CHECK-BE-NEXT: vins.f16 s1, s21094; CHECK-BE-NEXT: lsls r1, r1, #311095; CHECK-BE-NEXT: bne .LBB19_41096; CHECK-BE-NEXT: b .LBB19_51097entry:1098 %0 = load <4 x i16>, ptr %mask, align 21099 %1 = icmp sgt <4 x i16> %0, zeroinitializer1100 %2 = call <4 x half> @llvm.masked.load.v4f16.p0(ptr %src, i32 2, <4 x i1> %1, <4 x half> undef)1101 %3 = fpext <4 x half> %2 to <4 x float>1102 call void @llvm.masked.store.v4f32.p0(<4 x float> %3, ptr %dest, i32 1, <4 x i1> %1)1103 ret void1104}1105 1106declare void @llvm.masked.store.v4i32.p0(<4 x i32>, ptr, i32, <4 x i1>)1107declare void @llvm.masked.store.v8i16.p0(<8 x i16>, ptr, i32, <8 x i1>)1108declare void @llvm.masked.store.v16i8.p0(<16 x i8>, ptr, i32, <16 x i1>)1109declare void @llvm.masked.store.v8f16.p0(<8 x half>, ptr, i32, <8 x i1>)1110declare void @llvm.masked.store.v4f32.p0(<4 x float>, ptr, i32, <4 x i1>)1111declare <16 x i8> @llvm.masked.load.v16i8.p0(ptr, i32, <16 x i1>, <16 x i8>)1112declare <8 x i16> @llvm.masked.load.v8i16.p0(ptr, i32, <8 x i1>, <8 x i16>)1113declare <2 x i32> @llvm.masked.load.v2i32.p0(ptr, i32, <2 x i1>, <2 x i32>)1114declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32, <4 x i1>, <4 x i32>)1115declare <4 x float> @llvm.masked.load.v4f32.p0(ptr, i32, <4 x i1>, <4 x float>)1116declare <4 x half> @llvm.masked.load.v4f16.p0(ptr, i32, <4 x i1>, <4 x half>)1117declare <8 x half> @llvm.masked.load.v8f16.p0(ptr, i32, <8 x i1>, <8 x half>)1118 1119declare void @llvm.masked.store.v8i8.p0(<8 x i8>, ptr, i32, <8 x i1>)1120declare void @llvm.masked.store.v4i8.p0(<4 x i8>, ptr, i32, <4 x i1>)1121declare void @llvm.masked.store.v4i16.p0(<4 x i16>, ptr, i32, <4 x i1>)1122declare void @llvm.masked.store.v2i64.p0(<2 x i64>, ptr, i32, <2 x i1>)1123declare <4 x i16> @llvm.masked.load.v4i16.p0(ptr, i32, <4 x i1>, <4 x i16>)1124declare <4 x i8> @llvm.masked.load.v4i8.p0(ptr, i32, <4 x i1>, <4 x i8>)1125declare <8 x i8> @llvm.masked.load.v8i8.p0(ptr, i32, <8 x i1>, <8 x i8>)1126