499 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=arm-eabi -mattr=+v8.6a,+neon -float-abi=hard < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP163; RUN: llc -mtriple=arm-eabi -mattr=+v8.6a,+neon,+bf16 -float-abi=hard < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP164; RUN: llc -mtriple=arm-eabi -mattr=+v8.6a,+neon,+fullfp16,+bf16 -float-abi=hard < %s | FileCheck %s --check-prefixes=CHECK,CHECK-FP165 6%struct.float16x4x2_t = type { [2 x <4 x bfloat>] }7%struct.float16x8x2_t = type { [2 x <8 x bfloat>] }8 9define dso_local <4 x bfloat> @test_vbsl_bf16(<4 x i16> %a, <4 x bfloat> %b, <4 x bfloat> %c) {10; CHECK-LABEL: test_vbsl_bf16:11; CHECK: @ %bb.0: @ %entry12; CHECK-NEXT: vbsl d0, d1, d213; CHECK-NEXT: bx lr14entry:15 %0 = bitcast <4 x i16> %a to <8 x i8>16 %1 = bitcast <4 x bfloat> %b to <8 x i8>17 %2 = bitcast <4 x bfloat> %c to <8 x i8>18 %vbsl_v.i = tail call <8 x i8> @llvm.arm.neon.vbsl.v8i8(<8 x i8> %0, <8 x i8> %1, <8 x i8> %2)19 %3 = bitcast <8 x i8> %vbsl_v.i to <4 x bfloat>20 ret <4 x bfloat> %321}22 23define dso_local <8 x bfloat> @test_vbslq_bf16(<8 x i16> %a, <8 x bfloat> %b, <8 x bfloat> %c) {24; CHECK-LABEL: test_vbslq_bf16:25; CHECK: @ %bb.0: @ %entry26; CHECK-NEXT: vbsl q0, q1, q227; CHECK-NEXT: bx lr28entry:29 %0 = bitcast <8 x i16> %a to <16 x i8>30 %1 = bitcast <8 x bfloat> %b to <16 x i8>31 %2 = bitcast <8 x bfloat> %c to <16 x i8>32 %vbslq_v.i = tail call <16 x i8> @llvm.arm.neon.vbsl.v16i8(<16 x i8> %0, <16 x i8> %1, <16 x i8> %2)33 %3 = bitcast <16 x i8> %vbslq_v.i to <8 x bfloat>34 ret <8 x bfloat> %335}36 37define dso_local %struct.float16x4x2_t @test_vzip_bf16(<4 x bfloat> %a, <4 x bfloat> %b) {38; CHECK-LABEL: test_vzip_bf16:39; CHECK: @ %bb.0: @ %entry40; CHECK-NEXT: vzip.16 d0, d141; CHECK-NEXT: bx lr42entry:43 %vzip.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 0, i32 4, i32 1, i32 5>44 %vzip1.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 2, i32 6, i32 3, i32 7>45 %.fca.0.0.insert = insertvalue %struct.float16x4x2_t undef, <4 x bfloat> %vzip.i, 0, 046 %.fca.0.1.insert = insertvalue %struct.float16x4x2_t %.fca.0.0.insert, <4 x bfloat> %vzip1.i, 0, 147 ret %struct.float16x4x2_t %.fca.0.1.insert48}49 50define dso_local %struct.float16x8x2_t @test_vzipq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) {51; CHECK-LABEL: test_vzipq_bf16:52; CHECK: @ %bb.0: @ %entry53; CHECK-NEXT: vzip.16 q0, q154; CHECK-NEXT: bx lr55entry:56 %vzip.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>57 %vzip1.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>58 %.fca.0.0.insert = insertvalue %struct.float16x8x2_t undef, <8 x bfloat> %vzip.i, 0, 059 %.fca.0.1.insert = insertvalue %struct.float16x8x2_t %.fca.0.0.insert, <8 x bfloat> %vzip1.i, 0, 160 ret %struct.float16x8x2_t %.fca.0.1.insert61}62 63define dso_local %struct.float16x4x2_t @test_vuzp_bf16(<4 x bfloat> %a, <4 x bfloat> %b) {64; CHECK-LABEL: test_vuzp_bf16:65; CHECK: @ %bb.0: @ %entry66; CHECK-NEXT: vuzp.16 d0, d167; CHECK-NEXT: bx lr68entry:69 %vuzp.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 6>70 %vuzp1.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 1, i32 3, i32 5, i32 7>71 %.fca.0.0.insert = insertvalue %struct.float16x4x2_t undef, <4 x bfloat> %vuzp.i, 0, 072 %.fca.0.1.insert = insertvalue %struct.float16x4x2_t %.fca.0.0.insert, <4 x bfloat> %vuzp1.i, 0, 173 ret %struct.float16x4x2_t %.fca.0.1.insert74}75 76define dso_local %struct.float16x8x2_t @test_vuzpq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) {77; CHECK-LABEL: test_vuzpq_bf16:78; CHECK: @ %bb.0: @ %entry79; CHECK-NEXT: vuzp.16 q0, q180; CHECK-NEXT: bx lr81entry:82 %vuzp.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>83 %vuzp1.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>84 %.fca.0.0.insert = insertvalue %struct.float16x8x2_t undef, <8 x bfloat> %vuzp.i, 0, 085 %.fca.0.1.insert = insertvalue %struct.float16x8x2_t %.fca.0.0.insert, <8 x bfloat> %vuzp1.i, 0, 186 ret %struct.float16x8x2_t %.fca.0.1.insert87}88 89define dso_local %struct.float16x4x2_t @test_vtrn_bf16(<4 x bfloat> %a, <4 x bfloat> %b) {90; CHECK-LABEL: test_vtrn_bf16:91; CHECK: @ %bb.0: @ %entry92; CHECK-NEXT: vtrn.16 d0, d193; CHECK-NEXT: bx lr94entry:95 %vtrn.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 0, i32 4, i32 2, i32 6>96 %vtrn1.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 1, i32 5, i32 3, i32 7>97 %.fca.0.0.insert = insertvalue %struct.float16x4x2_t undef, <4 x bfloat> %vtrn.i, 0, 098 %.fca.0.1.insert = insertvalue %struct.float16x4x2_t %.fca.0.0.insert, <4 x bfloat> %vtrn1.i, 0, 199 ret %struct.float16x4x2_t %.fca.0.1.insert100}101 102define dso_local %struct.float16x8x2_t @test_vtrnq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) {103; CHECK-LABEL: test_vtrnq_bf16:104; CHECK: @ %bb.0: @ %entry105; CHECK-NEXT: vtrn.16 q0, q1106; CHECK-NEXT: bx lr107entry:108 %vtrn.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>109 %vtrn1.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>110 %.fca.0.0.insert = insertvalue %struct.float16x8x2_t undef, <8 x bfloat> %vtrn.i, 0, 0111 %.fca.0.1.insert = insertvalue %struct.float16x8x2_t %.fca.0.0.insert, <8 x bfloat> %vtrn1.i, 0, 1112 ret %struct.float16x8x2_t %.fca.0.1.insert113}114 115define dso_local <4 x bfloat> @test_vmov_n_bf16(float %a.coerce) {116; CHECK-NOFP16-LABEL: test_vmov_n_bf16:117; CHECK-NOFP16: @ %bb.0: @ %entry118; CHECK-NOFP16-NEXT: .pad #4119; CHECK-NOFP16-NEXT: sub sp, sp, #4120; CHECK-NOFP16-NEXT: vmov r0, s0121; CHECK-NOFP16-NEXT: strh r0, [sp, #2]122; CHECK-NOFP16-NEXT: add r0, sp, #2123; CHECK-NOFP16-NEXT: vld1.16 {d0[]}, [r0:16]124; CHECK-NOFP16-NEXT: add sp, sp, #4125; CHECK-NOFP16-NEXT: bx lr126;127; CHECK-FP16-LABEL: test_vmov_n_bf16:128; CHECK-FP16: @ %bb.0: @ %entry129; CHECK-FP16-NEXT: @ kill: def $s0 killed $s0 def $d0130; CHECK-FP16-NEXT: vdup.16 d0, d0[0]131; CHECK-FP16-NEXT: bx lr132entry:133 %0 = bitcast float %a.coerce to i32134 %tmp.0.extract.trunc = trunc i32 %0 to i16135 %1 = bitcast i16 %tmp.0.extract.trunc to bfloat136 %vecinit = insertelement <4 x bfloat> undef, bfloat %1, i32 0137 %vecinit4 = shufflevector <4 x bfloat> %vecinit, <4 x bfloat> undef, <4 x i32> zeroinitializer138 ret <4 x bfloat> %vecinit4139}140 141define dso_local <8 x bfloat> @test_vmovq_n_bf16(float %a.coerce) {142; CHECK-NOFP16-LABEL: test_vmovq_n_bf16:143; CHECK-NOFP16: @ %bb.0: @ %entry144; CHECK-NOFP16-NEXT: .pad #4145; CHECK-NOFP16-NEXT: sub sp, sp, #4146; CHECK-NOFP16-NEXT: vmov r0, s0147; CHECK-NOFP16-NEXT: strh r0, [sp, #2]148; CHECK-NOFP16-NEXT: add r0, sp, #2149; CHECK-NOFP16-NEXT: vld1.16 {d0[], d1[]}, [r0:16]150; CHECK-NOFP16-NEXT: add sp, sp, #4151; CHECK-NOFP16-NEXT: bx lr152;153; CHECK-FP16-LABEL: test_vmovq_n_bf16:154; CHECK-FP16: @ %bb.0: @ %entry155; CHECK-FP16-NEXT: @ kill: def $s0 killed $s0 def $d0156; CHECK-FP16-NEXT: vdup.16 q0, d0[0]157; CHECK-FP16-NEXT: bx lr158entry:159 %0 = bitcast float %a.coerce to i32160 %tmp.0.extract.trunc = trunc i32 %0 to i16161 %1 = bitcast i16 %tmp.0.extract.trunc to bfloat162 %vecinit = insertelement <8 x bfloat> undef, bfloat %1, i32 0163 %vecinit8 = shufflevector <8 x bfloat> %vecinit, <8 x bfloat> undef, <8 x i32> zeroinitializer164 ret <8 x bfloat> %vecinit8165}166 167define dso_local <4 x bfloat> @test_vdup_n_bf16(float %a.coerce) {168; CHECK-NOFP16-LABEL: test_vdup_n_bf16:169; CHECK-NOFP16: @ %bb.0: @ %entry170; CHECK-NOFP16-NEXT: .pad #4171; CHECK-NOFP16-NEXT: sub sp, sp, #4172; CHECK-NOFP16-NEXT: vmov r0, s0173; CHECK-NOFP16-NEXT: strh r0, [sp, #2]174; CHECK-NOFP16-NEXT: add r0, sp, #2175; CHECK-NOFP16-NEXT: vld1.16 {d0[]}, [r0:16]176; CHECK-NOFP16-NEXT: add sp, sp, #4177; CHECK-NOFP16-NEXT: bx lr178;179; CHECK-FP16-LABEL: test_vdup_n_bf16:180; CHECK-FP16: @ %bb.0: @ %entry181; CHECK-FP16-NEXT: @ kill: def $s0 killed $s0 def $d0182; CHECK-FP16-NEXT: vdup.16 d0, d0[0]183; CHECK-FP16-NEXT: bx lr184entry:185 %0 = bitcast float %a.coerce to i32186 %tmp.0.extract.trunc = trunc i32 %0 to i16187 %1 = bitcast i16 %tmp.0.extract.trunc to bfloat188 %vecinit = insertelement <4 x bfloat> undef, bfloat %1, i32 0189 %vecinit4 = shufflevector <4 x bfloat> %vecinit, <4 x bfloat> undef, <4 x i32> zeroinitializer190 ret <4 x bfloat> %vecinit4191}192 193define dso_local <8 x bfloat> @test_vdupq_n_bf16(float %a.coerce) {194; CHECK-NOFP16-LABEL: test_vdupq_n_bf16:195; CHECK-NOFP16: @ %bb.0: @ %entry196; CHECK-NOFP16-NEXT: .pad #4197; CHECK-NOFP16-NEXT: sub sp, sp, #4198; CHECK-NOFP16-NEXT: vmov r0, s0199; CHECK-NOFP16-NEXT: strh r0, [sp, #2]200; CHECK-NOFP16-NEXT: add r0, sp, #2201; CHECK-NOFP16-NEXT: vld1.16 {d0[], d1[]}, [r0:16]202; CHECK-NOFP16-NEXT: add sp, sp, #4203; CHECK-NOFP16-NEXT: bx lr204;205; CHECK-FP16-LABEL: test_vdupq_n_bf16:206; CHECK-FP16: @ %bb.0: @ %entry207; CHECK-FP16-NEXT: @ kill: def $s0 killed $s0 def $d0208; CHECK-FP16-NEXT: vdup.16 q0, d0[0]209; CHECK-FP16-NEXT: bx lr210entry:211 %0 = bitcast float %a.coerce to i32212 %tmp.0.extract.trunc = trunc i32 %0 to i16213 %1 = bitcast i16 %tmp.0.extract.trunc to bfloat214 %vecinit = insertelement <8 x bfloat> undef, bfloat %1, i32 0215 %vecinit8 = shufflevector <8 x bfloat> %vecinit, <8 x bfloat> undef, <8 x i32> zeroinitializer216 ret <8 x bfloat> %vecinit8217}218 219define dso_local <4 x bfloat> @test_vdup_lane_bf16(<4 x bfloat> %a) {220; CHECK-LABEL: test_vdup_lane_bf16:221; CHECK: @ %bb.0: @ %entry222; CHECK-NEXT: vdup.16 d0, d0[3]223; CHECK-NEXT: bx lr224entry:225 %shuffle = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>226 ret <4 x bfloat> %shuffle227}228 229define dso_local <8 x bfloat> @test_vdupq_lane_bf16(<4 x bfloat> %a) {230; CHECK-LABEL: test_vdupq_lane_bf16:231; CHECK: @ %bb.0: @ %entry232; CHECK-NEXT: vdup.16 q0, d0[3]233; CHECK-NEXT: bx lr234entry:235 %shuffle = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>236 ret <8 x bfloat> %shuffle237}238 239define dso_local <4 x bfloat> @test_vext_bf16(<4 x bfloat> %a, <4 x bfloat> %b) {240; CHECK-LABEL: test_vext_bf16:241; CHECK: @ %bb.0: @ %entry242; CHECK-NEXT: vext.16 d0, d0, d1, #2243; CHECK-NEXT: bx lr244entry:245 %vext = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 2, i32 3, i32 4, i32 5>246 ret <4 x bfloat> %vext247}248 249define dso_local <8 x bfloat> @test_vextq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) {250; CHECK-LABEL: test_vextq_bf16:251; CHECK: @ %bb.0: @ %entry252; CHECK-NEXT: vext.16 q0, q0, q1, #5253; CHECK-NEXT: bx lr254entry:255 %vext = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12>256 ret <8 x bfloat> %vext257}258 259define dso_local <4 x bfloat> @test_vext_aligned_bf16(<8 x bfloat> %a) {260; CHECK-LABEL: test_vext_aligned_bf16:261; CHECK: @ %bb.0: @ %entry262; CHECK-NEXT: vmov.f64 d0, d1263; CHECK-NEXT: bx lr264entry:265 %vext = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>266 ret <4 x bfloat> %vext267}268 269define dso_local <4 x bfloat> @test_vext_unaligned_bf16(<8 x bfloat> %a) {270; CHECK-LABEL: test_vext_unaligned_bf16:271; CHECK: @ %bb.0: @ %entry272; CHECK-NEXT: vext.16 d0, d0, d1, #3273; CHECK-NEXT: bx lr274entry:275 %vext = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <4 x i32> <i32 3, i32 4, i32 5, i32 6>276 ret <4 x bfloat> %vext277}278 279define arm_aapcs_vfpcc <8 x bfloat> @shuffle3step0_bf16(<32 x bfloat> %src) {280; CHECK-NOFP16-LABEL: shuffle3step0_bf16:281; CHECK-NOFP16: @ %bb.0: @ %entry282; CHECK-NOFP16-NEXT: vmov r1, s0283; CHECK-NOFP16-NEXT: vmov.u16 r0, d0[3]284; CHECK-NOFP16-NEXT: vrev32.16 d16, d3285; CHECK-NOFP16-NEXT: vext.16 d17, d4, d5, #2286; CHECK-NOFP16-NEXT: vext.16 d16, d16, d3, #1287; CHECK-NOFP16-NEXT: vext.16 d16, d17, d16, #2288; CHECK-NOFP16-NEXT: vext.16 d16, d16, d17, #1289; CHECK-NOFP16-NEXT: vext.16 d17, d16, d16, #1290; CHECK-NOFP16-NEXT: vmov.16 d16[0], r1291; CHECK-NOFP16-NEXT: vmov.16 d16[1], r0292; CHECK-NOFP16-NEXT: vmov r0, s3293; CHECK-NOFP16-NEXT: vmov.16 d16[2], r0294; CHECK-NOFP16-NEXT: vmov.u16 r0, d2[1]295; CHECK-NOFP16-NEXT: vmov.16 d16[3], r0296; CHECK-NOFP16-NEXT: vorr q0, q8, q8297; CHECK-NOFP16-NEXT: bx lr298;299; CHECK-FP16-LABEL: shuffle3step0_bf16:300; CHECK-FP16: @ %bb.0: @ %entry301; CHECK-FP16-NEXT: vmov r1, s0302; CHECK-FP16-NEXT: vext.16 d17, d4, d5, #2303; CHECK-FP16-NEXT: vmovx.f16 s8, s1304; CHECK-FP16-NEXT: vrev32.16 d16, d3305; CHECK-FP16-NEXT: vmov r0, s8306; CHECK-FP16-NEXT: vext.16 d16, d16, d3, #1307; CHECK-FP16-NEXT: vext.16 d16, d17, d16, #2308; CHECK-FP16-NEXT: vext.16 d16, d16, d17, #1309; CHECK-FP16-NEXT: vext.16 d17, d16, d16, #1310; CHECK-FP16-NEXT: vmov.16 d16[0], r1311; CHECK-FP16-NEXT: vmov.16 d16[1], r0312; CHECK-FP16-NEXT: vmov r0, s3313; CHECK-FP16-NEXT: vmovx.f16 s0, s4314; CHECK-FP16-NEXT: vmov.16 d16[2], r0315; CHECK-FP16-NEXT: vmov r0, s0316; CHECK-FP16-NEXT: vmov.16 d16[3], r0317; CHECK-FP16-NEXT: vorr q0, q8, q8318; CHECK-FP16-NEXT: bx lr319entry:320 %s1 = shufflevector <32 x bfloat> %src, <32 x bfloat> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>321 ret <8 x bfloat> %s1322}323 324define arm_aapcs_vfpcc <8 x bfloat> @shuffle3step1_bf16(<32 x bfloat> %src) {325; CHECK-NOFP16-LABEL: shuffle3step1_bf16:326; CHECK-NOFP16: @ %bb.0: @ %entry327; CHECK-NOFP16-NEXT: vorr q3, q0, q0328; CHECK-NOFP16-NEXT: vmov.u16 r1, d6[1]329; CHECK-NOFP16-NEXT: vmov r0, s14330; CHECK-NOFP16-NEXT: vmov.16 d0[0], r1331; CHECK-NOFP16-NEXT: vmov.16 d0[1], r0332; CHECK-NOFP16-NEXT: vmov.u16 r0, d7[3]333; CHECK-NOFP16-NEXT: vmov.16 d0[2], r0334; CHECK-NOFP16-NEXT: vmov r0, s5335; CHECK-NOFP16-NEXT: vdup.16 q1, d3[1]336; CHECK-NOFP16-NEXT: vmov r1, s4337; CHECK-NOFP16-NEXT: vmov.16 d0[3], r0338; CHECK-NOFP16-NEXT: vmov r0, s8339; CHECK-NOFP16-NEXT: vmov.16 d1[0], r1340; CHECK-NOFP16-NEXT: vmov.16 d1[1], r0341; CHECK-NOFP16-NEXT: vmov.u16 r0, d4[3]342; CHECK-NOFP16-NEXT: vmov.16 d1[2], r0343; CHECK-NOFP16-NEXT: vmov r0, s11344; CHECK-NOFP16-NEXT: vmov.16 d1[3], r0345; CHECK-NOFP16-NEXT: bx lr346;347; CHECK-FP16-LABEL: shuffle3step1_bf16:348; CHECK-FP16: @ %bb.0: @ %entry349; CHECK-FP16-NEXT: vorr q3, q0, q0350; CHECK-FP16-NEXT: vmovx.f16 s0, s12351; CHECK-FP16-NEXT: vmovx.f16 s12, s15352; CHECK-FP16-NEXT: vmov r1, s0353; CHECK-FP16-NEXT: vmov r0, s14354; CHECK-FP16-NEXT: vmov.16 d0[0], r1355; CHECK-FP16-NEXT: vmov.16 d0[1], r0356; CHECK-FP16-NEXT: vmov r0, s12357; CHECK-FP16-NEXT: vmov.16 d0[2], r0358; CHECK-FP16-NEXT: vmov r0, s5359; CHECK-FP16-NEXT: vdup.16 q1, d3[1]360; CHECK-FP16-NEXT: vmov r1, s4361; CHECK-FP16-NEXT: vmovx.f16 s4, s9362; CHECK-FP16-NEXT: vmov.16 d0[3], r0363; CHECK-FP16-NEXT: vmov r0, s8364; CHECK-FP16-NEXT: vmov.16 d1[0], r1365; CHECK-FP16-NEXT: vmov.16 d1[1], r0366; CHECK-FP16-NEXT: vmov r0, s4367; CHECK-FP16-NEXT: vmov.16 d1[2], r0368; CHECK-FP16-NEXT: vmov r0, s11369; CHECK-FP16-NEXT: vmov.16 d1[3], r0370; CHECK-FP16-NEXT: bx lr371entry:372 %s1 = shufflevector <32 x bfloat> %src, <32 x bfloat> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>373 ret <8 x bfloat> %s1374}375 376define arm_aapcs_vfpcc <8 x bfloat> @shuffle3step2_bf16(<32 x bfloat> %src) {377; CHECK-NOFP16-LABEL: shuffle3step2_bf16:378; CHECK-NOFP16: @ %bb.0: @ %entry379; CHECK-NOFP16-NEXT: vext.16 d16, d0, d1, #2380; CHECK-NOFP16-NEXT: vmov.u16 r0, d4[1]381; CHECK-NOFP16-NEXT: vext.16 d17, d16, d2, #3382; CHECK-NOFP16-NEXT: vext.16 d16, d2, d16, #1383; CHECK-NOFP16-NEXT: vdup.16 q1, d3[2]384; CHECK-NOFP16-NEXT: vext.16 d16, d16, d17, #2385; CHECK-NOFP16-NEXT: vmov r1, s4386; CHECK-NOFP16-NEXT: vext.16 d0, d16, d16, #1387; CHECK-NOFP16-NEXT: vmov.16 d1[0], r1388; CHECK-NOFP16-NEXT: vmov.16 d1[1], r0389; CHECK-NOFP16-NEXT: vmov r0, s10390; CHECK-NOFP16-NEXT: vmov.16 d1[2], r0391; CHECK-NOFP16-NEXT: vmov.u16 r0, d5[3]392; CHECK-NOFP16-NEXT: vmov.16 d1[3], r0393; CHECK-NOFP16-NEXT: bx lr394;395; CHECK-FP16-LABEL: shuffle3step2_bf16:396; CHECK-FP16: @ %bb.0: @ %entry397; CHECK-FP16-NEXT: vext.16 d16, d0, d1, #2398; CHECK-FP16-NEXT: vmovx.f16 s12, s8399; CHECK-FP16-NEXT: vmov r0, s12400; CHECK-FP16-NEXT: vext.16 d17, d16, d2, #3401; CHECK-FP16-NEXT: vext.16 d16, d2, d16, #1402; CHECK-FP16-NEXT: vdup.16 q1, d3[2]403; CHECK-FP16-NEXT: vext.16 d16, d16, d17, #2404; CHECK-FP16-NEXT: vmov r1, s4405; CHECK-FP16-NEXT: vmovx.f16 s4, s11406; CHECK-FP16-NEXT: vext.16 d0, d16, d16, #1407; CHECK-FP16-NEXT: vmov.16 d1[0], r1408; CHECK-FP16-NEXT: vmov.16 d1[1], r0409; CHECK-FP16-NEXT: vmov r0, s10410; CHECK-FP16-NEXT: vmov.16 d1[2], r0411; CHECK-FP16-NEXT: vmov r0, s4412; CHECK-FP16-NEXT: vmov.16 d1[3], r0413; CHECK-FP16-NEXT: bx lr414entry:415 %s1 = shufflevector <32 x bfloat> %src, <32 x bfloat> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>416 ret <8 x bfloat> %s1417}418 419 420define dso_local <4 x bfloat> @test_vrev64_bf16(<4 x bfloat> %a) {421; CHECK-LABEL: test_vrev64_bf16:422; CHECK: @ %bb.0: @ %entry423; CHECK-NEXT: vrev64.16 d0, d0424; CHECK-NEXT: bx lr425entry:426 %shuffle.i = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>427 ret <4 x bfloat> %shuffle.i428}429 430define dso_local <8 x bfloat> @test_vrev64q_bf16(<8 x bfloat> %a) {431; CHECK-LABEL: test_vrev64q_bf16:432; CHECK: @ %bb.0: @ %entry433; CHECK-NEXT: vrev64.16 q0, q0434; CHECK-NEXT: bx lr435entry:436 %shuffle.i = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>437 ret <8 x bfloat> %shuffle.i438}439 440define dso_local <4 x bfloat> @test_vrev32_bf16(<4 x bfloat> %a) {441; CHECK-LABEL: test_vrev32_bf16:442; CHECK: @ %bb.0: @ %entry443; CHECK-NEXT: vrev32.16 d0, d0444; CHECK-NEXT: bx lr445entry:446 %shuffle.i = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>447 ret <4 x bfloat> %shuffle.i448}449 450define dso_local <8 x bfloat> @test_vrev32q_bf16(<8 x bfloat> %a) {451; CHECK-LABEL: test_vrev32q_bf16:452; CHECK: @ %bb.0: @ %entry453; CHECK-NEXT: vrev32.16 q0, q0454; CHECK-NEXT: bx lr455entry:456 %shuffle.i = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>457 ret <8 x bfloat> %shuffle.i458}459 460define <4 x bfloat> @test_vld_dup1_4xbfloat(ptr %b) {461; CHECK-LABEL: test_vld_dup1_4xbfloat:462; CHECK: @ %bb.0: @ %entry463; CHECK-NEXT: vld1.16 {d0[]}, [r0:16]464; CHECK-NEXT: bx lr465entry:466 %b1 = load bfloat, ptr %b, align 2467 %vecinit = insertelement <4 x bfloat> undef, bfloat %b1, i32 0468 %vecinit2 = insertelement <4 x bfloat> %vecinit, bfloat %b1, i32 1469 %vecinit3 = insertelement <4 x bfloat> %vecinit2, bfloat %b1, i32 2470 %vecinit4 = insertelement <4 x bfloat> %vecinit3, bfloat %b1, i32 3471 ret <4 x bfloat> %vecinit4472}473 474define <8 x bfloat> @test_vld_dup1_8xbfloat(ptr %b) local_unnamed_addr {475; CHECK-LABEL: test_vld_dup1_8xbfloat:476; CHECK: @ %bb.0: @ %entry477; CHECK-NEXT: vld1.16 {d0[], d1[]}, [r0:16]478; CHECK-NEXT: bx lr479entry:480 %b1 = load bfloat, ptr %b, align 2481 %vecinit = insertelement <8 x bfloat> undef, bfloat %b1, i32 0482 %vecinit8 = shufflevector <8 x bfloat> %vecinit, <8 x bfloat> undef, <8 x i32> zeroinitializer483 ret <8 x bfloat> %vecinit8484}485 486define <8 x bfloat> @test_shufflevector8xbfloat(<4 x bfloat> %a) {487; CHECK-LABEL: test_shufflevector8xbfloat:488; CHECK: @ %bb.0: @ %entry489; CHECK-NEXT: @ kill: def $d0 killed $d0 def $q0490; CHECK-NEXT: vmov.f64 d1, d0491; CHECK-NEXT: bx lr492entry:493 %r = shufflevector <4 x bfloat> %a, <4 x bfloat> %a, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>494 ret <8 x bfloat> %r495}496 497declare <8 x i8> @llvm.arm.neon.vbsl.v8i8(<8 x i8>, <8 x i8>, <8 x i8>)498declare <16 x i8> @llvm.arm.neon.vbsl.v16i8(<16 x i8>, <16 x i8>, <16 x i8>)499