355 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64 -mattr=+v8.6a,+neon < %s | FileCheck %s3; RUN: llc -mtriple=aarch64 -mattr=+v8.6a,+neon,+bf16 < %s | FileCheck %s4; RUN: llc -mtriple=aarch64 -mattr=+v8.6a,+neon,+fullfp16,+bf16 < %s | FileCheck %s5 6%struct.float16x4x2_t = type { [2 x <4 x bfloat>] }7%struct.float16x8x2_t = type { [2 x <8 x bfloat>] }8 9define dso_local %struct.float16x4x2_t @test_vzip_bf16(<4 x bfloat> %a, <4 x bfloat> %b) {10; CHECK-LABEL: test_vzip_bf16:11; CHECK: // %bb.0: // %entry12; CHECK-NEXT: zip1 v2.4h, v0.4h, v1.4h13; CHECK-NEXT: zip2 v1.4h, v0.4h, v1.4h14; CHECK-NEXT: fmov d0, d215; CHECK-NEXT: ret16entry:17 %vzip.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 0, i32 4, i32 1, i32 5>18 %vzip1.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 2, i32 6, i32 3, i32 7>19 %.fca.0.0.insert = insertvalue %struct.float16x4x2_t undef, <4 x bfloat> %vzip.i, 0, 020 %.fca.0.1.insert = insertvalue %struct.float16x4x2_t %.fca.0.0.insert, <4 x bfloat> %vzip1.i, 0, 121 ret %struct.float16x4x2_t %.fca.0.1.insert22}23 24define dso_local %struct.float16x8x2_t @test_vzipq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) {25; CHECK-LABEL: test_vzipq_bf16:26; CHECK: // %bb.0: // %entry27; CHECK-NEXT: zip1 v2.8h, v0.8h, v1.8h28; CHECK-NEXT: zip2 v1.8h, v0.8h, v1.8h29; CHECK-NEXT: mov v0.16b, v2.16b30; CHECK-NEXT: ret31entry:32 %vzip.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>33 %vzip1.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>34 %.fca.0.0.insert = insertvalue %struct.float16x8x2_t undef, <8 x bfloat> %vzip.i, 0, 035 %.fca.0.1.insert = insertvalue %struct.float16x8x2_t %.fca.0.0.insert, <8 x bfloat> %vzip1.i, 0, 136 ret %struct.float16x8x2_t %.fca.0.1.insert37}38 39define dso_local %struct.float16x4x2_t @test_vuzp_bf16(<4 x bfloat> %a, <4 x bfloat> %b) {40; CHECK-LABEL: test_vuzp_bf16:41; CHECK: // %bb.0: // %entry42; CHECK-NEXT: uzp1 v2.4h, v0.4h, v1.4h43; CHECK-NEXT: uzp2 v1.4h, v0.4h, v1.4h44; CHECK-NEXT: fmov d0, d245; CHECK-NEXT: ret46entry:47 %vuzp.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 0, i32 2, i32 4, i32 6>48 %vuzp1.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 1, i32 3, i32 5, i32 7>49 %.fca.0.0.insert = insertvalue %struct.float16x4x2_t undef, <4 x bfloat> %vuzp.i, 0, 050 %.fca.0.1.insert = insertvalue %struct.float16x4x2_t %.fca.0.0.insert, <4 x bfloat> %vuzp1.i, 0, 151 ret %struct.float16x4x2_t %.fca.0.1.insert52}53 54define dso_local %struct.float16x8x2_t @test_vuzpq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) {55; CHECK-LABEL: test_vuzpq_bf16:56; CHECK: // %bb.0: // %entry57; CHECK-NEXT: uzp1 v2.8h, v0.8h, v1.8h58; CHECK-NEXT: uzp2 v1.8h, v0.8h, v1.8h59; CHECK-NEXT: mov v0.16b, v2.16b60; CHECK-NEXT: ret61entry:62 %vuzp.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>63 %vuzp1.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>64 %.fca.0.0.insert = insertvalue %struct.float16x8x2_t undef, <8 x bfloat> %vuzp.i, 0, 065 %.fca.0.1.insert = insertvalue %struct.float16x8x2_t %.fca.0.0.insert, <8 x bfloat> %vuzp1.i, 0, 166 ret %struct.float16x8x2_t %.fca.0.1.insert67}68 69define dso_local %struct.float16x4x2_t @test_vtrn_bf16(<4 x bfloat> %a, <4 x bfloat> %b) {70; CHECK-LABEL: test_vtrn_bf16:71; CHECK: // %bb.0: // %entry72; CHECK-NEXT: trn1 v2.4h, v0.4h, v1.4h73; CHECK-NEXT: trn2 v1.4h, v0.4h, v1.4h74; CHECK-NEXT: fmov d0, d275; CHECK-NEXT: ret76entry:77 %vtrn.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 0, i32 4, i32 2, i32 6>78 %vtrn1.i = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 1, i32 5, i32 3, i32 7>79 %.fca.0.0.insert = insertvalue %struct.float16x4x2_t undef, <4 x bfloat> %vtrn.i, 0, 080 %.fca.0.1.insert = insertvalue %struct.float16x4x2_t %.fca.0.0.insert, <4 x bfloat> %vtrn1.i, 0, 181 ret %struct.float16x4x2_t %.fca.0.1.insert82}83 84define dso_local %struct.float16x8x2_t @test_vtrnq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) {85; CHECK-LABEL: test_vtrnq_bf16:86; CHECK: // %bb.0: // %entry87; CHECK-NEXT: trn1 v2.8h, v0.8h, v1.8h88; CHECK-NEXT: trn2 v1.8h, v0.8h, v1.8h89; CHECK-NEXT: mov v0.16b, v2.16b90; CHECK-NEXT: ret91entry:92 %vtrn.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>93 %vtrn1.i = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>94 %.fca.0.0.insert = insertvalue %struct.float16x8x2_t undef, <8 x bfloat> %vtrn.i, 0, 095 %.fca.0.1.insert = insertvalue %struct.float16x8x2_t %.fca.0.0.insert, <8 x bfloat> %vtrn1.i, 0, 196 ret %struct.float16x8x2_t %.fca.0.1.insert97}98 99define dso_local <4 x bfloat> @test_vmov_n_bf16(float %a.coerce) {100; CHECK-LABEL: test_vmov_n_bf16:101; CHECK: // %bb.0: // %entry102; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0103; CHECK-NEXT: dup v0.4h, v0.h[0]104; CHECK-NEXT: ret105entry:106 %0 = bitcast float %a.coerce to i32107 %tmp.0.extract.trunc = trunc i32 %0 to i16108 %1 = bitcast i16 %tmp.0.extract.trunc to bfloat109 %vecinit = insertelement <4 x bfloat> undef, bfloat %1, i32 0110 %vecinit4 = shufflevector <4 x bfloat> %vecinit, <4 x bfloat> undef, <4 x i32> zeroinitializer111 ret <4 x bfloat> %vecinit4112}113 114define dso_local <8 x bfloat> @test_vmovq_n_bf16(float %a.coerce) {115; CHECK-LABEL: test_vmovq_n_bf16:116; CHECK: // %bb.0: // %entry117; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0118; CHECK-NEXT: dup v0.8h, v0.h[0]119; CHECK-NEXT: ret120entry:121 %0 = bitcast float %a.coerce to i32122 %tmp.0.extract.trunc = trunc i32 %0 to i16123 %1 = bitcast i16 %tmp.0.extract.trunc to bfloat124 %vecinit = insertelement <8 x bfloat> undef, bfloat %1, i32 0125 %vecinit8 = shufflevector <8 x bfloat> %vecinit, <8 x bfloat> undef, <8 x i32> zeroinitializer126 ret <8 x bfloat> %vecinit8127}128 129define dso_local <4 x bfloat> @test_vdup_n_bf16(float %a.coerce) {130; CHECK-LABEL: test_vdup_n_bf16:131; CHECK: // %bb.0: // %entry132; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0133; CHECK-NEXT: dup v0.4h, v0.h[0]134; CHECK-NEXT: ret135entry:136 %0 = bitcast float %a.coerce to i32137 %tmp.0.extract.trunc = trunc i32 %0 to i16138 %1 = bitcast i16 %tmp.0.extract.trunc to bfloat139 %vecinit = insertelement <4 x bfloat> undef, bfloat %1, i32 0140 %vecinit4 = shufflevector <4 x bfloat> %vecinit, <4 x bfloat> undef, <4 x i32> zeroinitializer141 ret <4 x bfloat> %vecinit4142}143 144define dso_local <8 x bfloat> @test_vdupq_n_bf16(float %a.coerce) {145; CHECK-LABEL: test_vdupq_n_bf16:146; CHECK: // %bb.0: // %entry147; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0148; CHECK-NEXT: dup v0.8h, v0.h[0]149; CHECK-NEXT: ret150entry:151 %0 = bitcast float %a.coerce to i32152 %tmp.0.extract.trunc = trunc i32 %0 to i16153 %1 = bitcast i16 %tmp.0.extract.trunc to bfloat154 %vecinit = insertelement <8 x bfloat> undef, bfloat %1, i32 0155 %vecinit8 = shufflevector <8 x bfloat> %vecinit, <8 x bfloat> undef, <8 x i32> zeroinitializer156 ret <8 x bfloat> %vecinit8157}158 159define dso_local <4 x bfloat> @test_vdup_lane_bf16(<4 x bfloat> %a) {160; CHECK-LABEL: test_vdup_lane_bf16:161; CHECK: // %bb.0: // %entry162; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0163; CHECK-NEXT: dup v0.4h, v0.h[3]164; CHECK-NEXT: ret165entry:166 %shuffle = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>167 ret <4 x bfloat> %shuffle168}169 170define dso_local <8 x bfloat> @test_vdupq_lane_bf16(<4 x bfloat> %a) {171; CHECK-LABEL: test_vdupq_lane_bf16:172; CHECK: // %bb.0: // %entry173; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0174; CHECK-NEXT: dup v0.8h, v0.h[3]175; CHECK-NEXT: ret176entry:177 %shuffle = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>178 ret <8 x bfloat> %shuffle179}180 181define dso_local <4 x bfloat> @test_vext_bf16(<4 x bfloat> %a, <4 x bfloat> %b) {182; CHECK-LABEL: test_vext_bf16:183; CHECK: // %bb.0: // %entry184; CHECK-NEXT: ext v0.8b, v0.8b, v1.8b, #4185; CHECK-NEXT: ret186entry:187 %vext = shufflevector <4 x bfloat> %a, <4 x bfloat> %b, <4 x i32> <i32 2, i32 3, i32 4, i32 5>188 ret <4 x bfloat> %vext189}190 191define dso_local <8 x bfloat> @test_vextq_bf16(<8 x bfloat> %a, <8 x bfloat> %b) {192; CHECK-LABEL: test_vextq_bf16:193; CHECK: // %bb.0: // %entry194; CHECK-NEXT: ext v0.16b, v0.16b, v1.16b, #10195; CHECK-NEXT: ret196entry:197 %vext = shufflevector <8 x bfloat> %a, <8 x bfloat> %b, <8 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12>198 ret <8 x bfloat> %vext199}200 201define dso_local <4 x bfloat> @test_vext_aligned_bf16(<8 x bfloat> %a) {202; CHECK-LABEL: test_vext_aligned_bf16:203; CHECK: // %bb.0: // %entry204; CHECK-NEXT: ext v0.16b, v0.16b, v0.16b, #8205; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0206; CHECK-NEXT: ret207entry:208 %vext = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>209 ret <4 x bfloat> %vext210}211 212define dso_local <4 x bfloat> @test_vext_unaligned_bf16(<8 x bfloat> %a) {213; CHECK-LABEL: test_vext_unaligned_bf16:214; CHECK: // %bb.0: // %entry215; CHECK-NEXT: ext v0.16b, v0.16b, v0.16b, #6216; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0217; CHECK-NEXT: ret218entry:219 %vext = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <4 x i32> <i32 3, i32 4, i32 5, i32 6>220 ret <4 x bfloat> %vext221}222 223define <8 x bfloat> @shuffle3step0_bf16(<32 x bfloat> %src) {224; CHECK-LABEL: shuffle3step0_bf16:225; CHECK: // %bb.0: // %entry226; CHECK-NEXT: adrp x8, .LCPI16_0227; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1228; CHECK-NEXT: mov v3.16b, v2.16b229; CHECK-NEXT: ldr q4, [x8, :lo12:.LCPI16_0]230; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1231; CHECK-NEXT: adrp x8, .LCPI16_1232; CHECK-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v4.16b233; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI16_1]234; CHECK-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b235; CHECK-NEXT: ret236entry:237 %s1 = shufflevector <32 x bfloat> %src, <32 x bfloat> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>238 ret <8 x bfloat> %s1239}240 241define <8 x bfloat> @shuffle3step1_bf16(<32 x bfloat> %src) {242; CHECK-LABEL: shuffle3step1_bf16:243; CHECK: // %bb.0: // %entry244; CHECK-NEXT: adrp x8, .LCPI17_0245; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1246; CHECK-NEXT: mov v3.16b, v2.16b247; CHECK-NEXT: ldr q4, [x8, :lo12:.LCPI17_0]248; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1249; CHECK-NEXT: adrp x8, .LCPI17_1250; CHECK-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v4.16b251; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI17_1]252; CHECK-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b253; CHECK-NEXT: ret254entry:255 %s1 = shufflevector <32 x bfloat> %src, <32 x bfloat> undef, <8 x i32> <i32 1, i32 4, i32 7, i32 10, i32 13, i32 16, i32 19, i32 22>256 ret <8 x bfloat> %s1257}258 259define <8 x bfloat> @shuffle3step2_bf16(<32 x bfloat> %src) {260; CHECK-LABEL: shuffle3step2_bf16:261; CHECK: // %bb.0: // %entry262; CHECK-NEXT: adrp x8, .LCPI18_0263; CHECK-NEXT: // kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1264; CHECK-NEXT: mov v3.16b, v2.16b265; CHECK-NEXT: ldr q4, [x8, :lo12:.LCPI18_0]266; CHECK-NEXT: // kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1267; CHECK-NEXT: adrp x8, .LCPI18_1268; CHECK-NEXT: tbl v2.16b, { v0.16b, v1.16b }, v4.16b269; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI18_1]270; CHECK-NEXT: tbl v0.16b, { v2.16b, v3.16b }, v0.16b271; CHECK-NEXT: ret272entry:273 %s1 = shufflevector <32 x bfloat> %src, <32 x bfloat> undef, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 14, i32 17, i32 20, i32 23>274 ret <8 x bfloat> %s1275}276 277 278define dso_local <4 x bfloat> @test_vrev64_bf16(<4 x bfloat> %a) {279; CHECK-LABEL: test_vrev64_bf16:280; CHECK: // %bb.0: // %entry281; CHECK-NEXT: rev64 v0.4h, v0.4h282; CHECK-NEXT: ret283entry:284 %shuffle.i = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>285 ret <4 x bfloat> %shuffle.i286}287 288define dso_local <8 x bfloat> @test_vrev64q_bf16(<8 x bfloat> %a) {289; CHECK-LABEL: test_vrev64q_bf16:290; CHECK: // %bb.0: // %entry291; CHECK-NEXT: rev64 v0.8h, v0.8h292; CHECK-NEXT: ret293entry:294 %shuffle.i = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>295 ret <8 x bfloat> %shuffle.i296}297 298define dso_local <4 x bfloat> @test_vrev32_bf16(<4 x bfloat> %a) {299; CHECK-LABEL: test_vrev32_bf16:300; CHECK: // %bb.0: // %entry301; CHECK-NEXT: rev32 v0.4h, v0.4h302; CHECK-NEXT: ret303entry:304 %shuffle.i = shufflevector <4 x bfloat> %a, <4 x bfloat> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>305 ret <4 x bfloat> %shuffle.i306}307 308define dso_local <8 x bfloat> @test_vrev32q_bf16(<8 x bfloat> %a) {309; CHECK-LABEL: test_vrev32q_bf16:310; CHECK: // %bb.0: // %entry311; CHECK-NEXT: rev32 v0.8h, v0.8h312; CHECK-NEXT: ret313entry:314 %shuffle.i = shufflevector <8 x bfloat> %a, <8 x bfloat> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>315 ret <8 x bfloat> %shuffle.i316}317 318define <4 x bfloat> @test_vld_dup1_4xbfloat(ptr %b) {319; CHECK-LABEL: test_vld_dup1_4xbfloat:320; CHECK: // %bb.0: // %entry321; CHECK-NEXT: ld1r { v0.4h }, [x0]322; CHECK-NEXT: ret323entry:324 %b1 = load bfloat, ptr %b, align 2325 %vecinit = insertelement <4 x bfloat> undef, bfloat %b1, i32 0326 %vecinit2 = insertelement <4 x bfloat> %vecinit, bfloat %b1, i32 1327 %vecinit3 = insertelement <4 x bfloat> %vecinit2, bfloat %b1, i32 2328 %vecinit4 = insertelement <4 x bfloat> %vecinit3, bfloat %b1, i32 3329 ret <4 x bfloat> %vecinit4330}331 332define <8 x bfloat> @test_vld_dup1_8xbfloat(ptr %b) local_unnamed_addr {333; CHECK-LABEL: test_vld_dup1_8xbfloat:334; CHECK: // %bb.0: // %entry335; CHECK-NEXT: ld1r { v0.8h }, [x0]336; CHECK-NEXT: ret337entry:338 %b1 = load bfloat, ptr %b, align 2339 %vecinit = insertelement <8 x bfloat> undef, bfloat %b1, i32 0340 %vecinit8 = shufflevector <8 x bfloat> %vecinit, <8 x bfloat> undef, <8 x i32> zeroinitializer341 ret <8 x bfloat> %vecinit8342}343 344define <8 x bfloat> @test_shufflevector8xbfloat(<4 x bfloat> %a) {345; CHECK-LABEL: test_shufflevector8xbfloat:346; CHECK: // %bb.0: // %entry347; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0348; CHECK-NEXT: mov v0.d[1], v0.d[0]349; CHECK-NEXT: ret350entry:351 %r = shufflevector <4 x bfloat> %a, <4 x bfloat> %a, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>352 ret <8 x bfloat> %r353}354 355