1029 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=armv8.6a-arm-none-eabi -mattr=+bf16,+neon,+fullfp16 < %s | FileCheck %s3; FIXME: Remove fullfp16 once bfloat arguments and returns lowering stops4; depending on it.5 6define arm_aapcs_vfpcc <4 x bfloat> @test_vld1_bf16(ptr nocapture readonly %ptr) {7; CHECK-LABEL: test_vld1_bf16:8; CHECK: @ %bb.0: @ %entry9; CHECK-NEXT: vld1.16 {d0}, [r0]10; CHECK-NEXT: bx lr11entry:12 %0 = load <4 x bfloat>, ptr %ptr, align 213 ret <4 x bfloat> %014}15 16define arm_aapcs_vfpcc <8 x bfloat> @test_vld1q_bf16(ptr nocapture readonly %ptr) {17; CHECK-LABEL: test_vld1q_bf16:18; CHECK: @ %bb.0: @ %entry19; CHECK-NEXT: vld1.16 {d0, d1}, [r0]20; CHECK-NEXT: bx lr21entry:22 %0 = load <8 x bfloat>, ptr %ptr, align 223 ret <8 x bfloat> %024}25 26define arm_aapcs_vfpcc <4 x bfloat> @test_vld1_lane_bf16(ptr nocapture readonly %ptr, <4 x bfloat> %src) {27; CHECK-LABEL: test_vld1_lane_bf16:28; CHECK: @ %bb.0: @ %entry29; CHECK-NEXT: vld1.16 {d0[0]}, [r0:16]30; CHECK-NEXT: bx lr31entry:32 %0 = load bfloat, ptr %ptr, align 233 %vld1_lane = insertelement <4 x bfloat> %src, bfloat %0, i32 034 ret <4 x bfloat> %vld1_lane35}36 37define arm_aapcs_vfpcc <8 x bfloat> @test_vld1q_lane_bf16(ptr nocapture readonly %ptr, <8 x bfloat> %src) {38; CHECK-LABEL: test_vld1q_lane_bf16:39; CHECK: @ %bb.0: @ %entry40; CHECK-NEXT: vld1.16 {d1[3]}, [r0:16]41; CHECK-NEXT: bx lr42entry:43 %0 = load bfloat, ptr %ptr, align 244 %vld1_lane = insertelement <8 x bfloat> %src, bfloat %0, i32 745 ret <8 x bfloat> %vld1_lane46}47 48define arm_aapcs_vfpcc <4 x bfloat> @test_vld1_dup_bf16(ptr nocapture readonly %ptr) {49; CHECK-LABEL: test_vld1_dup_bf16:50; CHECK: @ %bb.0: @ %entry51; CHECK-NEXT: vld1.16 {d0[]}, [r0:16]52; CHECK-NEXT: bx lr53entry:54 %0 = load bfloat, ptr %ptr, align 255 %1 = insertelement <4 x bfloat> undef, bfloat %0, i32 056 %lane = shufflevector <4 x bfloat> %1, <4 x bfloat> undef, <4 x i32> zeroinitializer57 ret <4 x bfloat> %lane58}59 60define arm_aapcs_vfpcc [2 x <2 x i32>] @test_vld1_bf16_x2(ptr %ptr) {61; CHECK-LABEL: test_vld1_bf16_x2:62; CHECK: @ %bb.0: @ %entry63; CHECK-NEXT: vld1.16 {d0, d1}, [r0]64; CHECK-NEXT: bx lr65entry:66 %vld1xN = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld1x2.v4bf16.p0(ptr %ptr)67 %vld1xN.fca.0.extract = extractvalue { <4 x bfloat>, <4 x bfloat> } %vld1xN, 068 %vld1xN.fca.1.extract = extractvalue { <4 x bfloat>, <4 x bfloat> } %vld1xN, 169 %0 = bitcast <4 x bfloat> %vld1xN.fca.0.extract to <2 x i32>70 %1 = bitcast <4 x bfloat> %vld1xN.fca.1.extract to <2 x i32>71 %.fca.0.insert = insertvalue [2 x <2 x i32>] undef, <2 x i32> %0, 072 %.fca.1.insert = insertvalue [2 x <2 x i32>] %.fca.0.insert, <2 x i32> %1, 173 ret [2 x <2 x i32>] %.fca.1.insert74}75 76define arm_aapcs_vfpcc [2 x <4 x i32>] @test_vld1q_bf16_x2(ptr %ptr) {77; CHECK-LABEL: test_vld1q_bf16_x2:78; CHECK: @ %bb.0: @ %entry79; CHECK-NEXT: vld1.16 {d0, d1, d2, d3}, [r0]80; CHECK-NEXT: bx lr81entry:82 %vld1xN = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld1x2.v8bf16.p0(ptr %ptr)83 %vld1xN.fca.0.extract = extractvalue { <8 x bfloat>, <8 x bfloat> } %vld1xN, 084 %vld1xN.fca.1.extract = extractvalue { <8 x bfloat>, <8 x bfloat> } %vld1xN, 185 %0 = bitcast <8 x bfloat> %vld1xN.fca.0.extract to <4 x i32>86 %1 = bitcast <8 x bfloat> %vld1xN.fca.1.extract to <4 x i32>87 %.fca.0.insert = insertvalue [2 x <4 x i32>] undef, <4 x i32> %0, 088 %.fca.1.insert = insertvalue [2 x <4 x i32>] %.fca.0.insert, <4 x i32> %1, 189 ret [2 x <4 x i32>] %.fca.1.insert90}91 92define arm_aapcs_vfpcc [3 x <2 x i32>] @test_vld1_bf16_x3(ptr %ptr) {93; CHECK-LABEL: test_vld1_bf16_x3:94; CHECK: @ %bb.0: @ %entry95; CHECK-NEXT: vld1.16 {d0, d1, d2}, [r0]96; CHECK-NEXT: bx lr97entry:98 %vld1xN = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld1x3.v4bf16.p0(ptr %ptr)99 %vld1xN.fca.0.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld1xN, 0100 %vld1xN.fca.1.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld1xN, 1101 %vld1xN.fca.2.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld1xN, 2102 %0 = bitcast <4 x bfloat> %vld1xN.fca.0.extract to <2 x i32>103 %1 = bitcast <4 x bfloat> %vld1xN.fca.1.extract to <2 x i32>104 %2 = bitcast <4 x bfloat> %vld1xN.fca.2.extract to <2 x i32>105 %.fca.0.insert = insertvalue [3 x <2 x i32>] undef, <2 x i32> %0, 0106 %.fca.1.insert = insertvalue [3 x <2 x i32>] %.fca.0.insert, <2 x i32> %1, 1107 %.fca.2.insert = insertvalue [3 x <2 x i32>] %.fca.1.insert, <2 x i32> %2, 2108 ret [3 x <2 x i32>] %.fca.2.insert109}110 111define arm_aapcs_vfpcc [3 x <4 x i32>] @test_vld1q_bf16_x3(ptr %ptr) {112; CHECK-LABEL: test_vld1q_bf16_x3:113; CHECK: @ %bb.0: @ %entry114; CHECK-NEXT: vld1.16 {d0, d1, d2}, [r0]!115; CHECK-NEXT: vld1.16 {d3, d4, d5}, [r0]116; CHECK-NEXT: bx lr117entry:118 %vld1xN = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld1x3.v8bf16.p0(ptr %ptr)119 %vld1xN.fca.0.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld1xN, 0120 %vld1xN.fca.1.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld1xN, 1121 %vld1xN.fca.2.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld1xN, 2122 %0 = bitcast <8 x bfloat> %vld1xN.fca.0.extract to <4 x i32>123 %1 = bitcast <8 x bfloat> %vld1xN.fca.1.extract to <4 x i32>124 %2 = bitcast <8 x bfloat> %vld1xN.fca.2.extract to <4 x i32>125 %.fca.0.insert = insertvalue [3 x <4 x i32>] undef, <4 x i32> %0, 0126 %.fca.1.insert = insertvalue [3 x <4 x i32>] %.fca.0.insert, <4 x i32> %1, 1127 %.fca.2.insert = insertvalue [3 x <4 x i32>] %.fca.1.insert, <4 x i32> %2, 2128 ret [3 x <4 x i32>] %.fca.2.insert129}130 131define arm_aapcs_vfpcc [4 x <2 x i32>] @test_vld1_bf16_x4(ptr %ptr) {132; CHECK-LABEL: test_vld1_bf16_x4:133; CHECK: @ %bb.0: @ %entry134; CHECK-NEXT: vld1.16 {d0, d1, d2, d3}, [r0]135; CHECK-NEXT: bx lr136entry:137 %vld1xN = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld1x4.v4bf16.p0(ptr %ptr)138 %vld1xN.fca.0.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld1xN, 0139 %vld1xN.fca.1.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld1xN, 1140 %vld1xN.fca.2.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld1xN, 2141 %vld1xN.fca.3.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld1xN, 3142 %0 = bitcast <4 x bfloat> %vld1xN.fca.0.extract to <2 x i32>143 %1 = bitcast <4 x bfloat> %vld1xN.fca.1.extract to <2 x i32>144 %2 = bitcast <4 x bfloat> %vld1xN.fca.2.extract to <2 x i32>145 %3 = bitcast <4 x bfloat> %vld1xN.fca.3.extract to <2 x i32>146 %.fca.0.insert = insertvalue [4 x <2 x i32>] undef, <2 x i32> %0, 0147 %.fca.1.insert = insertvalue [4 x <2 x i32>] %.fca.0.insert, <2 x i32> %1, 1148 %.fca.2.insert = insertvalue [4 x <2 x i32>] %.fca.1.insert, <2 x i32> %2, 2149 %.fca.3.insert = insertvalue [4 x <2 x i32>] %.fca.2.insert, <2 x i32> %3, 3150 ret [4 x <2 x i32>] %.fca.3.insert151}152 153define arm_aapcs_vfpcc [4 x <4 x i32>] @test_vld1q_bf16_x4(ptr %ptr) {154; CHECK-LABEL: test_vld1q_bf16_x4:155; CHECK: @ %bb.0: @ %entry156; CHECK-NEXT: vld1.16 {d0, d1, d2, d3}, [r0]!157; CHECK-NEXT: vld1.16 {d4, d5, d6, d7}, [r0]158; CHECK-NEXT: bx lr159entry:160 %vld1xN = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld1x4.v8bf16.p0(ptr %ptr)161 %vld1xN.fca.0.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld1xN, 0162 %vld1xN.fca.1.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld1xN, 1163 %vld1xN.fca.2.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld1xN, 2164 %vld1xN.fca.3.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld1xN, 3165 %0 = bitcast <8 x bfloat> %vld1xN.fca.0.extract to <4 x i32>166 %1 = bitcast <8 x bfloat> %vld1xN.fca.1.extract to <4 x i32>167 %2 = bitcast <8 x bfloat> %vld1xN.fca.2.extract to <4 x i32>168 %3 = bitcast <8 x bfloat> %vld1xN.fca.3.extract to <4 x i32>169 %.fca.0.insert = insertvalue [4 x <4 x i32>] undef, <4 x i32> %0, 0170 %.fca.1.insert = insertvalue [4 x <4 x i32>] %.fca.0.insert, <4 x i32> %1, 1171 %.fca.2.insert = insertvalue [4 x <4 x i32>] %.fca.1.insert, <4 x i32> %2, 2172 %.fca.3.insert = insertvalue [4 x <4 x i32>] %.fca.2.insert, <4 x i32> %3, 3173 ret [4 x <4 x i32>] %.fca.3.insert174}175 176define arm_aapcs_vfpcc <8 x bfloat> @test_vld1q_dup_bf16(ptr nocapture readonly %ptr) {177; CHECK-LABEL: test_vld1q_dup_bf16:178; CHECK: @ %bb.0: @ %entry179; CHECK-NEXT: vld1.16 {d0[], d1[]}, [r0:16]180; CHECK-NEXT: bx lr181entry:182 %0 = load bfloat, ptr %ptr, align 2183 %1 = insertelement <8 x bfloat> undef, bfloat %0, i32 0184 %lane = shufflevector <8 x bfloat> %1, <8 x bfloat> undef, <8 x i32> zeroinitializer185 ret <8 x bfloat> %lane186}187 188define arm_aapcs_vfpcc [2 x <2 x i32>] @test_vld2_bf16(ptr %ptr) {189; CHECK-LABEL: test_vld2_bf16:190; CHECK: @ %bb.0: @ %entry191; CHECK-NEXT: vld2.16 {d0, d1}, [r0]192; CHECK-NEXT: bx lr193entry:194 %vld2_v = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld2.v4bf16.p0(ptr %ptr, i32 2)195 %vld2_v.fca.0.extract = extractvalue { <4 x bfloat>, <4 x bfloat> } %vld2_v, 0196 %vld2_v.fca.1.extract = extractvalue { <4 x bfloat>, <4 x bfloat> } %vld2_v, 1197 %0 = bitcast <4 x bfloat> %vld2_v.fca.0.extract to <2 x i32>198 %1 = bitcast <4 x bfloat> %vld2_v.fca.1.extract to <2 x i32>199 %.fca.0.insert = insertvalue [2 x <2 x i32>] undef, <2 x i32> %0, 0200 %.fca.1.insert = insertvalue [2 x <2 x i32>] %.fca.0.insert, <2 x i32> %1, 1201 ret [2 x <2 x i32>] %.fca.1.insert202}203 204define arm_aapcs_vfpcc [2 x <4 x i32>] @test_vld2q_bf16(ptr %ptr) {205; CHECK-LABEL: test_vld2q_bf16:206; CHECK: @ %bb.0: @ %entry207; CHECK-NEXT: vld2.16 {d0, d1, d2, d3}, [r0]208; CHECK-NEXT: bx lr209entry:210 %vld2q_v = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld2.v8bf16.p0(ptr %ptr, i32 2)211 %vld2q_v.fca.0.extract = extractvalue { <8 x bfloat>, <8 x bfloat> } %vld2q_v, 0212 %vld2q_v.fca.1.extract = extractvalue { <8 x bfloat>, <8 x bfloat> } %vld2q_v, 1213 %0 = bitcast <8 x bfloat> %vld2q_v.fca.0.extract to <4 x i32>214 %1 = bitcast <8 x bfloat> %vld2q_v.fca.1.extract to <4 x i32>215 %.fca.0.insert = insertvalue [2 x <4 x i32>] undef, <4 x i32> %0, 0216 %.fca.1.insert = insertvalue [2 x <4 x i32>] %.fca.0.insert, <4 x i32> %1, 1217 ret [2 x <4 x i32>] %.fca.1.insert218}219 220define arm_aapcs_vfpcc [2 x <2 x i32>] @test_vld2_lane_bf16(ptr %ptr, [2 x <2 x i32>] %src.coerce) {221; CHECK-LABEL: test_vld2_lane_bf16:222; CHECK: @ %bb.0: @ %entry223; CHECK-NEXT: @ kill: def $d1 killed $d1 killed $q0 def $q0224; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0 def $q0225; CHECK-NEXT: vld2.16 {d0[1], d1[1]}, [r0]226; CHECK-NEXT: bx lr227entry:228 %src.coerce.fca.0.extract = extractvalue [2 x <2 x i32>] %src.coerce, 0229 %src.coerce.fca.1.extract = extractvalue [2 x <2 x i32>] %src.coerce, 1230 %0 = bitcast <2 x i32> %src.coerce.fca.0.extract to <4 x bfloat>231 %1 = bitcast <2 x i32> %src.coerce.fca.1.extract to <4 x bfloat>232 %vld2_lane_v = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld2lane.v4bf16.p0(ptr %ptr, <4 x bfloat> %0, <4 x bfloat> %1, i32 1, i32 2)233 %vld2_lane_v.fca.0.extract = extractvalue { <4 x bfloat>, <4 x bfloat> } %vld2_lane_v, 0234 %vld2_lane_v.fca.1.extract = extractvalue { <4 x bfloat>, <4 x bfloat> } %vld2_lane_v, 1235 %2 = bitcast <4 x bfloat> %vld2_lane_v.fca.0.extract to <2 x i32>236 %3 = bitcast <4 x bfloat> %vld2_lane_v.fca.1.extract to <2 x i32>237 %.fca.0.insert = insertvalue [2 x <2 x i32>] undef, <2 x i32> %2, 0238 %.fca.1.insert = insertvalue [2 x <2 x i32>] %.fca.0.insert, <2 x i32> %3, 1239 ret [2 x <2 x i32>] %.fca.1.insert240}241 242define arm_aapcs_vfpcc [2 x <4 x i32>] @test_vld2q_lane_bf16(ptr %ptr, [2 x <4 x i32>] %src.coerce) {243; CHECK-LABEL: test_vld2q_lane_bf16:244; CHECK: @ %bb.0: @ %entry245; CHECK-NEXT: @ kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1246; CHECK-NEXT: @ kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1247; CHECK-NEXT: vld2.16 {d1[3], d3[3]}, [r0]248; CHECK-NEXT: bx lr249entry:250 %src.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %src.coerce, 0251 %src.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %src.coerce, 1252 %0 = bitcast <4 x i32> %src.coerce.fca.0.extract to <8 x bfloat>253 %1 = bitcast <4 x i32> %src.coerce.fca.1.extract to <8 x bfloat>254 %vld2q_lane_v = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld2lane.v8bf16.p0(ptr %ptr, <8 x bfloat> %0, <8 x bfloat> %1, i32 7, i32 2)255 %vld2q_lane_v.fca.0.extract = extractvalue { <8 x bfloat>, <8 x bfloat> } %vld2q_lane_v, 0256 %vld2q_lane_v.fca.1.extract = extractvalue { <8 x bfloat>, <8 x bfloat> } %vld2q_lane_v, 1257 %2 = bitcast <8 x bfloat> %vld2q_lane_v.fca.0.extract to <4 x i32>258 %3 = bitcast <8 x bfloat> %vld2q_lane_v.fca.1.extract to <4 x i32>259 %.fca.0.insert = insertvalue [2 x <4 x i32>] undef, <4 x i32> %2, 0260 %.fca.1.insert = insertvalue [2 x <4 x i32>] %.fca.0.insert, <4 x i32> %3, 1261 ret [2 x <4 x i32>] %.fca.1.insert262}263 264define arm_aapcs_vfpcc [3 x <2 x i32>] @test_vld3_bf16(ptr %ptr) {265; CHECK-LABEL: test_vld3_bf16:266; CHECK: @ %bb.0: @ %entry267; CHECK-NEXT: vld3.16 {d0, d1, d2}, [r0]268; CHECK-NEXT: bx lr269entry:270 %vld3_v = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld3.v4bf16.p0(ptr %ptr, i32 2)271 %vld3_v.fca.0.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld3_v, 0272 %vld3_v.fca.1.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld3_v, 1273 %vld3_v.fca.2.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld3_v, 2274 %0 = bitcast <4 x bfloat> %vld3_v.fca.0.extract to <2 x i32>275 %1 = bitcast <4 x bfloat> %vld3_v.fca.1.extract to <2 x i32>276 %2 = bitcast <4 x bfloat> %vld3_v.fca.2.extract to <2 x i32>277 %.fca.0.insert = insertvalue [3 x <2 x i32>] undef, <2 x i32> %0, 0278 %.fca.1.insert = insertvalue [3 x <2 x i32>] %.fca.0.insert, <2 x i32> %1, 1279 %.fca.2.insert = insertvalue [3 x <2 x i32>] %.fca.1.insert, <2 x i32> %2, 2280 ret [3 x <2 x i32>] %.fca.2.insert281}282 283define arm_aapcs_vfpcc [3 x <4 x i32>] @test_vld3q_bf16(ptr %ptr) {284; CHECK-LABEL: test_vld3q_bf16:285; CHECK: @ %bb.0: @ %entry286; CHECK-NEXT: vld3.16 {d0, d2, d4}, [r0]!287; CHECK-NEXT: vld3.16 {d1, d3, d5}, [r0]288; CHECK-NEXT: bx lr289entry:290 %vld3q_v = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld3.v8bf16.p0(ptr %ptr, i32 2)291 %vld3q_v.fca.0.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld3q_v, 0292 %vld3q_v.fca.1.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld3q_v, 1293 %vld3q_v.fca.2.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld3q_v, 2294 %0 = bitcast <8 x bfloat> %vld3q_v.fca.0.extract to <4 x i32>295 %1 = bitcast <8 x bfloat> %vld3q_v.fca.1.extract to <4 x i32>296 %2 = bitcast <8 x bfloat> %vld3q_v.fca.2.extract to <4 x i32>297 %.fca.0.insert = insertvalue [3 x <4 x i32>] undef, <4 x i32> %0, 0298 %.fca.1.insert = insertvalue [3 x <4 x i32>] %.fca.0.insert, <4 x i32> %1, 1299 %.fca.2.insert = insertvalue [3 x <4 x i32>] %.fca.1.insert, <4 x i32> %2, 2300 ret [3 x <4 x i32>] %.fca.2.insert301}302 303define arm_aapcs_vfpcc [3 x <2 x i32>] @test_vld3_lane_bf16(ptr %ptr, [3 x <2 x i32>] %src.coerce) {304; CHECK-LABEL: test_vld3_lane_bf16:305; CHECK: @ %bb.0: @ %entry306; CHECK-NEXT: @ kill: def $d2 killed $d2 killed $q0_q1 def $q0_q1307; CHECK-NEXT: @ kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1308; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1309; CHECK-NEXT: vld3.16 {d0[1], d1[1], d2[1]}, [r0]310; CHECK-NEXT: bx lr311entry:312 %src.coerce.fca.0.extract = extractvalue [3 x <2 x i32>] %src.coerce, 0313 %src.coerce.fca.1.extract = extractvalue [3 x <2 x i32>] %src.coerce, 1314 %src.coerce.fca.2.extract = extractvalue [3 x <2 x i32>] %src.coerce, 2315 %0 = bitcast <2 x i32> %src.coerce.fca.0.extract to <4 x bfloat>316 %1 = bitcast <2 x i32> %src.coerce.fca.1.extract to <4 x bfloat>317 %2 = bitcast <2 x i32> %src.coerce.fca.2.extract to <4 x bfloat>318 %vld3_lane_v = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld3lane.v4bf16.p0(ptr %ptr, <4 x bfloat> %0, <4 x bfloat> %1, <4 x bfloat> %2, i32 1, i32 2)319 %vld3_lane_v.fca.0.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld3_lane_v, 0320 %vld3_lane_v.fca.1.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld3_lane_v, 1321 %vld3_lane_v.fca.2.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld3_lane_v, 2322 %3 = bitcast <4 x bfloat> %vld3_lane_v.fca.0.extract to <2 x i32>323 %4 = bitcast <4 x bfloat> %vld3_lane_v.fca.1.extract to <2 x i32>324 %5 = bitcast <4 x bfloat> %vld3_lane_v.fca.2.extract to <2 x i32>325 %.fca.0.insert = insertvalue [3 x <2 x i32>] undef, <2 x i32> %3, 0326 %.fca.1.insert = insertvalue [3 x <2 x i32>] %.fca.0.insert, <2 x i32> %4, 1327 %.fca.2.insert = insertvalue [3 x <2 x i32>] %.fca.1.insert, <2 x i32> %5, 2328 ret [3 x <2 x i32>] %.fca.2.insert329}330 331define arm_aapcs_vfpcc [3 x <4 x i32>] @test_vld3q_lane_bf16(ptr %ptr, [3 x <4 x i32>] %src.coerce) {332; CHECK-LABEL: test_vld3q_lane_bf16:333; CHECK: @ %bb.0: @ %entry334; CHECK-NEXT: @ kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3335; CHECK-NEXT: @ kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3336; CHECK-NEXT: @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3337; CHECK-NEXT: vld3.16 {d1[3], d3[3], d5[3]}, [r0]338; CHECK-NEXT: bx lr339entry:340 %src.coerce.fca.0.extract = extractvalue [3 x <4 x i32>] %src.coerce, 0341 %src.coerce.fca.1.extract = extractvalue [3 x <4 x i32>] %src.coerce, 1342 %src.coerce.fca.2.extract = extractvalue [3 x <4 x i32>] %src.coerce, 2343 %0 = bitcast <4 x i32> %src.coerce.fca.0.extract to <8 x bfloat>344 %1 = bitcast <4 x i32> %src.coerce.fca.1.extract to <8 x bfloat>345 %2 = bitcast <4 x i32> %src.coerce.fca.2.extract to <8 x bfloat>346 %vld3q_lane_v = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld3lane.v8bf16.p0(ptr %ptr, <8 x bfloat> %0, <8 x bfloat> %1, <8 x bfloat> %2, i32 7, i32 2)347 %vld3q_lane_v.fca.0.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld3q_lane_v, 0348 %vld3q_lane_v.fca.1.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld3q_lane_v, 1349 %vld3q_lane_v.fca.2.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld3q_lane_v, 2350 %3 = bitcast <8 x bfloat> %vld3q_lane_v.fca.0.extract to <4 x i32>351 %4 = bitcast <8 x bfloat> %vld3q_lane_v.fca.1.extract to <4 x i32>352 %5 = bitcast <8 x bfloat> %vld3q_lane_v.fca.2.extract to <4 x i32>353 %.fca.0.insert = insertvalue [3 x <4 x i32>] undef, <4 x i32> %3, 0354 %.fca.1.insert = insertvalue [3 x <4 x i32>] %.fca.0.insert, <4 x i32> %4, 1355 %.fca.2.insert = insertvalue [3 x <4 x i32>] %.fca.1.insert, <4 x i32> %5, 2356 ret [3 x <4 x i32>] %.fca.2.insert357}358 359define arm_aapcs_vfpcc [4 x <2 x i32>] @test_vld4_bf16(ptr %ptr) {360; CHECK-LABEL: test_vld4_bf16:361; CHECK: @ %bb.0: @ %entry362; CHECK-NEXT: vld4.16 {d0, d1, d2, d3}, [r0]363; CHECK-NEXT: bx lr364entry:365 %vld4_v = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld4.v4bf16.p0(ptr %ptr, i32 2)366 %vld4_v.fca.0.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld4_v, 0367 %vld4_v.fca.1.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld4_v, 1368 %vld4_v.fca.2.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld4_v, 2369 %vld4_v.fca.3.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld4_v, 3370 %0 = bitcast <4 x bfloat> %vld4_v.fca.0.extract to <2 x i32>371 %1 = bitcast <4 x bfloat> %vld4_v.fca.1.extract to <2 x i32>372 %2 = bitcast <4 x bfloat> %vld4_v.fca.2.extract to <2 x i32>373 %3 = bitcast <4 x bfloat> %vld4_v.fca.3.extract to <2 x i32>374 %.fca.0.insert = insertvalue [4 x <2 x i32>] undef, <2 x i32> %0, 0375 %.fca.1.insert = insertvalue [4 x <2 x i32>] %.fca.0.insert, <2 x i32> %1, 1376 %.fca.2.insert = insertvalue [4 x <2 x i32>] %.fca.1.insert, <2 x i32> %2, 2377 %.fca.3.insert = insertvalue [4 x <2 x i32>] %.fca.2.insert, <2 x i32> %3, 3378 ret [4 x <2 x i32>] %.fca.3.insert379}380 381define arm_aapcs_vfpcc [4 x <4 x i32>] @test_vld4q_bf16(ptr %ptr) {382; CHECK-LABEL: test_vld4q_bf16:383; CHECK: @ %bb.0: @ %entry384; CHECK-NEXT: vld4.16 {d0, d2, d4, d6}, [r0]!385; CHECK-NEXT: vld4.16 {d1, d3, d5, d7}, [r0]386; CHECK-NEXT: bx lr387entry:388 %vld4q_v = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld4.v8bf16.p0(ptr %ptr, i32 2)389 %vld4q_v.fca.0.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld4q_v, 0390 %vld4q_v.fca.1.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld4q_v, 1391 %vld4q_v.fca.2.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld4q_v, 2392 %vld4q_v.fca.3.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld4q_v, 3393 %0 = bitcast <8 x bfloat> %vld4q_v.fca.0.extract to <4 x i32>394 %1 = bitcast <8 x bfloat> %vld4q_v.fca.1.extract to <4 x i32>395 %2 = bitcast <8 x bfloat> %vld4q_v.fca.2.extract to <4 x i32>396 %3 = bitcast <8 x bfloat> %vld4q_v.fca.3.extract to <4 x i32>397 %.fca.0.insert = insertvalue [4 x <4 x i32>] undef, <4 x i32> %0, 0398 %.fca.1.insert = insertvalue [4 x <4 x i32>] %.fca.0.insert, <4 x i32> %1, 1399 %.fca.2.insert = insertvalue [4 x <4 x i32>] %.fca.1.insert, <4 x i32> %2, 2400 %.fca.3.insert = insertvalue [4 x <4 x i32>] %.fca.2.insert, <4 x i32> %3, 3401 ret [4 x <4 x i32>] %.fca.3.insert402}403 404define arm_aapcs_vfpcc [4 x <2 x i32>] @test_vld4_lane_bf16(ptr %ptr, [4 x <2 x i32>] %src.coerce) {405; CHECK-LABEL: test_vld4_lane_bf16:406; CHECK: @ %bb.0: @ %entry407; CHECK-NEXT: @ kill: def $d3 killed $d3 killed $q0_q1 def $q0_q1408; CHECK-NEXT: @ kill: def $d2 killed $d2 killed $q0_q1 def $q0_q1409; CHECK-NEXT: @ kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1410; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1411; CHECK-NEXT: vld4.16 {d0[1], d1[1], d2[1], d3[1]}, [r0]412; CHECK-NEXT: bx lr413entry:414 %src.coerce.fca.0.extract = extractvalue [4 x <2 x i32>] %src.coerce, 0415 %src.coerce.fca.1.extract = extractvalue [4 x <2 x i32>] %src.coerce, 1416 %src.coerce.fca.2.extract = extractvalue [4 x <2 x i32>] %src.coerce, 2417 %src.coerce.fca.3.extract = extractvalue [4 x <2 x i32>] %src.coerce, 3418 %0 = bitcast <2 x i32> %src.coerce.fca.0.extract to <4 x bfloat>419 %1 = bitcast <2 x i32> %src.coerce.fca.1.extract to <4 x bfloat>420 %2 = bitcast <2 x i32> %src.coerce.fca.2.extract to <4 x bfloat>421 %3 = bitcast <2 x i32> %src.coerce.fca.3.extract to <4 x bfloat>422 %vld4_lane_v = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld4lane.v4bf16.p0(ptr %ptr, <4 x bfloat> %0, <4 x bfloat> %1, <4 x bfloat> %2, <4 x bfloat> %3, i32 1, i32 2)423 %vld4_lane_v.fca.0.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld4_lane_v, 0424 %vld4_lane_v.fca.1.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld4_lane_v, 1425 %vld4_lane_v.fca.2.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld4_lane_v, 2426 %vld4_lane_v.fca.3.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld4_lane_v, 3427 %4 = bitcast <4 x bfloat> %vld4_lane_v.fca.0.extract to <2 x i32>428 %5 = bitcast <4 x bfloat> %vld4_lane_v.fca.1.extract to <2 x i32>429 %6 = bitcast <4 x bfloat> %vld4_lane_v.fca.2.extract to <2 x i32>430 %7 = bitcast <4 x bfloat> %vld4_lane_v.fca.3.extract to <2 x i32>431 %.fca.0.insert = insertvalue [4 x <2 x i32>] undef, <2 x i32> %4, 0432 %.fca.1.insert = insertvalue [4 x <2 x i32>] %.fca.0.insert, <2 x i32> %5, 1433 %.fca.2.insert = insertvalue [4 x <2 x i32>] %.fca.1.insert, <2 x i32> %6, 2434 %.fca.3.insert = insertvalue [4 x <2 x i32>] %.fca.2.insert, <2 x i32> %7, 3435 ret [4 x <2 x i32>] %.fca.3.insert436}437 438define arm_aapcs_vfpcc [4 x <4 x i32>] @test_vld4q_lane_bf16(ptr %ptr, [4 x <4 x i32>] %src.coerce) {439; CHECK-LABEL: test_vld4q_lane_bf16:440; CHECK: @ %bb.0: @ %entry441; CHECK-NEXT: @ kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3442; CHECK-NEXT: @ kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3443; CHECK-NEXT: @ kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3444; CHECK-NEXT: @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3445; CHECK-NEXT: vld4.16 {d1[3], d3[3], d5[3], d7[3]}, [r0]446; CHECK-NEXT: bx lr447entry:448 %src.coerce.fca.0.extract = extractvalue [4 x <4 x i32>] %src.coerce, 0449 %src.coerce.fca.1.extract = extractvalue [4 x <4 x i32>] %src.coerce, 1450 %src.coerce.fca.2.extract = extractvalue [4 x <4 x i32>] %src.coerce, 2451 %src.coerce.fca.3.extract = extractvalue [4 x <4 x i32>] %src.coerce, 3452 %0 = bitcast <4 x i32> %src.coerce.fca.0.extract to <8 x bfloat>453 %1 = bitcast <4 x i32> %src.coerce.fca.1.extract to <8 x bfloat>454 %2 = bitcast <4 x i32> %src.coerce.fca.2.extract to <8 x bfloat>455 %3 = bitcast <4 x i32> %src.coerce.fca.3.extract to <8 x bfloat>456 %vld4q_lane_v = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld4lane.v8bf16.p0(ptr %ptr, <8 x bfloat> %0, <8 x bfloat> %1, <8 x bfloat> %2, <8 x bfloat> %3, i32 7, i32 2)457 %vld4q_lane_v.fca.0.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld4q_lane_v, 0458 %vld4q_lane_v.fca.1.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld4q_lane_v, 1459 %vld4q_lane_v.fca.2.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld4q_lane_v, 2460 %vld4q_lane_v.fca.3.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld4q_lane_v, 3461 %4 = bitcast <8 x bfloat> %vld4q_lane_v.fca.0.extract to <4 x i32>462 %5 = bitcast <8 x bfloat> %vld4q_lane_v.fca.1.extract to <4 x i32>463 %6 = bitcast <8 x bfloat> %vld4q_lane_v.fca.2.extract to <4 x i32>464 %7 = bitcast <8 x bfloat> %vld4q_lane_v.fca.3.extract to <4 x i32>465 %.fca.0.insert = insertvalue [4 x <4 x i32>] undef, <4 x i32> %4, 0466 %.fca.1.insert = insertvalue [4 x <4 x i32>] %.fca.0.insert, <4 x i32> %5, 1467 %.fca.2.insert = insertvalue [4 x <4 x i32>] %.fca.1.insert, <4 x i32> %6, 2468 %.fca.3.insert = insertvalue [4 x <4 x i32>] %.fca.2.insert, <4 x i32> %7, 3469 ret [4 x <4 x i32>] %.fca.3.insert470}471 472define arm_aapcs_vfpcc [2 x <2 x i32>] @test_vld2_dup_bf16(ptr %ptr) {473; CHECK-LABEL: test_vld2_dup_bf16:474; CHECK: @ %bb.0: @ %entry475; CHECK-NEXT: vld2.16 {d0[], d1[]}, [r0]476; CHECK-NEXT: bx lr477entry:478 %vld2_dup_v = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld2dup.v4bf16.p0(ptr %ptr, i32 2)479 %vld2_dup_v.fca.0.extract = extractvalue { <4 x bfloat>, <4 x bfloat> } %vld2_dup_v, 0480 %vld2_dup_v.fca.1.extract = extractvalue { <4 x bfloat>, <4 x bfloat> } %vld2_dup_v, 1481 %0 = bitcast <4 x bfloat> %vld2_dup_v.fca.0.extract to <2 x i32>482 %1 = bitcast <4 x bfloat> %vld2_dup_v.fca.1.extract to <2 x i32>483 %.fca.0.insert = insertvalue [2 x <2 x i32>] undef, <2 x i32> %0, 0484 %.fca.1.insert = insertvalue [2 x <2 x i32>] %.fca.0.insert, <2 x i32> %1, 1485 ret [2 x <2 x i32>] %.fca.1.insert486}487 488define arm_aapcs_vfpcc [2 x <4 x i32>] @test_vld2q_dup_bf16(ptr %ptr) {489; CHECK-LABEL: test_vld2q_dup_bf16:490; CHECK: @ %bb.0: @ %entry491; CHECK-NEXT: vld2.16 {d0[], d2[]}, [r0]492; CHECK-NEXT: vld2.16 {d1[], d3[]}, [r0]493; CHECK-NEXT: bx lr494entry:495 %vld2q_dup_v = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld2dup.v8bf16.p0(ptr %ptr, i32 2)496 %vld2q_dup_v.fca.0.extract = extractvalue { <8 x bfloat>, <8 x bfloat> } %vld2q_dup_v, 0497 %vld2q_dup_v.fca.1.extract = extractvalue { <8 x bfloat>, <8 x bfloat> } %vld2q_dup_v, 1498 %0 = bitcast <8 x bfloat> %vld2q_dup_v.fca.0.extract to <4 x i32>499 %1 = bitcast <8 x bfloat> %vld2q_dup_v.fca.1.extract to <4 x i32>500 %.fca.0.insert = insertvalue [2 x <4 x i32>] undef, <4 x i32> %0, 0501 %.fca.1.insert = insertvalue [2 x <4 x i32>] %.fca.0.insert, <4 x i32> %1, 1502 ret [2 x <4 x i32>] %.fca.1.insert503}504 505define arm_aapcs_vfpcc [3 x <2 x i32>] @test_vld3_dup_bf16(ptr %ptr) {506; CHECK-LABEL: test_vld3_dup_bf16:507; CHECK: @ %bb.0: @ %entry508; CHECK-NEXT: vld3.16 {d0[], d1[], d2[]}, [r0]509; CHECK-NEXT: bx lr510entry:511 %vld3_dup_v = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld3dup.v4bf16.p0(ptr %ptr, i32 2)512 %vld3_dup_v.fca.0.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld3_dup_v, 0513 %vld3_dup_v.fca.1.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld3_dup_v, 1514 %vld3_dup_v.fca.2.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld3_dup_v, 2515 %0 = bitcast <4 x bfloat> %vld3_dup_v.fca.0.extract to <2 x i32>516 %1 = bitcast <4 x bfloat> %vld3_dup_v.fca.1.extract to <2 x i32>517 %2 = bitcast <4 x bfloat> %vld3_dup_v.fca.2.extract to <2 x i32>518 %.fca.0.insert = insertvalue [3 x <2 x i32>] undef, <2 x i32> %0, 0519 %.fca.1.insert = insertvalue [3 x <2 x i32>] %.fca.0.insert, <2 x i32> %1, 1520 %.fca.2.insert = insertvalue [3 x <2 x i32>] %.fca.1.insert, <2 x i32> %2, 2521 ret [3 x <2 x i32>] %.fca.2.insert522}523 524define arm_aapcs_vfpcc [3 x <4 x i32>] @test_vld3q_dup_bf16(ptr %ptr) {525; CHECK-LABEL: test_vld3q_dup_bf16:526; CHECK: @ %bb.0: @ %entry527; CHECK-NEXT: vld3.16 {d0[], d2[], d4[]}, [r0]528; CHECK-NEXT: vld3.16 {d1[], d3[], d5[]}, [r0]529; CHECK-NEXT: bx lr530entry:531 %vld3q_dup_v = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld3dup.v8bf16.p0(ptr %ptr, i32 2)532 %vld3q_dup_v.fca.0.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld3q_dup_v, 0533 %vld3q_dup_v.fca.1.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld3q_dup_v, 1534 %vld3q_dup_v.fca.2.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld3q_dup_v, 2535 %0 = bitcast <8 x bfloat> %vld3q_dup_v.fca.0.extract to <4 x i32>536 %1 = bitcast <8 x bfloat> %vld3q_dup_v.fca.1.extract to <4 x i32>537 %2 = bitcast <8 x bfloat> %vld3q_dup_v.fca.2.extract to <4 x i32>538 %.fca.0.insert = insertvalue [3 x <4 x i32>] undef, <4 x i32> %0, 0539 %.fca.1.insert = insertvalue [3 x <4 x i32>] %.fca.0.insert, <4 x i32> %1, 1540 %.fca.2.insert = insertvalue [3 x <4 x i32>] %.fca.1.insert, <4 x i32> %2, 2541 ret [3 x <4 x i32>] %.fca.2.insert542}543 544define arm_aapcs_vfpcc [4 x <2 x i32>] @test_vld4_dup_bf16(ptr %ptr) {545; CHECK-LABEL: test_vld4_dup_bf16:546; CHECK: @ %bb.0: @ %entry547; CHECK-NEXT: vld4.16 {d0[], d1[], d2[], d3[]}, [r0]548; CHECK-NEXT: bx lr549entry:550 %vld4_dup_v = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld4dup.v4bf16.p0(ptr %ptr, i32 2)551 %vld4_dup_v.fca.0.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld4_dup_v, 0552 %vld4_dup_v.fca.1.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld4_dup_v, 1553 %vld4_dup_v.fca.2.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld4_dup_v, 2554 %vld4_dup_v.fca.3.extract = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } %vld4_dup_v, 3555 %0 = bitcast <4 x bfloat> %vld4_dup_v.fca.0.extract to <2 x i32>556 %1 = bitcast <4 x bfloat> %vld4_dup_v.fca.1.extract to <2 x i32>557 %2 = bitcast <4 x bfloat> %vld4_dup_v.fca.2.extract to <2 x i32>558 %3 = bitcast <4 x bfloat> %vld4_dup_v.fca.3.extract to <2 x i32>559 %.fca.0.insert = insertvalue [4 x <2 x i32>] undef, <2 x i32> %0, 0560 %.fca.1.insert = insertvalue [4 x <2 x i32>] %.fca.0.insert, <2 x i32> %1, 1561 %.fca.2.insert = insertvalue [4 x <2 x i32>] %.fca.1.insert, <2 x i32> %2, 2562 %.fca.3.insert = insertvalue [4 x <2 x i32>] %.fca.2.insert, <2 x i32> %3, 3563 ret [4 x <2 x i32>] %.fca.3.insert564}565 566define arm_aapcs_vfpcc [4 x <4 x i32>] @test_vld4q_dup_bf16(ptr %ptr) {567; CHECK-LABEL: test_vld4q_dup_bf16:568; CHECK: @ %bb.0: @ %entry569; CHECK-NEXT: vld4.16 {d0[], d2[], d4[], d6[]}, [r0]570; CHECK-NEXT: vld4.16 {d1[], d3[], d5[], d7[]}, [r0]571; CHECK-NEXT: bx lr572entry:573 %vld4q_dup_v = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld4dup.v8bf16.p0(ptr %ptr, i32 2)574 %vld4q_dup_v.fca.0.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld4q_dup_v, 0575 %vld4q_dup_v.fca.1.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld4q_dup_v, 1576 %vld4q_dup_v.fca.2.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld4q_dup_v, 2577 %vld4q_dup_v.fca.3.extract = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } %vld4q_dup_v, 3578 %0 = bitcast <8 x bfloat> %vld4q_dup_v.fca.0.extract to <4 x i32>579 %1 = bitcast <8 x bfloat> %vld4q_dup_v.fca.1.extract to <4 x i32>580 %2 = bitcast <8 x bfloat> %vld4q_dup_v.fca.2.extract to <4 x i32>581 %3 = bitcast <8 x bfloat> %vld4q_dup_v.fca.3.extract to <4 x i32>582 %.fca.0.insert = insertvalue [4 x <4 x i32>] undef, <4 x i32> %0, 0583 %.fca.1.insert = insertvalue [4 x <4 x i32>] %.fca.0.insert, <4 x i32> %1, 1584 %.fca.2.insert = insertvalue [4 x <4 x i32>] %.fca.1.insert, <4 x i32> %2, 2585 %.fca.3.insert = insertvalue [4 x <4 x i32>] %.fca.2.insert, <4 x i32> %3, 3586 ret [4 x <4 x i32>] %.fca.3.insert587}588 589define arm_aapcs_vfpcc void @test_vst1_bf16(ptr %ptr, <4 x bfloat> %val) {590; CHECK-LABEL: test_vst1_bf16:591; CHECK: @ %bb.0: @ %entry592; CHECK-NEXT: vst1.16 {d0}, [r0]593; CHECK-NEXT: bx lr594entry:595 tail call void @llvm.arm.neon.vst1.p0.v4bf16(ptr %ptr, <4 x bfloat> %val, i32 2)596 ret void597}598 599define arm_aapcs_vfpcc void @test_vst1q_bf16(ptr %ptr, <8 x bfloat> %val) {600; CHECK-LABEL: test_vst1q_bf16:601; CHECK: @ %bb.0: @ %entry602; CHECK-NEXT: vst1.16 {d0, d1}, [r0]603; CHECK-NEXT: bx lr604entry:605 tail call void @llvm.arm.neon.vst1.p0.v8bf16(ptr %ptr, <8 x bfloat> %val, i32 2)606 ret void607}608 609define arm_aapcs_vfpcc void @test_vst1_lane_bf16(ptr nocapture %ptr, <4 x bfloat> %val) {610; CHECK-LABEL: test_vst1_lane_bf16:611; CHECK: @ %bb.0: @ %entry612; CHECK-NEXT: vmovx.f16 s0, s0613; CHECK-NEXT: vstr.16 s0, [r0]614; CHECK-NEXT: bx lr615entry:616 %0 = extractelement <4 x bfloat> %val, i32 1617 store bfloat %0, ptr %ptr, align 2618 ret void619}620 621define arm_aapcs_vfpcc void @test_vst1q_lane_bf16(ptr nocapture %ptr, <8 x bfloat> %val) {622; CHECK-LABEL: test_vst1q_lane_bf16:623; CHECK: @ %bb.0: @ %entry624; CHECK-NEXT: vmovx.f16 s0, s3625; CHECK-NEXT: vstr.16 s0, [r0]626; CHECK-NEXT: bx lr627entry:628 %0 = extractelement <8 x bfloat> %val, i32 7629 store bfloat %0, ptr %ptr, align 2630 ret void631}632 633define arm_aapcs_vfpcc void @test_vst1_bf16_x2(ptr nocapture %ptr, [2 x <2 x i32>] %val.coerce) {634; CHECK-LABEL: test_vst1_bf16_x2:635; CHECK: @ %bb.0: @ %entry636; CHECK-NEXT: @ kill: def $d1 killed $d1 killed $q0 def $q0637; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0 def $q0638; CHECK-NEXT: vst1.16 {d0, d1}, [r0]639; CHECK-NEXT: bx lr640entry:641 %val.coerce.fca.0.extract = extractvalue [2 x <2 x i32>] %val.coerce, 0642 %val.coerce.fca.1.extract = extractvalue [2 x <2 x i32>] %val.coerce, 1643 %0 = bitcast <2 x i32> %val.coerce.fca.0.extract to <4 x bfloat>644 %1 = bitcast <2 x i32> %val.coerce.fca.1.extract to <4 x bfloat>645 tail call void @llvm.arm.neon.vst1x2.p0.v4bf16(ptr %ptr, <4 x bfloat> %0, <4 x bfloat> %1)646 ret void647}648 649define arm_aapcs_vfpcc void @test_vst1q_bf16_x2(ptr nocapture %ptr, [2 x <4 x i32>] %val.coerce) {650; CHECK-LABEL: test_vst1q_bf16_x2:651; CHECK: @ %bb.0: @ %entry652; CHECK-NEXT: @ kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1653; CHECK-NEXT: @ kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1654; CHECK-NEXT: vst1.16 {d0, d1, d2, d3}, [r0]655; CHECK-NEXT: bx lr656entry:657 %val.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %val.coerce, 0658 %val.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %val.coerce, 1659 %0 = bitcast <4 x i32> %val.coerce.fca.0.extract to <8 x bfloat>660 %1 = bitcast <4 x i32> %val.coerce.fca.1.extract to <8 x bfloat>661 tail call void @llvm.arm.neon.vst1x2.p0.v8bf16(ptr %ptr, <8 x bfloat> %0, <8 x bfloat> %1)662 ret void663}664 665define arm_aapcs_vfpcc void @test_vst1_bf16_x3(ptr nocapture %ptr, [3 x <2 x i32>] %val.coerce) {666; CHECK-LABEL: test_vst1_bf16_x3:667; CHECK: @ %bb.0: @ %entry668; CHECK-NEXT: @ kill: def $d2 killed $d2 killed $q0_q1 def $q0_q1669; CHECK-NEXT: @ kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1670; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1671; CHECK-NEXT: vst1.16 {d0, d1, d2}, [r0]672; CHECK-NEXT: bx lr673entry:674 %val.coerce.fca.0.extract = extractvalue [3 x <2 x i32>] %val.coerce, 0675 %val.coerce.fca.1.extract = extractvalue [3 x <2 x i32>] %val.coerce, 1676 %val.coerce.fca.2.extract = extractvalue [3 x <2 x i32>] %val.coerce, 2677 %0 = bitcast <2 x i32> %val.coerce.fca.0.extract to <4 x bfloat>678 %1 = bitcast <2 x i32> %val.coerce.fca.1.extract to <4 x bfloat>679 %2 = bitcast <2 x i32> %val.coerce.fca.2.extract to <4 x bfloat>680 tail call void @llvm.arm.neon.vst1x3.p0.v4bf16(ptr %ptr, <4 x bfloat> %0, <4 x bfloat> %1, <4 x bfloat> %2)681 ret void682}683 684define arm_aapcs_vfpcc void @test_vst1q_bf16_x3(ptr nocapture %ptr, [3 x <4 x i32>] %val.coerce) {685; CHECK-LABEL: test_vst1q_bf16_x3:686; CHECK: @ %bb.0: @ %entry687; CHECK-NEXT: @ kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3688; CHECK-NEXT: @ kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3689; CHECK-NEXT: @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3690; CHECK-NEXT: vst1.16 {d0, d1, d2}, [r0]!691; CHECK-NEXT: vst1.16 {d3, d4, d5}, [r0]692; CHECK-NEXT: bx lr693entry:694 %val.coerce.fca.0.extract = extractvalue [3 x <4 x i32>] %val.coerce, 0695 %val.coerce.fca.1.extract = extractvalue [3 x <4 x i32>] %val.coerce, 1696 %val.coerce.fca.2.extract = extractvalue [3 x <4 x i32>] %val.coerce, 2697 %0 = bitcast <4 x i32> %val.coerce.fca.0.extract to <8 x bfloat>698 %1 = bitcast <4 x i32> %val.coerce.fca.1.extract to <8 x bfloat>699 %2 = bitcast <4 x i32> %val.coerce.fca.2.extract to <8 x bfloat>700 tail call void @llvm.arm.neon.vst1x3.p0.v8bf16(ptr %ptr, <8 x bfloat> %0, <8 x bfloat> %1, <8 x bfloat> %2)701 ret void702}703 704define arm_aapcs_vfpcc void @test_vst1_bf16_x4(ptr nocapture %ptr, [4 x <2 x i32>] %val.coerce) {705; CHECK-LABEL: test_vst1_bf16_x4:706; CHECK: @ %bb.0: @ %entry707; CHECK-NEXT: @ kill: def $d3 killed $d3 killed $q0_q1 def $q0_q1708; CHECK-NEXT: @ kill: def $d2 killed $d2 killed $q0_q1 def $q0_q1709; CHECK-NEXT: @ kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1710; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1711; CHECK-NEXT: vst1.16 {d0, d1, d2, d3}, [r0]712; CHECK-NEXT: bx lr713entry:714 %val.coerce.fca.0.extract = extractvalue [4 x <2 x i32>] %val.coerce, 0715 %val.coerce.fca.1.extract = extractvalue [4 x <2 x i32>] %val.coerce, 1716 %val.coerce.fca.2.extract = extractvalue [4 x <2 x i32>] %val.coerce, 2717 %val.coerce.fca.3.extract = extractvalue [4 x <2 x i32>] %val.coerce, 3718 %0 = bitcast <2 x i32> %val.coerce.fca.0.extract to <4 x bfloat>719 %1 = bitcast <2 x i32> %val.coerce.fca.1.extract to <4 x bfloat>720 %2 = bitcast <2 x i32> %val.coerce.fca.2.extract to <4 x bfloat>721 %3 = bitcast <2 x i32> %val.coerce.fca.3.extract to <4 x bfloat>722 tail call void @llvm.arm.neon.vst1x4.p0.v4bf16(ptr %ptr, <4 x bfloat> %0, <4 x bfloat> %1, <4 x bfloat> %2, <4 x bfloat> %3)723 ret void724}725 726define arm_aapcs_vfpcc void @test_vst1q_bf16_x4(ptr nocapture %ptr, [4 x <4 x i32>] %val.coerce) {727; CHECK-LABEL: test_vst1q_bf16_x4:728; CHECK: @ %bb.0: @ %entry729; CHECK-NEXT: @ kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3730; CHECK-NEXT: @ kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3731; CHECK-NEXT: @ kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3732; CHECK-NEXT: @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3733; CHECK-NEXT: vst1.16 {d0, d1, d2, d3}, [r0]!734; CHECK-NEXT: vst1.16 {d4, d5, d6, d7}, [r0]735; CHECK-NEXT: bx lr736entry:737 %val.coerce.fca.0.extract = extractvalue [4 x <4 x i32>] %val.coerce, 0738 %val.coerce.fca.1.extract = extractvalue [4 x <4 x i32>] %val.coerce, 1739 %val.coerce.fca.2.extract = extractvalue [4 x <4 x i32>] %val.coerce, 2740 %val.coerce.fca.3.extract = extractvalue [4 x <4 x i32>] %val.coerce, 3741 %0 = bitcast <4 x i32> %val.coerce.fca.0.extract to <8 x bfloat>742 %1 = bitcast <4 x i32> %val.coerce.fca.1.extract to <8 x bfloat>743 %2 = bitcast <4 x i32> %val.coerce.fca.2.extract to <8 x bfloat>744 %3 = bitcast <4 x i32> %val.coerce.fca.3.extract to <8 x bfloat>745 tail call void @llvm.arm.neon.vst1x4.p0.v8bf16(ptr %ptr, <8 x bfloat> %0, <8 x bfloat> %1, <8 x bfloat> %2, <8 x bfloat> %3)746 ret void747}748 749define arm_aapcs_vfpcc void @test_vst2_bf16(ptr %ptr, [2 x <2 x i32>] %val.coerce) {750; CHECK-LABEL: test_vst2_bf16:751; CHECK: @ %bb.0: @ %entry752; CHECK-NEXT: @ kill: def $d1 killed $d1 killed $q0 def $q0753; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0 def $q0754; CHECK-NEXT: vst2.16 {d0, d1}, [r0]755; CHECK-NEXT: bx lr756entry:757 %val.coerce.fca.0.extract = extractvalue [2 x <2 x i32>] %val.coerce, 0758 %val.coerce.fca.1.extract = extractvalue [2 x <2 x i32>] %val.coerce, 1759 %0 = bitcast <2 x i32> %val.coerce.fca.0.extract to <4 x bfloat>760 %1 = bitcast <2 x i32> %val.coerce.fca.1.extract to <4 x bfloat>761 tail call void @llvm.arm.neon.vst2.p0.v4bf16(ptr %ptr, <4 x bfloat> %0, <4 x bfloat> %1, i32 2)762 ret void763}764 765define arm_aapcs_vfpcc void @test_vst2q_bf16(ptr %ptr, [2 x <4 x i32>] %val.coerce) {766; CHECK-LABEL: test_vst2q_bf16:767; CHECK: @ %bb.0: @ %entry768; CHECK-NEXT: @ kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1769; CHECK-NEXT: @ kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1770; CHECK-NEXT: vst2.16 {d0, d1, d2, d3}, [r0]771; CHECK-NEXT: bx lr772entry:773 %val.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %val.coerce, 0774 %val.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %val.coerce, 1775 %0 = bitcast <4 x i32> %val.coerce.fca.0.extract to <8 x bfloat>776 %1 = bitcast <4 x i32> %val.coerce.fca.1.extract to <8 x bfloat>777 tail call void @llvm.arm.neon.vst2.p0.v8bf16(ptr %ptr, <8 x bfloat> %0, <8 x bfloat> %1, i32 2)778 ret void779}780 781define arm_aapcs_vfpcc void @test_vst2_lane_bf16(ptr %ptr, [2 x <2 x i32>] %val.coerce) {782; CHECK-LABEL: test_vst2_lane_bf16:783; CHECK: @ %bb.0: @ %entry784; CHECK-NEXT: @ kill: def $d1 killed $d1 killed $q0 def $q0785; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0 def $q0786; CHECK-NEXT: vst2.16 {d0[1], d1[1]}, [r0]787; CHECK-NEXT: bx lr788entry:789 %val.coerce.fca.0.extract = extractvalue [2 x <2 x i32>] %val.coerce, 0790 %val.coerce.fca.1.extract = extractvalue [2 x <2 x i32>] %val.coerce, 1791 %0 = bitcast <2 x i32> %val.coerce.fca.0.extract to <4 x bfloat>792 %1 = bitcast <2 x i32> %val.coerce.fca.1.extract to <4 x bfloat>793 tail call void @llvm.arm.neon.vst2lane.p0.v4bf16(ptr %ptr, <4 x bfloat> %0, <4 x bfloat> %1, i32 1, i32 2)794 ret void795}796 797define arm_aapcs_vfpcc void @test_vst2q_lane_bf16(ptr %ptr, [2 x <4 x i32>] %val.coerce) {798; CHECK-LABEL: test_vst2q_lane_bf16:799; CHECK: @ %bb.0: @ %entry800; CHECK-NEXT: @ kill: def $q1 killed $q1 killed $q0_q1 def $q0_q1801; CHECK-NEXT: @ kill: def $q0 killed $q0 killed $q0_q1 def $q0_q1802; CHECK-NEXT: vst2.16 {d1[3], d3[3]}, [r0]803; CHECK-NEXT: bx lr804entry:805 %val.coerce.fca.0.extract = extractvalue [2 x <4 x i32>] %val.coerce, 0806 %val.coerce.fca.1.extract = extractvalue [2 x <4 x i32>] %val.coerce, 1807 %0 = bitcast <4 x i32> %val.coerce.fca.0.extract to <8 x bfloat>808 %1 = bitcast <4 x i32> %val.coerce.fca.1.extract to <8 x bfloat>809 tail call void @llvm.arm.neon.vst2lane.p0.v8bf16(ptr %ptr, <8 x bfloat> %0, <8 x bfloat> %1, i32 7, i32 2)810 ret void811}812 813define arm_aapcs_vfpcc void @test_vst3_bf16(ptr %ptr, [3 x <2 x i32>] %val.coerce) {814; CHECK-LABEL: test_vst3_bf16:815; CHECK: @ %bb.0: @ %entry816; CHECK-NEXT: @ kill: def $d2 killed $d2 killed $q0_q1 def $q0_q1817; CHECK-NEXT: @ kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1818; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1819; CHECK-NEXT: vst3.16 {d0, d1, d2}, [r0]820; CHECK-NEXT: bx lr821entry:822 %val.coerce.fca.0.extract = extractvalue [3 x <2 x i32>] %val.coerce, 0823 %val.coerce.fca.1.extract = extractvalue [3 x <2 x i32>] %val.coerce, 1824 %val.coerce.fca.2.extract = extractvalue [3 x <2 x i32>] %val.coerce, 2825 %0 = bitcast <2 x i32> %val.coerce.fca.0.extract to <4 x bfloat>826 %1 = bitcast <2 x i32> %val.coerce.fca.1.extract to <4 x bfloat>827 %2 = bitcast <2 x i32> %val.coerce.fca.2.extract to <4 x bfloat>828 tail call void @llvm.arm.neon.vst3.p0.v4bf16(ptr %ptr, <4 x bfloat> %0, <4 x bfloat> %1, <4 x bfloat> %2, i32 2)829 ret void830}831 832define arm_aapcs_vfpcc void @test_vst3q_bf16(ptr %ptr, [3 x <4 x i32>] %val.coerce) {833; CHECK-LABEL: test_vst3q_bf16:834; CHECK: @ %bb.0: @ %entry835; CHECK-NEXT: @ kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3836; CHECK-NEXT: @ kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3837; CHECK-NEXT: @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3838; CHECK-NEXT: vst3.16 {d0, d2, d4}, [r0]!839; CHECK-NEXT: vst3.16 {d1, d3, d5}, [r0]840; CHECK-NEXT: bx lr841entry:842 %val.coerce.fca.0.extract = extractvalue [3 x <4 x i32>] %val.coerce, 0843 %val.coerce.fca.1.extract = extractvalue [3 x <4 x i32>] %val.coerce, 1844 %val.coerce.fca.2.extract = extractvalue [3 x <4 x i32>] %val.coerce, 2845 %0 = bitcast <4 x i32> %val.coerce.fca.0.extract to <8 x bfloat>846 %1 = bitcast <4 x i32> %val.coerce.fca.1.extract to <8 x bfloat>847 %2 = bitcast <4 x i32> %val.coerce.fca.2.extract to <8 x bfloat>848 tail call void @llvm.arm.neon.vst3.p0.v8bf16(ptr %ptr, <8 x bfloat> %0, <8 x bfloat> %1, <8 x bfloat> %2, i32 2)849 ret void850}851 852define arm_aapcs_vfpcc void @test_vst3_lane_bf16(ptr %ptr, [3 x <2 x i32>] %val.coerce) {853; CHECK-LABEL: test_vst3_lane_bf16:854; CHECK: @ %bb.0: @ %entry855; CHECK-NEXT: @ kill: def $d2 killed $d2 killed $q0_q1 def $q0_q1856; CHECK-NEXT: @ kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1857; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1858; CHECK-NEXT: vst3.16 {d0[1], d1[1], d2[1]}, [r0]859; CHECK-NEXT: bx lr860entry:861 %val.coerce.fca.0.extract = extractvalue [3 x <2 x i32>] %val.coerce, 0862 %val.coerce.fca.1.extract = extractvalue [3 x <2 x i32>] %val.coerce, 1863 %val.coerce.fca.2.extract = extractvalue [3 x <2 x i32>] %val.coerce, 2864 %0 = bitcast <2 x i32> %val.coerce.fca.0.extract to <4 x bfloat>865 %1 = bitcast <2 x i32> %val.coerce.fca.1.extract to <4 x bfloat>866 %2 = bitcast <2 x i32> %val.coerce.fca.2.extract to <4 x bfloat>867 tail call void @llvm.arm.neon.vst3lane.p0.v4bf16(ptr %ptr, <4 x bfloat> %0, <4 x bfloat> %1, <4 x bfloat> %2, i32 1, i32 2)868 ret void869}870 871define arm_aapcs_vfpcc void @test_vst3q_lane_bf16(ptr %ptr, [3 x <4 x i32>] %val.coerce) {872; CHECK-LABEL: test_vst3q_lane_bf16:873; CHECK: @ %bb.0: @ %entry874; CHECK-NEXT: @ kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3875; CHECK-NEXT: @ kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3876; CHECK-NEXT: @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3877; CHECK-NEXT: vst3.16 {d1[3], d3[3], d5[3]}, [r0]878; CHECK-NEXT: bx lr879entry:880 %val.coerce.fca.0.extract = extractvalue [3 x <4 x i32>] %val.coerce, 0881 %val.coerce.fca.1.extract = extractvalue [3 x <4 x i32>] %val.coerce, 1882 %val.coerce.fca.2.extract = extractvalue [3 x <4 x i32>] %val.coerce, 2883 %0 = bitcast <4 x i32> %val.coerce.fca.0.extract to <8 x bfloat>884 %1 = bitcast <4 x i32> %val.coerce.fca.1.extract to <8 x bfloat>885 %2 = bitcast <4 x i32> %val.coerce.fca.2.extract to <8 x bfloat>886 tail call void @llvm.arm.neon.vst3lane.p0.v8bf16(ptr %ptr, <8 x bfloat> %0, <8 x bfloat> %1, <8 x bfloat> %2, i32 7, i32 2)887 ret void888}889 890define arm_aapcs_vfpcc void @test_vst4_bf16(ptr %ptr, [4 x <2 x i32>] %val.coerce) {891; CHECK-LABEL: test_vst4_bf16:892; CHECK: @ %bb.0: @ %entry893; CHECK-NEXT: @ kill: def $d3 killed $d3 killed $q0_q1 def $q0_q1894; CHECK-NEXT: @ kill: def $d2 killed $d2 killed $q0_q1 def $q0_q1895; CHECK-NEXT: @ kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1896; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1897; CHECK-NEXT: vst4.16 {d0, d1, d2, d3}, [r0]898; CHECK-NEXT: bx lr899entry:900 %val.coerce.fca.0.extract = extractvalue [4 x <2 x i32>] %val.coerce, 0901 %val.coerce.fca.1.extract = extractvalue [4 x <2 x i32>] %val.coerce, 1902 %val.coerce.fca.2.extract = extractvalue [4 x <2 x i32>] %val.coerce, 2903 %val.coerce.fca.3.extract = extractvalue [4 x <2 x i32>] %val.coerce, 3904 %0 = bitcast <2 x i32> %val.coerce.fca.0.extract to <4 x bfloat>905 %1 = bitcast <2 x i32> %val.coerce.fca.1.extract to <4 x bfloat>906 %2 = bitcast <2 x i32> %val.coerce.fca.2.extract to <4 x bfloat>907 %3 = bitcast <2 x i32> %val.coerce.fca.3.extract to <4 x bfloat>908 tail call void @llvm.arm.neon.vst4.p0.v4bf16(ptr %ptr, <4 x bfloat> %0, <4 x bfloat> %1, <4 x bfloat> %2, <4 x bfloat> %3, i32 2)909 ret void910}911 912define arm_aapcs_vfpcc void @test_vst4q_bf16(ptr %ptr, [4 x <4 x i32>] %val.coerce) {913; CHECK-LABEL: test_vst4q_bf16:914; CHECK: @ %bb.0: @ %entry915; CHECK-NEXT: @ kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3916; CHECK-NEXT: @ kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3917; CHECK-NEXT: @ kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3918; CHECK-NEXT: @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3919; CHECK-NEXT: vst4.16 {d0, d2, d4, d6}, [r0]!920; CHECK-NEXT: vst4.16 {d1, d3, d5, d7}, [r0]921; CHECK-NEXT: bx lr922entry:923 %val.coerce.fca.0.extract = extractvalue [4 x <4 x i32>] %val.coerce, 0924 %val.coerce.fca.1.extract = extractvalue [4 x <4 x i32>] %val.coerce, 1925 %val.coerce.fca.2.extract = extractvalue [4 x <4 x i32>] %val.coerce, 2926 %val.coerce.fca.3.extract = extractvalue [4 x <4 x i32>] %val.coerce, 3927 %0 = bitcast <4 x i32> %val.coerce.fca.0.extract to <8 x bfloat>928 %1 = bitcast <4 x i32> %val.coerce.fca.1.extract to <8 x bfloat>929 %2 = bitcast <4 x i32> %val.coerce.fca.2.extract to <8 x bfloat>930 %3 = bitcast <4 x i32> %val.coerce.fca.3.extract to <8 x bfloat>931 tail call void @llvm.arm.neon.vst4.p0.v8bf16(ptr %ptr, <8 x bfloat> %0, <8 x bfloat> %1, <8 x bfloat> %2, <8 x bfloat> %3, i32 2)932 ret void933}934 935define arm_aapcs_vfpcc void @test_vst4_lane_bf16(ptr %ptr, [4 x <2 x i32>] %val.coerce) {936; CHECK-LABEL: test_vst4_lane_bf16:937; CHECK: @ %bb.0: @ %entry938; CHECK-NEXT: @ kill: def $d3 killed $d3 killed $q0_q1 def $q0_q1939; CHECK-NEXT: @ kill: def $d2 killed $d2 killed $q0_q1 def $q0_q1940; CHECK-NEXT: @ kill: def $d1 killed $d1 killed $q0_q1 def $q0_q1941; CHECK-NEXT: @ kill: def $d0 killed $d0 killed $q0_q1 def $q0_q1942; CHECK-NEXT: vst4.16 {d0[1], d1[1], d2[1], d3[1]}, [r0]943; CHECK-NEXT: bx lr944entry:945 %val.coerce.fca.0.extract = extractvalue [4 x <2 x i32>] %val.coerce, 0946 %val.coerce.fca.1.extract = extractvalue [4 x <2 x i32>] %val.coerce, 1947 %val.coerce.fca.2.extract = extractvalue [4 x <2 x i32>] %val.coerce, 2948 %val.coerce.fca.3.extract = extractvalue [4 x <2 x i32>] %val.coerce, 3949 %0 = bitcast <2 x i32> %val.coerce.fca.0.extract to <4 x bfloat>950 %1 = bitcast <2 x i32> %val.coerce.fca.1.extract to <4 x bfloat>951 %2 = bitcast <2 x i32> %val.coerce.fca.2.extract to <4 x bfloat>952 %3 = bitcast <2 x i32> %val.coerce.fca.3.extract to <4 x bfloat>953 tail call void @llvm.arm.neon.vst4lane.p0.v4bf16(ptr %ptr, <4 x bfloat> %0, <4 x bfloat> %1, <4 x bfloat> %2, <4 x bfloat> %3, i32 1, i32 2)954 ret void955}956 957define arm_aapcs_vfpcc void @test_vst4q_lane_bf16(ptr %ptr, [4 x <4 x i32>] %val.coerce) {958; CHECK-LABEL: test_vst4q_lane_bf16:959; CHECK: @ %bb.0: @ %entry960; CHECK-NEXT: @ kill: def $q3 killed $q3 killed $q0_q1_q2_q3 def $q0_q1_q2_q3961; CHECK-NEXT: @ kill: def $q2 killed $q2 killed $q0_q1_q2_q3 def $q0_q1_q2_q3962; CHECK-NEXT: @ kill: def $q1 killed $q1 killed $q0_q1_q2_q3 def $q0_q1_q2_q3963; CHECK-NEXT: @ kill: def $q0 killed $q0 killed $q0_q1_q2_q3 def $q0_q1_q2_q3964; CHECK-NEXT: vst4.16 {d1[3], d3[3], d5[3], d7[3]}, [r0]965; CHECK-NEXT: bx lr966entry:967 %val.coerce.fca.0.extract = extractvalue [4 x <4 x i32>] %val.coerce, 0968 %val.coerce.fca.1.extract = extractvalue [4 x <4 x i32>] %val.coerce, 1969 %val.coerce.fca.2.extract = extractvalue [4 x <4 x i32>] %val.coerce, 2970 %val.coerce.fca.3.extract = extractvalue [4 x <4 x i32>] %val.coerce, 3971 %0 = bitcast <4 x i32> %val.coerce.fca.0.extract to <8 x bfloat>972 %1 = bitcast <4 x i32> %val.coerce.fca.1.extract to <8 x bfloat>973 %2 = bitcast <4 x i32> %val.coerce.fca.2.extract to <8 x bfloat>974 %3 = bitcast <4 x i32> %val.coerce.fca.3.extract to <8 x bfloat>975 tail call void @llvm.arm.neon.vst4lane.p0.v8bf16(ptr %ptr, <8 x bfloat> %0, <8 x bfloat> %1, <8 x bfloat> %2, <8 x bfloat> %3, i32 7, i32 2)976 ret void977}978 979declare { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld2.v4bf16.p0(ptr, i32)980declare { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld2.v8bf16.p0(ptr, i32)981declare { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld3.v4bf16.p0(ptr, i32)982declare { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld3.v8bf16.p0(ptr, i32)983declare { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld4.v4bf16.p0(ptr, i32)984declare { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld4.v8bf16.p0(ptr, i32)985 986declare { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld2dup.v4bf16.p0(ptr, i32)987declare { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld2dup.v8bf16.p0(ptr, i32)988declare { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld3dup.v4bf16.p0(ptr, i32)989declare { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld3dup.v8bf16.p0(ptr, i32)990declare { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld4dup.v4bf16.p0(ptr, i32)991declare { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld4dup.v8bf16.p0(ptr, i32)992 993declare { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld1x2.v4bf16.p0(ptr)994declare { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld1x2.v8bf16.p0(ptr)995declare { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld1x3.v4bf16.p0(ptr)996declare { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld1x3.v8bf16.p0(ptr)997declare { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld1x4.v4bf16.p0(ptr)998declare { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld1x4.v8bf16.p0(ptr)999 1000declare { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld2lane.v4bf16.p0(ptr, <4 x bfloat>, <4 x bfloat>, i32, i32)1001declare { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld2lane.v8bf16.p0(ptr, <8 x bfloat>, <8 x bfloat>, i32, i32)1002declare { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld3lane.v4bf16.p0(ptr, <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, i32, i32)1003declare { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld3lane.v8bf16.p0(ptr, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, i32, i32)1004declare { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld4lane.v4bf16.p0(ptr, <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, i32, i32)1005declare { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld4lane.v8bf16.p0(ptr, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, i32, i32)1006 1007declare void @llvm.arm.neon.vst1.p0.v4bf16(ptr, <4 x bfloat>, i32)1008declare void @llvm.arm.neon.vst1.p0.v8bf16(ptr, <8 x bfloat>, i32)1009declare void @llvm.arm.neon.vst2.p0.v4bf16(ptr, <4 x bfloat>, <4 x bfloat>, i32)1010declare void @llvm.arm.neon.vst2.p0.v8bf16(ptr, <8 x bfloat>, <8 x bfloat>, i32)1011declare void @llvm.arm.neon.vst3.p0.v4bf16(ptr, <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, i32)1012declare void @llvm.arm.neon.vst3.p0.v8bf16(ptr, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, i32)1013declare void @llvm.arm.neon.vst4.p0.v4bf16(ptr, <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, i32)1014declare void @llvm.arm.neon.vst4.p0.v8bf16(ptr, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, i32)1015 1016declare void @llvm.arm.neon.vst1x2.p0.v4bf16(ptr nocapture, <4 x bfloat>, <4 x bfloat>)1017declare void @llvm.arm.neon.vst1x2.p0.v8bf16(ptr nocapture, <8 x bfloat>, <8 x bfloat>)1018declare void @llvm.arm.neon.vst1x3.p0.v4bf16(ptr nocapture, <4 x bfloat>, <4 x bfloat>, <4 x bfloat>)1019declare void @llvm.arm.neon.vst1x3.p0.v8bf16(ptr nocapture, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>)1020declare void @llvm.arm.neon.vst1x4.p0.v4bf16(ptr nocapture, <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat>)1021declare void @llvm.arm.neon.vst1x4.p0.v8bf16(ptr nocapture, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>)1022 1023declare void @llvm.arm.neon.vst2lane.p0.v4bf16(ptr, <4 x bfloat>, <4 x bfloat>, i32, i32)1024declare void @llvm.arm.neon.vst2lane.p0.v8bf16(ptr, <8 x bfloat>, <8 x bfloat>, i32, i32)1025declare void @llvm.arm.neon.vst3lane.p0.v4bf16(ptr, <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, i32, i32)1026declare void @llvm.arm.neon.vst3lane.p0.v8bf16(ptr, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, i32, i32)1027declare void @llvm.arm.neon.vst4lane.p0.v4bf16(ptr, <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, i32, i32)1028declare void @llvm.arm.neon.vst4lane.p0.v8bf16(ptr, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, i32, i32)1029