brintos / llvm-project-archived public Read only

1// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py2// RUN: %clang_cc1 -triple aarch64 -target-feature +neon -target-feature +bf16 \3// RUN:  -O2 -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK644// RUN: %clang_cc1 -triple armv8.6a-arm-none-eabi -target-feature +neon -target-feature +bf16 -mfloat-abi hard \5// RUN:  -O2 -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK326 7// REQUIRES: arm-registered-target,aarch64-registered-target8 9#include "arm_neon.h"10 11// CHECK-LABEL: @test_vld1_bf16(12// CHECK-NEXT:  entry:13// CHECK-NEXT:    [[TMP1:%.*]] = load <4 x bfloat>, ptr [[PTR:%.*]], align 214// CHECK-NEXT:    ret <4 x bfloat> [[TMP1]]15//16bfloat16x4_t test_vld1_bf16(bfloat16_t const *ptr) {17  return vld1_bf16(ptr);18}19 20// CHECK-LABEL: @test_vld1q_bf16(21// CHECK-NEXT:  entry:22// CHECK-NEXT:    [[TMP1:%.*]] = load <8 x bfloat>, ptr [[PTR:%.*]], align 223// CHECK-NEXT:    ret <8 x bfloat> [[TMP1]]24//25bfloat16x8_t test_vld1q_bf16(bfloat16_t const *ptr) {26  return vld1q_bf16(ptr);27}28 29// CHECK-LABEL: @test_vld1_lane_bf16(30// CHECK-NEXT:  entry:31// CHECK-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[PTR:%.*]], align 232// CHECK-NEXT:    [[VLD1_LANE:%.*]] = insertelement <4 x bfloat> [[SRC:%.*]], bfloat [[TMP0]], i64 033// CHECK-NEXT:    ret <4 x bfloat> [[VLD1_LANE]]34//35bfloat16x4_t test_vld1_lane_bf16(bfloat16_t const *ptr, bfloat16x4_t src) {36  return vld1_lane_bf16(ptr, src, 0);37}38 39// CHECK-LABEL: @test_vld1q_lane_bf16(40// CHECK-NEXT:  entry:41// CHECK-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[PTR:%.*]], align 242// CHECK-NEXT:    [[VLD1_LANE:%.*]] = insertelement <8 x bfloat> [[SRC:%.*]], bfloat [[TMP0]], i64 743// CHECK-NEXT:    ret <8 x bfloat> [[VLD1_LANE]]44//45bfloat16x8_t test_vld1q_lane_bf16(bfloat16_t const *ptr, bfloat16x8_t src) {46  return vld1q_lane_bf16(ptr, src, 7);47}48 49// CHECK-LABEL: @test_vld1_dup_bf16(50// CHECK-NEXT:  entry:51// CHECK-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[PTR:%.*]], align 252// CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x bfloat> poison, bfloat [[TMP0]], i64 053// CHECK-NEXT:    [[LANE:%.*]] = shufflevector <4 x bfloat> [[TMP1]], <4 x bfloat> poison, <4 x i32> zeroinitializer54// CHECK-NEXT:    ret <4 x bfloat> [[LANE]]55//56bfloat16x4_t test_vld1_dup_bf16(bfloat16_t const *ptr) {57  return vld1_dup_bf16(ptr);58}59 60// CHECK64-LABEL: @test_vld1_bf16_x2(61// CHECK64-NEXT:  entry:62// CHECK64-NEXT:    [[VLD1XN:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld1x2.v4bf16.p0(ptr [[PTR:%.*]])63// CHECK64-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 064// CHECK64-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 165// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T:%.*]] poison, <4 x bfloat> [[VLD1XN_FCA_0_EXTRACT]], 0, 066// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD1XN_FCA_1_EXTRACT]], 0, 167// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_1_INSERT]]68//69// CHECK32-LABEL: @test_vld1_bf16_x2(70// CHECK32-NEXT:  entry:71// CHECK32-NEXT:    [[VLD1XN:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld1x2.v4bf16.p0(ptr [[PTR:%.*]])72// CHECK32-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 073// CHECK32-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 174// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_0_EXTRACT]] to <2 x i32>75// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_1_EXTRACT]] to <2 x i32>76// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <2 x i32>] poison, <2 x i32> [[TMP0]], 077// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP1]], 178// CHECK32-NEXT:    ret [2 x <2 x i32>] [[DOTFCA_1_INSERT]]79//80bfloat16x4x2_t test_vld1_bf16_x2(bfloat16_t const *ptr) {81  return vld1_bf16_x2(ptr);82}83 84// CHECK64-LABEL: @test_vld1q_bf16_x2(85// CHECK64-NEXT:  entry:86// CHECK64-NEXT:    [[VLD1XN:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld1x2.v8bf16.p0(ptr [[PTR:%.*]])87// CHECK64-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 088// CHECK64-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 189// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T:%.*]] poison, <8 x bfloat> [[VLD1XN_FCA_0_EXTRACT]], 0, 090// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD1XN_FCA_1_EXTRACT]], 0, 191// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]]92//93// CHECK32-LABEL: @test_vld1q_bf16_x2(94// CHECK32-NEXT:  entry:95// CHECK32-NEXT:    [[VLD1XN:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld1x2.v8bf16.p0(ptr [[PTR:%.*]])96// CHECK32-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 097// CHECK32-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 198// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_0_EXTRACT]] to <4 x i32>99// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_1_EXTRACT]] to <4 x i32>100// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <4 x i32>] poison, <4 x i32> [[TMP0]], 0101// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP1]], 1102// CHECK32-NEXT:    ret [2 x <4 x i32>] [[DOTFCA_1_INSERT]]103//104bfloat16x8x2_t test_vld1q_bf16_x2(bfloat16_t const *ptr) {105  return vld1q_bf16_x2(ptr);106}107 108// CHECK64-LABEL: @test_vld1_bf16_x3(109// CHECK64-NEXT:  entry:110// CHECK64-NEXT:    [[VLD1XN:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld1x3.v4bf16.p0(ptr [[PTR:%.*]])111// CHECK64-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 0112// CHECK64-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 1113// CHECK64-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 2114// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T:%.*]] poison, <4 x bfloat> [[VLD1XN_FCA_0_EXTRACT]], 0, 0115// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD1XN_FCA_1_EXTRACT]], 0, 1116// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD1XN_FCA_2_EXTRACT]], 0, 2117// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_2_INSERT]]118//119// CHECK32-LABEL: @test_vld1_bf16_x3(120// CHECK32-NEXT:  entry:121// CHECK32-NEXT:    [[VLD1XN:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld1x3.v4bf16.p0(ptr [[PTR:%.*]])122// CHECK32-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 0123// CHECK32-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 1124// CHECK32-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 2125// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_0_EXTRACT]] to <2 x i32>126// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_1_EXTRACT]] to <2 x i32>127// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_2_EXTRACT]] to <2 x i32>128// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <2 x i32>] poison, <2 x i32> [[TMP0]], 0129// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP1]], 1130// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP2]], 2131// CHECK32-NEXT:    ret [3 x <2 x i32>] [[DOTFCA_2_INSERT]]132//133bfloat16x4x3_t test_vld1_bf16_x3(bfloat16_t const *ptr) {134  return vld1_bf16_x3(ptr);135}136 137// CHECK64-LABEL: @test_vld1q_bf16_x3(138// CHECK64-NEXT:  entry:139// CHECK64-NEXT:    [[VLD1XN:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld1x3.v8bf16.p0(ptr [[PTR:%.*]])140// CHECK64-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 0141// CHECK64-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 1142// CHECK64-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 2143// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T:%.*]] poison, <8 x bfloat> [[VLD1XN_FCA_0_EXTRACT]], 0, 0144// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD1XN_FCA_1_EXTRACT]], 0, 1145// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD1XN_FCA_2_EXTRACT]], 0, 2146// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_2_INSERT]]147//148// CHECK32-LABEL: @test_vld1q_bf16_x3(149// CHECK32-NEXT:  entry:150// CHECK32-NEXT:    [[VLD1XN:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld1x3.v8bf16.p0(ptr [[PTR:%.*]])151// CHECK32-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 0152// CHECK32-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 1153// CHECK32-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 2154// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_0_EXTRACT]] to <4 x i32>155// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_1_EXTRACT]] to <4 x i32>156// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_2_EXTRACT]] to <4 x i32>157// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <4 x i32>] poison, <4 x i32> [[TMP0]], 0158// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP1]], 1159// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP2]], 2160// CHECK32-NEXT:    ret [3 x <4 x i32>] [[DOTFCA_2_INSERT]]161//162bfloat16x8x3_t test_vld1q_bf16_x3(bfloat16_t const *ptr) {163  return vld1q_bf16_x3(ptr);164}165 166// CHECK64-LABEL: @test_vld1_bf16_x4(167// CHECK64-NEXT:  entry:168// CHECK64-NEXT:    [[VLD1XN:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld1x4.v4bf16.p0(ptr [[PTR:%.*]])169// CHECK64-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 0170// CHECK64-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 1171// CHECK64-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 2172// CHECK64-NEXT:    [[VLD1XN_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 3173// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T:%.*]] poison, <4 x bfloat> [[VLD1XN_FCA_0_EXTRACT]], 0, 0174// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD1XN_FCA_1_EXTRACT]], 0, 1175// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD1XN_FCA_2_EXTRACT]], 0, 2176// CHECK64-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_2_INSERT]], <4 x bfloat> [[VLD1XN_FCA_3_EXTRACT]], 0, 3177// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_3_INSERT]]178//179// CHECK32-LABEL: @test_vld1_bf16_x4(180// CHECK32-NEXT:  entry:181// CHECK32-NEXT:    [[VLD1XN:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld1x4.v4bf16.p0(ptr [[PTR:%.*]])182// CHECK32-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 0183// CHECK32-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 1184// CHECK32-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 2185// CHECK32-NEXT:    [[VLD1XN_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD1XN]], 3186// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_0_EXTRACT]] to <2 x i32>187// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_1_EXTRACT]] to <2 x i32>188// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_2_EXTRACT]] to <2 x i32>189// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <4 x bfloat> [[VLD1XN_FCA_3_EXTRACT]] to <2 x i32>190// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <2 x i32>] poison, <2 x i32> [[TMP0]], 0191// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP1]], 1192// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP2]], 2193// CHECK32-NEXT:    [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_2_INSERT]], <2 x i32> [[TMP3]], 3194// CHECK32-NEXT:    ret [4 x <2 x i32>] [[DOTFCA_3_INSERT]]195//196bfloat16x4x4_t test_vld1_bf16_x4(bfloat16_t const *ptr) {197  return vld1_bf16_x4(ptr);198}199 200// CHECK64-LABEL: @test_vld1q_bf16_x4(201// CHECK64-NEXT:  entry:202// CHECK64-NEXT:    [[VLD1XN:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld1x4.v8bf16.p0(ptr [[PTR:%.*]])203// CHECK64-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 0204// CHECK64-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 1205// CHECK64-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 2206// CHECK64-NEXT:    [[VLD1XN_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 3207// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T:%.*]] poison, <8 x bfloat> [[VLD1XN_FCA_0_EXTRACT]], 0, 0208// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD1XN_FCA_1_EXTRACT]], 0, 1209// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD1XN_FCA_2_EXTRACT]], 0, 2210// CHECK64-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x bfloat> [[VLD1XN_FCA_3_EXTRACT]], 0, 3211// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_3_INSERT]]212//213// CHECK32-LABEL: @test_vld1q_bf16_x4(214// CHECK32-NEXT:  entry:215// CHECK32-NEXT:    [[VLD1XN:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld1x4.v8bf16.p0(ptr [[PTR:%.*]])216// CHECK32-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 0217// CHECK32-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 1218// CHECK32-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 2219// CHECK32-NEXT:    [[VLD1XN_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD1XN]], 3220// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_0_EXTRACT]] to <4 x i32>221// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_1_EXTRACT]] to <4 x i32>222// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_2_EXTRACT]] to <4 x i32>223// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <8 x bfloat> [[VLD1XN_FCA_3_EXTRACT]] to <4 x i32>224// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <4 x i32>] poison, <4 x i32> [[TMP0]], 0225// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP1]], 1226// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP2]], 2227// CHECK32-NEXT:    [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_2_INSERT]], <4 x i32> [[TMP3]], 3228// CHECK32-NEXT:    ret [4 x <4 x i32>] [[DOTFCA_3_INSERT]]229//230bfloat16x8x4_t test_vld1q_bf16_x4(bfloat16_t const *ptr) {231  return vld1q_bf16_x4(ptr);232}233 234// CHECK-LABEL: @test_vld1q_dup_bf16(235// CHECK-NEXT:  entry:236// CHECK-NEXT:    [[TMP0:%.*]] = load bfloat, ptr [[PTR:%.*]], align 2237// CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x bfloat> poison, bfloat [[TMP0]], i64 0238// CHECK-NEXT:    [[LANE:%.*]] = shufflevector <8 x bfloat> [[TMP1]], <8 x bfloat> poison, <8 x i32> zeroinitializer239// CHECK-NEXT:    ret <8 x bfloat> [[LANE]]240//241bfloat16x8_t test_vld1q_dup_bf16(bfloat16_t const *ptr) {242  return vld1q_dup_bf16(ptr);243}244 245// CHECK64-LABEL: @test_vld2_bf16(246// CHECK64-NEXT:  entry:247// CHECK64-NEXT:    [[VLD2:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld2.v4bf16.p0(ptr [[PTR:%.*]])248// CHECK64-NEXT:    [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2]], 0249// CHECK64-NEXT:    [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2]], 1250// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T:%.*]] poison, <4 x bfloat> [[VLD2_FCA_0_EXTRACT]], 0, 0251// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD2_FCA_1_EXTRACT]], 0, 1252// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_1_INSERT]]253//254// CHECK32-LABEL: @test_vld2_bf16(255// CHECK32-NEXT:  entry:256// CHECK32-NEXT:    [[VLD2_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld2.v4bf16.p0(ptr [[PTR:%.*]], i32 2)257// CHECK32-NEXT:    [[VLD2_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_V]], 0258// CHECK32-NEXT:    [[VLD2_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_V]], 1259// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD2_V_FCA_0_EXTRACT]] to <2 x i32>260// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD2_V_FCA_1_EXTRACT]] to <2 x i32>261// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <2 x i32>] poison, <2 x i32> [[TMP1]], 0262// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP2]], 1263// CHECK32-NEXT:    ret [2 x <2 x i32>] [[DOTFCA_1_INSERT]]264//265bfloat16x4x2_t test_vld2_bf16(bfloat16_t const *ptr) {266  return vld2_bf16(ptr);267}268 269// CHECK64-LABEL: @test_vld2q_bf16(270// CHECK64-NEXT:  entry:271// CHECK64-NEXT:    [[VLD2:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld2.v8bf16.p0(ptr [[PTR:%.*]])272// CHECK64-NEXT:    [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2]], 0273// CHECK64-NEXT:    [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2]], 1274// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T:%.*]] poison, <8 x bfloat> [[VLD2_FCA_0_EXTRACT]], 0, 0275// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD2_FCA_1_EXTRACT]], 0, 1276// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]]277//278// CHECK32-LABEL: @test_vld2q_bf16(279// CHECK32-NEXT:  entry:280// CHECK32-NEXT:    [[VLD2Q_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld2.v8bf16.p0(ptr [[PTR:%.*]], i32 2)281// CHECK32-NEXT:    [[VLD2Q_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2Q_V]], 0282// CHECK32-NEXT:    [[VLD2Q_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2Q_V]], 1283// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD2Q_V_FCA_0_EXTRACT]] to <4 x i32>284// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD2Q_V_FCA_1_EXTRACT]] to <4 x i32>285// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <4 x i32>] poison, <4 x i32> [[TMP1]], 0286// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP2]], 1287// CHECK32-NEXT:    ret [2 x <4 x i32>] [[DOTFCA_1_INSERT]]288//289bfloat16x8x2_t test_vld2q_bf16(bfloat16_t const *ptr) {290  return vld2q_bf16(ptr);291}292 293// CHECK64-LABEL: @test_vld2_lane_bf16(294// CHECK64-NEXT:  entry:295// CHECK64-NEXT:    [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[SRC_COERCE:%.*]], 0296// CHECK64-NEXT:    [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[SRC_COERCE]], 1297// CHECK64-NEXT:    [[VLD2_LANE:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld2lane.v4bf16.p0(<4 x bfloat> [[SRC_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[SRC_COERCE_FCA_1_EXTRACT]], i64 1, ptr [[PTR:%.*]])298// CHECK64-NEXT:    [[VLD2_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_LANE]], 0299// CHECK64-NEXT:    [[VLD2_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_LANE]], 1300// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T:%.*]] poison, <4 x bfloat> [[VLD2_LANE_FCA_0_EXTRACT]], 0, 0301// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD2_LANE_FCA_1_EXTRACT]], 0, 1302// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_1_INSERT]]303//304// CHECK32-LABEL: @test_vld2_lane_bf16(305// CHECK32-NEXT:  entry:306// CHECK32-NEXT:    [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[SRC_COERCE:%.*]], 0307// CHECK32-NEXT:    [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[SRC_COERCE]], 1308// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>309// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>310// CHECK32-NEXT:    [[VLD2_LANE_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld2lane.v4bf16.p0(ptr [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], i32 1, i32 2)311// CHECK32-NEXT:    [[VLD2_LANE_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_LANE_V]], 0312// CHECK32-NEXT:    [[VLD2_LANE_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_LANE_V]], 1313// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <4 x bfloat> [[VLD2_LANE_V_FCA_0_EXTRACT]] to <2 x i32>314// CHECK32-NEXT:    [[TMP4:%.*]] = bitcast <4 x bfloat> [[VLD2_LANE_V_FCA_1_EXTRACT]] to <2 x i32>315// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <2 x i32>] poison, <2 x i32> [[TMP3]], 0316// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP4]], 1317// CHECK32-NEXT:    ret [2 x <2 x i32>] [[DOTFCA_1_INSERT]]318//319bfloat16x4x2_t test_vld2_lane_bf16(bfloat16_t const *ptr, bfloat16x4x2_t src) {320  return vld2_lane_bf16(ptr, src, 1);321}322 323// CHECK64-LABEL: @test_vld2q_lane_bf16(324// CHECK64-NEXT:  entry:325// CHECK64-NEXT:    [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[SRC_COERCE:%.*]], 0326// CHECK64-NEXT:    [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[SRC_COERCE]], 1327// CHECK64-NEXT:    [[VLD2_LANE:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld2lane.v8bf16.p0(<8 x bfloat> [[SRC_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[SRC_COERCE_FCA_1_EXTRACT]], i64 7, ptr [[PTR:%.*]])328// CHECK64-NEXT:    [[VLD2_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2_LANE]], 0329// CHECK64-NEXT:    [[VLD2_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2_LANE]], 1330// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T:%.*]] poison, <8 x bfloat> [[VLD2_LANE_FCA_0_EXTRACT]], 0, 0331// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD2_LANE_FCA_1_EXTRACT]], 0, 1332// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]]333//334// CHECK32-LABEL: @test_vld2q_lane_bf16(335// CHECK32-NEXT:  entry:336// CHECK32-NEXT:    [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[SRC_COERCE:%.*]], 0337// CHECK32-NEXT:    [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[SRC_COERCE]], 1338// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>339// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>340// CHECK32-NEXT:    [[VLD2Q_LANE_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld2lane.v8bf16.p0(ptr [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], i32 7, i32 2)341// CHECK32-NEXT:    [[VLD2Q_LANE_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2Q_LANE_V]], 0342// CHECK32-NEXT:    [[VLD2Q_LANE_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2Q_LANE_V]], 1343// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <8 x bfloat> [[VLD2Q_LANE_V_FCA_0_EXTRACT]] to <4 x i32>344// CHECK32-NEXT:    [[TMP4:%.*]] = bitcast <8 x bfloat> [[VLD2Q_LANE_V_FCA_1_EXTRACT]] to <4 x i32>345// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <4 x i32>] poison, <4 x i32> [[TMP3]], 0346// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP4]], 1347// CHECK32-NEXT:    ret [2 x <4 x i32>] [[DOTFCA_1_INSERT]]348//349bfloat16x8x2_t test_vld2q_lane_bf16(bfloat16_t const *ptr, bfloat16x8x2_t src) {350  return vld2q_lane_bf16(ptr, src, 7);351}352 353// CHECK64-LABEL: @test_vld3_bf16(354// CHECK64-NEXT:  entry:355// CHECK64-NEXT:    [[VLD3:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld3.v4bf16.p0(ptr [[PTR:%.*]])356// CHECK64-NEXT:    [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3]], 0357// CHECK64-NEXT:    [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3]], 1358// CHECK64-NEXT:    [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3]], 2359// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T:%.*]] poison, <4 x bfloat> [[VLD3_FCA_0_EXTRACT]], 0, 0360// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD3_FCA_1_EXTRACT]], 0, 1361// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD3_FCA_2_EXTRACT]], 0, 2362// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_2_INSERT]]363//364// CHECK32-LABEL: @test_vld3_bf16(365// CHECK32-NEXT:  entry:366// CHECK32-NEXT:    [[VLD3_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld3.v4bf16.p0(ptr [[PTR:%.*]], i32 2)367// CHECK32-NEXT:    [[VLD3_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_V]], 0368// CHECK32-NEXT:    [[VLD3_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_V]], 1369// CHECK32-NEXT:    [[VLD3_V_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_V]], 2370// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD3_V_FCA_0_EXTRACT]] to <2 x i32>371// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD3_V_FCA_1_EXTRACT]] to <2 x i32>372// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <4 x bfloat> [[VLD3_V_FCA_2_EXTRACT]] to <2 x i32>373// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <2 x i32>] poison, <2 x i32> [[TMP1]], 0374// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP2]], 1375// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP3]], 2376// CHECK32-NEXT:    ret [3 x <2 x i32>] [[DOTFCA_2_INSERT]]377//378bfloat16x4x3_t test_vld3_bf16(bfloat16_t const *ptr) {379  return vld3_bf16(ptr);380}381 382// CHECK64-LABEL: @test_vld3q_bf16(383// CHECK64-NEXT:  entry:384// CHECK64-NEXT:    [[VLD3:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld3.v8bf16.p0(ptr [[PTR:%.*]])385// CHECK64-NEXT:    [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3]], 0386// CHECK64-NEXT:    [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3]], 1387// CHECK64-NEXT:    [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3]], 2388// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T:%.*]] poison, <8 x bfloat> [[VLD3_FCA_0_EXTRACT]], 0, 0389// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD3_FCA_1_EXTRACT]], 0, 1390// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD3_FCA_2_EXTRACT]], 0, 2391// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_2_INSERT]]392//393// CHECK32-LABEL: @test_vld3q_bf16(394// CHECK32-NEXT:  entry:395// CHECK32-NEXT:    [[VLD3Q_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld3.v8bf16.p0(ptr [[PTR:%.*]], i32 2)396// CHECK32-NEXT:    [[VLD3Q_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_V]], 0397// CHECK32-NEXT:    [[VLD3Q_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_V]], 1398// CHECK32-NEXT:    [[VLD3Q_V_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_V]], 2399// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD3Q_V_FCA_0_EXTRACT]] to <4 x i32>400// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD3Q_V_FCA_1_EXTRACT]] to <4 x i32>401// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <8 x bfloat> [[VLD3Q_V_FCA_2_EXTRACT]] to <4 x i32>402// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <4 x i32>] poison, <4 x i32> [[TMP1]], 0403// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP2]], 1404// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP3]], 2405// CHECK32-NEXT:    ret [3 x <4 x i32>] [[DOTFCA_2_INSERT]]406//407bfloat16x8x3_t test_vld3q_bf16(bfloat16_t const *ptr) {408  return vld3q_bf16(ptr);409}410 411// CHECK64-LABEL: @test_vld3_lane_bf16(412// CHECK64-NEXT:  entry:413// CHECK64-NEXT:    [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[SRC_COERCE:%.*]], 0414// CHECK64-NEXT:    [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[SRC_COERCE]], 1415// CHECK64-NEXT:    [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[SRC_COERCE]], 2416// CHECK64-NEXT:    [[VLD3_LANE:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld3lane.v4bf16.p0(<4 x bfloat> [[SRC_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[SRC_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[SRC_COERCE_FCA_2_EXTRACT]], i64 1, ptr [[PTR:%.*]])417// CHECK64-NEXT:    [[VLD3_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_LANE]], 0418// CHECK64-NEXT:    [[VLD3_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_LANE]], 1419// CHECK64-NEXT:    [[VLD3_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_LANE]], 2420// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T:%.*]] poison, <4 x bfloat> [[VLD3_LANE_FCA_0_EXTRACT]], 0, 0421// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD3_LANE_FCA_1_EXTRACT]], 0, 1422// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD3_LANE_FCA_2_EXTRACT]], 0, 2423// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_2_INSERT]]424//425// CHECK32-LABEL: @test_vld3_lane_bf16(426// CHECK32-NEXT:  entry:427// CHECK32-NEXT:    [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[SRC_COERCE:%.*]], 0428// CHECK32-NEXT:    [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[SRC_COERCE]], 1429// CHECK32-NEXT:    [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[SRC_COERCE]], 2430// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>431// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>432// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>433// CHECK32-NEXT:    [[VLD3_LANE_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld3lane.v4bf16.p0(ptr [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]], i32 1, i32 2)434// CHECK32-NEXT:    [[VLD3_LANE_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_LANE_V]], 0435// CHECK32-NEXT:    [[VLD3_LANE_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_LANE_V]], 1436// CHECK32-NEXT:    [[VLD3_LANE_V_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_LANE_V]], 2437// CHECK32-NEXT:    [[TMP4:%.*]] = bitcast <4 x bfloat> [[VLD3_LANE_V_FCA_0_EXTRACT]] to <2 x i32>438// CHECK32-NEXT:    [[TMP5:%.*]] = bitcast <4 x bfloat> [[VLD3_LANE_V_FCA_1_EXTRACT]] to <2 x i32>439// CHECK32-NEXT:    [[TMP6:%.*]] = bitcast <4 x bfloat> [[VLD3_LANE_V_FCA_2_EXTRACT]] to <2 x i32>440// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <2 x i32>] poison, <2 x i32> [[TMP4]], 0441// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP5]], 1442// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP6]], 2443// CHECK32-NEXT:    ret [3 x <2 x i32>] [[DOTFCA_2_INSERT]]444//445bfloat16x4x3_t test_vld3_lane_bf16(bfloat16_t const *ptr, bfloat16x4x3_t src) {446  return vld3_lane_bf16(ptr, src, 1);447}448 449// CHECK64-LABEL: @test_vld3q_lane_bf16(450// CHECK64-NEXT:  entry:451// CHECK64-NEXT:    [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[SRC_COERCE:%.*]], 0452// CHECK64-NEXT:    [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[SRC_COERCE]], 1453// CHECK64-NEXT:    [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[SRC_COERCE]], 2454// CHECK64-NEXT:    [[VLD3_LANE:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld3lane.v8bf16.p0(<8 x bfloat> [[SRC_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[SRC_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[SRC_COERCE_FCA_2_EXTRACT]], i64 7, ptr [[PTR:%.*]])455// CHECK64-NEXT:    [[VLD3_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3_LANE]], 0456// CHECK64-NEXT:    [[VLD3_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3_LANE]], 1457// CHECK64-NEXT:    [[VLD3_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3_LANE]], 2458// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T:%.*]] poison, <8 x bfloat> [[VLD3_LANE_FCA_0_EXTRACT]], 0, 0459// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD3_LANE_FCA_1_EXTRACT]], 0, 1460// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD3_LANE_FCA_2_EXTRACT]], 0, 2461// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_2_INSERT]]462//463// CHECK32-LABEL: @test_vld3q_lane_bf16(464// CHECK32-NEXT:  entry:465// CHECK32-NEXT:    [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[SRC_COERCE:%.*]], 0466// CHECK32-NEXT:    [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[SRC_COERCE]], 1467// CHECK32-NEXT:    [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[SRC_COERCE]], 2468// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>469// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>470// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>471// CHECK32-NEXT:    [[VLD3Q_LANE_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld3lane.v8bf16.p0(ptr [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]], i32 7, i32 2)472// CHECK32-NEXT:    [[VLD3Q_LANE_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_LANE_V]], 0473// CHECK32-NEXT:    [[VLD3Q_LANE_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_LANE_V]], 1474// CHECK32-NEXT:    [[VLD3Q_LANE_V_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_LANE_V]], 2475// CHECK32-NEXT:    [[TMP4:%.*]] = bitcast <8 x bfloat> [[VLD3Q_LANE_V_FCA_0_EXTRACT]] to <4 x i32>476// CHECK32-NEXT:    [[TMP5:%.*]] = bitcast <8 x bfloat> [[VLD3Q_LANE_V_FCA_1_EXTRACT]] to <4 x i32>477// CHECK32-NEXT:    [[TMP6:%.*]] = bitcast <8 x bfloat> [[VLD3Q_LANE_V_FCA_2_EXTRACT]] to <4 x i32>478// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <4 x i32>] poison, <4 x i32> [[TMP4]], 0479// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP5]], 1480// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP6]], 2481// CHECK32-NEXT:    ret [3 x <4 x i32>] [[DOTFCA_2_INSERT]]482//483bfloat16x8x3_t test_vld3q_lane_bf16(bfloat16_t const *ptr, bfloat16x8x3_t src) {484  return vld3q_lane_bf16(ptr, src, 7);485  // return vld3q_lane_bf16(ptr, src, 8);486}487 488// CHECK64-LABEL: @test_vld4_bf16(489// CHECK64-NEXT:  entry:490// CHECK64-NEXT:    [[VLD4:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld4.v4bf16.p0(ptr [[PTR:%.*]])491// CHECK64-NEXT:    [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 0492// CHECK64-NEXT:    [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 1493// CHECK64-NEXT:    [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 2494// CHECK64-NEXT:    [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 3495// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T:%.*]] poison, <4 x bfloat> [[VLD4_FCA_0_EXTRACT]], 0, 0496// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD4_FCA_1_EXTRACT]], 0, 1497// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD4_FCA_2_EXTRACT]], 0, 2498// CHECK64-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_2_INSERT]], <4 x bfloat> [[VLD4_FCA_3_EXTRACT]], 0, 3499// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_3_INSERT]]500//501// CHECK32-LABEL: @test_vld4_bf16(502// CHECK32-NEXT:  entry:503// CHECK32-NEXT:    [[VLD4_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld4.v4bf16.p0(ptr [[PTR:%.*]], i32 2)504// CHECK32-NEXT:    [[VLD4_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_V]], 0505// CHECK32-NEXT:    [[VLD4_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_V]], 1506// CHECK32-NEXT:    [[VLD4_V_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_V]], 2507// CHECK32-NEXT:    [[VLD4_V_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_V]], 3508// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD4_V_FCA_0_EXTRACT]] to <2 x i32>509// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD4_V_FCA_1_EXTRACT]] to <2 x i32>510// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <4 x bfloat> [[VLD4_V_FCA_2_EXTRACT]] to <2 x i32>511// CHECK32-NEXT:    [[TMP4:%.*]] = bitcast <4 x bfloat> [[VLD4_V_FCA_3_EXTRACT]] to <2 x i32>512// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <2 x i32>] poison, <2 x i32> [[TMP1]], 0513// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP2]], 1514// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP3]], 2515// CHECK32-NEXT:    [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_2_INSERT]], <2 x i32> [[TMP4]], 3516// CHECK32-NEXT:    ret [4 x <2 x i32>] [[DOTFCA_3_INSERT]]517//518bfloat16x4x4_t test_vld4_bf16(bfloat16_t const *ptr) {519  return vld4_bf16(ptr);520}521 522// CHECK64-LABEL: @test_vld4q_bf16(523// CHECK64-NEXT:  entry:524// CHECK64-NEXT:    [[VLD4:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld4.v8bf16.p0(ptr [[PTR:%.*]])525// CHECK64-NEXT:    [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 0526// CHECK64-NEXT:    [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 1527// CHECK64-NEXT:    [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 2528// CHECK64-NEXT:    [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 3529// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T:%.*]] poison, <8 x bfloat> [[VLD4_FCA_0_EXTRACT]], 0, 0530// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD4_FCA_1_EXTRACT]], 0, 1531// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD4_FCA_2_EXTRACT]], 0, 2532// CHECK64-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x bfloat> [[VLD4_FCA_3_EXTRACT]], 0, 3533// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_3_INSERT]]534//535// CHECK32-LABEL: @test_vld4q_bf16(536// CHECK32-NEXT:  entry:537// CHECK32-NEXT:    [[VLD4Q_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld4.v8bf16.p0(ptr [[PTR:%.*]], i32 2)538// CHECK32-NEXT:    [[VLD4Q_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_V]], 0539// CHECK32-NEXT:    [[VLD4Q_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_V]], 1540// CHECK32-NEXT:    [[VLD4Q_V_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_V]], 2541// CHECK32-NEXT:    [[VLD4Q_V_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_V]], 3542// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD4Q_V_FCA_0_EXTRACT]] to <4 x i32>543// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD4Q_V_FCA_1_EXTRACT]] to <4 x i32>544// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <8 x bfloat> [[VLD4Q_V_FCA_2_EXTRACT]] to <4 x i32>545// CHECK32-NEXT:    [[TMP4:%.*]] = bitcast <8 x bfloat> [[VLD4Q_V_FCA_3_EXTRACT]] to <4 x i32>546// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <4 x i32>] poison, <4 x i32> [[TMP1]], 0547// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP2]], 1548// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP3]], 2549// CHECK32-NEXT:    [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_2_INSERT]], <4 x i32> [[TMP4]], 3550// CHECK32-NEXT:    ret [4 x <4 x i32>] [[DOTFCA_3_INSERT]]551//552bfloat16x8x4_t test_vld4q_bf16(bfloat16_t const *ptr) {553  return vld4q_bf16(ptr);554}555 556// CHECK64-LABEL: @test_vld4_lane_bf16(557// CHECK64-NEXT:  entry:558// CHECK64-NEXT:    [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[SRC_COERCE:%.*]], 0559// CHECK64-NEXT:    [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[SRC_COERCE]], 1560// CHECK64-NEXT:    [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[SRC_COERCE]], 2561// CHECK64-NEXT:    [[SRC_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[SRC_COERCE]], 3562// CHECK64-NEXT:    [[VLD4_LANE:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld4lane.v4bf16.p0(<4 x bfloat> [[SRC_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[SRC_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[SRC_COERCE_FCA_2_EXTRACT]], <4 x bfloat> [[SRC_COERCE_FCA_3_EXTRACT]], i64 1, ptr [[PTR:%.*]])563// CHECK64-NEXT:    [[VLD4_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE]], 0564// CHECK64-NEXT:    [[VLD4_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE]], 1565// CHECK64-NEXT:    [[VLD4_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE]], 2566// CHECK64-NEXT:    [[VLD4_LANE_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE]], 3567// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T:%.*]] poison, <4 x bfloat> [[VLD4_LANE_FCA_0_EXTRACT]], 0, 0568// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD4_LANE_FCA_1_EXTRACT]], 0, 1569// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD4_LANE_FCA_2_EXTRACT]], 0, 2570// CHECK64-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_2_INSERT]], <4 x bfloat> [[VLD4_LANE_FCA_3_EXTRACT]], 0, 3571// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_3_INSERT]]572//573// CHECK32-LABEL: @test_vld4_lane_bf16(574// CHECK32-NEXT:  entry:575// CHECK32-NEXT:    [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[SRC_COERCE:%.*]], 0576// CHECK32-NEXT:    [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[SRC_COERCE]], 1577// CHECK32-NEXT:    [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[SRC_COERCE]], 2578// CHECK32-NEXT:    [[SRC_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[SRC_COERCE]], 3579// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>580// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>581// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>582// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[SRC_COERCE_FCA_3_EXTRACT]] to <4 x bfloat>583// CHECK32-NEXT:    [[VLD4_LANE_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld4lane.v4bf16.p0(ptr [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]], <4 x bfloat> [[TMP3]], i32 1, i32 2)584// CHECK32-NEXT:    [[VLD4_LANE_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE_V]], 0585// CHECK32-NEXT:    [[VLD4_LANE_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE_V]], 1586// CHECK32-NEXT:    [[VLD4_LANE_V_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE_V]], 2587// CHECK32-NEXT:    [[VLD4_LANE_V_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_LANE_V]], 3588// CHECK32-NEXT:    [[TMP5:%.*]] = bitcast <4 x bfloat> [[VLD4_LANE_V_FCA_0_EXTRACT]] to <2 x i32>589// CHECK32-NEXT:    [[TMP6:%.*]] = bitcast <4 x bfloat> [[VLD4_LANE_V_FCA_1_EXTRACT]] to <2 x i32>590// CHECK32-NEXT:    [[TMP7:%.*]] = bitcast <4 x bfloat> [[VLD4_LANE_V_FCA_2_EXTRACT]] to <2 x i32>591// CHECK32-NEXT:    [[TMP8:%.*]] = bitcast <4 x bfloat> [[VLD4_LANE_V_FCA_3_EXTRACT]] to <2 x i32>592// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <2 x i32>] poison, <2 x i32> [[TMP5]], 0593// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP6]], 1594// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP7]], 2595// CHECK32-NEXT:    [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_2_INSERT]], <2 x i32> [[TMP8]], 3596// CHECK32-NEXT:    ret [4 x <2 x i32>] [[DOTFCA_3_INSERT]]597//598bfloat16x4x4_t test_vld4_lane_bf16(bfloat16_t const *ptr, bfloat16x4x4_t src) {599  return vld4_lane_bf16(ptr, src, 1);600}601 602// CHECK64-LABEL: @test_vld4q_lane_bf16(603// CHECK64-NEXT:  entry:604// CHECK64-NEXT:    [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[SRC_COERCE:%.*]], 0605// CHECK64-NEXT:    [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[SRC_COERCE]], 1606// CHECK64-NEXT:    [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[SRC_COERCE]], 2607// CHECK64-NEXT:    [[SRC_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[SRC_COERCE]], 3608// CHECK64-NEXT:    [[VLD4_LANE:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld4lane.v8bf16.p0(<8 x bfloat> [[SRC_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[SRC_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[SRC_COERCE_FCA_2_EXTRACT]], <8 x bfloat> [[SRC_COERCE_FCA_3_EXTRACT]], i64 7, ptr [[PTR:%.*]])609// CHECK64-NEXT:    [[VLD4_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4_LANE]], 0610// CHECK64-NEXT:    [[VLD4_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4_LANE]], 1611// CHECK64-NEXT:    [[VLD4_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4_LANE]], 2612// CHECK64-NEXT:    [[VLD4_LANE_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4_LANE]], 3613// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T:%.*]] poison, <8 x bfloat> [[VLD4_LANE_FCA_0_EXTRACT]], 0, 0614// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD4_LANE_FCA_1_EXTRACT]], 0, 1615// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD4_LANE_FCA_2_EXTRACT]], 0, 2616// CHECK64-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x bfloat> [[VLD4_LANE_FCA_3_EXTRACT]], 0, 3617// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_3_INSERT]]618//619// CHECK32-LABEL: @test_vld4q_lane_bf16(620// CHECK32-NEXT:  entry:621// CHECK32-NEXT:    [[SRC_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[SRC_COERCE:%.*]], 0622// CHECK32-NEXT:    [[SRC_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[SRC_COERCE]], 1623// CHECK32-NEXT:    [[SRC_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[SRC_COERCE]], 2624// CHECK32-NEXT:    [[SRC_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[SRC_COERCE]], 3625// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>626// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>627// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>628// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[SRC_COERCE_FCA_3_EXTRACT]] to <8 x bfloat>629// CHECK32-NEXT:    [[VLD4Q_LANE_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld4lane.v8bf16.p0(ptr [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]], <8 x bfloat> [[TMP3]], i32 7, i32 2)630// CHECK32-NEXT:    [[VLD4Q_LANE_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_LANE_V]], 0631// CHECK32-NEXT:    [[VLD4Q_LANE_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_LANE_V]], 1632// CHECK32-NEXT:    [[VLD4Q_LANE_V_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_LANE_V]], 2633// CHECK32-NEXT:    [[VLD4Q_LANE_V_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_LANE_V]], 3634// CHECK32-NEXT:    [[TMP5:%.*]] = bitcast <8 x bfloat> [[VLD4Q_LANE_V_FCA_0_EXTRACT]] to <4 x i32>635// CHECK32-NEXT:    [[TMP6:%.*]] = bitcast <8 x bfloat> [[VLD4Q_LANE_V_FCA_1_EXTRACT]] to <4 x i32>636// CHECK32-NEXT:    [[TMP7:%.*]] = bitcast <8 x bfloat> [[VLD4Q_LANE_V_FCA_2_EXTRACT]] to <4 x i32>637// CHECK32-NEXT:    [[TMP8:%.*]] = bitcast <8 x bfloat> [[VLD4Q_LANE_V_FCA_3_EXTRACT]] to <4 x i32>638// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <4 x i32>] poison, <4 x i32> [[TMP5]], 0639// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP6]], 1640// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP7]], 2641// CHECK32-NEXT:    [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_2_INSERT]], <4 x i32> [[TMP8]], 3642// CHECK32-NEXT:    ret [4 x <4 x i32>] [[DOTFCA_3_INSERT]]643//644bfloat16x8x4_t test_vld4q_lane_bf16(bfloat16_t const *ptr, bfloat16x8x4_t src) {645  return vld4q_lane_bf16(ptr, src, 7);646}647 648// CHECK64-LABEL: @test_vld2_dup_bf16(649// CHECK64-NEXT:  entry:650// CHECK64-NEXT:    [[VLD2:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld2r.v4bf16.p0(ptr [[PTR:%.*]])651// CHECK64-NEXT:    [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2]], 0652// CHECK64-NEXT:    [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2]], 1653// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T:%.*]] poison, <4 x bfloat> [[VLD2_FCA_0_EXTRACT]], 0, 0654// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD2_FCA_1_EXTRACT]], 0, 1655// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X4X2_T]] [[DOTFCA_0_1_INSERT]]656//657// CHECK32-LABEL: @test_vld2_dup_bf16(658// CHECK32-NEXT:  entry:659// CHECK32-NEXT:    [[VLD2_DUP_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld2dup.v4bf16.p0(ptr [[PTR:%.*]], i32 2)660// CHECK32-NEXT:    [[VLD2_DUP_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_DUP_V]], 0661// CHECK32-NEXT:    [[VLD2_DUP_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat> } [[VLD2_DUP_V]], 1662// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD2_DUP_V_FCA_0_EXTRACT]] to <2 x i32>663// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD2_DUP_V_FCA_1_EXTRACT]] to <2 x i32>664// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <2 x i32>] poison, <2 x i32> [[TMP1]], 0665// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP2]], 1666// CHECK32-NEXT:    ret [2 x <2 x i32>] [[DOTFCA_1_INSERT]]667//668bfloat16x4x2_t test_vld2_dup_bf16(bfloat16_t const *ptr) {669  return vld2_dup_bf16(ptr);670}671 672// CHECK64-LABEL: @test_vld2q_dup_bf16(673// CHECK64-NEXT:  entry:674// CHECK64-NEXT:    [[VLD2:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld2r.v8bf16.p0(ptr [[PTR:%.*]])675// CHECK64-NEXT:    [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2]], 0676// CHECK64-NEXT:    [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2]], 1677// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T:%.*]] poison, <8 x bfloat> [[VLD2_FCA_0_EXTRACT]], 0, 0678// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD2_FCA_1_EXTRACT]], 0, 1679// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]]680//681// CHECK32-LABEL: @test_vld2q_dup_bf16(682// CHECK32-NEXT:  entry:683// CHECK32-NEXT:    [[VLD2Q_DUP_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld2dup.v8bf16.p0(ptr [[PTR:%.*]], i32 2)684// CHECK32-NEXT:    [[VLD2Q_DUP_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2Q_DUP_V]], 0685// CHECK32-NEXT:    [[VLD2Q_DUP_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLD2Q_DUP_V]], 1686// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD2Q_DUP_V_FCA_0_EXTRACT]] to <4 x i32>687// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD2Q_DUP_V_FCA_1_EXTRACT]] to <4 x i32>688// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [2 x <4 x i32>] poison, <4 x i32> [[TMP1]], 0689// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [2 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP2]], 1690// CHECK32-NEXT:    ret [2 x <4 x i32>] [[DOTFCA_1_INSERT]]691//692bfloat16x8x2_t test_vld2q_dup_bf16(bfloat16_t const *ptr) {693  return vld2q_dup_bf16(ptr);694}695 696// CHECK64-LABEL: @test_vld3_dup_bf16(697// CHECK64-NEXT:  entry:698// CHECK64-NEXT:    [[VLD3:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld3r.v4bf16.p0(ptr [[PTR:%.*]])699// CHECK64-NEXT:    [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3]], 0700// CHECK64-NEXT:    [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3]], 1701// CHECK64-NEXT:    [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3]], 2702// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T:%.*]] poison, <4 x bfloat> [[VLD3_FCA_0_EXTRACT]], 0, 0703// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD3_FCA_1_EXTRACT]], 0, 1704// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD3_FCA_2_EXTRACT]], 0, 2705// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X4X3_T]] [[DOTFCA_0_2_INSERT]]706//707// CHECK32-LABEL: @test_vld3_dup_bf16(708// CHECK32-NEXT:  entry:709// CHECK32-NEXT:    [[VLD3_DUP_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld3dup.v4bf16.p0(ptr [[PTR:%.*]], i32 2)710// CHECK32-NEXT:    [[VLD3_DUP_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_DUP_V]], 0711// CHECK32-NEXT:    [[VLD3_DUP_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_DUP_V]], 1712// CHECK32-NEXT:    [[VLD3_DUP_V_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD3_DUP_V]], 2713// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD3_DUP_V_FCA_0_EXTRACT]] to <2 x i32>714// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD3_DUP_V_FCA_1_EXTRACT]] to <2 x i32>715// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <4 x bfloat> [[VLD3_DUP_V_FCA_2_EXTRACT]] to <2 x i32>716// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <2 x i32>] poison, <2 x i32> [[TMP1]], 0717// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP2]], 1718// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP3]], 2719// CHECK32-NEXT:    ret [3 x <2 x i32>] [[DOTFCA_2_INSERT]]720//721bfloat16x4x3_t test_vld3_dup_bf16(bfloat16_t const *ptr) {722  return vld3_dup_bf16(ptr);723}724 725// CHECK64-LABEL: @test_vld3q_dup_bf16(726// CHECK64-NEXT:  entry:727// CHECK64-NEXT:    [[VLD3:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld3r.v8bf16.p0(ptr [[PTR:%.*]])728// CHECK64-NEXT:    [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3]], 0729// CHECK64-NEXT:    [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3]], 1730// CHECK64-NEXT:    [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3]], 2731// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T:%.*]] poison, <8 x bfloat> [[VLD3_FCA_0_EXTRACT]], 0, 0732// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD3_FCA_1_EXTRACT]], 0, 1733// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD3_FCA_2_EXTRACT]], 0, 2734// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X8X3_T]] [[DOTFCA_0_2_INSERT]]735//736// CHECK32-LABEL: @test_vld3q_dup_bf16(737// CHECK32-NEXT:  entry:738// CHECK32-NEXT:    [[VLD3Q_DUP_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld3dup.v8bf16.p0(ptr [[PTR:%.*]], i32 2)739// CHECK32-NEXT:    [[VLD3Q_DUP_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_DUP_V]], 0740// CHECK32-NEXT:    [[VLD3Q_DUP_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_DUP_V]], 1741// CHECK32-NEXT:    [[VLD3Q_DUP_V_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD3Q_DUP_V]], 2742// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD3Q_DUP_V_FCA_0_EXTRACT]] to <4 x i32>743// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD3Q_DUP_V_FCA_1_EXTRACT]] to <4 x i32>744// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <8 x bfloat> [[VLD3Q_DUP_V_FCA_2_EXTRACT]] to <4 x i32>745// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [3 x <4 x i32>] poison, <4 x i32> [[TMP1]], 0746// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP2]], 1747// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [3 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP3]], 2748// CHECK32-NEXT:    ret [3 x <4 x i32>] [[DOTFCA_2_INSERT]]749//750bfloat16x8x3_t test_vld3q_dup_bf16(bfloat16_t const *ptr) {751  return vld3q_dup_bf16(ptr);752}753 754// CHECK64-LABEL: @test_vld4_dup_bf16(755// CHECK64-NEXT:  entry:756// CHECK64-NEXT:    [[VLD4:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.aarch64.neon.ld4r.v4bf16.p0(ptr [[PTR:%.*]])757// CHECK64-NEXT:    [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 0758// CHECK64-NEXT:    [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 1759// CHECK64-NEXT:    [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 2760// CHECK64-NEXT:    [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4]], 3761// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T:%.*]] poison, <4 x bfloat> [[VLD4_FCA_0_EXTRACT]], 0, 0762// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_0_INSERT]], <4 x bfloat> [[VLD4_FCA_1_EXTRACT]], 0, 1763// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_1_INSERT]], <4 x bfloat> [[VLD4_FCA_2_EXTRACT]], 0, 2764// CHECK64-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_2_INSERT]], <4 x bfloat> [[VLD4_FCA_3_EXTRACT]], 0, 3765// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X4X4_T]] [[DOTFCA_0_3_INSERT]]766//767// CHECK32-LABEL: @test_vld4_dup_bf16(768// CHECK32-NEXT:  entry:769// CHECK32-NEXT:    [[VLD4_DUP_V:%.*]] = tail call { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } @llvm.arm.neon.vld4dup.v4bf16.p0(ptr [[PTR:%.*]], i32 2)770// CHECK32-NEXT:    [[VLD4_DUP_V_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_DUP_V]], 0771// CHECK32-NEXT:    [[VLD4_DUP_V_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_DUP_V]], 1772// CHECK32-NEXT:    [[VLD4_DUP_V_FCA_2_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_DUP_V]], 2773// CHECK32-NEXT:    [[VLD4_DUP_V_FCA_3_EXTRACT:%.*]] = extractvalue { <4 x bfloat>, <4 x bfloat>, <4 x bfloat>, <4 x bfloat> } [[VLD4_DUP_V]], 3774// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x bfloat> [[VLD4_DUP_V_FCA_0_EXTRACT]] to <2 x i32>775// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x bfloat> [[VLD4_DUP_V_FCA_1_EXTRACT]] to <2 x i32>776// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <4 x bfloat> [[VLD4_DUP_V_FCA_2_EXTRACT]] to <2 x i32>777// CHECK32-NEXT:    [[TMP4:%.*]] = bitcast <4 x bfloat> [[VLD4_DUP_V_FCA_3_EXTRACT]] to <2 x i32>778// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <2 x i32>] poison, <2 x i32> [[TMP1]], 0779// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_0_INSERT]], <2 x i32> [[TMP2]], 1780// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_1_INSERT]], <2 x i32> [[TMP3]], 2781// CHECK32-NEXT:    [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <2 x i32>] [[DOTFCA_2_INSERT]], <2 x i32> [[TMP4]], 3782// CHECK32-NEXT:    ret [4 x <2 x i32>] [[DOTFCA_3_INSERT]]783//784bfloat16x4x4_t test_vld4_dup_bf16(bfloat16_t const *ptr) {785  return vld4_dup_bf16(ptr);786}787 788// CHECK64-LABEL: @test_vld4q_dup_bf16(789// CHECK64-NEXT:  entry:790// CHECK64-NEXT:    [[VLD4:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.ld4r.v8bf16.p0(ptr [[PTR:%.*]])791// CHECK64-NEXT:    [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 0792// CHECK64-NEXT:    [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 1793// CHECK64-NEXT:    [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 2794// CHECK64-NEXT:    [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4]], 3795// CHECK64-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T:%.*]] poison, <8 x bfloat> [[VLD4_FCA_0_EXTRACT]], 0, 0796// CHECK64-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLD4_FCA_1_EXTRACT]], 0, 1797// CHECK64-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x bfloat> [[VLD4_FCA_2_EXTRACT]], 0, 2798// CHECK64-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x bfloat> [[VLD4_FCA_3_EXTRACT]], 0, 3799// CHECK64-NEXT:    ret [[STRUCT_BFLOAT16X8X4_T]] [[DOTFCA_0_3_INSERT]]800//801// CHECK32-LABEL: @test_vld4q_dup_bf16(802// CHECK32-NEXT:  entry:803// CHECK32-NEXT:    [[VLD4Q_DUP_V:%.*]] = tail call { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } @llvm.arm.neon.vld4dup.v8bf16.p0(ptr [[PTR:%.*]], i32 2)804// CHECK32-NEXT:    [[VLD4Q_DUP_V_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_DUP_V]], 0805// CHECK32-NEXT:    [[VLD4Q_DUP_V_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_DUP_V]], 1806// CHECK32-NEXT:    [[VLD4Q_DUP_V_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_DUP_V]], 2807// CHECK32-NEXT:    [[VLD4Q_DUP_V_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat> } [[VLD4Q_DUP_V]], 3808// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <8 x bfloat> [[VLD4Q_DUP_V_FCA_0_EXTRACT]] to <4 x i32>809// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <8 x bfloat> [[VLD4Q_DUP_V_FCA_1_EXTRACT]] to <4 x i32>810// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <8 x bfloat> [[VLD4Q_DUP_V_FCA_2_EXTRACT]] to <4 x i32>811// CHECK32-NEXT:    [[TMP4:%.*]] = bitcast <8 x bfloat> [[VLD4Q_DUP_V_FCA_3_EXTRACT]] to <4 x i32>812// CHECK32-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [4 x <4 x i32>] poison, <4 x i32> [[TMP1]], 0813// CHECK32-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_0_INSERT]], <4 x i32> [[TMP2]], 1814// CHECK32-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_1_INSERT]], <4 x i32> [[TMP3]], 2815// CHECK32-NEXT:    [[DOTFCA_3_INSERT:%.*]] = insertvalue [4 x <4 x i32>] [[DOTFCA_2_INSERT]], <4 x i32> [[TMP4]], 3816// CHECK32-NEXT:    ret [4 x <4 x i32>] [[DOTFCA_3_INSERT]]817//818bfloat16x8x4_t test_vld4q_dup_bf16(bfloat16_t const *ptr) {819  return vld4q_dup_bf16(ptr);820}821 822// CHECK64-LABEL: @test_vst1_bf16(823// CHECK64-NEXT:  entry:824// CHECK64-NEXT:    store <4 x bfloat> [[VAL:%.*]], ptr [[PTR:%.*]], align 2825// CHECK64-NEXT:    ret void826//827// CHECK32-LABEL: @test_vst1_bf16(828// CHECK32-NEXT:  entry:829// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst1.p0.v4bf16(ptr [[PTR:%.*]], <4 x bfloat> [[VAL:%.*]], i32 2)830// CHECK32-NEXT:    ret void831//832void test_vst1_bf16(bfloat16_t *ptr, bfloat16x4_t val) {833  vst1_bf16(ptr, val);834}835 836// CHECK64-LABEL: @test_vst1q_bf16(837// CHECK64-NEXT:  entry:838// CHECK64-NEXT:    store <8 x bfloat> [[VAL:%.*]], ptr [[PTR:%.*]], align 2839// CHECK64-NEXT:    ret void840//841// CHECK32-LABEL: @test_vst1q_bf16(842// CHECK32-NEXT:  entry:843// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst1.p0.v8bf16(ptr [[PTR:%.*]], <8 x bfloat> [[VAL:%.*]], i32 2)844// CHECK32-NEXT:    ret void845//846void test_vst1q_bf16(bfloat16_t *ptr, bfloat16x8_t val) {847  vst1q_bf16(ptr, val);848}849 850// CHECK-LABEL: @test_vst1_lane_bf16(851// CHECK-NEXT:  entry:852// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x bfloat> [[VAL:%.*]], i64 1853// CHECK-NEXT:    store bfloat [[TMP0]], ptr [[PTR:%.*]], align 2854// CHECK-NEXT:    ret void855//856void test_vst1_lane_bf16(bfloat16_t *ptr, bfloat16x4_t val) {857  vst1_lane_bf16(ptr, val, 1);858}859 860// CHECK-LABEL: @test_vst1q_lane_bf16(861// CHECK-NEXT:  entry:862// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <8 x bfloat> [[VAL:%.*]], i64 7863// CHECK-NEXT:    store bfloat [[TMP0]], ptr [[PTR:%.*]], align 2864// CHECK-NEXT:    ret void865//866void test_vst1q_lane_bf16(bfloat16_t *ptr, bfloat16x8_t val) {867  vst1q_lane_bf16(ptr, val, 7);868}869 870// CHECK64-LABEL: @test_vst1_bf16_x2(871// CHECK64-NEXT:  entry:872// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[VAL_COERCE:%.*]], 0873// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[VAL_COERCE]], 1874// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st1x2.v4bf16.p0(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR:%.*]])875// CHECK64-NEXT:    ret void876//877// CHECK32-LABEL: @test_vst1_bf16_x2(878// CHECK32-NEXT:  entry:879// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[VAL_COERCE:%.*]], 0880// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[VAL_COERCE]], 1881// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>882// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>883// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst1x2.p0.v4bf16(ptr [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]])884// CHECK32-NEXT:    ret void885//886void test_vst1_bf16_x2(bfloat16_t *ptr, bfloat16x4x2_t val) {887  vst1_bf16_x2(ptr, val);888}889 890// CHECK64-LABEL: @test_vst1q_bf16_x2(891// CHECK64-NEXT:  entry:892// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE:%.*]], 0893// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE]], 1894// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st1x2.v8bf16.p0(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR:%.*]])895// CHECK64-NEXT:    ret void896//897// CHECK32-LABEL: @test_vst1q_bf16_x2(898// CHECK32-NEXT:  entry:899// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE:%.*]], 0900// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE]], 1901// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>902// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>903// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst1x2.p0.v8bf16(ptr [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]])904// CHECK32-NEXT:    ret void905//906void test_vst1q_bf16_x2(bfloat16_t *ptr, bfloat16x8x2_t val) {907  vst1q_bf16_x2(ptr, val);908}909 910// CHECK64-LABEL: @test_vst1_bf16_x3(911// CHECK64-NEXT:  entry:912// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE:%.*]], 0913// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE]], 1914// CHECK64-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE]], 2915// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st1x3.v4bf16.p0(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], ptr [[PTR:%.*]])916// CHECK64-NEXT:    ret void917//918// CHECK32-LABEL: @test_vst1_bf16_x3(919// CHECK32-NEXT:  entry:920// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE:%.*]], 0921// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE]], 1922// CHECK32-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE]], 2923// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>924// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>925// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>926// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst1x3.p0.v4bf16(ptr [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]])927// CHECK32-NEXT:    ret void928//929void test_vst1_bf16_x3(bfloat16_t *ptr, bfloat16x4x3_t val) {930  vst1_bf16_x3(ptr, val);931}932 933// CHECK64-LABEL: @test_vst1q_bf16_x3(934// CHECK64-NEXT:  entry:935// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE:%.*]], 0936// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE]], 1937// CHECK64-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE]], 2938// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st1x3.v8bf16.p0(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], ptr [[PTR:%.*]])939// CHECK64-NEXT:    ret void940//941// CHECK32-LABEL: @test_vst1q_bf16_x3(942// CHECK32-NEXT:  entry:943// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE:%.*]], 0944// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE]], 1945// CHECK32-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE]], 2946// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>947// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>948// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>949// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst1x3.p0.v8bf16(ptr [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]])950// CHECK32-NEXT:    ret void951//952void test_vst1q_bf16_x3(bfloat16_t *ptr, bfloat16x8x3_t val) {953  vst1q_bf16_x3(ptr, val);954}955 956// CHECK64-LABEL: @test_vst1_bf16_x4(957// CHECK64-NEXT:  entry:958// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE:%.*]], 0959// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 1960// CHECK64-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 2961// CHECK64-NEXT:    [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 3962// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st1x4.v4bf16.p0(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_3_EXTRACT]], ptr [[PTR:%.*]])963// CHECK64-NEXT:    ret void964//965// CHECK32-LABEL: @test_vst1_bf16_x4(966// CHECK32-NEXT:  entry:967// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE:%.*]], 0968// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 1969// CHECK32-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 2970// CHECK32-NEXT:    [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 3971// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>972// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>973// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>974// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_3_EXTRACT]] to <4 x bfloat>975// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst1x4.p0.v4bf16(ptr [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]], <4 x bfloat> [[TMP3]])976// CHECK32-NEXT:    ret void977//978void test_vst1_bf16_x4(bfloat16_t *ptr, bfloat16x4x4_t val) {979  vst1_bf16_x4(ptr, val);980}981 982// CHECK64-LABEL: @test_vst1q_bf16_x4(983// CHECK64-NEXT:  entry:984// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE:%.*]], 0985// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 1986// CHECK64-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 2987// CHECK64-NEXT:    [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 3988// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st1x4.v8bf16.p0(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_3_EXTRACT]], ptr [[PTR:%.*]])989// CHECK64-NEXT:    ret void990//991// CHECK32-LABEL: @test_vst1q_bf16_x4(992// CHECK32-NEXT:  entry:993// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE:%.*]], 0994// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 1995// CHECK32-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 2996// CHECK32-NEXT:    [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 3997// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>998// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>999// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>1000// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_3_EXTRACT]] to <8 x bfloat>1001// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst1x4.p0.v8bf16(ptr [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]], <8 x bfloat> [[TMP3]])1002// CHECK32-NEXT:    ret void1003//1004void test_vst1q_bf16_x4(bfloat16_t *ptr, bfloat16x8x4_t val) {1005  vst1q_bf16_x4(ptr, val);1006}1007 1008// CHECK64-LABEL: @test_vst2_bf16(1009// CHECK64-NEXT:  entry:1010// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[VAL_COERCE:%.*]], 01011// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[VAL_COERCE]], 11012// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st2.v4bf16.p0(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR:%.*]])1013// CHECK64-NEXT:    ret void1014//1015// CHECK32-LABEL: @test_vst2_bf16(1016// CHECK32-NEXT:  entry:1017// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[VAL_COERCE:%.*]], 01018// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[VAL_COERCE]], 11019// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>1020// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>1021// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst2.p0.v4bf16(ptr [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], i32 2)1022// CHECK32-NEXT:    ret void1023//1024void test_vst2_bf16(bfloat16_t *ptr, bfloat16x4x2_t val) {1025  vst2_bf16(ptr, val);1026}1027 1028// CHECK64-LABEL: @test_vst2q_bf16(1029// CHECK64-NEXT:  entry:1030// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE:%.*]], 01031// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE]], 11032// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st2.v8bf16.p0(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR:%.*]])1033// CHECK64-NEXT:    ret void1034//1035// CHECK32-LABEL: @test_vst2q_bf16(1036// CHECK32-NEXT:  entry:1037// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE:%.*]], 01038// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE]], 11039// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>1040// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>1041// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst2.p0.v8bf16(ptr [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], i32 2)1042// CHECK32-NEXT:    ret void1043//1044void test_vst2q_bf16(bfloat16_t *ptr, bfloat16x8x2_t val) {1045  vst2q_bf16(ptr, val);1046}1047 1048// CHECK64-LABEL: @test_vst2_lane_bf16(1049// CHECK64-NEXT:  entry:1050// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[VAL_COERCE:%.*]], 01051// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x bfloat>] [[VAL_COERCE]], 11052// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st2lane.v4bf16.p0(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], i64 1, ptr [[PTR:%.*]])1053// CHECK64-NEXT:    ret void1054//1055// CHECK32-LABEL: @test_vst2_lane_bf16(1056// CHECK32-NEXT:  entry:1057// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[VAL_COERCE:%.*]], 01058// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <2 x i32>] [[VAL_COERCE]], 11059// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>1060// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>1061// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst2lane.p0.v4bf16(ptr [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], i32 1, i32 2)1062// CHECK32-NEXT:    ret void1063//1064void test_vst2_lane_bf16(bfloat16_t *ptr, bfloat16x4x2_t val) {1065  vst2_lane_bf16(ptr, val, 1);1066}1067 1068// CHECK64-LABEL: @test_vst2q_lane_bf16(1069// CHECK64-NEXT:  entry:1070// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE:%.*]], 01071// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE]], 11072// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st2lane.v8bf16.p0(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], i64 7, ptr [[PTR:%.*]])1073// CHECK64-NEXT:    ret void1074//1075// CHECK32-LABEL: @test_vst2q_lane_bf16(1076// CHECK32-NEXT:  entry:1077// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE:%.*]], 01078// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE]], 11079// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>1080// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>1081// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst2lane.p0.v8bf16(ptr [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], i32 7, i32 2)1082// CHECK32-NEXT:    ret void1083//1084void test_vst2q_lane_bf16(bfloat16_t *ptr, bfloat16x8x2_t val) {1085  vst2q_lane_bf16(ptr, val, 7);1086}1087 1088// CHECK64-LABEL: @test_vst3_bf16(1089// CHECK64-NEXT:  entry:1090// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE:%.*]], 01091// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE]], 11092// CHECK64-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE]], 21093// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st3.v4bf16.p0(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], ptr [[PTR:%.*]])1094// CHECK64-NEXT:    ret void1095//1096// CHECK32-LABEL: @test_vst3_bf16(1097// CHECK32-NEXT:  entry:1098// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE:%.*]], 01099// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE]], 11100// CHECK32-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE]], 21101// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>1102// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>1103// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>1104// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst3.p0.v4bf16(ptr [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]], i32 2)1105// CHECK32-NEXT:    ret void1106//1107void test_vst3_bf16(bfloat16_t *ptr, bfloat16x4x3_t val) {1108  vst3_bf16(ptr, val);1109}1110 1111// CHECK64-LABEL: @test_vst3q_bf16(1112// CHECK64-NEXT:  entry:1113// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE:%.*]], 01114// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE]], 11115// CHECK64-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE]], 21116// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st3.v8bf16.p0(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], ptr [[PTR:%.*]])1117// CHECK64-NEXT:    ret void1118//1119// CHECK32-LABEL: @test_vst3q_bf16(1120// CHECK32-NEXT:  entry:1121// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE:%.*]], 01122// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE]], 11123// CHECK32-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE]], 21124// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>1125// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>1126// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>1127// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst3.p0.v8bf16(ptr [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]], i32 2)1128// CHECK32-NEXT:    ret void1129//1130void test_vst3q_bf16(bfloat16_t *ptr, bfloat16x8x3_t val) {1131  vst3q_bf16(ptr, val);1132}1133 1134// CHECK64-LABEL: @test_vst3_lane_bf16(1135// CHECK64-NEXT:  entry:1136// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE:%.*]], 01137// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE]], 11138// CHECK64-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x bfloat>] [[VAL_COERCE]], 21139// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st3lane.v4bf16.p0(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], i64 1, ptr [[PTR:%.*]])1140// CHECK64-NEXT:    ret void1141//1142// CHECK32-LABEL: @test_vst3_lane_bf16(1143// CHECK32-NEXT:  entry:1144// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE:%.*]], 01145// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE]], 11146// CHECK32-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <2 x i32>] [[VAL_COERCE]], 21147// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>1148// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>1149// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>1150// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst3lane.p0.v4bf16(ptr [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]], i32 1, i32 2)1151// CHECK32-NEXT:    ret void1152//1153void test_vst3_lane_bf16(bfloat16_t *ptr, bfloat16x4x3_t val) {1154  vst3_lane_bf16(ptr, val, 1);1155}1156 1157// CHECK64-LABEL: @test_vst3q_lane_bf16(1158// CHECK64-NEXT:  entry:1159// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE:%.*]], 01160// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE]], 11161// CHECK64-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x bfloat>] [[VAL_COERCE]], 21162// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st3lane.v8bf16.p0(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], i64 7, ptr [[PTR:%.*]])1163// CHECK64-NEXT:    ret void1164//1165// CHECK32-LABEL: @test_vst3q_lane_bf16(1166// CHECK32-NEXT:  entry:1167// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE:%.*]], 01168// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE]], 11169// CHECK32-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <4 x i32>] [[VAL_COERCE]], 21170// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>1171// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>1172// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>1173// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst3lane.p0.v8bf16(ptr [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]], i32 7, i32 2)1174// CHECK32-NEXT:    ret void1175//1176void test_vst3q_lane_bf16(bfloat16_t *ptr, bfloat16x8x3_t val) {1177  vst3q_lane_bf16(ptr, val, 7);1178}1179 1180// CHECK64-LABEL: @test_vst4_bf16(1181// CHECK64-NEXT:  entry:1182// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE:%.*]], 01183// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 11184// CHECK64-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 21185// CHECK64-NEXT:    [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 31186// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st4.v4bf16.p0(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_3_EXTRACT]], ptr [[PTR:%.*]])1187// CHECK64-NEXT:    ret void1188//1189// CHECK32-LABEL: @test_vst4_bf16(1190// CHECK32-NEXT:  entry:1191// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE:%.*]], 01192// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 11193// CHECK32-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 21194// CHECK32-NEXT:    [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 31195// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>1196// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>1197// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>1198// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_3_EXTRACT]] to <4 x bfloat>1199// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst4.p0.v4bf16(ptr [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]], <4 x bfloat> [[TMP3]], i32 2)1200// CHECK32-NEXT:    ret void1201//1202void test_vst4_bf16(bfloat16_t *ptr, bfloat16x4x4_t val) {1203  vst4_bf16(ptr, val);1204}1205 1206// CHECK64-LABEL: @test_vst4q_bf16(1207// CHECK64-NEXT:  entry:1208// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE:%.*]], 01209// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 11210// CHECK64-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 21211// CHECK64-NEXT:    [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 31212// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st4.v8bf16.p0(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_3_EXTRACT]], ptr [[PTR:%.*]])1213// CHECK64-NEXT:    ret void1214//1215// CHECK32-LABEL: @test_vst4q_bf16(1216// CHECK32-NEXT:  entry:1217// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE:%.*]], 01218// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 11219// CHECK32-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 21220// CHECK32-NEXT:    [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 31221// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>1222// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>1223// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>1224// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_3_EXTRACT]] to <8 x bfloat>1225// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst4.p0.v8bf16(ptr [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]], <8 x bfloat> [[TMP3]], i32 2)1226// CHECK32-NEXT:    ret void1227//1228void test_vst4q_bf16(bfloat16_t *ptr, bfloat16x8x4_t val) {1229  vst4q_bf16(ptr, val);1230}1231 1232// CHECK64-LABEL: @test_vst4_lane_bf16(1233// CHECK64-NEXT:  entry:1234// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE:%.*]], 01235// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 11236// CHECK64-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 21237// CHECK64-NEXT:    [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x bfloat>] [[VAL_COERCE]], 31238// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st4lane.v4bf16.p0(<4 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], <4 x bfloat> [[VAL_COERCE_FCA_3_EXTRACT]], i64 1, ptr [[PTR:%.*]])1239// CHECK64-NEXT:    ret void1240//1241// CHECK32-LABEL: @test_vst4_lane_bf16(1242// CHECK32-NEXT:  entry:1243// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE:%.*]], 01244// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 11245// CHECK32-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 21246// CHECK32-NEXT:    [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <2 x i32>] [[VAL_COERCE]], 31247// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <4 x bfloat>1248// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <4 x bfloat>1249// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <4 x bfloat>1250// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[VAL_COERCE_FCA_3_EXTRACT]] to <4 x bfloat>1251// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst4lane.p0.v4bf16(ptr [[PTR:%.*]], <4 x bfloat> [[TMP0]], <4 x bfloat> [[TMP1]], <4 x bfloat> [[TMP2]], <4 x bfloat> [[TMP3]], i32 1, i32 2)1252// CHECK32-NEXT:    ret void1253//1254void test_vst4_lane_bf16(bfloat16_t *ptr, bfloat16x4x4_t val) {1255  vst4_lane_bf16(ptr, val, 1);1256}1257 1258// CHECK64-LABEL: @test_vst4q_lane_bf16(1259// CHECK64-NEXT:  entry:1260// CHECK64-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE:%.*]], 01261// CHECK64-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 11262// CHECK64-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 21263// CHECK64-NEXT:    [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x bfloat>] [[VAL_COERCE]], 31264// CHECK64-NEXT:    tail call void @llvm.aarch64.neon.st4lane.v8bf16.p0(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_2_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_3_EXTRACT]], i64 7, ptr [[PTR:%.*]])1265// CHECK64-NEXT:    ret void1266//1267// CHECK32-LABEL: @test_vst4q_lane_bf16(1268// CHECK32-NEXT:  entry:1269// CHECK32-NEXT:    [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE:%.*]], 01270// CHECK32-NEXT:    [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 11271// CHECK32-NEXT:    [[VAL_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 21272// CHECK32-NEXT:    [[VAL_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <4 x i32>] [[VAL_COERCE]], 31273// CHECK32-NEXT:    [[TMP0:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]] to <8 x bfloat>1274// CHECK32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]] to <8 x bfloat>1275// CHECK32-NEXT:    [[TMP2:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_2_EXTRACT]] to <8 x bfloat>1276// CHECK32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[VAL_COERCE_FCA_3_EXTRACT]] to <8 x bfloat>1277// CHECK32-NEXT:    tail call void @llvm.arm.neon.vst4lane.p0.v8bf16(ptr [[PTR:%.*]], <8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], <8 x bfloat> [[TMP2]], <8 x bfloat> [[TMP3]], i32 7, i32 2)1278// CHECK32-NEXT:    ret void1279//1280void test_vst4q_lane_bf16(bfloat16_t *ptr, bfloat16x8x4_t val) {1281  vst4q_lane_bf16(ptr, val, 7);1282}1283