359 lines · c
1// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py2// RUN: %clang_cc1 -triple aarch64 -target-feature +neon -target-feature +bf16 \3// RUN: -disable-O0-optnone -emit-llvm %s -o - | opt -S -passes=mem2reg,sroa | FileCheck %s4 5// REQUIRES: aarch64-registered-target || arm-registered-target6 7#include <arm_neon.h>8 9// CHECK-LABEL: @test_vbfdot_f32(10// CHECK-NEXT: entry:11// CHECK-NEXT: [[TMP0:%.*]] = bitcast <2 x float> [[R:%.*]] to <2 x i32>12// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x bfloat> [[A:%.*]] to <4 x i16>13// CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x bfloat> [[B:%.*]] to <4 x i16>14// CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>15// CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8>16// CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to <8 x i8>17// CHECK-NEXT: [[VBFDOT_I:%.*]] = bitcast <8 x i8> [[TMP3]] to <2 x float>18// CHECK-NEXT: [[VBFDOT1_I:%.*]] = bitcast <8 x i8> [[TMP4]] to <4 x bfloat>19// CHECK-NEXT: [[VBFDOT2_I:%.*]] = bitcast <8 x i8> [[TMP5]] to <4 x bfloat>20// CHECK-NEXT: [[VBFDOT3_I:%.*]] = call <2 x float> @llvm.aarch64.neon.bfdot.v2f32.v4bf16(<2 x float> [[VBFDOT_I]], <4 x bfloat> [[VBFDOT1_I]], <4 x bfloat> [[VBFDOT2_I]])21// CHECK-NEXT: ret <2 x float> [[VBFDOT3_I]]22//23float32x2_t test_vbfdot_f32(float32x2_t r, bfloat16x4_t a, bfloat16x4_t b) {24 return vbfdot_f32(r, a, b);25}26 27// CHECK-LABEL: @test_vbfdotq_f32(28// CHECK-NEXT: entry:29// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x float> [[R:%.*]] to <4 x i32>30// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <8 x i16>31// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[B:%.*]] to <8 x i16>32// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>33// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>34// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>35// CHECK-NEXT: [[VBFDOT_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>36// CHECK-NEXT: [[VBFDOT1_I:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x bfloat>37// CHECK-NEXT: [[VBFDOT2_I:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x bfloat>38// CHECK-NEXT: [[VBFDOT3_I:%.*]] = call <4 x float> @llvm.aarch64.neon.bfdot.v4f32.v8bf16(<4 x float> [[VBFDOT_I]], <8 x bfloat> [[VBFDOT1_I]], <8 x bfloat> [[VBFDOT2_I]])39// CHECK-NEXT: ret <4 x float> [[VBFDOT3_I]]40//41float32x4_t test_vbfdotq_f32(float32x4_t r, bfloat16x8_t a, bfloat16x8_t b){42 return vbfdotq_f32(r, a, b);43}44 45// CHECK-LABEL: @test_vbfdot_lane_f32(46// CHECK-NEXT: entry:47// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x bfloat> [[B:%.*]] to <2 x i32>48// CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <2 x float>49// CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x float> [[TMP1]] to <2 x i32>50// CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <8 x i8>51// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i8> [[TMP3]] to <2 x float>52// CHECK-NEXT: [[LANE:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> [[TMP4]], <2 x i32> zeroinitializer53// CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x float> [[LANE]] to <2 x i32>54// CHECK-NEXT: [[TMP6:%.*]] = bitcast <2 x i32> [[TMP5]] to <4 x bfloat>55// CHECK-NEXT: [[TMP7:%.*]] = bitcast <2 x float> [[R:%.*]] to <2 x i32>56// CHECK-NEXT: [[TMP8:%.*]] = bitcast <4 x bfloat> [[A:%.*]] to <4 x i16>57// CHECK-NEXT: [[TMP9:%.*]] = bitcast <4 x bfloat> [[TMP6]] to <4 x i16>58// CHECK-NEXT: [[TMP10:%.*]] = bitcast <2 x i32> [[TMP7]] to <8 x i8>59// CHECK-NEXT: [[TMP11:%.*]] = bitcast <4 x i16> [[TMP8]] to <8 x i8>60// CHECK-NEXT: [[TMP12:%.*]] = bitcast <4 x i16> [[TMP9]] to <8 x i8>61// CHECK-NEXT: [[VBFDOT_I:%.*]] = bitcast <8 x i8> [[TMP10]] to <2 x float>62// CHECK-NEXT: [[VBFDOT1_I:%.*]] = bitcast <8 x i8> [[TMP11]] to <4 x bfloat>63// CHECK-NEXT: [[VBFDOT2_I:%.*]] = bitcast <8 x i8> [[TMP12]] to <4 x bfloat>64// CHECK-NEXT: [[VBFDOT3_I:%.*]] = call <2 x float> @llvm.aarch64.neon.bfdot.v2f32.v4bf16(<2 x float> [[VBFDOT_I]], <4 x bfloat> [[VBFDOT1_I]], <4 x bfloat> [[VBFDOT2_I]])65// CHECK-NEXT: ret <2 x float> [[VBFDOT3_I]]66//67float32x2_t test_vbfdot_lane_f32(float32x2_t r, bfloat16x4_t a, bfloat16x4_t b){68 return vbfdot_lane_f32(r, a, b, 0);69}70 71// CHECK-LABEL: @test_vbfdotq_laneq_f32(72// CHECK-NEXT: entry:73// CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x bfloat> [[B:%.*]] to <4 x i32>74// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to <4 x float>75// CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>76// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <16 x i8>77// CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>78// CHECK-NEXT: [[LANE:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP4]], <4 x i32> <i32 3, i32 3, i32 3, i32 3>79// CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x float> [[LANE]] to <4 x i32>80// CHECK-NEXT: [[TMP6:%.*]] = bitcast <4 x i32> [[TMP5]] to <8 x bfloat>81// CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x float> [[R:%.*]] to <4 x i32>82// CHECK-NEXT: [[TMP8:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <8 x i16>83// CHECK-NEXT: [[TMP9:%.*]] = bitcast <8 x bfloat> [[TMP6]] to <8 x i16>84// CHECK-NEXT: [[TMP10:%.*]] = bitcast <4 x i32> [[TMP7]] to <16 x i8>85// CHECK-NEXT: [[TMP11:%.*]] = bitcast <8 x i16> [[TMP8]] to <16 x i8>86// CHECK-NEXT: [[TMP12:%.*]] = bitcast <8 x i16> [[TMP9]] to <16 x i8>87// CHECK-NEXT: [[VBFDOT_I:%.*]] = bitcast <16 x i8> [[TMP10]] to <4 x float>88// CHECK-NEXT: [[VBFDOT1_I:%.*]] = bitcast <16 x i8> [[TMP11]] to <8 x bfloat>89// CHECK-NEXT: [[VBFDOT2_I:%.*]] = bitcast <16 x i8> [[TMP12]] to <8 x bfloat>90// CHECK-NEXT: [[VBFDOT3_I:%.*]] = call <4 x float> @llvm.aarch64.neon.bfdot.v4f32.v8bf16(<4 x float> [[VBFDOT_I]], <8 x bfloat> [[VBFDOT1_I]], <8 x bfloat> [[VBFDOT2_I]])91// CHECK-NEXT: ret <4 x float> [[VBFDOT3_I]]92//93float32x4_t test_vbfdotq_laneq_f32(float32x4_t r, bfloat16x8_t a, bfloat16x8_t b) {94 return vbfdotq_laneq_f32(r, a, b, 3);95}96 97// CHECK-LABEL: @test_vbfdot_laneq_f32(98// CHECK-NEXT: entry:99// CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x bfloat> [[B:%.*]] to <4 x i32>100// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to <4 x float>101// CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>102// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <16 x i8>103// CHECK-NEXT: [[TMP4:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>104// CHECK-NEXT: [[LANE:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP4]], <2 x i32> <i32 3, i32 3>105// CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x float> [[LANE]] to <2 x i32>106// CHECK-NEXT: [[TMP6:%.*]] = bitcast <2 x i32> [[TMP5]] to <4 x bfloat>107// CHECK-NEXT: [[TMP7:%.*]] = bitcast <2 x float> [[R:%.*]] to <2 x i32>108// CHECK-NEXT: [[TMP8:%.*]] = bitcast <4 x bfloat> [[A:%.*]] to <4 x i16>109// CHECK-NEXT: [[TMP9:%.*]] = bitcast <4 x bfloat> [[TMP6]] to <4 x i16>110// CHECK-NEXT: [[TMP10:%.*]] = bitcast <2 x i32> [[TMP7]] to <8 x i8>111// CHECK-NEXT: [[TMP11:%.*]] = bitcast <4 x i16> [[TMP8]] to <8 x i8>112// CHECK-NEXT: [[TMP12:%.*]] = bitcast <4 x i16> [[TMP9]] to <8 x i8>113// CHECK-NEXT: [[VBFDOT_I:%.*]] = bitcast <8 x i8> [[TMP10]] to <2 x float>114// CHECK-NEXT: [[VBFDOT1_I:%.*]] = bitcast <8 x i8> [[TMP11]] to <4 x bfloat>115// CHECK-NEXT: [[VBFDOT2_I:%.*]] = bitcast <8 x i8> [[TMP12]] to <4 x bfloat>116// CHECK-NEXT: [[VBFDOT3_I:%.*]] = call <2 x float> @llvm.aarch64.neon.bfdot.v2f32.v4bf16(<2 x float> [[VBFDOT_I]], <4 x bfloat> [[VBFDOT1_I]], <4 x bfloat> [[VBFDOT2_I]])117// CHECK-NEXT: ret <2 x float> [[VBFDOT3_I]]118//119float32x2_t test_vbfdot_laneq_f32(float32x2_t r, bfloat16x4_t a, bfloat16x8_t b) {120 return vbfdot_laneq_f32(r, a, b, 3);121}122 123// CHECK-LABEL: @test_vbfdotq_lane_f32(124// CHECK-NEXT: entry:125// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x bfloat> [[B:%.*]] to <2 x i32>126// CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <2 x float>127// CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x float> [[TMP1]] to <2 x i32>128// CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <8 x i8>129// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i8> [[TMP3]] to <2 x float>130// CHECK-NEXT: [[LANE:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> [[TMP4]], <4 x i32> zeroinitializer131// CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x float> [[LANE]] to <4 x i32>132// CHECK-NEXT: [[TMP6:%.*]] = bitcast <4 x i32> [[TMP5]] to <8 x bfloat>133// CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x float> [[R:%.*]] to <4 x i32>134// CHECK-NEXT: [[TMP8:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <8 x i16>135// CHECK-NEXT: [[TMP9:%.*]] = bitcast <8 x bfloat> [[TMP6]] to <8 x i16>136// CHECK-NEXT: [[TMP10:%.*]] = bitcast <4 x i32> [[TMP7]] to <16 x i8>137// CHECK-NEXT: [[TMP11:%.*]] = bitcast <8 x i16> [[TMP8]] to <16 x i8>138// CHECK-NEXT: [[TMP12:%.*]] = bitcast <8 x i16> [[TMP9]] to <16 x i8>139// CHECK-NEXT: [[VBFDOT_I:%.*]] = bitcast <16 x i8> [[TMP10]] to <4 x float>140// CHECK-NEXT: [[VBFDOT1_I:%.*]] = bitcast <16 x i8> [[TMP11]] to <8 x bfloat>141// CHECK-NEXT: [[VBFDOT2_I:%.*]] = bitcast <16 x i8> [[TMP12]] to <8 x bfloat>142// CHECK-NEXT: [[VBFDOT3_I:%.*]] = call <4 x float> @llvm.aarch64.neon.bfdot.v4f32.v8bf16(<4 x float> [[VBFDOT_I]], <8 x bfloat> [[VBFDOT1_I]], <8 x bfloat> [[VBFDOT2_I]])143// CHECK-NEXT: ret <4 x float> [[VBFDOT3_I]]144//145float32x4_t test_vbfdotq_lane_f32(float32x4_t r, bfloat16x8_t a, bfloat16x4_t b) {146 return vbfdotq_lane_f32(r, a, b, 0);147}148 149// CHECK-LABEL: @test_vbfmmlaq_f32(150// CHECK-NEXT: entry:151// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x float> [[R:%.*]] to <4 x i32>152// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <8 x i16>153// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[B:%.*]] to <8 x i16>154// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>155// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>156// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>157// CHECK-NEXT: [[VBFMMLAQ_F32_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>158// CHECK-NEXT: [[VBFMMLAQ_F321_I:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x bfloat>159// CHECK-NEXT: [[VBFMMLAQ_F322_I:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x bfloat>160// CHECK-NEXT: [[VBFMMLAQ_F323_I:%.*]] = call <4 x float> @llvm.aarch64.neon.bfmmla(<4 x float> [[VBFMMLAQ_F32_I]], <8 x bfloat> [[VBFMMLAQ_F321_I]], <8 x bfloat> [[VBFMMLAQ_F322_I]])161// CHECK-NEXT: [[VBFMMLAQ_F324_I:%.*]] = bitcast <4 x float> [[VBFMMLAQ_F323_I]] to <16 x i8>162// CHECK-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[VBFMMLAQ_F324_I]] to <4 x i32>163// CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x i32> [[TMP6]] to <4 x float>164// CHECK-NEXT: ret <4 x float> [[TMP7]]165//166float32x4_t test_vbfmmlaq_f32(float32x4_t r, bfloat16x8_t a, bfloat16x8_t b) {167 return vbfmmlaq_f32(r, a, b);168}169 170// CHECK-LABEL: @test_vbfmlalbq_f32(171// CHECK-NEXT: entry:172// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x float> [[R:%.*]] to <4 x i32>173// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <8 x i16>174// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[B:%.*]] to <8 x i16>175// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>176// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>177// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>178// CHECK-NEXT: [[VBFMLALBQ_F32_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>179// CHECK-NEXT: [[VBFMLALBQ_F321_I:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x bfloat>180// CHECK-NEXT: [[VBFMLALBQ_F322_I:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x bfloat>181// CHECK-NEXT: [[VBFMLALBQ_F323_I:%.*]] = call <4 x float> @llvm.aarch64.neon.bfmlalb(<4 x float> [[VBFMLALBQ_F32_I]], <8 x bfloat> [[VBFMLALBQ_F321_I]], <8 x bfloat> [[VBFMLALBQ_F322_I]])182// CHECK-NEXT: [[VBFMLALBQ_F324_I:%.*]] = bitcast <4 x float> [[VBFMLALBQ_F323_I]] to <16 x i8>183// CHECK-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[VBFMLALBQ_F324_I]] to <4 x i32>184// CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x i32> [[TMP6]] to <4 x float>185// CHECK-NEXT: ret <4 x float> [[TMP7]]186//187float32x4_t test_vbfmlalbq_f32(float32x4_t r, bfloat16x8_t a, bfloat16x8_t b) {188 return vbfmlalbq_f32(r, a, b);189}190 191// CHECK-LABEL: @test_vbfmlaltq_f32(192// CHECK-NEXT: entry:193// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x float> [[R:%.*]] to <4 x i32>194// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <8 x i16>195// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[B:%.*]] to <8 x i16>196// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>197// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>198// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>199// CHECK-NEXT: [[VBFMLALTQ_F32_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>200// CHECK-NEXT: [[VBFMLALTQ_F321_I:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x bfloat>201// CHECK-NEXT: [[VBFMLALTQ_F322_I:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x bfloat>202// CHECK-NEXT: [[VBFMLALTQ_F323_I:%.*]] = call <4 x float> @llvm.aarch64.neon.bfmlalt(<4 x float> [[VBFMLALTQ_F32_I]], <8 x bfloat> [[VBFMLALTQ_F321_I]], <8 x bfloat> [[VBFMLALTQ_F322_I]])203// CHECK-NEXT: [[VBFMLALTQ_F324_I:%.*]] = bitcast <4 x float> [[VBFMLALTQ_F323_I]] to <16 x i8>204// CHECK-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[VBFMLALTQ_F324_I]] to <4 x i32>205// CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x i32> [[TMP6]] to <4 x float>206// CHECK-NEXT: ret <4 x float> [[TMP7]]207//208float32x4_t test_vbfmlaltq_f32(float32x4_t r, bfloat16x8_t a, bfloat16x8_t b) {209 return vbfmlaltq_f32(r, a, b);210}211 212// CHECK-LABEL: @test_vbfmlalbq_lane_f32(213// CHECK-NEXT: entry:214// CHECK-NEXT: [[VGET_LANE:%.*]] = extractelement <4 x bfloat> [[B:%.*]], i32 0215// CHECK-NEXT: [[VECINIT:%.*]] = insertelement <8 x bfloat> poison, bfloat [[VGET_LANE]], i32 0216// CHECK-NEXT: [[VGET_LANE4:%.*]] = extractelement <4 x bfloat> [[B]], i32 0217// CHECK-NEXT: [[VECINIT6:%.*]] = insertelement <8 x bfloat> [[VECINIT]], bfloat [[VGET_LANE4]], i32 1218// CHECK-NEXT: [[VGET_LANE10:%.*]] = extractelement <4 x bfloat> [[B]], i32 0219// CHECK-NEXT: [[VECINIT12:%.*]] = insertelement <8 x bfloat> [[VECINIT6]], bfloat [[VGET_LANE10]], i32 2220// CHECK-NEXT: [[VGET_LANE16:%.*]] = extractelement <4 x bfloat> [[B]], i32 0221// CHECK-NEXT: [[VECINIT18:%.*]] = insertelement <8 x bfloat> [[VECINIT12]], bfloat [[VGET_LANE16]], i32 3222// CHECK-NEXT: [[VGET_LANE22:%.*]] = extractelement <4 x bfloat> [[B]], i32 0223// CHECK-NEXT: [[VECINIT24:%.*]] = insertelement <8 x bfloat> [[VECINIT18]], bfloat [[VGET_LANE22]], i32 4224// CHECK-NEXT: [[VGET_LANE28:%.*]] = extractelement <4 x bfloat> [[B]], i32 0225// CHECK-NEXT: [[VECINIT30:%.*]] = insertelement <8 x bfloat> [[VECINIT24]], bfloat [[VGET_LANE28]], i32 5226// CHECK-NEXT: [[VGET_LANE34:%.*]] = extractelement <4 x bfloat> [[B]], i32 0227// CHECK-NEXT: [[VECINIT36:%.*]] = insertelement <8 x bfloat> [[VECINIT30]], bfloat [[VGET_LANE34]], i32 6228// CHECK-NEXT: [[VGET_LANE40:%.*]] = extractelement <4 x bfloat> [[B]], i32 0229// CHECK-NEXT: [[VECINIT42:%.*]] = insertelement <8 x bfloat> [[VECINIT36]], bfloat [[VGET_LANE40]], i32 7230// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x float> [[R:%.*]] to <4 x i32>231// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <8 x i16>232// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[VECINIT42]] to <8 x i16>233// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>234// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>235// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>236// CHECK-NEXT: [[VBFMLALBQ_F32_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>237// CHECK-NEXT: [[VBFMLALBQ_F321_I:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x bfloat>238// CHECK-NEXT: [[VBFMLALBQ_F322_I:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x bfloat>239// CHECK-NEXT: [[VBFMLALBQ_F323_I:%.*]] = call <4 x float> @llvm.aarch64.neon.bfmlalb(<4 x float> [[VBFMLALBQ_F32_I]], <8 x bfloat> [[VBFMLALBQ_F321_I]], <8 x bfloat> [[VBFMLALBQ_F322_I]])240// CHECK-NEXT: [[VBFMLALBQ_F324_I:%.*]] = bitcast <4 x float> [[VBFMLALBQ_F323_I]] to <16 x i8>241// CHECK-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[VBFMLALBQ_F324_I]] to <4 x i32>242// CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x i32> [[TMP6]] to <4 x float>243// CHECK-NEXT: ret <4 x float> [[TMP7]]244//245float32x4_t test_vbfmlalbq_lane_f32(float32x4_t r, bfloat16x8_t a, bfloat16x4_t b) {246 return vbfmlalbq_lane_f32(r, a, b, 0);247}248 249// CHECK-LABEL: @test_vbfmlalbq_laneq_f32(250// CHECK-NEXT: entry:251// CHECK-NEXT: [[VGETQ_LANE:%.*]] = extractelement <8 x bfloat> [[B:%.*]], i32 3252// CHECK-NEXT: [[VECINIT:%.*]] = insertelement <8 x bfloat> poison, bfloat [[VGETQ_LANE]], i32 0253// CHECK-NEXT: [[VGETQ_LANE4:%.*]] = extractelement <8 x bfloat> [[B]], i32 3254// CHECK-NEXT: [[VECINIT6:%.*]] = insertelement <8 x bfloat> [[VECINIT]], bfloat [[VGETQ_LANE4]], i32 1255// CHECK-NEXT: [[VGETQ_LANE10:%.*]] = extractelement <8 x bfloat> [[B]], i32 3256// CHECK-NEXT: [[VECINIT12:%.*]] = insertelement <8 x bfloat> [[VECINIT6]], bfloat [[VGETQ_LANE10]], i32 2257// CHECK-NEXT: [[VGETQ_LANE16:%.*]] = extractelement <8 x bfloat> [[B]], i32 3258// CHECK-NEXT: [[VECINIT18:%.*]] = insertelement <8 x bfloat> [[VECINIT12]], bfloat [[VGETQ_LANE16]], i32 3259// CHECK-NEXT: [[VGETQ_LANE22:%.*]] = extractelement <8 x bfloat> [[B]], i32 3260// CHECK-NEXT: [[VECINIT24:%.*]] = insertelement <8 x bfloat> [[VECINIT18]], bfloat [[VGETQ_LANE22]], i32 4261// CHECK-NEXT: [[VGETQ_LANE28:%.*]] = extractelement <8 x bfloat> [[B]], i32 3262// CHECK-NEXT: [[VECINIT30:%.*]] = insertelement <8 x bfloat> [[VECINIT24]], bfloat [[VGETQ_LANE28]], i32 5263// CHECK-NEXT: [[VGETQ_LANE34:%.*]] = extractelement <8 x bfloat> [[B]], i32 3264// CHECK-NEXT: [[VECINIT36:%.*]] = insertelement <8 x bfloat> [[VECINIT30]], bfloat [[VGETQ_LANE34]], i32 6265// CHECK-NEXT: [[VGETQ_LANE40:%.*]] = extractelement <8 x bfloat> [[B]], i32 3266// CHECK-NEXT: [[VECINIT42:%.*]] = insertelement <8 x bfloat> [[VECINIT36]], bfloat [[VGETQ_LANE40]], i32 7267// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x float> [[R:%.*]] to <4 x i32>268// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <8 x i16>269// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[VECINIT42]] to <8 x i16>270// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>271// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>272// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>273// CHECK-NEXT: [[VBFMLALBQ_F32_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>274// CHECK-NEXT: [[VBFMLALBQ_F321_I:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x bfloat>275// CHECK-NEXT: [[VBFMLALBQ_F322_I:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x bfloat>276// CHECK-NEXT: [[VBFMLALBQ_F323_I:%.*]] = call <4 x float> @llvm.aarch64.neon.bfmlalb(<4 x float> [[VBFMLALBQ_F32_I]], <8 x bfloat> [[VBFMLALBQ_F321_I]], <8 x bfloat> [[VBFMLALBQ_F322_I]])277// CHECK-NEXT: [[VBFMLALBQ_F324_I:%.*]] = bitcast <4 x float> [[VBFMLALBQ_F323_I]] to <16 x i8>278// CHECK-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[VBFMLALBQ_F324_I]] to <4 x i32>279// CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x i32> [[TMP6]] to <4 x float>280// CHECK-NEXT: ret <4 x float> [[TMP7]]281//282float32x4_t test_vbfmlalbq_laneq_f32(float32x4_t r, bfloat16x8_t a, bfloat16x8_t b) {283 return vbfmlalbq_laneq_f32(r, a, b, 3);284}285 286// CHECK-LABEL: @test_vbfmlaltq_lane_f32(287// CHECK-NEXT: entry:288// CHECK-NEXT: [[VGET_LANE:%.*]] = extractelement <4 x bfloat> [[B:%.*]], i32 0289// CHECK-NEXT: [[VECINIT:%.*]] = insertelement <8 x bfloat> poison, bfloat [[VGET_LANE]], i32 0290// CHECK-NEXT: [[VGET_LANE4:%.*]] = extractelement <4 x bfloat> [[B]], i32 0291// CHECK-NEXT: [[VECINIT6:%.*]] = insertelement <8 x bfloat> [[VECINIT]], bfloat [[VGET_LANE4]], i32 1292// CHECK-NEXT: [[VGET_LANE10:%.*]] = extractelement <4 x bfloat> [[B]], i32 0293// CHECK-NEXT: [[VECINIT12:%.*]] = insertelement <8 x bfloat> [[VECINIT6]], bfloat [[VGET_LANE10]], i32 2294// CHECK-NEXT: [[VGET_LANE16:%.*]] = extractelement <4 x bfloat> [[B]], i32 0295// CHECK-NEXT: [[VECINIT18:%.*]] = insertelement <8 x bfloat> [[VECINIT12]], bfloat [[VGET_LANE16]], i32 3296// CHECK-NEXT: [[VGET_LANE22:%.*]] = extractelement <4 x bfloat> [[B]], i32 0297// CHECK-NEXT: [[VECINIT24:%.*]] = insertelement <8 x bfloat> [[VECINIT18]], bfloat [[VGET_LANE22]], i32 4298// CHECK-NEXT: [[VGET_LANE28:%.*]] = extractelement <4 x bfloat> [[B]], i32 0299// CHECK-NEXT: [[VECINIT30:%.*]] = insertelement <8 x bfloat> [[VECINIT24]], bfloat [[VGET_LANE28]], i32 5300// CHECK-NEXT: [[VGET_LANE34:%.*]] = extractelement <4 x bfloat> [[B]], i32 0301// CHECK-NEXT: [[VECINIT36:%.*]] = insertelement <8 x bfloat> [[VECINIT30]], bfloat [[VGET_LANE34]], i32 6302// CHECK-NEXT: [[VGET_LANE40:%.*]] = extractelement <4 x bfloat> [[B]], i32 0303// CHECK-NEXT: [[VECINIT42:%.*]] = insertelement <8 x bfloat> [[VECINIT36]], bfloat [[VGET_LANE40]], i32 7304// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x float> [[R:%.*]] to <4 x i32>305// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <8 x i16>306// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[VECINIT42]] to <8 x i16>307// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>308// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>309// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>310// CHECK-NEXT: [[VBFMLALTQ_F32_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>311// CHECK-NEXT: [[VBFMLALTQ_F321_I:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x bfloat>312// CHECK-NEXT: [[VBFMLALTQ_F322_I:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x bfloat>313// CHECK-NEXT: [[VBFMLALTQ_F323_I:%.*]] = call <4 x float> @llvm.aarch64.neon.bfmlalt(<4 x float> [[VBFMLALTQ_F32_I]], <8 x bfloat> [[VBFMLALTQ_F321_I]], <8 x bfloat> [[VBFMLALTQ_F322_I]])314// CHECK-NEXT: [[VBFMLALTQ_F324_I:%.*]] = bitcast <4 x float> [[VBFMLALTQ_F323_I]] to <16 x i8>315// CHECK-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[VBFMLALTQ_F324_I]] to <4 x i32>316// CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x i32> [[TMP6]] to <4 x float>317// CHECK-NEXT: ret <4 x float> [[TMP7]]318//319float32x4_t test_vbfmlaltq_lane_f32(float32x4_t r, bfloat16x8_t a, bfloat16x4_t b) {320 return vbfmlaltq_lane_f32(r, a, b, 0);321}322 323// CHECK-LABEL: @test_vbfmlaltq_laneq_f32(324// CHECK-NEXT: entry:325// CHECK-NEXT: [[VGETQ_LANE:%.*]] = extractelement <8 x bfloat> [[B:%.*]], i32 3326// CHECK-NEXT: [[VECINIT:%.*]] = insertelement <8 x bfloat> poison, bfloat [[VGETQ_LANE]], i32 0327// CHECK-NEXT: [[VGETQ_LANE4:%.*]] = extractelement <8 x bfloat> [[B]], i32 3328// CHECK-NEXT: [[VECINIT6:%.*]] = insertelement <8 x bfloat> [[VECINIT]], bfloat [[VGETQ_LANE4]], i32 1329// CHECK-NEXT: [[VGETQ_LANE10:%.*]] = extractelement <8 x bfloat> [[B]], i32 3330// CHECK-NEXT: [[VECINIT12:%.*]] = insertelement <8 x bfloat> [[VECINIT6]], bfloat [[VGETQ_LANE10]], i32 2331// CHECK-NEXT: [[VGETQ_LANE16:%.*]] = extractelement <8 x bfloat> [[B]], i32 3332// CHECK-NEXT: [[VECINIT18:%.*]] = insertelement <8 x bfloat> [[VECINIT12]], bfloat [[VGETQ_LANE16]], i32 3333// CHECK-NEXT: [[VGETQ_LANE22:%.*]] = extractelement <8 x bfloat> [[B]], i32 3334// CHECK-NEXT: [[VECINIT24:%.*]] = insertelement <8 x bfloat> [[VECINIT18]], bfloat [[VGETQ_LANE22]], i32 4335// CHECK-NEXT: [[VGETQ_LANE28:%.*]] = extractelement <8 x bfloat> [[B]], i32 3336// CHECK-NEXT: [[VECINIT30:%.*]] = insertelement <8 x bfloat> [[VECINIT24]], bfloat [[VGETQ_LANE28]], i32 5337// CHECK-NEXT: [[VGETQ_LANE34:%.*]] = extractelement <8 x bfloat> [[B]], i32 3338// CHECK-NEXT: [[VECINIT36:%.*]] = insertelement <8 x bfloat> [[VECINIT30]], bfloat [[VGETQ_LANE34]], i32 6339// CHECK-NEXT: [[VGETQ_LANE40:%.*]] = extractelement <8 x bfloat> [[B]], i32 3340// CHECK-NEXT: [[VECINIT42:%.*]] = insertelement <8 x bfloat> [[VECINIT36]], bfloat [[VGETQ_LANE40]], i32 7341// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x float> [[R:%.*]] to <4 x i32>342// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x bfloat> [[A:%.*]] to <8 x i16>343// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x bfloat> [[VECINIT42]] to <8 x i16>344// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>345// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>346// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>347// CHECK-NEXT: [[VBFMLALTQ_F32_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>348// CHECK-NEXT: [[VBFMLALTQ_F321_I:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x bfloat>349// CHECK-NEXT: [[VBFMLALTQ_F322_I:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x bfloat>350// CHECK-NEXT: [[VBFMLALTQ_F323_I:%.*]] = call <4 x float> @llvm.aarch64.neon.bfmlalt(<4 x float> [[VBFMLALTQ_F32_I]], <8 x bfloat> [[VBFMLALTQ_F321_I]], <8 x bfloat> [[VBFMLALTQ_F322_I]])351// CHECK-NEXT: [[VBFMLALTQ_F324_I:%.*]] = bitcast <4 x float> [[VBFMLALTQ_F323_I]] to <16 x i8>352// CHECK-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[VBFMLALTQ_F324_I]] to <4 x i32>353// CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x i32> [[TMP6]] to <4 x float>354// CHECK-NEXT: ret <4 x float> [[TMP7]]355//356float32x4_t test_vbfmlaltq_laneq_f32(float32x4_t r, bfloat16x8_t a, bfloat16x8_t b) {357 return vbfmlaltq_laneq_f32(r, a, b, 3);358}359